Support for private certificate authorities during DB curation (#494)

* Add injectable HTTP client to file getter

Signed-off-by: Dan Luhring <dan.luhring@anchore.com>

* WIP: Map config for custom CA certs

Signed-off-by: Dan Luhring <dan.luhring@anchore.com>

* update curator and add tests

Signed-off-by: Alex Goodman <alex.goodman@anchore.com>

* add TLS helper scripts

Signed-off-by: Alex Goodman <alex.goodman@anchore.com>

* remove grype-db local mod edit

Signed-off-by: Alex Goodman <alex.goodman@anchore.com>

* tidy go modules

Signed-off-by: Alex Goodman <alex.goodman@anchore.com>

* use ssl.context over deprecated fn

Signed-off-by: Alex Goodman <alex.goodman@anchore.com>

* disallow tls 1 and 1.1

Signed-off-by: Alex Goodman <alex.goodman@anchore.com>

* suppress non-archive sources for fetch-to-dir capability

Signed-off-by: Alex Goodman <alex.goodman@anchore.com>

* ensure DB load failure does not panic

Signed-off-by: Alex Goodman <alex.goodman@anchore.com>

* address review comments

Signed-off-by: Alex Goodman <alex.goodman@anchore.com>

Co-authored-by: Alex Goodman <alex.goodman@anchore.com>
This commit is contained in:
Dan Luhring 2021-11-22 11:59:38 -05:00 committed by GitHub
parent 1e35cbf20b
commit 70ec3bfb71
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
23 changed files with 936 additions and 72 deletions

View file

@ -19,7 +19,10 @@ func init() {
}
func runDBCheckCmd(_ *cobra.Command, _ []string) error {
dbCurator := db.NewCurator(appConfig.DB.ToCuratorConfig())
dbCurator, err := db.NewCurator(appConfig.DB.ToCuratorConfig())
if err != nil {
return err
}
updateAvailable, _, err := dbCurator.IsUpdateAvailable()
if err != nil {

View file

@ -19,7 +19,10 @@ func init() {
}
func runDBDeleteCmd(_ *cobra.Command, _ []string) error {
dbCurator := db.NewCurator(appConfig.DB.ToCuratorConfig())
dbCurator, err := db.NewCurator(appConfig.DB.ToCuratorConfig())
if err != nil {
return err
}
if err := dbCurator.Delete(); err != nil {
return fmt.Errorf("unable to delete vulnerability database: %+v", err)

View file

@ -22,7 +22,10 @@ func init() {
}
func runDBImportCmd(_ *cobra.Command, args []string) error {
dbCurator := db.NewCurator(appConfig.DB.ToCuratorConfig())
dbCurator, err := db.NewCurator(appConfig.DB.ToCuratorConfig())
if err != nil {
return err
}
if err := dbCurator.ImportFrom(args[0]); err != nil {
return fmt.Errorf("unable to import vulnerability database: %+v", err)

View file

@ -20,7 +20,11 @@ func init() {
}
func runDBStatusCmd(_ *cobra.Command, _ []string) error {
dbCurator := db.NewCurator(appConfig.DB.ToCuratorConfig())
dbCurator, err := db.NewCurator(appConfig.DB.ToCuratorConfig())
if err != nil {
return err
}
status := dbCurator.Status()
statusStr := "valid"

View file

@ -19,7 +19,10 @@ func init() {
}
func runDBUpdateCmd(_ *cobra.Command, _ []string) error {
dbCurator := db.NewCurator(appConfig.DB.ToCuratorConfig())
dbCurator, err := db.NewCurator(appConfig.DB.ToCuratorConfig())
if err != nil {
return err
}
updated, err := dbCurator.Update()
if err != nil {

View file

@ -233,6 +233,7 @@ func startWorker(userInput string, failOnSeverity *vulnerability.Severity) <-cha
var packages []pkg.Package
var context pkg.Context
var wg = &sync.WaitGroup{}
var loadedDB, gatheredPackages bool
wg.Add(2)
@ -240,12 +241,11 @@ func startWorker(userInput string, failOnSeverity *vulnerability.Severity) <-cha
defer wg.Done()
log.Debug("loading DB")
provider, metadataProvider, dbStatus, err = grype.LoadVulnerabilityDB(appConfig.DB.ToCuratorConfig(), appConfig.DB.AutoUpdate)
if err != nil {
errs <- fmt.Errorf("failed to load vulnerability db: %w", err)
}
if dbStatus == nil {
errs <- fmt.Errorf("unable to determine DB status")
if err = validateDBLoad(err, dbStatus); err != nil {
errs <- err
return
}
loadedDB = true
}()
go func() {
@ -254,11 +254,13 @@ func startWorker(userInput string, failOnSeverity *vulnerability.Severity) <-cha
packages, context, err = pkg.Provide(userInput, appConfig.ScopeOpt, appConfig.Registry.ToOptions())
if err != nil {
errs <- fmt.Errorf("failed to catalog: %w", err)
return
}
gatheredPackages = true
}()
wg.Wait()
if err != nil {
if !loadedDB || !gatheredPackages {
return
}
@ -270,7 +272,7 @@ func startWorker(userInput string, failOnSeverity *vulnerability.Severity) <-cha
remainingMatches, ignoredMatches := match.ApplyIgnoreRules(allMatches, appConfig.Ignore)
if count := len(ignoredMatches); count > 0 {
log.Infof("Ignoring %d matches due to user-provided ignore rules", count)
log.Infof("ignoring %d matches due to user-provided ignore rules", count)
}
// determine if there are any severities >= to the max allowable severity (which is optional).
@ -288,6 +290,19 @@ func startWorker(userInput string, failOnSeverity *vulnerability.Severity) <-cha
return errs
}
func validateDBLoad(loadErr error, status *db.Status) error {
if loadErr != nil {
return fmt.Errorf("failed to load vulnerability db: %w", loadErr)
}
if status == nil {
return fmt.Errorf("unable to determine DB status")
}
if status.Err != nil {
return fmt.Errorf("db could not be loaded: %w", status.Err)
}
return nil
}
func validateRootArgs(cmd *cobra.Command, args []string) error {
isPipedInput, err := internal.IsPipedInput()
if err != nil {

5
go.mod
View file

@ -7,7 +7,7 @@ require (
github.com/adrg/xdg v0.2.1
github.com/anchore/go-testutils v0.0.0-20200925183923-d5f45b0d3c04
github.com/anchore/go-version v1.2.2-0.20210903204242-51efa5b487c4
github.com/anchore/grype-db v0.0.0-20211115233739-c0e91d61ed51
github.com/anchore/grype-db v0.0.0-20211119195714-911ff7162dc6
github.com/anchore/stereoscope v0.0.0-20211024152658-003132a67c10
github.com/anchore/syft v0.30.1
github.com/bmatcuk/doublestar/v2 v2.0.4
@ -19,7 +19,8 @@ require (
github.com/google/go-cmp v0.4.1
github.com/google/uuid v1.2.0
github.com/gookit/color v1.4.2
github.com/hashicorp/go-getter v1.4.1
github.com/hashicorp/go-cleanhttp v0.5.2
github.com/hashicorp/go-getter v1.5.9
github.com/hashicorp/go-multierror v1.1.0
github.com/jinzhu/copier v0.0.0-20190924061706-b57f9002281a
github.com/knqyf263/go-apk-version v0.0.0-20200609155635-041fdbb8563f

16
go.sum
View file

@ -128,14 +128,16 @@ github.com/anchore/go-version v1.2.2-0.20210903204242-51efa5b487c4 h1:rmZG77uXgE
github.com/anchore/go-version v1.2.2-0.20210903204242-51efa5b487c4/go.mod h1:Bkc+JYWjMCF8OyZ340IMSIi2Ebf3uwByOk6ho4wne1E=
github.com/anchore/grype v0.14.1-0.20210702143224-05ade7bbbf70/go.mod h1:yPh9WHflzInB/INwPrDs2wLKmRsa8owAuojmv4K8H6I=
github.com/anchore/grype v0.24.2-0.20211115221156-a2762bbbf043/go.mod h1:/MP3vfuCUqCfBSrXymncltcJNd8/dn85rXGtJdR83Ks=
github.com/anchore/grype v0.24.2-0.20211115221156-a2762bbbf043/go.mod h1:/MP3vfuCUqCfBSrXymncltcJNd8/dn85rXGtJdR83Ks=
github.com/anchore/grype-db v0.0.0-20210527140125-6f881b00e927/go.mod h1:XSlPf1awNrMpah+rHbWrzgUvnmWLgn/KkdicxERVClg=
github.com/anchore/grype-db v0.0.0-20210527140125-6f881b00e927/go.mod h1:XSlPf1awNrMpah+rHbWrzgUvnmWLgn/KkdicxERVClg=
github.com/anchore/grype-db v0.0.0-20210928194208-f146397d6cd0/go.mod h1:GniMuMokZ2iAX67Qrd5fJW7BstX8a+4U48LyypGC2g0=
github.com/anchore/grype-db v0.0.0-20211115233739-c0e91d61ed51 h1:39XbpqI17fDF0MGscxUXZFft7S4pY94SmEdSRK30ef0=
github.com/anchore/grype-db v0.0.0-20211115233739-c0e91d61ed51/go.mod h1:PANGB2KzOSPq4lBmLgF9mKjLEHuQYEpUUAP5qcuEA/8=
github.com/anchore/grype-db v0.0.0-20210928194208-f146397d6cd0/go.mod h1:GniMuMokZ2iAX67Qrd5fJW7BstX8a+4U48LyypGC2g0=
github.com/anchore/grype-db v0.0.0-20211119195714-911ff7162dc6 h1:Tgj23PcbHBgcpsOXlXwy5aC8dC5cEujvkbQ0soygLaQ=
github.com/anchore/grype-db v0.0.0-20211119195714-911ff7162dc6/go.mod h1:/sgH+Lxc8E6wHQt1w64336mWr8QWrt0yjcRsriijIkg=
github.com/anchore/packageurl-go v0.0.0-20210922164639-b3fa992ebd29 h1:K9LfnxwhqvihqU0+MF325FNy7fsKV9EGaUxdfR4gnWk=
github.com/anchore/packageurl-go v0.0.0-20210922164639-b3fa992ebd29/go.mod h1:Oc1UkGaJwY6ND6vtAqPSlYrptKRJngHwkwB6W7l1uP0=
github.com/anchore/stereoscope v0.0.0-20210524175238-3b7662f3a66f/go.mod h1:vhh1M99rfWx5ejMvz1lkQiFZUrC5wu32V12R4JXH+ZI=
github.com/anchore/stereoscope v0.0.0-20210524175238-3b7662f3a66f/go.mod h1:vhh1M99rfWx5ejMvz1lkQiFZUrC5wu32V12R4JXH+ZI=
github.com/anchore/stereoscope v0.0.0-20210817160504-0f4abc2a5a5a/go.mod h1:165DfE5jApgEkHTWwu7Bijeml9fofudrgcpuWaD9+tk=
github.com/anchore/stereoscope v0.0.0-20211024152658-003132a67c10 h1:BmK/CgNlu+X9foWK2ZAIehxzYws760AZSGVNamQZpiw=
github.com/anchore/stereoscope v0.0.0-20211024152658-003132a67c10/go.mod h1:Rqltg0KqVKoDy4kCuJMLPFDfUg7pEmSqUEA9pOO1L7c=
@ -459,10 +461,12 @@ github.com/hashicorp/consul/sdk v0.1.1/go.mod h1:VKf9jXwCTEY1QZP2MOLRhb5i/I/ssyN
github.com/hashicorp/errwrap v1.0.0 h1:hLrqtEDnRye3+sgx6z4qVLNuviH3MR5aQ0ykNJa/UYA=
github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4=
github.com/hashicorp/go-cleanhttp v0.5.0/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80=
github.com/hashicorp/go-cleanhttp v0.5.1 h1:dH3aiDG9Jvb5r5+bYHsikaOUIpcM0xvgMXVoDkXMzJM=
github.com/hashicorp/go-cleanhttp v0.5.1/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80=
github.com/hashicorp/go-getter v1.4.1 h1:3A2Mh8smGFcf5M+gmcv898mZdrxpseik45IpcyISLsA=
github.com/hashicorp/go-cleanhttp v0.5.2 h1:035FKYIWjmULyFRBKPs8TBQoi0x6d9G4xc9neXJWAZQ=
github.com/hashicorp/go-cleanhttp v0.5.2/go.mod h1:kO/YDlP8L1346E6Sodw+PrpBSV4/SoxCXGY6BqNFT48=
github.com/hashicorp/go-getter v1.4.1/go.mod h1:7qxyCd8rBfcShwsvxgIguu4KbS3l8bUCwg2Umn7RjeY=
github.com/hashicorp/go-getter v1.5.9 h1:b7ahZW50iQiUek/at3CvZhPK1/jiV6CtKcsJiR6E4R0=
github.com/hashicorp/go-getter v1.5.9/go.mod h1:BrrV/1clo8cCYu6mxvboYg+KutTiFnXjMEgDD8+i7ZI=
github.com/hashicorp/go-hclog v0.9.2/go.mod h1:5CU+agLiy3J7N7QjHK5d05KxGsuXiQLrjA0H7acj2lQ=
github.com/hashicorp/go-immutable-radix v1.0.0/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60=
github.com/hashicorp/go-msgpack v0.5.3/go.mod h1:ahLV/dePpqEmjfWmKiqvPkv/twdG7iPBM1vqhUKIvfM=
@ -535,6 +539,8 @@ github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI
github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
github.com/klauspost/compress v1.4.0/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A=
github.com/klauspost/compress v1.4.1/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A=
github.com/klauspost/compress v1.11.2 h1:MiK62aErc3gIiVEtyzKfeOHgW7atJb5g/KNX5m3c2nQ=
github.com/klauspost/compress v1.11.2/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs=
github.com/klauspost/cpuid v0.0.0-20180405133222-e7e905edc00e/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek=
github.com/klauspost/cpuid v1.2.0/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek=
github.com/knqyf263/go-apk-version v0.0.0-20200609155635-041fdbb8563f h1:GvCU5GXhHq+7LeOzx/haG7HSIZokl3/0GkoUFzsRJjg=

View file

@ -1,8 +1,11 @@
package db
import (
"crypto/tls"
"crypto/x509"
"fmt"
"io/ioutil"
"net/http"
"os"
"path"
"strconv"
@ -15,6 +18,7 @@ import (
"github.com/anchore/grype/internal/bus"
"github.com/anchore/grype/internal/file"
"github.com/anchore/grype/internal/log"
"github.com/hashicorp/go-cleanhttp"
"github.com/spf13/afero"
"github.com/wagoodman/go-partybus"
"github.com/wagoodman/go-progress"
@ -27,6 +31,7 @@ const (
type Config struct {
DBRootDir string
ListingURL string
CACert string
ValidateByHashOnGet bool
}
@ -40,17 +45,24 @@ type Curator struct {
validateByHashOnGet bool
}
func NewCurator(cfg Config) Curator {
func NewCurator(cfg Config) (Curator, error) {
dbDir := path.Join(cfg.DBRootDir, strconv.Itoa(vulnerability.SchemaVersion))
fs := afero.NewOsFs()
httpClient, err := defaultHTTPClient(fs, cfg.CACert)
if err != nil {
return Curator{}, err
}
return Curator{
fs: afero.NewOsFs(),
fs: fs,
targetSchema: vulnerability.SchemaVersion,
downloader: file.NewGetter(),
downloader: file.NewGetter(httpClient),
dbDir: dbDir,
dbPath: path.Join(dbDir, FileName),
listingURL: cfg.ListingURL,
validateByHashOnGet: cfg.ValidateByHashOnGet,
}
}, nil
}
func (c *Curator) GetStore() (*reader.Reader, error) {
@ -122,16 +134,16 @@ func (c *Curator) Update() (bool, error) {
updateAvailable, updateEntry, err := c.IsUpdateAvailable()
if err != nil {
// we want to continue if possible even if we can't check for an update
log.Infof("unable to check for vulnerability database update")
log.Warnf("unable to check for vulnerability database update")
log.Debugf("check for vulnerability update failed: %+v", err)
}
if updateAvailable {
log.Infof("Downloading new vulnerability DB")
log.Infof("downloading new vulnerability DB")
err = c.UpdateTo(updateEntry, downloadProgress, importProgress, stage)
if err != nil {
return false, fmt.Errorf("unable to update vulnerability database: %w", err)
}
log.Infof("Updated vulnerability DB to version=%d built=%q", updateEntry.Version, updateEntry.Built.String())
log.Infof("updated vulnerability DB to version=%d built=%q", updateEntry.Version, updateEntry.Built.String())
return true, nil
}
stage.Current = "no update available"
@ -143,7 +155,7 @@ func (c *Curator) Update() (bool, error) {
func (c *Curator) IsUpdateAvailable() (bool, *curation.ListingEntry, error) {
log.Debugf("checking for available database updates")
listing, err := curation.NewListingFromURL(c.fs, c.listingURL)
listing, err := c.ListingFromURL()
if err != nil {
return false, nil, err
}
@ -312,3 +324,49 @@ func (c *Curator) activate(dbDirPath string) error {
// activate the new db cache
return file.CopyDir(c.fs, dbDirPath, c.dbDir)
}
// ListingFromURL loads a Listing from a URL.
func (c Curator) ListingFromURL() (curation.Listing, error) {
tempFile, err := afero.TempFile(c.fs, "", "grype-db-listing")
if err != nil {
return curation.Listing{}, fmt.Errorf("unable to create listing temp file: %w", err)
}
defer func() {
err := c.fs.RemoveAll(tempFile.Name())
if err != nil {
log.Errorf("failed to remove file (%s): %w", tempFile.Name(), err)
}
}()
// download the listing file
err = c.downloader.GetFile(tempFile.Name(), c.listingURL)
if err != nil {
return curation.Listing{}, fmt.Errorf("unable to download listing: %w", err)
}
// parse the listing file
listing, err := curation.NewListingFromFile(c.fs, tempFile.Name())
if err != nil {
return curation.Listing{}, err
}
return listing, nil
}
func defaultHTTPClient(fs afero.Fs, caCertPath string) (*http.Client, error) {
httpClient := cleanhttp.DefaultClient()
if caCertPath != "" {
rootCAs := x509.NewCertPool()
pemBytes, err := afero.ReadFile(fs, caCertPath)
if err != nil {
return nil, fmt.Errorf("unable to configure root CAs for curator: %w", err)
}
rootCAs.AppendCertsFromPEM(pemBytes)
httpClient.Transport.(*http.Transport).TLSClientConfig = &tls.Config{
MinVersion: tls.VersionTLS12,
RootCAs: rootCAs,
}
}
return httpClient, nil
}

View file

@ -1,21 +1,28 @@
package db
import (
"bufio"
"fmt"
"io"
"net/http"
"net/url"
"os"
"os/exec"
"path"
"path/filepath"
"strconv"
"syscall"
"testing"
"time"
"github.com/stretchr/testify/assert"
"github.com/wagoodman/go-progress"
"github.com/anchore/grype-db/pkg/curation"
"github.com/anchore/grype/internal"
"github.com/anchore/grype/internal/file"
"github.com/gookit/color"
"github.com/spf13/afero"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"github.com/wagoodman/go-progress"
)
func mustUrl(u *url.URL, err error) *url.URL {
@ -59,18 +66,118 @@ func (g *testGetter) GetToDir(dst, src string, _ ...*progress.Manual) error {
return afero.WriteFile(g.fs, dst, []byte(g.dir[src]), 0755)
}
func newTestCurator(fs afero.Fs, getter file.Getter, dbDir, metadataUrl string, validateDbHash bool) Curator {
c := NewCurator(Config{
func newTestCurator(tb testing.TB, fs afero.Fs, getter file.Getter, dbDir, metadataUrl string, validateDbHash bool) Curator {
c, err := NewCurator(Config{
DBRootDir: dbDir,
ListingURL: metadataUrl,
ValidateByHashOnGet: validateDbHash,
})
require.NoError(tb, err)
c.downloader = getter
c.fs = fs
return c
}
func Test_defaultHTTPClient(t *testing.T) {
tests := []struct {
name string
hasCert bool
}{
{
name: "no custom cert should use default system root certs",
hasCert: false,
},
{
name: "should use single custom cert",
hasCert: true,
},
}
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
var certPath string
if test.hasCert {
certPath = generateCertFixture(t)
}
httpClient, err := defaultHTTPClient(afero.NewOsFs(), certPath)
require.NoError(t, err)
if test.hasCert {
require.NotNil(t, httpClient.Transport.(*http.Transport).TLSClientConfig)
assert.Len(t, httpClient.Transport.(*http.Transport).TLSClientConfig.RootCAs.Subjects(), 1)
} else {
assert.Nil(t, httpClient.Transport.(*http.Transport).TLSClientConfig)
}
})
}
}
func generateCertFixture(t *testing.T) string {
path := "test-fixtures/tls/server.crt"
if _, err := os.Stat(path); !os.IsNotExist(err) {
// fixture already exists...
return path
}
t.Logf(color.Bold.Sprint("Generating Key/Cert Fixture"))
cwd, err := os.Getwd()
if err != nil {
t.Errorf("unable to get cwd: %+v", err)
}
cmd := exec.Command("make", "server.crt")
cmd.Dir = filepath.Join(cwd, "test-fixtures/tls")
stderr, err := cmd.StderrPipe()
if err != nil {
t.Fatalf("could not get stderr: %+v", err)
}
stdout, err := cmd.StdoutPipe()
if err != nil {
t.Fatalf("could not get stdout: %+v", err)
}
err = cmd.Start()
if err != nil {
t.Fatalf("failed to start cmd: %+v", err)
}
show := func(label string, reader io.ReadCloser) {
scanner := bufio.NewScanner(reader)
scanner.Split(bufio.ScanLines)
for scanner.Scan() {
t.Logf("%s: %s", label, scanner.Text())
}
}
go show("out", stdout)
go show("err", stderr)
if err := cmd.Wait(); err != nil {
if exiterr, ok := err.(*exec.ExitError); ok {
// The program has exited with an exit code != 0
// This works on both Unix and Windows. Although package
// syscall is generally platform dependent, WaitStatus is
// defined for both Unix and Windows and in both cases has
// an ExitStatus() method with the same signature.
if status, ok := exiterr.Sys().(syscall.WaitStatus); ok {
if status.ExitStatus() != 0 {
t.Fatalf("failed to generate fixture: rc=%d", status.ExitStatus())
}
}
} else {
t.Fatalf("unable to get generate fixture result: %+v", err)
}
}
return path
}
func TestCuratorDownload(t *testing.T) {
tests := []struct {
name string
@ -99,7 +206,7 @@ func TestCuratorDownload(t *testing.T) {
}
fs := afero.NewMemMapFs()
getter := newTestGetter(fs, files, dirs)
cur := newTestCurator(fs, getter, "/tmp/dbdir", metadataUrl, false)
cur := newTestCurator(t, fs, getter, "/tmp/dbdir", metadataUrl, false)
path, err := cur.download(test.entry, &progress.Manual{})
@ -120,7 +227,6 @@ func TestCuratorDownload(t *testing.T) {
if string(actual) != contents {
t.Fatalf("bad contents: %+v", string(actual))
}
})
}
}
@ -176,7 +282,7 @@ func TestCuratorValidate(t *testing.T) {
fs := afero.NewOsFs()
getter := newTestGetter(fs, nil, nil)
cur := newTestCurator(fs, getter, "/tmp/dbdir", metadataUrl, test.cfgValidateDbHash)
cur := newTestCurator(t, fs, getter, "/tmp/dbdir", metadataUrl, test.cfgValidateDbHash)
cur.targetSchema = test.constraint
@ -192,12 +298,10 @@ func TestCuratorValidate(t *testing.T) {
}
func TestCuratorDBPathHasSchemaVersion(t *testing.T) {
fs := afero.NewMemMapFs()
dbRootPath := "/tmp/dbdir"
cur := newTestCurator(fs, nil, dbRootPath, "http://metadata.io", false)
cur := newTestCurator(t, fs, nil, dbRootPath, "http://metadata.io", false)
assert.Equal(t, path.Join(dbRootPath, strconv.Itoa(cur.targetSchema)), cur.dbDir, "unexpected dir")
assert.Contains(t, cur.dbPath, path.Join(dbRootPath, strconv.Itoa(cur.targetSchema)), "unexpected path")
}

5
grype/db/test-fixtures/tls/.gitignore vendored Executable file
View file

@ -0,0 +1,5 @@
server.key
server.crt
www/
listing.json
dbdir/

View file

@ -0,0 +1,45 @@
all: clean serve
.PHONY: serve
serve: www/listing.json www/db.tar.gz server.crt
python3 serve.py
.PHONY: grype-test-fail
grype-test-fail: clean-dbdir dbdir
GRYPE_DB_CACHE_DIR=$(shell pwd)/dbdir \
GRYPE_DB_UPDATE_URL=https://$(shell hostname).local/listing.json \
go run ../../../../main.go -vv alpine:latest
.PHONY: grype-test-pass
grype-test-pass: clean-dbdir dbdir
GRYPE_DB_CA_CERT=$(shell pwd)/server.crt \
GRYPE_DB_CACHE_DIR=$(shell pwd)/dbdir \
GRYPE_DB_UPDATE_URL=https://$(shell hostname).local/listing.json \
go run ../../../../main.go -vv alpine:latest
dbdir:
mkdir -p dbdir
server.crt server.key:
./generate-x509-cert-pair.sh
www:
mkdir -p www
listing.json:
curl -L -O https://toolbox-data.anchore.io/grype/databases/listing.json
www/listing.json www/db.tar.gz: www listing.json
$(eval location=$(shell python3 listing.py))
curl -L -o www/db.tar.gz $(location)
.PHONY: clean
clean: clean-dbdir
rm -rf www
rm -f server.crt
rm -f server.key
.PHONY: clean-dbdir
clean-dbdir:
rm -rf dbdir/

View file

@ -0,0 +1,24 @@
# TLS test utils
Note: Makefile, server.crt, and server.key are used in automated testing, the remaining files are for convenience in manual verification.
You will require Python 3 to run these utils.
To standup a test server:
```
make serve
```
To test grype against this server:
```
# without the custom cert configured (thus will fail)
make grype-test-fail
# with the custom cert configured
make grype-test-pass
```
To remove all temp files:
```
make clean
```

View file

@ -0,0 +1,16 @@
#!/usr/bin/env bash
set -eux
# create private key
openssl genrsa -out server.key 2048
# generate self-signed public key (cert) based on the private key
openssl req -new -x509 -sha256 \
-key server.key \
-out server.crt \
-days 3650 \
-reqexts SAN \
-extensions SAN \
-config <(cat /etc/ssl/openssl.cnf <(printf "[SAN]\nsubjectAltName=DNS:$(hostname).local")) \
-subj "/C=US/ST=Test/L=Test/O=Test/CN=$(hostname).local"

View file

@ -0,0 +1,27 @@
import urllib.request
import json
import os
with open('listing.json', 'r') as fh:
data = json.loads(fh.read())
entry = data["available"]["3"][-1]
hostname = os.popen('hostname').read().strip()
with open('www/listing.json', 'w') as fh:
json.dump(
{
"available": {
entry["version"]: [
{
"built": entry["built"],
"version": entry["version"],
"url": f"https://{hostname}.local/db.tar.gz",
"checksum": entry["checksum"]
}
]
}
}, fh)
print(entry["url"])

View file

@ -0,0 +1,25 @@
from http.server import HTTPServer, SimpleHTTPRequestHandler
import ssl
import logging
port = 443
directory = "www"
class Handler(SimpleHTTPRequestHandler):
def __init__(self, *args, **kwargs):
super().__init__(*args, directory=directory, **kwargs)
def do_GET(self):
logging.error(self.headers)
SimpleHTTPRequestHandler.do_GET(self)
httpd = HTTPServer(('0.0.0.0', port), Handler)
sslctx = ssl.SSLContext()
sslctx.options |= ssl.OP_NO_TLSv1 | ssl.OP_NO_TLSv1_1
sslctx.load_cert_chain(certfile='server.crt', keyfile="server.key")
httpd.socket = sslctx.wrap_socket(httpd.socket, server_side=True)
print(f"Server running on https://0.0.0.0:{port}")
httpd.serve_forever()

View file

@ -29,7 +29,10 @@ func FindVulnerabilitiesForPackage(provider vulnerability.Provider, d *distro.Di
}
func LoadVulnerabilityDB(cfg db.Config, update bool) (vulnerability.Provider, vulnerability.MetadataProvider, *db.Status, error) {
dbCurator := db.NewCurator(cfg)
dbCurator, err := db.NewCurator(cfg)
if err != nil {
return nil, nil, nil, err
}
if update {
_, err := dbCurator.Update()

View file

@ -12,6 +12,7 @@ import (
type database struct {
Dir string `yaml:"cache-dir" json:"cache-dir" mapstructure:"cache-dir"`
UpdateURL string `yaml:"update-url" json:"update-url" mapstructure:"update-url"`
CACert string `yaml:"ca-cert" json:"ca-cert" mapstructure:"ca-cert"`
AutoUpdate bool `yaml:"auto-update" json:"auto-update" mapstructure:"auto-update"`
ValidateByHashOnStart bool `yaml:"validate-by-hash-on-start" json:"validate-by-hash-on-start" mapstructure:"validate-by-hash-on-start"`
}
@ -19,6 +20,7 @@ type database struct {
func (cfg database) loadDefaultValues(v *viper.Viper) {
v.SetDefault("db.cache-dir", path.Join(xdg.CacheHome, internal.ApplicationName, "db"))
v.SetDefault("db.update-url", internal.DBUpdateURL)
v.SetDefault("db.ca-cert", "")
v.SetDefault("db.auto-update", true)
v.SetDefault("db.validate-by-hash-on-start", false)
}
@ -27,6 +29,7 @@ func (cfg database) ToCuratorConfig() db.Config {
return db.Config{
DBRootDir: cfg.Dir,
ListingURL: cfg.UpdateURL,
CACert: cfg.CACert,
ValidateByHashOnGet: cfg.ValidateByHashOnStart,
}
}

View file

@ -3,63 +3,124 @@ package file
import (
"fmt"
"io"
"net/http"
"github.com/hashicorp/go-getter/helper/url"
"github.com/anchore/grype/internal"
"github.com/hashicorp/go-getter"
"github.com/wagoodman/go-progress"
)
var (
archiveExtensions = []string{
".tar",
".tar.gz",
".tgz",
".zip",
}
ErrNonArchiveSource = fmt.Errorf("non-archive sources are not supported for directory destinations")
)
type Getter interface {
// GetFile downloads the give URL into the given path. The URL must reference a single file.
GetFile(dst, src string, monitor ...*progress.Manual) error
// Get downloads the given URL into the given directory. The directory must already exist.
// GetToDir downloads the resource found at the `src` URL into the given `dst` directory.
// The directory must already exist, and the remote resource MUST BE AN ARCHIVE (e.g. `.tar.gz`).
GetToDir(dst, src string, monitor ...*progress.Manual) error
}
type HashiGoGetter struct {
httpGetter getter.HttpGetter
}
type progressAdapter struct {
monitor *progress.Manual
}
func NewGetter() *HashiGoGetter {
return &HashiGoGetter{}
// NewGetter creates and returns a new Getter. Providing an http.Client is optional. If one is provided,
// it will be used for all HTTP(S) getting; otherwise, go-getter's default getters will be used.
func NewGetter(httpClient *http.Client) *HashiGoGetter {
return &HashiGoGetter{
httpGetter: getter.HttpGetter{
Client: httpClient,
},
}
}
func (g HashiGoGetter) GetFile(dst, src string, monitors ...*progress.Manual) error {
switch len(monitors) {
case 0:
return getter.GetFile(dst, src)
case 1:
return getter.GetFile(dst, src,
getter.WithProgress(
&progressAdapter{
monitor: monitors[0],
},
),
)
default:
if len(monitors) > 1 {
return fmt.Errorf("multiple monitors provided, which is not allowed")
}
return getterClient(dst, src, false, g.httpGetter, monitors).Get()
}
func (g HashiGoGetter) GetToDir(dst, src string, monitors ...*progress.Manual) error {
switch len(monitors) {
case 0:
return getter.Get(dst, src)
case 1:
return getter.Get(dst, src,
getter.WithProgress(
&progressAdapter{
monitor: monitors[0],
},
),
)
default:
// though there are multiple getters, only the http/https getter requires extra validation
if err := validateHTTPSource(src); err != nil {
return err
}
if len(monitors) > 1 {
return fmt.Errorf("multiple monitors provided, which is not allowed")
}
return getterClient(dst, src, true, g.httpGetter, monitors).Get()
}
func validateHTTPSource(src string) error {
// we are ignoring any sources that are not destined to use the http getter object
if !internal.HasAnyOfPrefixes(src, "http://", "https://") {
return nil
}
u, err := url.Parse(src)
if err != nil {
return fmt.Errorf("bad URL provided %q: %w", src, err)
}
// only allow for sources with archive extensions
if !internal.HasAnyOfSuffixes(u.Path, archiveExtensions...) {
return ErrNonArchiveSource
}
return nil
}
func getterClient(dst, src string, dir bool, httpGetter getter.HttpGetter, monitors []*progress.Manual) *getter.Client {
client := &getter.Client{
Src: src,
Dst: dst,
Dir: dir,
Getters: map[string]getter.Getter{
"http": &httpGetter,
"https": &httpGetter,
// note: these are the default getters from https://github.com/hashicorp/go-getter/blob/v1.5.9/get.go#L68-L74
// it is possible that other implementations need to account for custom httpclient injection, however,
// that has not been accounted for at this time.
"file": new(getter.FileGetter),
"git": new(getter.GitGetter),
"gcs": new(getter.GCSGetter),
"hg": new(getter.HgGetter),
"s3": new(getter.S3Getter),
},
Options: mapToGetterClientOptions(monitors),
}
return client
}
func withProgress(monitor *progress.Manual) func(client *getter.Client) error {
return getter.WithProgress(
&progressAdapter{monitor: monitor},
)
}
func mapToGetterClientOptions(monitors []*progress.Manual) []getter.ClientOption {
// TODO: This function is no longer needed once a generic `map` method is available.
var result []getter.ClientOption
for _, monitor := range monitors {
result = append(result, withProgress(monitor))
}
return result
}
type readCloser struct {
@ -68,6 +129,10 @@ type readCloser struct {
func (c *readCloser) Close() error { return nil }
type progressAdapter struct {
monitor *progress.Manual
}
func (a *progressAdapter) TrackProgress(_ string, currentSize, totalSize int64, stream io.ReadCloser) io.ReadCloser {
a.monitor.N = currentSize
a.monitor.Total = totalSize

View file

@ -0,0 +1,261 @@
package file
import (
"archive/tar"
"bytes"
"context"
"crypto/x509"
"fmt"
"net"
"net/http"
"net/http/httptest"
"net/url"
"path"
"testing"
"github.com/stretchr/testify/assert"
)
func TestGetter_GetFile(t *testing.T) {
testCases := []struct {
name string
prepareClient func(*http.Client)
assert assert.ErrorAssertionFunc
}{
{
name: "client trusts server's CA",
assert: assert.NoError,
},
{
name: "client doesn't trust server's CA",
prepareClient: removeTrustedCAs,
assert: assertUnknownAuthorityError,
},
}
for _, tc := range testCases {
t.Run(tc.name, func(t *testing.T) {
requestPath := "/foo"
server := newTestServer(t, withResponseForPath(t, requestPath, testFileContent))
t.Cleanup(server.Close)
httpClient := getClient(t, server)
if tc.prepareClient != nil {
tc.prepareClient(httpClient)
}
getter := NewGetter(httpClient)
requestURL := createRequestURL(t, server, requestPath)
tempDir := t.TempDir()
tempFile := path.Join(tempDir, "some-destination-file")
err := getter.GetFile(tempFile, requestURL)
tc.assert(t, err)
})
}
}
func TestGetter_GetToDir_FilterNonArchivesWired(t *testing.T) {
testCases := []struct {
name string
source string
assert assert.ErrorAssertionFunc
}{
{
name: "error out on non-archive sources",
source: "http://localhost/something.txt",
assert: assertErrNonArchiveSource,
},
}
for _, test := range testCases {
t.Run(test.name, func(t *testing.T) {
test.assert(t, NewGetter(nil).GetToDir(t.TempDir(), test.source))
})
}
}
func TestGetter_validateHttpSource(t *testing.T) {
testCases := []struct {
name string
source string
assert assert.ErrorAssertionFunc
}{
{
name: "error out on non-archive sources",
source: "http://localhost/something.txt",
assert: assertErrNonArchiveSource,
},
{
name: "filter out non-archive sources with get param",
source: "https://localhost/vulnerability-db_v3_2021-11-21T08:15:44Z.txt?checksum=sha256%3Ac402d01fa909a3fa85a5c6733ef27a3a51a9105b6c62b9152adbd24c08358911",
assert: assertErrNonArchiveSource,
},
{
name: "ignore non http-https input",
source: "s3://bucket/something.txt",
assert: assert.NoError,
},
}
for _, test := range testCases {
t.Run(test.name, func(t *testing.T) {
test.assert(t, validateHTTPSource(test.source))
})
}
}
func TestGetter_GetToDir_CertConcerns(t *testing.T) {
testCases := []struct {
name string
prepareClient func(*http.Client)
assert assert.ErrorAssertionFunc
}{
{
name: "client trusts server's CA",
assert: assert.NoError,
},
{
name: "client doesn't trust server's CA",
prepareClient: removeTrustedCAs,
assert: assertUnknownAuthorityError,
},
}
for _, tc := range testCases {
t.Run(tc.name, func(t *testing.T) {
requestPath := "/foo.tar"
tarball := createTarball("foo", testFileContent)
server := newTestServer(t, withResponseForPath(t, requestPath, tarball))
t.Cleanup(server.Close)
httpClient := getClient(t, server)
if tc.prepareClient != nil {
tc.prepareClient(httpClient)
}
getter := NewGetter(httpClient)
requestURL := createRequestURL(t, server, requestPath)
tempDir := t.TempDir()
err := getter.GetToDir(tempDir, requestURL)
tc.assert(t, err)
})
}
}
func assertUnknownAuthorityError(t assert.TestingT, err error, _ ...interface{}) bool {
return assert.ErrorAs(t, err, &x509.UnknownAuthorityError{})
}
func assertErrNonArchiveSource(t assert.TestingT, err error, _ ...interface{}) bool {
return assert.ErrorIs(t, err, ErrNonArchiveSource)
}
func removeTrustedCAs(client *http.Client) {
client.Transport.(*http.Transport).TLSClientConfig.RootCAs = nil
}
// createTarball makes a single-file tarball and returns it as a byte slice.
func createTarball(filename string, content []byte) []byte {
tarBuffer := new(bytes.Buffer)
tarWriter := tar.NewWriter(tarBuffer)
tarWriter.WriteHeader(&tar.Header{
Name: filename,
Size: int64(len(content)),
Mode: 0600,
})
tarWriter.Write(content)
tarWriter.Close()
return tarBuffer.Bytes()
}
type muxOption func(mux *http.ServeMux)
func withResponseForPath(t *testing.T, path string, response []byte) muxOption {
t.Helper()
return func(mux *http.ServeMux) {
mux.HandleFunc(path, func(w http.ResponseWriter, req *http.Request) {
t.Logf("server handling request: %s %s", req.Method, req.URL)
_, err := w.Write(response)
if err != nil {
t.Fatal(err)
}
})
}
}
func newTestServer(t *testing.T, muxOptions ...muxOption) *httptest.Server {
t.Helper()
mux := http.NewServeMux()
for _, option := range muxOptions {
option(mux)
}
server := httptest.NewTLSServer(mux)
t.Logf("new TLS server listening at %s", getHost(t, server))
return server
}
func createRequestURL(t *testing.T, server *httptest.Server, path string) string {
t.Helper()
// TODO: Figure out how to get this value from the server without hardcoding it here
const testServerCertificateName = "example.com"
serverURL, err := url.Parse(server.URL)
if err != nil {
t.Fatal(err)
}
// Set URL hostname to value from TLS certificate
serverURL.Host = fmt.Sprintf("%s:%s", testServerCertificateName, serverURL.Port())
serverURL.Path = path
return serverURL.String()
}
// getClient returns an http.Client that can be used to contact the test TLS server.
func getClient(t *testing.T, server *httptest.Server) *http.Client {
t.Helper()
httpClient := server.Client()
transport := httpClient.Transport.(*http.Transport)
serverHost := getHost(t, server)
transport.DialContext = func(_ context.Context, _, addr string) (net.Conn, error) {
t.Logf("client dialing %q for host %q", serverHost, addr)
// Ensure the client dials our test server
return net.Dial("tcp", serverHost)
}
return httpClient
}
// getHost extracts the host value from a server URL string.
// e.g. given a server with URL "http://1.2.3.4:5000/foo", getHost returns "1.2.3.4:5000"
func getHost(t *testing.T, server *httptest.Server) string {
t.Helper()
u, err := url.Parse(server.URL)
if err != nil {
t.Fatal(err)
}
return u.Hostname() + ":" + u.Port()
}
var testFileContent = []byte("This is the content of a test file!\n")

View file

@ -0,0 +1,25 @@
package internal
import "strings"
// HasAnyOfSuffixes returns an indication if the given string has any of the given suffixes.
func HasAnyOfSuffixes(input string, suffixes ...string) bool {
for _, suffix := range suffixes {
if strings.HasSuffix(input, suffix) {
return true
}
}
return false
}
// HasAnyOfPrefixes returns an indication if the given string has any of the given prefixes.
func HasAnyOfPrefixes(input string, prefixes ...string) bool {
for _, prefix := range prefixes {
if strings.HasPrefix(input, prefix) {
return true
}
}
return false
}

View file

@ -0,0 +1,122 @@
package internal
import (
"testing"
"github.com/stretchr/testify/assert"
)
func TestHasAnyOfSuffixes(t *testing.T) {
tests := []struct {
name string
input string
suffixes []string
expected bool
}{
{
name: "go case",
input: "this has something",
suffixes: []string{
"has something",
"has NOT something",
},
expected: true,
},
{
name: "no match",
input: "this has something",
suffixes: []string{
"has NOT something",
},
expected: false,
},
{
name: "empty",
input: "this has something",
suffixes: []string{},
expected: false,
},
{
name: "positive match last",
input: "this has something",
suffixes: []string{
"that does not have",
"something",
},
expected: true,
},
{
name: "empty input",
input: "",
suffixes: []string{
"that does not have",
"this has",
},
expected: false,
},
}
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
assert.Equal(t, test.expected, HasAnyOfSuffixes(test.input, test.suffixes...))
})
}
}
func TestHasAnyOfPrefixes(t *testing.T) {
tests := []struct {
name string
input string
prefixes []string
expected bool
}{
{
name: "go case",
input: "this has something",
prefixes: []string{
"this has",
"that does not have",
},
expected: true,
},
{
name: "no match",
input: "this has something",
prefixes: []string{
"this DOES NOT has",
"that does not have",
},
expected: false,
},
{
name: "empty",
input: "this has something",
prefixes: []string{},
expected: false,
},
{
name: "positive match last",
input: "this has something",
prefixes: []string{
"that does not have",
"this has",
},
expected: true,
},
{
name: "empty input",
input: "",
prefixes: []string{
"that does not have",
"this has",
},
expected: false,
},
}
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
assert.Equal(t, test.expected, HasAnyOfPrefixes(test.input, test.prefixes...))
})
}
}

View file

@ -0,0 +1,43 @@
package cli
import (
"strings"
"testing"
)
func TestDBValidations(t *testing.T) {
tests := []struct {
name string
args []string
env map[string]string
assertions []traitAssertion
}{
{
// regression: do not panic on bad DB load
name: "fail on bad DB load",
args: []string{"-vv", "dir:."},
env: map[string]string{
"GRYPE_DB_CACHE_DIR": t.TempDir(),
"GRYPE_DB_CA_CERT": "./does-not-exist.crt",
},
assertions: []traitAssertion{
assertInOutput("failed to load vulnerability db"),
assertFailingReturnCode,
},
},
}
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
cmd, stdout, stderr := runGrype(t, test.env, test.args...)
for _, traitAssertionFn := range test.assertions {
traitAssertionFn(t, stdout, stderr, cmd.ProcessState.ExitCode())
}
if t.Failed() {
t.Log("STDOUT:\n", stdout)
t.Log("STDERR:\n", stderr)
t.Log("COMMAND:", strings.Join(cmd.Args, " "))
}
})
}
}