From d420368ba9db3dda5d1f41b0348048ef95ff9fd3 Mon Sep 17 00:00:00 2001 From: Alex Goodman Date: Tue, 23 Mar 2021 11:00:59 -0400 Subject: [PATCH] add tests around new file metadata cataloger Signed-off-by: Alex Goodman --- go.mod | 3 +- go.sum | 4 +- syft/file/digest_cataloger.go | 18 ++- syft/file/digest_cataloger_test.go | 98 ++++++++++++++ syft/file/metadata_cataloger.go | 13 +- syft/file/metadata_cataloger_test.go | 125 ++++++++++++++++++ syft/file/test-fixtures/a-path.txt | 1 + syft/file/test-fixtures/another-path.txt | 1 + .../image-file-type-mix/Dockerfile | 10 ++ .../image-file-type-mix/file-1.txt | 1 + syft/file/test-fixtures/last/path.txt | 1 + syft/source/all_layers_resolver.go | 2 +- syft/source/file_metadata_test.go | 122 +++++++++++++++++ syft/source/image_squash_resolver.go | 2 +- syft/source/mock_resolver.go | 19 ++- .../image-file-type-mix/Dockerfile | 10 ++ .../image-file-type-mix/file-1.txt | 1 + 17 files changed, 407 insertions(+), 24 deletions(-) create mode 100644 syft/file/digest_cataloger_test.go create mode 100644 syft/file/metadata_cataloger_test.go create mode 100644 syft/file/test-fixtures/a-path.txt create mode 100644 syft/file/test-fixtures/another-path.txt create mode 100644 syft/file/test-fixtures/image-file-type-mix/Dockerfile create mode 100644 syft/file/test-fixtures/image-file-type-mix/file-1.txt create mode 100644 syft/file/test-fixtures/last/path.txt create mode 100644 syft/source/file_metadata_test.go create mode 100644 syft/source/test-fixtures/image-file-type-mix/Dockerfile create mode 100644 syft/source/test-fixtures/image-file-type-mix/file-1.txt diff --git a/go.mod b/go.mod index f2138d29d..95354b280 100644 --- a/go.mod +++ b/go.mod @@ -10,7 +10,7 @@ require ( github.com/anchore/go-rpmdb v0.0.0-20201106153645-0043963c2e12 github.com/anchore/go-testutils v0.0.0-20200925183923-d5f45b0d3c04 github.com/anchore/go-version v1.2.2-0.20200701162849-18adb9c92b9b - github.com/anchore/stereoscope v0.0.0-20210317203852-f77bbcbede40 + github.com/anchore/stereoscope v0.0.0-20210323145922-1f45cd8849b4 github.com/antihax/optional v1.0.0 github.com/bmatcuk/doublestar/v2 v2.0.4 github.com/docker/docker v17.12.0-ce-rc1.0.20200309214505-aa6a9891b09c+incompatible @@ -34,6 +34,7 @@ require ( github.com/spf13/cobra v1.0.1-0.20200909172742-8a63648dd905 github.com/spf13/pflag v1.0.5 github.com/spf13/viper v1.7.0 + github.com/stretchr/testify v1.6.0 github.com/wagoodman/go-partybus v0.0.0-20200526224238-eb215533f07d github.com/wagoodman/go-progress v0.0.0-20200731105512-1020f39e6240 github.com/wagoodman/jotframe v0.0.0-20200730190914-3517092dd163 diff --git a/go.sum b/go.sum index ba334d43d..eecc235eb 100644 --- a/go.sum +++ b/go.sum @@ -115,8 +115,8 @@ github.com/anchore/go-testutils v0.0.0-20200925183923-d5f45b0d3c04 h1:VzprUTpc0v github.com/anchore/go-testutils v0.0.0-20200925183923-d5f45b0d3c04/go.mod h1:6dK64g27Qi1qGQZ67gFmBFvEHScy0/C8qhQhNe5B5pQ= github.com/anchore/go-version v1.2.2-0.20200701162849-18adb9c92b9b h1:e1bmaoJfZVsCYMrIZBpFxwV26CbsuoEh5muXD5I1Ods= github.com/anchore/go-version v1.2.2-0.20200701162849-18adb9c92b9b/go.mod h1:Bkc+JYWjMCF8OyZ340IMSIi2Ebf3uwByOk6ho4wne1E= -github.com/anchore/stereoscope v0.0.0-20210317203852-f77bbcbede40 h1:k3/JigkYl7NjMad9eDBBcsg9qiXJFreW6rKNgE0aMUI= -github.com/anchore/stereoscope v0.0.0-20210317203852-f77bbcbede40/go.mod h1:T/OUHXgngXFlo2vknQsDx+n/jErCLPt5o0H1ZXFBpig= +github.com/anchore/stereoscope v0.0.0-20210323145922-1f45cd8849b4 h1:Uuvne+/Mgeyu3fR1JCxiFUQQo2Gp5vXTInum3GbhbwM= +github.com/anchore/stereoscope v0.0.0-20210323145922-1f45cd8849b4/go.mod h1:G7tFR0iI9r6AvibmXKA9v010pRS1IIJgd0t6fOMDxCw= github.com/andreyvit/diff v0.0.0-20170406064948-c7f18ee00883/go.mod h1:rCTlJbsFo29Kk6CurOXKm700vrz8f0KW0JNfpkRJY/8= github.com/antihax/optional v1.0.0 h1:xK2lYat7ZLaVVcIuj82J8kIro4V6kDe0AUDFboUCwcg= github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY= diff --git a/syft/file/digest_cataloger.go b/syft/file/digest_cataloger.go index 37a8ff1b9..10a5bba42 100644 --- a/syft/file/digest_cataloger.go +++ b/syft/file/digest_cataloger.go @@ -13,8 +13,7 @@ import ( var supportedHashAlgorithms = make(map[string]crypto.Hash) type DigestsCataloger struct { - resolver source.FileResolver - hashes []crypto.Hash + hashes []crypto.Hash } func init() { @@ -27,7 +26,7 @@ func init() { } } -func NewDigestsCataloger(resolver source.FileResolver, hashAlgorithms []string) (*DigestsCataloger, error) { +func NewDigestsCataloger(hashAlgorithms []string) (*DigestsCataloger, error) { var hashes []crypto.Hash for _, hashStr := range hashAlgorithms { name := cleanAlgorithmName(hashStr) @@ -39,15 +38,14 @@ func NewDigestsCataloger(resolver source.FileResolver, hashAlgorithms []string) } return &DigestsCataloger{ - resolver: resolver, - hashes: hashes, + hashes: hashes, }, nil } -func (i *DigestsCataloger) Catalog() (map[source.Location][]Digest, error) { +func (i *DigestsCataloger) Catalog(resolver source.FileResolver) (map[source.Location][]Digest, error) { results := make(map[source.Location][]Digest) - for location := range i.resolver.AllLocations() { - result, err := i.catalogLocation(location) + for location := range resolver.AllLocations() { + result, err := i.catalogLocation(resolver, location) if err != nil { return nil, err } @@ -56,8 +54,8 @@ func (i *DigestsCataloger) Catalog() (map[source.Location][]Digest, error) { return results, nil } -func (i *DigestsCataloger) catalogLocation(location source.Location) ([]Digest, error) { - contentReader, err := i.resolver.FileContentsByLocation(location) +func (i *DigestsCataloger) catalogLocation(resolver source.FileResolver, location source.Location) ([]Digest, error) { + contentReader, err := resolver.FileContentsByLocation(location) if err != nil { return nil, err } diff --git a/syft/file/digest_cataloger_test.go b/syft/file/digest_cataloger_test.go new file mode 100644 index 000000000..ad21f9184 --- /dev/null +++ b/syft/file/digest_cataloger_test.go @@ -0,0 +1,98 @@ +package file + +import ( + "crypto" + "fmt" + "io/ioutil" + "os" + "testing" + + "github.com/stretchr/testify/assert" + + "github.com/anchore/syft/syft/source" +) + +func testDigests(t testing.TB, files []string, hashes ...crypto.Hash) map[source.Location][]Digest { + digests := make(map[source.Location][]Digest) + + for _, f := range files { + fh, err := os.Open(f) + if err != nil { + t.Fatalf("could not open %q : %+v", f, err) + } + b, err := ioutil.ReadAll(fh) + if err != nil { + t.Fatalf("could not read %q : %+v", f, err) + } + + for _, hash := range hashes { + h := hash.New() + h.Write(b) + digests[source.NewLocation(f)] = append(digests[source.NewLocation(f)], Digest{ + Algorithm: cleanAlgorithmName(hash.String()), + Value: fmt.Sprintf("%x", h.Sum(nil)), + }) + } + } + + return digests +} + +func TestDigestsCataloger(t *testing.T) { + files := []string{"test-fixtures/last/path.txt", "test-fixtures/another-path.txt", "test-fixtures/a-path.txt"} + + tests := []struct { + name string + algorithms []string + expected map[source.Location][]Digest + constructorErr bool + catalogErr bool + }{ + { + name: "bad algorithm", + algorithms: []string{"sha-nothing"}, + constructorErr: true, + }, + { + name: "unsupported algorithm", + algorithms: []string{"sha512"}, + constructorErr: true, + }, + { + name: "md5-sha1-sha256", + algorithms: []string{"md5"}, + expected: testDigests(t, files, crypto.MD5), + }, + { + name: "md5-sha1-sha256", + algorithms: []string{"md5", "sha1", "sha256"}, + expected: testDigests(t, files, crypto.MD5, crypto.SHA1, crypto.SHA256), + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + c, err := NewDigestsCataloger(test.algorithms) + if err != nil && !test.constructorErr { + t.Fatalf("could not create cataloger (but should have been able to): %+v", err) + } else if err == nil && test.constructorErr { + t.Fatalf("expected constructor error but did not get one") + } else if test.constructorErr && err != nil { + return + } + + resolver := source.NewMockResolverForPaths(files...) + actual, err := c.Catalog(resolver) + if err != nil && !test.catalogErr { + t.Fatalf("could not catalog (but should have been able to): %+v", err) + } else if err == nil && test.catalogErr { + t.Fatalf("expected catalog error but did not get one") + } else if test.catalogErr && err != nil { + return + } + + assert.Equal(t, actual, test.expected, "mismatched digests") + + }) + } +} diff --git a/syft/file/metadata_cataloger.go b/syft/file/metadata_cataloger.go index 8f0565902..7ffd41168 100644 --- a/syft/file/metadata_cataloger.go +++ b/syft/file/metadata_cataloger.go @@ -5,19 +5,16 @@ import ( ) type MetadataCataloger struct { - resolver source.FileResolver } -func NewMetadataCataloger(resolver source.FileResolver) *MetadataCataloger { - return &MetadataCataloger{ - resolver: resolver, - } +func NewMetadataCataloger() *MetadataCataloger { + return &MetadataCataloger{} } -func (i *MetadataCataloger) Catalog() (map[source.Location]source.FileMetadata, error) { +func (i *MetadataCataloger) Catalog(resolver source.FileResolver) (map[source.Location]source.FileMetadata, error) { results := make(map[source.Location]source.FileMetadata) - for location := range i.resolver.AllLocations() { - metadata, err := i.resolver.FileMetadataByLocation(location) + for location := range resolver.AllLocations() { + metadata, err := resolver.FileMetadataByLocation(location) if err != nil { return nil, err } diff --git a/syft/file/metadata_cataloger_test.go b/syft/file/metadata_cataloger_test.go new file mode 100644 index 000000000..91c89e85e --- /dev/null +++ b/syft/file/metadata_cataloger_test.go @@ -0,0 +1,125 @@ +package file + +import ( + "os" + "testing" + + "github.com/anchore/stereoscope/pkg/file" + "github.com/anchore/stereoscope/pkg/imagetest" + "github.com/anchore/syft/syft/source" + "github.com/stretchr/testify/assert" +) + +func TestFileMetadataFetch(t *testing.T) { + img := imagetest.GetFixtureImage(t, "docker-archive", "image-file-type-mix") + + c := NewMetadataCataloger() + + src, err := source.NewFromImage(img, "---") + if err != nil { + t.Fatalf("could not create source: %+v", err) + } + + resolver, err := src.FileResolver(source.SquashedScope) + if err != nil { + t.Fatalf("could not create resolver: %+v", err) + } + + actual, err := c.Catalog(resolver) + if err != nil { + t.Fatalf("could not catalog: %+v", err) + } + + tests := []struct { + path string + exists bool + expected source.FileMetadata + err bool + }{ + { + path: "/file-1.txt", + exists: true, + expected: source.FileMetadata{ + Mode: 0644, + Type: "regularFile", + UserID: 1, + GroupID: 2, + }, + }, + { + path: "/hardlink-1", + exists: true, + expected: source.FileMetadata{ + Mode: 0644, + Type: "hardLink", + UserID: 1, + GroupID: 2, + }, + }, + { + path: "/symlink-1", + exists: true, + expected: source.FileMetadata{ + Mode: 0777 | os.ModeSymlink, + Type: "symbolicLink", + UserID: 0, + GroupID: 0, + }, + }, + { + path: "/char-device-1", + exists: true, + expected: source.FileMetadata{ + Mode: 0644 | os.ModeDevice | os.ModeCharDevice, + Type: "characterDevice", + UserID: 0, + GroupID: 0, + }, + }, + { + path: "/block-device-1", + exists: true, + expected: source.FileMetadata{ + Mode: 0644 | os.ModeDevice, + Type: "blockDevice", + UserID: 0, + GroupID: 0, + }, + }, + { + path: "/fifo-1", + exists: true, + expected: source.FileMetadata{ + Mode: 0644 | os.ModeNamedPipe, + Type: "fifoNode", + UserID: 0, + GroupID: 0, + }, + }, + { + path: "/bin", + exists: true, + expected: source.FileMetadata{ + Mode: 0755 | os.ModeDir, + Type: "directory", + UserID: 0, + GroupID: 0, + }, + }, + } + + for _, test := range tests { + t.Run(test.path, func(t *testing.T) { + _, ref, err := img.SquashedTree().File(file.Path(test.path)) + if err != nil { + t.Fatalf("unable to get file: %+v", err) + } + + l := source.NewLocationFromImage(test.path, *ref, img) + + assert.Equal(t, actual[l], test.expected, "mismatched metadata") + + }) + } + +} diff --git a/syft/file/test-fixtures/a-path.txt b/syft/file/test-fixtures/a-path.txt new file mode 100644 index 000000000..67e954034 --- /dev/null +++ b/syft/file/test-fixtures/a-path.txt @@ -0,0 +1 @@ +test-fixtures/a-path.txt file contents! \ No newline at end of file diff --git a/syft/file/test-fixtures/another-path.txt b/syft/file/test-fixtures/another-path.txt new file mode 100644 index 000000000..0d654f8fe --- /dev/null +++ b/syft/file/test-fixtures/another-path.txt @@ -0,0 +1 @@ +test-fixtures/another-path.txt file contents! \ No newline at end of file diff --git a/syft/file/test-fixtures/image-file-type-mix/Dockerfile b/syft/file/test-fixtures/image-file-type-mix/Dockerfile new file mode 100644 index 000000000..218e369d0 --- /dev/null +++ b/syft/file/test-fixtures/image-file-type-mix/Dockerfile @@ -0,0 +1,10 @@ +FROM busybox:latest + +ADD file-1.txt . +RUN chmod 644 file-1.txt +RUN chown 1:2 file-1.txt +RUN ln -s file-1.txt symlink-1 +RUN ln file-1.txt hardlink-1 +RUN mknod char-device-1 c 89 1 +RUN mknod block-device-1 b 0 1 +RUN mknod fifo-1 p \ No newline at end of file diff --git a/syft/file/test-fixtures/image-file-type-mix/file-1.txt b/syft/file/test-fixtures/image-file-type-mix/file-1.txt new file mode 100644 index 000000000..d86db8155 --- /dev/null +++ b/syft/file/test-fixtures/image-file-type-mix/file-1.txt @@ -0,0 +1 @@ +file 1! \ No newline at end of file diff --git a/syft/file/test-fixtures/last/path.txt b/syft/file/test-fixtures/last/path.txt new file mode 100644 index 000000000..3d4a165ab --- /dev/null +++ b/syft/file/test-fixtures/last/path.txt @@ -0,0 +1 @@ +test-fixtures/last/path.txt file contents! \ No newline at end of file diff --git a/syft/source/all_layers_resolver.go b/syft/source/all_layers_resolver.go index 6437fb4bc..b446eccbf 100644 --- a/syft/source/all_layers_resolver.go +++ b/syft/source/all_layers_resolver.go @@ -194,7 +194,7 @@ func (r *allLayersResolver) AllLocations() <-chan Location { defer close(results) for _, layerIdx := range r.layers { tree := r.img.Layers[layerIdx].Tree - for _, ref := range tree.AllFiles() { + for _, ref := range tree.AllFiles(file.AllTypes...) { results <- NewLocationFromImage(string(ref.RealPath), ref, r.img) } } diff --git a/syft/source/file_metadata_test.go b/syft/source/file_metadata_test.go new file mode 100644 index 000000000..4092e4374 --- /dev/null +++ b/syft/source/file_metadata_test.go @@ -0,0 +1,122 @@ +package source + +import ( + "os" + "testing" + + "github.com/anchore/stereoscope/pkg/file" + "github.com/anchore/stereoscope/pkg/imagetest" + "github.com/stretchr/testify/assert" +) + +func TestFileMetadataFetch(t *testing.T) { + img := imagetest.GetFixtureImage(t, "docker-archive", "image-file-type-mix") + + tests := []struct { + path string + exists bool + expected FileMetadata + err bool + }{ + { + path: "/file-1.txt", + exists: true, + expected: FileMetadata{ + Mode: 0644, + Type: "regularFile", + UserID: 1, + GroupID: 2, + }, + }, + { + path: "/hardlink-1", + exists: true, + expected: FileMetadata{ + Mode: 0644, + Type: "hardLink", + UserID: 1, + GroupID: 2, + }, + }, + { + path: "/symlink-1", + exists: true, + expected: FileMetadata{ + Mode: 0777 | os.ModeSymlink, + Type: "symbolicLink", + UserID: 0, + GroupID: 0, + }, + }, + { + path: "/char-device-1", + exists: true, + expected: FileMetadata{ + Mode: 0644 | os.ModeDevice | os.ModeCharDevice, + Type: "characterDevice", + UserID: 0, + GroupID: 0, + }, + }, + { + path: "/block-device-1", + exists: true, + expected: FileMetadata{ + Mode: 0644 | os.ModeDevice, + Type: "blockDevice", + UserID: 0, + GroupID: 0, + }, + }, + { + path: "/fifo-1", + exists: true, + expected: FileMetadata{ + Mode: 0644 | os.ModeNamedPipe, + Type: "fifoNode", + UserID: 0, + GroupID: 0, + }, + }, + { + path: "/bin", + exists: true, + expected: FileMetadata{ + Mode: 0755 | os.ModeDir, + Type: "directory", + UserID: 0, + GroupID: 0, + }, + }, + } + + for _, test := range tests { + t.Run(test.path, func(t *testing.T) { + exists, ref, err := img.SquashedTree().File(file.Path(test.path)) + if err != nil { + t.Fatalf("unable to get file: %+v", err) + } + + if exists && !test.exists { + t.Fatalf("file=%q exists but shouldn't", test.path) + } else if !exists && test.exists { + t.Fatalf("file=%q does not exist but should", test.path) + } else if !exists && !test.exists { + return + } + + actual, err := fileMetadataByLocation(img, NewLocationFromImage(test.path, *ref, img)) + if err != nil && !test.err { + t.Fatalf("could not fetch (but should have been able to): %+v", err) + } else if err == nil && test.err { + t.Fatalf("expected fetch error but did not get one") + } else if test.err && err != nil { + return + } + + assert.Equal(t, test.expected, actual, "file metadata mismatch") + + }) + } + +} diff --git a/syft/source/image_squash_resolver.go b/syft/source/image_squash_resolver.go index 3500a80a2..de51edf11 100644 --- a/syft/source/image_squash_resolver.go +++ b/syft/source/image_squash_resolver.go @@ -144,7 +144,7 @@ func (r *imageSquashResolver) AllLocations() <-chan Location { results := make(chan Location) go func() { defer close(results) - for _, ref := range r.img.SquashedTree().AllFiles() { + for _, ref := range r.img.SquashedTree().AllFiles(file.AllTypes...) { results <- NewLocationFromImage(string(ref.RealPath), ref, r.img) } }() diff --git a/syft/source/mock_resolver.go b/syft/source/mock_resolver.go index 31d129449..b574ccd14 100644 --- a/syft/source/mock_resolver.go +++ b/syft/source/mock_resolver.go @@ -15,6 +15,7 @@ var _ FileResolver = (*MockResolver)(nil) // paths, which are typically paths to test fixtures. type MockResolver struct { Locations []Location + Metadata map[Location]FileMetadata } // NewMockResolverForPaths creates a new MockResolver, where the only resolvable @@ -28,6 +29,15 @@ func NewMockResolverForPaths(paths ...string) *MockResolver { return &MockResolver{Locations: locations} } +func NewMockResolverForPathsWithMetadata(metadata map[Location]FileMetadata) *MockResolver { + var locations []Location + for p := range metadata { + locations = append(locations, p) + } + + return &MockResolver{Locations: locations, Metadata: metadata} +} + // HasPath indicates if the given path exists in the underlying source. func (r MockResolver) HasPath(path string) bool { for _, l := range r.Locations { @@ -98,7 +108,14 @@ func (r MockResolver) RelativeFileByPath(_ Location, path string) *Location { } func (r MockResolver) AllLocations() <-chan Location { - panic("not implemented") + results := make(chan Location) + go func() { + defer close(results) + for _, l := range r.Locations { + results <- l + } + }() + return results } func (r MockResolver) FileMetadataByLocation(Location) (FileMetadata, error) { diff --git a/syft/source/test-fixtures/image-file-type-mix/Dockerfile b/syft/source/test-fixtures/image-file-type-mix/Dockerfile new file mode 100644 index 000000000..218e369d0 --- /dev/null +++ b/syft/source/test-fixtures/image-file-type-mix/Dockerfile @@ -0,0 +1,10 @@ +FROM busybox:latest + +ADD file-1.txt . +RUN chmod 644 file-1.txt +RUN chown 1:2 file-1.txt +RUN ln -s file-1.txt symlink-1 +RUN ln file-1.txt hardlink-1 +RUN mknod char-device-1 c 89 1 +RUN mknod block-device-1 b 0 1 +RUN mknod fifo-1 p \ No newline at end of file diff --git a/syft/source/test-fixtures/image-file-type-mix/file-1.txt b/syft/source/test-fixtures/image-file-type-mix/file-1.txt new file mode 100644 index 000000000..d86db8155 --- /dev/null +++ b/syft/source/test-fixtures/image-file-type-mix/file-1.txt @@ -0,0 +1 @@ +file 1! \ No newline at end of file