Mirrors/syft
2
0
Fork 0
mirror of https://github.com/anchore/syft synced 2024-11-10 06:14:16 +00:00
Code Issues Projects Releases Packages Wiki Activity

add tests around new file metadata cataloger

Browse source 
Signed-off-by: Alex Goodman <alex.goodman@anchore.com>
This commit is contained in:
Alex Goodman 2021-03-23 11:00:59 -04:00
parent 40199096e9
commit d420368ba9
No known key found for this signature in database
GPG key ID: 5CB45AE22BAB7EA7
17 changed files with 407 additions and 24 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

3
go.mod
View file

@ -10,7 +10,7 @@ require (
github.com/anchore/go-rpmdb v0.0.0-20201106153645-0043963c2e12
github.com/anchore/go-testutils v0.0.0-20200925183923-d5f45b0d3c04
github.com/anchore/go-version v1.2.2-0.20200701162849-18adb9c92b9b
github.com/anchore/stereoscope v0.0.0-20210317203852-f77bbcbede40
github.com/anchore/stereoscope v0.0.0-20210323145922-1f45cd8849b4
github.com/antihax/optional v1.0.0
github.com/bmatcuk/doublestar/v2 v2.0.4
github.com/docker/docker v17.12.0-ce-rc1.0.20200309214505-aa6a9891b09c+incompatible
@ -34,6 +34,7 @@ require (
github.com/spf13/cobra v1.0.1-0.20200909172742-8a63648dd905
github.com/spf13/pflag v1.0.5
github.com/spf13/viper v1.7.0
github.com/stretchr/testify v1.6.0
github.com/wagoodman/go-partybus v0.0.0-20200526224238-eb215533f07d
github.com/wagoodman/go-progress v0.0.0-20200731105512-1020f39e6240
github.com/wagoodman/jotframe v0.0.0-20200730190914-3517092dd163

4
go.sum
View file

@ -115,8 +115,8 @@ github.com/anchore/go-testutils v0.0.0-20200925183923-d5f45b0d3c04 h1:VzprUTpc0v
github.com/anchore/go-testutils v0.0.0-20200925183923-d5f45b0d3c04/go.mod h1:6dK64g27Qi1qGQZ67gFmBFvEHScy0/C8qhQhNe5B5pQ=
github.com/anchore/go-version v1.2.2-0.20200701162849-18adb9c92b9b h1:e1bmaoJfZVsCYMrIZBpFxwV26CbsuoEh5muXD5I1Ods=
github.com/anchore/go-version v1.2.2-0.20200701162849-18adb9c92b9b/go.mod h1:Bkc+JYWjMCF8OyZ340IMSIi2Ebf3uwByOk6ho4wne1E=
github.com/anchore/stereoscope v0.0.0-20210317203852-f77bbcbede40 h1:k3/JigkYl7NjMad9eDBBcsg9qiXJFreW6rKNgE0aMUI=
github.com/anchore/stereoscope v0.0.0-20210317203852-f77bbcbede40/go.mod h1:T/OUHXgngXFlo2vknQsDx+n/jErCLPt5o0H1ZXFBpig=
github.com/anchore/stereoscope v0.0.0-20210323145922-1f45cd8849b4 h1:Uuvne+/Mgeyu3fR1JCxiFUQQo2Gp5vXTInum3GbhbwM=
github.com/anchore/stereoscope v0.0.0-20210323145922-1f45cd8849b4/go.mod h1:G7tFR0iI9r6AvibmXKA9v010pRS1IIJgd0t6fOMDxCw=
github.com/andreyvit/diff v0.0.0-20170406064948-c7f18ee00883/go.mod h1:rCTlJbsFo29Kk6CurOXKm700vrz8f0KW0JNfpkRJY/8=
github.com/antihax/optional v1.0.0 h1:xK2lYat7ZLaVVcIuj82J8kIro4V6kDe0AUDFboUCwcg=
github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY=

18
syft/file/digest_cataloger.go
View file

@ -13,8 +13,7 @@ import (
var supportedHashAlgorithms = make(map[string]crypto.Hash)
type DigestsCataloger struct {
resolver source.FileResolver
hashes []crypto.Hash
hashes []crypto.Hash
}
func init() {
@ -27,7 +26,7 @@ func init() {
}
}
func NewDigestsCataloger(resolver source.FileResolver, hashAlgorithms []string) (*DigestsCataloger, error) {
func NewDigestsCataloger(hashAlgorithms []string) (*DigestsCataloger, error) {
var hashes []crypto.Hash
for _, hashStr := range hashAlgorithms {
name := cleanAlgorithmName(hashStr)
@ -39,15 +38,14 @@ func NewDigestsCataloger(resolver source.FileResolver, hashAlgorithms []string)
}
return &DigestsCataloger{
resolver: resolver,
hashes: hashes,
hashes: hashes,
}, nil
}
func (i *DigestsCataloger) Catalog() (map[source.Location][]Digest, error) {
func (i *DigestsCataloger) Catalog(resolver source.FileResolver) (map[source.Location][]Digest, error) {
results := make(map[source.Location][]Digest)
for location := range i.resolver.AllLocations() {
result, err := i.catalogLocation(location)
for location := range resolver.AllLocations() {
result, err := i.catalogLocation(resolver, location)
if err != nil {
return nil, err
}
@ -56,8 +54,8 @@ func (i *DigestsCataloger) Catalog() (map[source.Location][]Digest, error) {
return results, nil
}
func (i *DigestsCataloger) catalogLocation(location source.Location) ([]Digest, error) {
contentReader, err := i.resolver.FileContentsByLocation(location)
func (i *DigestsCataloger) catalogLocation(resolver source.FileResolver, location source.Location) ([]Digest, error) {
contentReader, err := resolver.FileContentsByLocation(location)
if err != nil {
return nil, err
}

98
syft/file/digest_cataloger_test.go Normal file
View file

@ -0,0 +1,98 @@
package file
import (
"crypto"
"fmt"
"io/ioutil"
"os"
"testing"
"github.com/stretchr/testify/assert"
"github.com/anchore/syft/syft/source"
)
func testDigests(t testing.TB, files []string, hashes ...crypto.Hash) map[source.Location][]Digest {
digests := make(map[source.Location][]Digest)
for _, f := range files {
fh, err := os.Open(f)
if err != nil {
t.Fatalf("could not open %q : %+v", f, err)
}
b, err := ioutil.ReadAll(fh)
if err != nil {
t.Fatalf("could not read %q : %+v", f, err)
}
for _, hash := range hashes {
h := hash.New()
h.Write(b)
digests[source.NewLocation(f)] = append(digests[source.NewLocation(f)], Digest{
Algorithm: cleanAlgorithmName(hash.String()),
Value: fmt.Sprintf("%x", h.Sum(nil)),
})
}
}
return digests
}
func TestDigestsCataloger(t *testing.T) {
files := []string{"test-fixtures/last/path.txt", "test-fixtures/another-path.txt", "test-fixtures/a-path.txt"}
tests := []struct {
name string
algorithms []string
expected map[source.Location][]Digest
constructorErr bool
catalogErr bool
}{
{
name: "bad algorithm",
algorithms: []string{"sha-nothing"},
constructorErr: true,
},
{
name: "unsupported algorithm",
algorithms: []string{"sha512"},
constructorErr: true,
},
{
name: "md5-sha1-sha256",
algorithms: []string{"md5"},
expected: testDigests(t, files, crypto.MD5),
},
{
name: "md5-sha1-sha256",
algorithms: []string{"md5", "sha1", "sha256"},
expected: testDigests(t, files, crypto.MD5, crypto.SHA1, crypto.SHA256),
},
}
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
c, err := NewDigestsCataloger(test.algorithms)
if err != nil && !test.constructorErr {
t.Fatalf("could not create cataloger (but should have been able to): %+v", err)
} else if err == nil && test.constructorErr {
t.Fatalf("expected constructor error but did not get one")
} else if test.constructorErr && err != nil {
return
}
resolver := source.NewMockResolverForPaths(files...)
actual, err := c.Catalog(resolver)
if err != nil && !test.catalogErr {
t.Fatalf("could not catalog (but should have been able to): %+v", err)
} else if err == nil && test.catalogErr {
t.Fatalf("expected catalog error but did not get one")
} else if test.catalogErr && err != nil {
return
}
assert.Equal(t, actual, test.expected, "mismatched digests")
})
}
}

13
syft/file/metadata_cataloger.go
View file

@ -5,19 +5,16 @@ import (
)
type MetadataCataloger struct {
resolver source.FileResolver
}
func NewMetadataCataloger(resolver source.FileResolver) *MetadataCataloger {
return &MetadataCataloger{
resolver: resolver,
}
func NewMetadataCataloger() *MetadataCataloger {
return &MetadataCataloger{}
}
func (i *MetadataCataloger) Catalog() (map[source.Location]source.FileMetadata, error) {
func (i *MetadataCataloger) Catalog(resolver source.FileResolver) (map[source.Location]source.FileMetadata, error) {
results := make(map[source.Location]source.FileMetadata)
for location := range i.resolver.AllLocations() {
metadata, err := i.resolver.FileMetadataByLocation(location)
for location := range resolver.AllLocations() {
metadata, err := resolver.FileMetadataByLocation(location)
if err != nil {
return nil, err
}

125
syft/file/metadata_cataloger_test.go Normal file
View file

@ -0,0 +1,125 @@
package file
import (
"os"
"testing"
"github.com/anchore/stereoscope/pkg/file"
"github.com/anchore/stereoscope/pkg/imagetest"
"github.com/anchore/syft/syft/source"
"github.com/stretchr/testify/assert"
)
func TestFileMetadataFetch(t *testing.T) {
img := imagetest.GetFixtureImage(t, "docker-archive", "image-file-type-mix")
c := NewMetadataCataloger()
src, err := source.NewFromImage(img, "---")
if err != nil {
t.Fatalf("could not create source: %+v", err)
}
resolver, err := src.FileResolver(source.SquashedScope)
if err != nil {
t.Fatalf("could not create resolver: %+v", err)
}
actual, err := c.Catalog(resolver)
if err != nil {
t.Fatalf("could not catalog: %+v", err)
}
tests := []struct {
path string
exists bool
expected source.FileMetadata
err bool
}{
{
path: "/file-1.txt",
exists: true,
expected: source.FileMetadata{
Mode: 0644,
Type: "regularFile",
UserID: 1,
GroupID: 2,
},
},
{
path: "/hardlink-1",
exists: true,
expected: source.FileMetadata{
Mode: 0644,
Type: "hardLink",
UserID: 1,
GroupID: 2,
},
},
{
path: "/symlink-1",
exists: true,
expected: source.FileMetadata{
Mode: 0777 | os.ModeSymlink,
Type: "symbolicLink",
UserID: 0,
GroupID: 0,
},
},
{
path: "/char-device-1",
exists: true,
expected: source.FileMetadata{
Mode: 0644 | os.ModeDevice | os.ModeCharDevice,
Type: "characterDevice",
UserID: 0,
GroupID: 0,
},
},
{
path: "/block-device-1",
exists: true,
expected: source.FileMetadata{
Mode: 0644 | os.ModeDevice,
Type: "blockDevice",
UserID: 0,
GroupID: 0,
},
},
{
path: "/fifo-1",
exists: true,
expected: source.FileMetadata{
Mode: 0644 | os.ModeNamedPipe,
Type: "fifoNode",
UserID: 0,
GroupID: 0,
},
},
{
path: "/bin",
exists: true,
expected: source.FileMetadata{
Mode: 0755 | os.ModeDir,
Type: "directory",
UserID: 0,
GroupID: 0,
},
},
}
for _, test := range tests {
t.Run(test.path, func(t *testing.T) {
_, ref, err := img.SquashedTree().File(file.Path(test.path))
if err != nil {
t.Fatalf("unable to get file: %+v", err)
}
l := source.NewLocationFromImage(test.path, *ref, img)
assert.Equal(t, actual[l], test.expected, "mismatched metadata")
})
}
}

1
syft/file/test-fixtures/a-path.txt Normal file
View file

@ -0,0 +1 @@
test-fixtures/a-path.txt file contents!

1
syft/file/test-fixtures/another-path.txt Normal file
View file

@ -0,0 +1 @@
test-fixtures/another-path.txt file contents!

10
syft/file/test-fixtures/image-file-type-mix/Dockerfile Normal file
View file

@ -0,0 +1,10 @@
FROM busybox:latest
ADD file-1.txt .
RUN chmod 644 file-1.txt
RUN chown 1:2 file-1.txt
RUN ln -s file-1.txt symlink-1
RUN ln file-1.txt hardlink-1
RUN mknod char-device-1 c 89 1
RUN mknod block-device-1 b 0 1
RUN mknod fifo-1 p

1
syft/file/test-fixtures/image-file-type-mix/file-1.txt Normal file
View file

@ -0,0 +1 @@
file 1!

1
syft/file/test-fixtures/last/path.txt Normal file
View file

@ -0,0 +1 @@
test-fixtures/last/path.txt file contents!

2
syft/source/all_layers_resolver.go
View file

@ -194,7 +194,7 @@ func (r *allLayersResolver) AllLocations() <-chan Location {
defer close(results)
for _, layerIdx := range r.layers {
tree := r.img.Layers[layerIdx].Tree
for _, ref := range tree.AllFiles() {
for _, ref := range tree.AllFiles(file.AllTypes...) {
results <- NewLocationFromImage(string(ref.RealPath), ref, r.img)
}
}

122
syft/source/file_metadata_test.go Normal file
View file

@ -0,0 +1,122 @@
package source
import (
"os"
"testing"
"github.com/anchore/stereoscope/pkg/file"
"github.com/anchore/stereoscope/pkg/imagetest"
"github.com/stretchr/testify/assert"
)
func TestFileMetadataFetch(t *testing.T) {
img := imagetest.GetFixtureImage(t, "docker-archive", "image-file-type-mix")
tests := []struct {
path string
exists bool
expected FileMetadata
err bool
}{
{
path: "/file-1.txt",
exists: true,
expected: FileMetadata{
Mode: 0644,
Type: "regularFile",
UserID: 1,
GroupID: 2,
},
},
{
path: "/hardlink-1",
exists: true,
expected: FileMetadata{
Mode: 0644,
Type: "hardLink",
UserID: 1,
GroupID: 2,
},
},
{
path: "/symlink-1",
exists: true,
expected: FileMetadata{
Mode: 0777 | os.ModeSymlink,
Type: "symbolicLink",
UserID: 0,
GroupID: 0,
},
},
{
path: "/char-device-1",
exists: true,
expected: FileMetadata{
Mode: 0644 | os.ModeDevice | os.ModeCharDevice,
Type: "characterDevice",
UserID: 0,
GroupID: 0,
},
},
{
path: "/block-device-1",
exists: true,
expected: FileMetadata{
Mode: 0644 | os.ModeDevice,
Type: "blockDevice",
UserID: 0,
GroupID: 0,
},
},
{
path: "/fifo-1",
exists: true,
expected: FileMetadata{
Mode: 0644 | os.ModeNamedPipe,
Type: "fifoNode",
UserID: 0,
GroupID: 0,
},
},
{
path: "/bin",
exists: true,
expected: FileMetadata{
Mode: 0755 | os.ModeDir,
Type: "directory",
UserID: 0,
GroupID: 0,
},
},
}
for _, test := range tests {
t.Run(test.path, func(t *testing.T) {
exists, ref, err := img.SquashedTree().File(file.Path(test.path))
if err != nil {
t.Fatalf("unable to get file: %+v", err)
}
if exists && !test.exists {
t.Fatalf("file=%q exists but shouldn't", test.path)
} else if !exists && test.exists {
t.Fatalf("file=%q does not exist but should", test.path)
} else if !exists && !test.exists {
return
}
actual, err := fileMetadataByLocation(img, NewLocationFromImage(test.path, *ref, img))
if err != nil && !test.err {
t.Fatalf("could not fetch (but should have been able to): %+v", err)
} else if err == nil && test.err {
t.Fatalf("expected fetch error but did not get one")
} else if test.err && err != nil {
return
}
assert.Equal(t, test.expected, actual, "file metadata mismatch")
})
}
}

2
syft/source/image_squash_resolver.go
View file

@ -144,7 +144,7 @@ func (r *imageSquashResolver) AllLocations() <-chan Location {
results := make(chan Location)
go func() {
defer close(results)
for _, ref := range r.img.SquashedTree().AllFiles() {
for _, ref := range r.img.SquashedTree().AllFiles(file.AllTypes...) {
results <- NewLocationFromImage(string(ref.RealPath), ref, r.img)
}
}()

19
syft/source/mock_resolver.go
View file

@ -15,6 +15,7 @@ var _ FileResolver = (*MockResolver)(nil)
// paths, which are typically paths to test fixtures.
type MockResolver struct {
Locations []Location
Metadata map[Location]FileMetadata
}
// NewMockResolverForPaths creates a new MockResolver, where the only resolvable
@ -28,6 +29,15 @@ func NewMockResolverForPaths(paths ...string) *MockResolver {
return &MockResolver{Locations: locations}
}
func NewMockResolverForPathsWithMetadata(metadata map[Location]FileMetadata) *MockResolver {
var locations []Location
for p := range metadata {
locations = append(locations, p)
}
return &MockResolver{Locations: locations, Metadata: metadata}
}
// HasPath indicates if the given path exists in the underlying source.
func (r MockResolver) HasPath(path string) bool {
for _, l := range r.Locations {
@ -98,7 +108,14 @@ func (r MockResolver) RelativeFileByPath(_ Location, path string) *Location {
}
func (r MockResolver) AllLocations() <-chan Location {
panic("not implemented")
results := make(chan Location)
go func() {
defer close(results)
for _, l := range r.Locations {
results <- l
}
}()
return results
}
func (r MockResolver) FileMetadataByLocation(Location) (FileMetadata, error) {

10
syft/source/test-fixtures/image-file-type-mix/Dockerfile Normal file
View file

@ -0,0 +1,10 @@
FROM busybox:latest
ADD file-1.txt .
RUN chmod 644 file-1.txt
RUN chown 1:2 file-1.txt
RUN ln -s file-1.txt symlink-1
RUN ln file-1.txt hardlink-1
RUN mknod char-device-1 c 89 1
RUN mknod block-device-1 b 0 1
RUN mknod fifo-1 p

1
syft/source/test-fixtures/image-file-type-mix/file-1.txt Normal file
View file

@ -0,0 +1 @@
file 1!