Mirrors/syft
2
0
Fork 0
mirror of https://github.com/anchore/syft synced 2024-11-10 06:14:16 +00:00
Code Issues Projects Releases Packages Wiki Activity

bump stereoscope to pull in content API refactors

Browse source 
Signed-off-by: Alex Goodman <alex.goodman@anchore.com>
This commit is contained in:
Alex Goodman 2020-12-15 16:07:11 -05:00 committed by Dan Luhring
parent e299a5355f
commit d475e6280a
No known key found for this signature in database
GPG key ID: 9CEE23D079426CEF
18 changed files with 99 additions and 94 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
go.mod
View file

@ -9,7 +9,7 @@ require (
github.com/anchore/go-rpmdb v0.0.0-20201106153645-0043963c2e12
github.com/anchore/go-testutils v0.0.0-20200925183923-d5f45b0d3c04
github.com/anchore/go-version v1.2.2-0.20200701162849-18adb9c92b9b
github.com/anchore/stereoscope v0.0.0-20201210022249-091f9bddb42e
github.com/anchore/stereoscope v0.0.0-20201219143203-af397ece87bf
github.com/antihax/optional v1.0.0
github.com/bmatcuk/doublestar v1.3.3
github.com/docker/docker v17.12.0-ce-rc1.0.20200309214505-aa6a9891b09c+incompatible

2
go.sum
View file

@ -138,6 +138,8 @@ github.com/anchore/go-version v1.2.2-0.20200701162849-18adb9c92b9b h1:e1bmaoJfZV
github.com/anchore/go-version v1.2.2-0.20200701162849-18adb9c92b9b/go.mod h1:Bkc+JYWjMCF8OyZ340IMSIi2Ebf3uwByOk6ho4wne1E=
github.com/anchore/stereoscope v0.0.0-20201210022249-091f9bddb42e h1:vHUqHTvH9/oxdDDh1fxS9Ls9gWGytKO7XbbzcQ9MBwI=
github.com/anchore/stereoscope v0.0.0-20201210022249-091f9bddb42e/go.mod h1:/dHAFjYflH/1tzhdHAcnMCjprMch+YzHJKi59m/1KCM=
github.com/anchore/stereoscope v0.0.0-20201219143203-af397ece87bf h1:4sN/HJ6whcrK/HxORFGAQUWM58Q7EFiPmoxRKcEs76A=
github.com/anchore/stereoscope v0.0.0-20201219143203-af397ece87bf/go.mod h1:/dHAFjYflH/1tzhdHAcnMCjprMch+YzHJKi59m/1KCM=
github.com/andreyvit/diff v0.0.0-20170406064948-c7f18ee00883/go.mod h1:rCTlJbsFo29Kk6CurOXKm700vrz8f0KW0JNfpkRJY/8=
github.com/antihax/optional v1.0.0 h1:xK2lYat7ZLaVVcIuj82J8kIro4V6kDe0AUDFboUCwcg=
github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY=

6
syft/cataloger/common/generic_cataloger.go
View file

@ -4,7 +4,7 @@ Package common provides generic utilities used by multiple catalogers.
package common
import (
"strings"
"io"
"github.com/anchore/syft/internal/log"
"github.com/anchore/syft/syft/pkg"
@ -89,7 +89,7 @@ func (c *GenericCataloger) selectFiles(resolver source.FileResolver) []source.Lo
}
// catalog takes a set of file contents and uses any configured parser functions to resolve and return discovered packages
func (c *GenericCataloger) catalog(contents map[source.Location]string) ([]pkg.Package, error) {
func (c *GenericCataloger) catalog(contents map[source.Location]io.ReadCloser) ([]pkg.Package, error) {
defer c.clear()
packages := make([]pkg.Package, 0)
@ -101,7 +101,7 @@ func (c *GenericCataloger) catalog(contents map[source.Location]string) ([]pkg.P
continue
}
entries, err := parser(location.Path, strings.NewReader(content))
entries, err := parser(location.Path, content)
if err != nil {
// TODO: should we fail? or only log?
log.Warnf("cataloger '%s' failed to parse entries (location=%+v): %+v", c.upstreamCataloger, location, err)

18
syft/cataloger/common/generic_cataloger_test.go
View file

@ -4,28 +4,28 @@ import (
"fmt"
"io"
"io/ioutil"
"strings"
"testing"
"github.com/anchore/syft/syft/source"
"github.com/anchore/syft/syft/pkg"
"github.com/anchore/syft/syft/source"
)
type testResolverMock struct {
contents map[source.Location]string
contents map[source.Location]io.ReadCloser
}
func newTestResolver() *testResolverMock {
return &testResolverMock{
contents: make(map[source.Location]string),
contents: make(map[source.Location]io.ReadCloser),
}
}
func (r *testResolverMock) FileContentsByLocation(_ source.Location) (string, error) {
return "", fmt.Errorf("not implemented")
func (r *testResolverMock) FileContentsByLocation(_ source.Location) (io.ReadCloser, error) {
return nil, fmt.Errorf("not implemented")
}
func (r *testResolverMock) MultipleFileContentsByLocation([]source.Location) (map[source.Location]string, error) {
func (r *testResolverMock) MultipleFileContentsByLocation([]source.Location) (map[source.Location]io.ReadCloser, error) {
return r.contents, nil
}
@ -34,7 +34,7 @@ func (r *testResolverMock) FilesByPath(paths ...string) ([]source.Location, erro
for idx, p := range paths {
results[idx] = source.NewLocation(p)
r.contents[results[idx]] = fmt.Sprintf("%s file contents!", p)
r.contents[results[idx]] = ioutil.NopCloser(strings.NewReader(fmt.Sprintf("%s file contents!", p)))
}
return results, nil
@ -43,7 +43,7 @@ func (r *testResolverMock) FilesByPath(paths ...string) ([]source.Location, erro
func (r *testResolverMock) FilesByGlob(_ ...string) ([]source.Location, error) {
path := "/a-path.txt"
location := source.NewLocation(path)
r.contents[location] = fmt.Sprintf("%s file contents!", path)
r.contents[location] = ioutil.NopCloser(strings.NewReader(fmt.Sprintf("%s file contents!", path)))
return []source.Location{location}, nil
}

7
syft/cataloger/deb/cataloger.go
View file

@ -8,7 +8,6 @@ import (
"io"
"path"
"path/filepath"
"strings"
"github.com/anchore/syft/syft/pkg"
"github.com/anchore/syft/syft/source"
@ -47,7 +46,7 @@ func (c *Cataloger) Catalog(resolver source.Resolver) ([]pkg.Package, error) {
return nil, err
}
pkgs, err = parseDpkgStatus(strings.NewReader(dbContents))
pkgs, err = parseDpkgStatus(dbContents)
if err != nil {
return nil, fmt.Errorf("unable to catalog dpkg package=%+v: %w", dbLocation.Path, err)
}
@ -138,7 +137,7 @@ func fetchMd5Contents(resolver source.Resolver, dbLocation source.Location, pkgs
var refsByName = make(map[string]source.Location)
for location, contents := range md5ContentsByLocation {
name := nameByRef[location]
contentsByName[name] = strings.NewReader(contents)
contentsByName[name] = contents
refsByName[name] = location
}
@ -174,7 +173,7 @@ func fetchCopyrightContents(resolver source.Resolver, dbLocation source.Location
var refsByName = make(map[string]source.Location)
for location, contents := range copyrightContentsByLocation {
name := nameByLocation[location]
contentsByName[name] = strings.NewReader(contents)
contentsByName[name] = contents
refsByName[name] = location
}

7
syft/cataloger/python/package_cataloger.go
View file

@ -3,7 +3,6 @@ package python
import (
"bufio"
"fmt"
"strings"
"github.com/anchore/syft/syft/pkg"
@ -119,7 +118,7 @@ func (c *PackageCataloger) catalogEggOrWheel(entry *packageEntry) (*pkg.Package,
func (c *PackageCataloger) assembleEggOrWheelMetadata(entry *packageEntry) (*pkg.PythonPackageMetadata, []source.Location, error) {
var sources = []source.Location{entry.Metadata.Location}
metadata, err := parseWheelOrEggMetadata(entry.Metadata.Location.Path, strings.NewReader(entry.Metadata.Contents))
metadata, err := parseWheelOrEggMetadata(entry.Metadata.Location.Path, entry.Metadata.Contents)
if err != nil {
return nil, nil, err
}
@ -153,7 +152,7 @@ func (c *PackageCataloger) processRecordFiles(entry *source.FileData) (files []p
sources = append(sources, entry.Location)
// parse the record contents
records, err := parseWheelOrEggRecord(strings.NewReader(entry.Contents))
records, err := parseWheelOrEggRecord(entry.Contents)
if err != nil {
return nil, nil, err
}
@ -171,7 +170,7 @@ func (c *PackageCataloger) processTopLevelPackages(entry *source.FileData) (pkgs
sources = append(sources, entry.Location)
scanner := bufio.NewScanner(strings.NewReader(entry.Contents))
scanner := bufio.NewScanner(entry.Contents)
for scanner.Scan() {
pkgs = append(pkgs, scanner.Text())
}

5
syft/cataloger/rpmdb/cataloger.go
View file

@ -5,7 +5,6 @@ package rpmdb
import (
"fmt"
"strings"
"github.com/anchore/syft/syft/pkg"
"github.com/anchore/syft/syft/source"
@ -37,12 +36,12 @@ func (c *Cataloger) Catalog(resolver source.Resolver) ([]pkg.Package, error) {
var pkgs []pkg.Package
for _, location := range fileMatches {
dbContents, err := resolver.FileContentsByLocation(location)
dbContentReader, err := resolver.FileContentsByLocation(location)
if err != nil {
return nil, err
}
pkgs, err = parseRpmDB(resolver, location, strings.NewReader(dbContents))
pkgs, err = parseRpmDB(resolver, location, dbContentReader)
if err != nil {
return nil, fmt.Errorf("unable to catalog rpmdb package=%+v: %w", location.Path, err)
}

49
syft/distro/identify.go
View file

@ -1,6 +1,7 @@
package distro
import (
"io/ioutil"
"regexp"
"strings"
@ -16,28 +17,28 @@ type parseEntry struct {
fn parseFunc
}
var identityFiles = []parseEntry{
{
// most distros provide a link at this location
path: "/etc/os-release",
fn: parseOsRelease,
},
{
// standard location for rhel & debian distros
path: "/usr/lib/os-release",
fn: parseOsRelease,
},
{
// check for busybox (important to check this last since other distros contain the busybox binary)
path: "/bin/busybox",
fn: parseBusyBox,
},
}
// Identify parses distro-specific files to determine distro metadata like version and release.
func Identify(resolver source.Resolver) *Distro {
var distro *Distro
identityFiles := []parseEntry{
{
// most distros provide a link at this location
path: "/etc/os-release",
fn: parseOsRelease,
},
{
// standard location for rhel & debian distros
path: "/usr/lib/os-release",
fn: parseOsRelease,
},
{
// check for busybox (important to check this last since other distros contain the busybox binary)
path: "/bin/busybox",
fn: parseBusyBox,
},
}
identifyLoop:
for _, entry := range identityFiles {
locations, err := resolver.FilesByPath(entry.path)
@ -52,19 +53,25 @@ identifyLoop:
}
for _, location := range locations {
content, err := resolver.FileContentsByLocation(location)
contentReader, err := resolver.FileContentsByLocation(location)
if err != nil {
log.Debugf("unable to get contents from %s: %s", entry.path, err)
continue
}
if content == "" {
content, err := ioutil.ReadAll(contentReader)
if err != nil {
log.Errorf("unable to read %q: %+v", location.Path, err)
break
}
if len(content) == 0 {
log.Debugf("no contents in file, skipping: %s", entry.path)
continue
}
if candidateDistro := entry.fn(content); candidateDistro != nil {
if candidateDistro := entry.fn(string(content)); candidateDistro != nil {
distro = candidateDistro
break identifyLoop
}

13
syft/source/all_layers_resolver.go
View file

@ -3,6 +3,7 @@ package source
import (
"archive/tar"
"fmt"
"io"
"github.com/anchore/stereoscope/pkg/file"
"github.com/anchore/stereoscope/pkg/image"
@ -150,7 +151,7 @@ func (r *AllLayersResolver) RelativeFileByPath(location Location, path string) *
return nil
}
relativeRef := entry.Source.SquashedTree.File(file.Path(path))
relativeRef := entry.Layer.SquashedTree.File(file.Path(path))
if relativeRef == nil {
return nil
}
@ -162,22 +163,22 @@ func (r *AllLayersResolver) RelativeFileByPath(location Location, path string) *
// MultipleFileContentsByLocation returns the file contents for all file.References relative to the image. Note that a
// file.Reference is a path relative to a particular layer.
func (r *AllLayersResolver) MultipleFileContentsByLocation(locations []Location) (map[Location]string, error) {
func (r *AllLayersResolver) MultipleFileContentsByLocation(locations []Location) (map[Location]io.ReadCloser, error) {
return mapLocationRefs(r.img.MultipleFileContentsByRef, locations)
}
// FileContentsByLocation fetches file contents for a single file reference, irregardless of the source layer.
// If the path does not exist an error is returned.
func (r *AllLayersResolver) FileContentsByLocation(location Location) (string, error) {
func (r *AllLayersResolver) FileContentsByLocation(location Location) (io.ReadCloser, error) {
return r.img.FileContentsByRef(location.ref)
}
type multiContentFetcher func(refs ...file.Reference) (map[file.Reference]string, error)
type multiContentFetcher func(refs ...file.Reference) (map[file.Reference]io.ReadCloser, error)
func mapLocationRefs(callback multiContentFetcher, locations []Location) (map[Location]string, error) {
func mapLocationRefs(callback multiContentFetcher, locations []Location) (map[Location]io.ReadCloser, error) {
var fileRefs = make([]file.Reference, len(locations))
var locationByRefs = make(map[file.Reference]Location)
var results = make(map[Location]string)
var results = make(map[Location]io.ReadCloser)
for i, location := range locations {
locationByRefs[location.ref] = location

8
syft/source/all_layers_resolver_test.go
View file

@ -116,8 +116,8 @@ func TestAllLayersResolver_FilesByPath(t *testing.T) {
t.Fatalf("failed to get metadata: %+v", err)
}
if entry.Source.Metadata.Index != expected.layer {
t.Errorf("bad resolve layer: '%d'!='%d'", entry.Source.Metadata.Index, expected.layer)
if entry.Layer.Metadata.Index != expected.layer {
t.Errorf("bad resolve layer: '%d'!='%d'", entry.Layer.Metadata.Index, expected.layer)
}
}
})
@ -229,8 +229,8 @@ func TestAllLayersResolver_FilesByGlob(t *testing.T) {
t.Fatalf("failed to get metadata: %+v", err)
}
if entry.Source.Metadata.Index != expected.layer {
t.Errorf("bad resolve layer: '%d'!='%d'", entry.Source.Metadata.Index, expected.layer)
if entry.Layer.Metadata.Index != expected.layer {
t.Errorf("bad resolve layer: '%d'!='%d'", entry.Layer.Metadata.Index, expected.layer)
}
}
})

9
syft/source/content_requester_test.go
View file

@ -1,6 +1,7 @@
package source
import (
"io/ioutil"
"testing"
"github.com/anchore/stereoscope/pkg/imagetest"
@ -54,10 +55,14 @@ func TestContentRequester(t *testing.T) {
for _, entry := range data {
if expected, ok := test.expectedContents[entry.Location.Path]; ok {
for expected != entry.Contents {
actualBytes, err := ioutil.ReadAll(entry.Contents)
if err != nil {
t.Fatalf("could not read %q: %+v", entry.Location.Path, err)
}
for expected != string(actualBytes) {
t.Errorf("mismatched contents for %q", entry.Location.Path)
dmp := diffmatchpatch.New()
diffs := dmp.DiffMain(expected, entry.Contents, true)
diffs := dmp.DiffMain(expected, string(actualBytes), true)
t.Errorf("diff: %s", dmp.DiffPrettyText(diffs))
}
continue

33
syft/source/directory_resolver.go
View file

@ -2,11 +2,13 @@ package source
import (
"fmt"
"io/ioutil"
"io"
"os"
"path"
"path/filepath"
"github.com/anchore/stereoscope/pkg/file"
"github.com/anchore/syft/internal/log"
"github.com/bmatcuk/doublestar"
)
@ -96,35 +98,16 @@ func (s *DirectoryResolver) RelativeFileByPath(_ Location, path string) *Locatio
}
// MultipleFileContentsByLocation returns the file contents for all file.References relative a directory.
func (s DirectoryResolver) MultipleFileContentsByLocation(locations []Location) (map[Location]string, error) {
refContents := make(map[Location]string)
func (s DirectoryResolver) MultipleFileContentsByLocation(locations []Location) (map[Location]io.ReadCloser, error) {
refContents := make(map[Location]io.ReadCloser)
for _, location := range locations {
contents, err := fileContents(location.Path)
if err != nil {
return nil, fmt.Errorf("could not read contents of file: %s", location.Path)
}
refContents[location] = string(contents)
refContents[location] = file.NewDeferredReadCloser(location.Path)
}
return refContents, nil
}
// FileContentsByLocation fetches file contents for a single file reference relative to a directory.
// If the path does not exist an error is returned.
func (s DirectoryResolver) FileContentsByLocation(location Location) (string, error) {
contents, err := fileContents(location.Path)
if err != nil {
return "", fmt.Errorf("could not read contents of file: %s", location.Path)
}
return string(contents), nil
}
func fileContents(path string) ([]byte, error) {
contents, err := ioutil.ReadFile(path)
if err != nil {
return nil, err
}
return contents, nil
func (s DirectoryResolver) FileContentsByLocation(location Location) (io.ReadCloser, error) {
return file.NewDeferredReadCloser(location.Path), nil
}

4
syft/source/file_data.go
View file

@ -1,6 +1,8 @@
package source
import "io"
type FileData struct {
Location Location
Contents string
Contents io.ReadCloser
}

5
syft/source/image_squash_resolver.go
View file

@ -2,6 +2,7 @@ package source
import (
"fmt"
"io"
"github.com/anchore/stereoscope/pkg/file"
"github.com/anchore/stereoscope/pkg/image"
@ -121,12 +122,12 @@ func (r *ImageSquashResolver) RelativeFileByPath(_ Location, path string) *Locat
// MultipleFileContentsByLocation returns the file contents for all file.References relative to the image. Note that a
// file.Reference is a path relative to a particular layer, in this case only from the squashed representation.
func (r *ImageSquashResolver) MultipleFileContentsByLocation(locations []Location) (map[Location]string, error) {
func (r *ImageSquashResolver) MultipleFileContentsByLocation(locations []Location) (map[Location]io.ReadCloser, error) {
return mapLocationRefs(r.img.MultipleFileContentsByRef, locations)
}
// FileContentsByLocation fetches file contents for a single file reference, irregardless of the source layer.
// If the path does not exist an error is returned.
func (r *ImageSquashResolver) FileContentsByLocation(location Location) (string, error) {
func (r *ImageSquashResolver) FileContentsByLocation(location Location) (io.ReadCloser, error) {
return r.img.FileContentsByRef(location.ref)
}

8
syft/source/image_squash_resolver_test.go
View file

@ -89,8 +89,8 @@ func TestImageSquashResolver_FilesByPath(t *testing.T) {
t.Fatalf("failed to get metadata: %+v", err)
}
if entry.Source.Metadata.Index != c.resolveLayer {
t.Errorf("bad resolve layer: '%d'!='%d'", entry.Source.Metadata.Index, c.resolveLayer)
if entry.Layer.Metadata.Index != c.resolveLayer {
t.Errorf("bad resolve layer: '%d'!='%d'", entry.Layer.Metadata.Index, c.resolveLayer)
}
})
}
@ -179,8 +179,8 @@ func TestImageSquashResolver_FilesByGlob(t *testing.T) {
t.Fatalf("failed to get metadata: %+v", err)
}
if entry.Source.Metadata.Index != c.resolveLayer {
t.Errorf("bad resolve layer: '%d'!='%d'", entry.Source.Metadata.Index, c.resolveLayer)
if entry.Layer.Metadata.Index != c.resolveLayer {
t.Errorf("bad resolve layer: '%d'!='%d'", entry.Layer.Metadata.Index, c.resolveLayer)
}
})
}

2
syft/source/location.go
View file

@ -34,7 +34,7 @@ func NewLocationFromImage(ref file.Reference, img *image.Image) Location {
return Location{
Path: string(ref.Path),
FileSystemID: entry.Source.Metadata.Digest,
FileSystemID: entry.Layer.Metadata.Digest,
ref: ref,
}
}

5
syft/source/resolver.go
View file

@ -2,6 +2,7 @@ package source
import (
"fmt"
"io"
"github.com/anchore/stereoscope/pkg/image"
)
@ -14,8 +15,8 @@ type Resolver interface {
// ContentResolver knows how to get file content for given file.References
type ContentResolver interface {
FileContentsByLocation(Location) (string, error)
MultipleFileContentsByLocation([]Location) (map[Location]string, error)
FileContentsByLocation(Location) (io.ReadCloser, error)
MultipleFileContentsByLocation([]Location) (map[Location]io.ReadCloser, error)
// TODO: we should consider refactoring to return a set of io.Readers or file.Openers instead of the full contents themselves (allow for optional buffering).
}

10
syft/source/source_test.go
View file

@ -1,6 +1,7 @@
package source
import (
"io/ioutil"
"os"
"testing"
@ -137,9 +138,14 @@ func TestMultipleFileContentsByLocation(t *testing.T) {
location := locations[0]
contents, err := p.Resolver.MultipleFileContentsByLocation([]Location{location})
content := contents[location]
contentReader := contents[location]
if content != test.expected {
content, err := ioutil.ReadAll(contentReader)
if err != nil {
t.Fatalf("cannot read contents: %+v", err)
}
if string(content) != test.expected {
t.Errorf("unexpected contents from file: '%s' != '%s'", content, test.expected)
}