Mirrors/syft
2
0
Fork 0
mirror of https://github.com/anchore/syft synced 2024-11-10 06:14:16 +00:00
Code Issues Projects Releases Packages Wiki Activity

allow file metadata digests to be optional + add link destination

Browse source 
Signed-off-by: Alex Goodman <alex.goodman@anchore.com>
This commit is contained in:
Alex Goodman 2021-03-26 17:32:37 -04:00
parent 8551168702
commit db35186c7d
No known key found for this signature in database
GPG key ID: 5CB45AE22BAB7EA7
8 changed files with 523 additions and 51 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

28
internal/presenter/poweruser/json_file_metadata.go
View file

@ -16,11 +16,12 @@ type JSONFileMetadata struct {
}
type JSONFileMetadataEntry struct {
Mode int `json:"mode"`
Type source.FileType `json:"type"`
UserID int `json:"userID"`
GroupID int `json:"groupID"`
Digests []file.Digest `json:"digests"`
Mode int `json:"mode"`
Type source.FileType `json:"type"`
LinkDestination string `json:"linkDestination,omitempty"`
UserID int `json:"userID"`
GroupID int `json:"groupID"`
Digests []file.Digest `json:"digests,omitempty"`
}
func NewJSONFileMetadata(data map[source.Location]source.FileMetadata, digests map[source.Location][]file.Digest) ([]JSONFileMetadata, error) {
@ -31,7 +32,7 @@ func NewJSONFileMetadata(data map[source.Location]source.FileMetadata, digests m
return nil, fmt.Errorf("invalid mode found in file catalog @ location=%+v mode=%q: %w", location, metadata.Mode, err)
}
digestResults := make([]file.Digest, 0)
var digestResults []file.Digest
if digestsForLocation, exists := digests[location]; exists {
digestResults = digestsForLocation
}
@ -39,21 +40,22 @@ func NewJSONFileMetadata(data map[source.Location]source.FileMetadata, digests m
results = append(results, JSONFileMetadata{
Location: location,
Metadata: JSONFileMetadataEntry{
Mode: mode,
Type: metadata.Type,
UserID: metadata.UserID,
GroupID: metadata.GroupID,
Digests: digestResults,
Mode: mode,
Type: metadata.Type,
LinkDestination: metadata.LinkDestination,
UserID: metadata.UserID,
GroupID: metadata.GroupID,
Digests: digestResults,
},
})
}
// sort by real path then virtual path to ensure the result is stable across multiple runs
sort.SliceStable(results, func(i, j int) bool {
if results[i].Location.RealPath != results[j].Location.RealPath {
if results[i].Location.RealPath == results[j].Location.RealPath {
return results[i].Location.VirtualPath < results[j].Location.VirtualPath
}
return false
return results[i].Location.RealPath < results[j].Location.RealPath
})
return results, nil
}

172
internal/presenter/poweruser/json_presenter_test.go Normal file
View file

@ -0,0 +1,172 @@
package poweruser
import (
"bytes"
"flag"
"testing"
"github.com/sergi/go-diff/diffmatchpatch"
"github.com/anchore/syft/syft/file"
"github.com/anchore/go-testutils"
"github.com/anchore/syft/internal/config"
"github.com/anchore/syft/syft/distro"
"github.com/anchore/syft/syft/pkg"
"github.com/anchore/syft/syft/source"
)
var updateJSONGoldenFiles = flag.Bool("update-json", false, "update the *.golden files for json presenters")
func must(c pkg.CPE, e error) pkg.CPE {
if e != nil {
panic(e)
}
return c
}
func TestJSONPresenter(t *testing.T) {
var buffer bytes.Buffer
catalog := pkg.NewCatalog()
catalog.Add(pkg.Package{
ID: "package-1-id",
Name: "package-1",
Version: "1.0.1",
Locations: []source.Location{
{
RealPath: "/a/place/a",
},
},
Type: pkg.PythonPkg,
FoundBy: "the-cataloger-1",
Language: pkg.Python,
MetadataType: pkg.PythonPackageMetadataType,
Licenses: []string{"MIT"},
Metadata: pkg.PythonPackageMetadata{
Name: "package-1",
Version: "1.0.1",
},
PURL: "a-purl-1",
CPEs: []pkg.CPE{
must(pkg.NewCPE("cpe:2.3:*:some:package:1:*:*:*:*:*:*:*")),
},
})
catalog.Add(pkg.Package{
ID: "package-2-id",
Name: "package-2",
Version: "2.0.1",
Locations: []source.Location{
{
RealPath: "/b/place/b",
},
},
Type: pkg.DebPkg,
FoundBy: "the-cataloger-2",
MetadataType: pkg.DpkgMetadataType,
Metadata: pkg.DpkgMetadata{
Package: "package-2",
Version: "2.0.1",
},
PURL: "a-purl-2",
CPEs: []pkg.CPE{
must(pkg.NewCPE("cpe:2.3:*:some:package:2:*:*:*:*:*:*:*")),
},
})
cfg := JSONDocumentConfig{
ApplicationConfig: config.Application{},
PackageCatalog: catalog,
FileMetadata: map[source.Location]source.FileMetadata{
source.NewLocation("/a/place"): {
Mode: 0775,
Type: "directory",
UserID: 0,
GroupID: 0,
},
source.NewLocation("/a/place/a"): {
Mode: 0775,
Type: "regularFile",
UserID: 0,
GroupID: 0,
},
source.NewLocation("/b"): {
Mode: 0775,
Type: "symbolicLink",
LinkDestination: "/c",
UserID: 0,
GroupID: 0,
},
source.NewLocation("/b/place/b"): {
Mode: 0644,
Type: "regularFile",
UserID: 1,
GroupID: 2,
},
},
FileDigests: map[source.Location][]file.Digest{
source.NewLocation("/a/place/a"): {
{
Algorithm: "sha256",
Value: "366a3f5653e34673b875891b021647440d0127c2ef041e3b1a22da2a7d4f3703",
},
},
source.NewLocation("/b/place/b"): {
{
Algorithm: "sha256",
Value: "1b3722da2a7d90d033b87581a2a3f12021647445653e34666ef041e3b4f3707c",
},
},
},
Distro: &distro.Distro{
Type: distro.RedHat,
RawVersion: "7",
IDLike: "rhel",
},
SourceMetadata: source.Metadata{
Scheme: source.ImageScheme,
ImageMetadata: source.ImageMetadata{
UserInput: "user-image-input",
ID: "sha256:c2b46b4eb06296933b7cf0722683964e9ecbd93265b9ef6ae9642e3952afbba0",
ManifestDigest: "sha256:2731251dc34951c0e50fcc643b4c5f74922dad1a5d98f302b504cf46cd5d9368",
MediaType: "application/vnd.docker.distribution.manifest.v2+json",
Tags: []string{
"stereoscope-fixture-image-simple:85066c51088bdd274f7a89e99e00490f666c49e72ffc955707cd6e18f0e22c5b",
},
Size: 38,
Layers: []source.LayerMetadata{
{
MediaType: "application/vnd.docker.image.rootfs.diff.tar.gzip",
Digest: "sha256:3de16c5b8659a2e8d888b8ded8427be7a5686a3c8c4e4dd30de20f362827285b",
Size: 22,
},
{
MediaType: "application/vnd.docker.image.rootfs.diff.tar.gzip",
Digest: "sha256:366a3f5653e34673b875891b021647440d0127c2ef041e3b1a22da2a7d4f3703",
Size: 16,
},
},
RawManifest: []byte("eyJzY2hlbWFWZXJzaW9uIjoyLCJtZWRpYVR5cGUiOiJh..."),
RawConfig: []byte("eyJhcmNoaXRlY3R1cmUiOiJhbWQ2NCIsImNvbmZp..."),
},
},
}
if err := NewJSONPresenter(cfg).Present(&buffer); err != nil {
t.Fatal(err)
}
actual := buffer.Bytes()
if *updateJSONGoldenFiles {
testutils.UpdateGoldenFileContents(t, actual)
}
var expected = testutils.GetGoldenFileContents(t)
if !bytes.Equal(expected, actual) {
dmp := diffmatchpatch.New()
diffs := dmp.DiffMain(string(expected), string(actual), true)
t.Errorf("mismatched output:\n%s", dmp.DiffPrettyText(diffs))
}
}

198
internal/presenter/poweruser/test-fixtures/snapshot/TestJSONPresenter.golden Normal file
View file

@ -0,0 +1,198 @@
{
"fileMetadata": [
{
"location": {
"path": "/a/place"
},
"metadata": {
"mode": 775,
"type": "directory",
"userID": 0,
"groupID": 0
}
},
{
"location": {
"path": "/a/place/a"
},
"metadata": {
"mode": 775,
"type": "regularFile",
"userID": 0,
"groupID": 0,
"digests": [
{
"algorithm": "sha256",
"value": "366a3f5653e34673b875891b021647440d0127c2ef041e3b1a22da2a7d4f3703"
}
]
}
},
{
"location": {
"path": "/b"
},
"metadata": {
"mode": 775,
"type": "symbolicLink",
"linkDestination": "/c",
"userID": 0,
"groupID": 0
}
},
{
"location": {
"path": "/b/place/b"
},
"metadata": {
"mode": 644,
"type": "regularFile",
"userID": 1,
"groupID": 2,
"digests": [
{
"algorithm": "sha256",
"value": "1b3722da2a7d90d033b87581a2a3f12021647445653e34666ef041e3b4f3707c"
}
]
}
}
],
"artifacts": [
{
"id": "package-1-id",
"name": "package-1",
"version": "1.0.1",
"type": "python",
"foundBy": "the-cataloger-1",
"locations": [
{
"path": "/a/place/a"
}
],
"licenses": [
"MIT"
],
"language": "python",
"cpes": [
"cpe:2.3:*:some:package:1:*:*:*:*:*:*:*"
],
"purl": "a-purl-1",
"metadataType": "PythonPackageMetadata",
"metadata": {
"name": "package-1",
"version": "1.0.1",
"license": "",
"author": "",
"authorEmail": "",
"platform": "",
"sitePackagesRootPath": ""
}
},
{
"id": "package-2-id",
"name": "package-2",
"version": "2.0.1",
"type": "deb",
"foundBy": "the-cataloger-2",
"locations": [
{
"path": "/b/place/b"
}
],
"licenses": [],
"language": "",
"cpes": [
"cpe:2.3:*:some:package:2:*:*:*:*:*:*:*"
],
"purl": "a-purl-2",
"metadataType": "DpkgMetadata",
"metadata": {
"package": "package-2",
"source": "",
"version": "2.0.1",
"sourceVersion": "",
"architecture": "",
"maintainer": "",
"installedSize": 0,
"files": null
}
}
],
"artifactRelationships": [],
"source": {
"type": "image",
"target": {
"userInput": "user-image-input",
"imageID": "sha256:c2b46b4eb06296933b7cf0722683964e9ecbd93265b9ef6ae9642e3952afbba0",
"manifestDigest": "sha256:2731251dc34951c0e50fcc643b4c5f74922dad1a5d98f302b504cf46cd5d9368",
"mediaType": "application/vnd.docker.distribution.manifest.v2+json",
"tags": [
"stereoscope-fixture-image-simple:85066c51088bdd274f7a89e99e00490f666c49e72ffc955707cd6e18f0e22c5b"
],
"imageSize": 38,
"layers": [
{
"mediaType": "application/vnd.docker.image.rootfs.diff.tar.gzip",
"digest": "sha256:3de16c5b8659a2e8d888b8ded8427be7a5686a3c8c4e4dd30de20f362827285b",
"size": 22
},
{
"mediaType": "application/vnd.docker.image.rootfs.diff.tar.gzip",
"digest": "sha256:366a3f5653e34673b875891b021647440d0127c2ef041e3b1a22da2a7d4f3703",
"size": 16
}
],
"manifest": "ZXlKelkyaGxiV0ZXWlhKemFXOXVJam95TENKdFpXUnBZVlI1Y0dVaU9pSmguLi4=",
"config": "ZXlKaGNtTm9hWFJsWTNSMWNtVWlPaUpoYldRMk5DSXNJbU52Ym1acC4uLg==",
"scope": ""
}
},
"distro": {
"name": "redhat",
"version": "7",
"idLike": "rhel"
},
"descriptor": {
"name": "syft",
"version": "[not provided]",
"configuration": {
"configPath": "",
"output": "",
"quiet": false,
"log": {
"structured": false,
"level": "",
"file-location": ""
},
"dev": {
"profile-cpu": false,
"profile-mem": false
},
"check-for-app-update": false,
"anchore": {
"host": "",
"path": "",
"dockerfile": "",
"overwrite-existing-image": false
},
"package": {
"cataloger": {
"enabled": false,
"scope": ""
}
},
"file-metadata": {
"cataloger": {
"enabled": false,
"scope": ""
},
"digests": null
}
}
},
"schema": {
"version": "1.0.4",
"url": "https://raw.githubusercontent.com/anchore/syft/main/schema/json/schema-1.0.4.json"
}
}

6
schema/json/schema-1.0.4.json
View file

@ -345,8 +345,7 @@
"mode",
"type",
"userID",
"groupID",
"digests"
"groupID"
],
"properties": {
"mode": {
@ -355,6 +354,9 @@
"type": {
"type": "string"
},
"linkDestination": {
"type": "string"
},
"userID": {
"type": "integer"
},

20
syft/file/digest_cataloger.go
View file

@ -74,16 +74,18 @@ func (i *DigestsCataloger) catalogLocation(resolver source.FileResolver, locatio
return nil, fmt.Errorf("unable to observe contents of %+v: %+v", location.RealPath, err)
}
if size == 0 {
return make([]Digest, 0), nil
}
result := make([]Digest, len(i.hashes))
if size > 0 {
// only capture digests when there is content. It is important to do this based on SIZE and not
// FILE TYPE. The reasoning is that it is possible for a tar to be crafted with a header-only
// file type but a body is still allowed.
for idx, hasher := range hashers {
result[idx] = Digest{
Algorithm: cleanAlgorithmName(i.hashes[idx].String()),
Value: fmt.Sprintf("%+x", hasher.Sum(nil)),
}
// only capture digests when there is content. It is important to do this based on SIZE and not
// FILE TYPE. The reasoning is that it is possible for a tar to be crafted with a header-only
// file type but a body is still allowed.
for idx, hasher := range hashers {
result[idx] = Digest{
Algorithm: cleanAlgorithmName(i.hashes[idx].String()),
Value: fmt.Sprintf("%+x", hasher.Sum(nil)),
}
}

104
syft/file/digest_cataloger_test.go
View file

@ -7,6 +7,10 @@ import (
"os"
"testing"
"github.com/anchore/stereoscope/pkg/file"
"github.com/anchore/stereoscope/pkg/imagetest"
"github.com/stretchr/testify/assert"
"github.com/anchore/syft/syft/source"
@ -38,12 +42,13 @@ func testDigests(t testing.TB, files []string, hashes ...crypto.Hash) map[source
return digests
}
func TestDigestsCataloger(t *testing.T) {
files := []string{"test-fixtures/last/path.txt", "test-fixtures/another-path.txt", "test-fixtures/a-path.txt"}
func TestDigestsCataloger_SimpleContents(t *testing.T) {
regularFiles := []string{"test-fixtures/last/path.txt", "test-fixtures/another-path.txt", "test-fixtures/a-path.txt"}
tests := []struct {
name string
algorithms []string
files []string
expected map[source.Location][]Digest
constructorErr bool
catalogErr bool
@ -51,22 +56,32 @@ func TestDigestsCataloger(t *testing.T) {
{
name: "bad algorithm",
algorithms: []string{"sha-nothing"},
files: regularFiles,
constructorErr: true,
},
{
name: "unsupported algorithm",
algorithms: []string{"sha512"},
files: regularFiles,
constructorErr: true,
},
{
name: "md5-sha1-sha256",
name: "md5",
algorithms: []string{"md5"},
expected: testDigests(t, files, crypto.MD5),
files: regularFiles,
expected: testDigests(t, regularFiles, crypto.MD5),
},
{
name: "md5-sha1-sha256",
algorithms: []string{"md5", "sha1", "sha256"},
expected: testDigests(t, files, crypto.MD5, crypto.SHA1, crypto.SHA256),
files: regularFiles,
expected: testDigests(t, regularFiles, crypto.MD5, crypto.SHA1, crypto.SHA256),
},
{
name: "directory returns error",
algorithms: []string{"md5"},
files: []string{"test-fixtures/last"},
catalogErr: true,
},
}
@ -81,7 +96,7 @@ func TestDigestsCataloger(t *testing.T) {
return
}
resolver := source.NewMockResolverForPaths(files...)
resolver := source.NewMockResolverForPaths(test.files...)
actual, err := c.Catalog(resolver)
if err != nil && !test.catalogErr {
t.Fatalf("could not catalog (but should have been able to): %+v", err)
@ -96,3 +111,80 @@ func TestDigestsCataloger(t *testing.T) {
})
}
}
func TestDigestsCataloger_MixFileTypes(t *testing.T) {
testImage := "image-file-type-mix"
if *updateImageGoldenFiles {
imagetest.UpdateGoldenFixtureImage(t, testImage)
}
img := imagetest.GetGoldenFixtureImage(t, testImage)
src, err := source.NewFromImage(img, "---")
if err != nil {
t.Fatalf("could not create source: %+v", err)
}
resolver, err := src.FileResolver(source.SquashedScope)
if err != nil {
t.Fatalf("could not create resolver: %+v", err)
}
tests := []struct {
path string
expected string
}{
{
path: "/file-1.txt",
expected: "888c139e550867814eb7c33b84d76e4d",
},
{
path: "/hardlink-1",
},
{
path: "/symlink-1",
},
{
path: "/char-device-1",
},
{
path: "/block-device-1",
},
{
path: "/fifo-1",
},
{
path: "/bin",
},
}
for _, test := range tests {
t.Run(test.path, func(t *testing.T) {
c, err := NewDigestsCataloger([]string{"md5"})
if err != nil {
t.Fatalf("unable to get cataloger: %+v", err)
}
actual, err := c.Catalog(resolver)
if err != nil {
t.Fatalf("could not catalog: %+v", err)
}
_, ref, err := img.SquashedTree().File(file.Path(test.path))
if err != nil {
t.Fatalf("unable to get file=%q : %+v", test.path, err)
}
l := source.NewLocationFromImage(test.path, *ref, img)
if len(actual[l]) == 0 {
if test.expected != "" {
t.Fatalf("no digest found, but expected one")
}
} else {
assert.Equal(t, actual[l][0].Value, test.expected, "mismatched digests")
}
})
}
}

28
syft/file/metadata_cataloger_test.go
View file

@ -50,7 +50,7 @@ func TestFileMetadataCataloger(t *testing.T) {
exists: true,
expected: source.FileMetadata{
Mode: 0644,
Type: "regularFile",
Type: "RegularFile",
UserID: 1,
GroupID: 2,
},
@ -59,20 +59,22 @@ func TestFileMetadataCataloger(t *testing.T) {
path: "/hardlink-1",
exists: true,
expected: source.FileMetadata{
Mode: 0644,
Type: "hardLink",
UserID: 1,
GroupID: 2,
Mode: 0644,
Type: "HardLink",
LinkDestination: "file-1.txt",
UserID: 1,
GroupID: 2,
},
},
{
path: "/symlink-1",
exists: true,
expected: source.FileMetadata{
Mode: 0777 | os.ModeSymlink,
Type: "symbolicLink",
UserID: 0,
GroupID: 0,
Mode: 0777 | os.ModeSymlink,
Type: "SymbolicLink",
LinkDestination: "file-1.txt",
UserID: 0,
GroupID: 0,
},
},
{
@ -80,7 +82,7 @@ func TestFileMetadataCataloger(t *testing.T) {
exists: true,
expected: source.FileMetadata{
Mode: 0644 | os.ModeDevice | os.ModeCharDevice,
Type: "characterDevice",
Type: "CharacterDevice",
UserID: 0,
GroupID: 0,
},
@ -90,7 +92,7 @@ func TestFileMetadataCataloger(t *testing.T) {
exists: true,
expected: source.FileMetadata{
Mode: 0644 | os.ModeDevice,
Type: "blockDevice",
Type: "BlockDevice",
UserID: 0,
GroupID: 0,
},
@ -100,7 +102,7 @@ func TestFileMetadataCataloger(t *testing.T) {
exists: true,
expected: source.FileMetadata{
Mode: 0644 | os.ModeNamedPipe,
Type: "fifoNode",
Type: "FIFONode",
UserID: 0,
GroupID: 0,
},
@ -110,7 +112,7 @@ func TestFileMetadataCataloger(t *testing.T) {
exists: true,
expected: source.FileMetadata{
Mode: 0755 | os.ModeDir,
Type: "directory",
Type: "Directory",
UserID: 0,
GroupID: 0,
},

18
syft/source/file_metadata.go
View file

@ -7,10 +7,11 @@ import (
)
type FileMetadata struct {
Mode os.FileMode
Type FileType
UserID int
GroupID int
Mode os.FileMode
Type FileType
UserID int
GroupID int
LinkDestination string
}
func fileMetadataByLocation(img *image.Image, location Location) (FileMetadata, error) {
@ -20,9 +21,10 @@ func fileMetadataByLocation(img *image.Image, location Location) (FileMetadata,
}
return FileMetadata{
Mode: entry.Metadata.Mode,
Type: newFileTypeFromTarHeaderTypeFlag(entry.Metadata.TypeFlag),
UserID: entry.Metadata.UserID,
GroupID: entry.Metadata.GroupID,
Mode: entry.Metadata.Mode,
Type: newFileTypeFromTarHeaderTypeFlag(entry.Metadata.TypeFlag),
UserID: entry.Metadata.UserID,
GroupID: entry.Metadata.GroupID,
LinkDestination: entry.Metadata.Linkname,
}, nil
}