feat: add RelationshipsBySourceOwnership to syft json output (#1248)

This commit is contained in:
Christopher Angelo Phillips 2022-10-11 15:11:03 -04:00 committed by GitHub
parent fa0b3c0438
commit 89575199b8
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
16 changed files with 1677 additions and 36 deletions

View file

@ -6,5 +6,5 @@ const (
// JSONSchemaVersion is the current schema version output by the JSON encoder
// This is roughly following the "SchemaVer" guidelines for versioning the JSON schema. Please see schema/json/README.md for details on how to increment.
JSONSchemaVersion = "4.0.0"
JSONSchemaVersion = "4.1.0"
)

File diff suppressed because it is too large Load diff

View file

@ -156,6 +156,7 @@ func TestEncodeFullJSONDocument(t *testing.T) {
},
},
Source: source.Metadata{
ID: "c2b46b4eb06296933b7cf0722683964e9ecbd93265b9ef6ae9642e3952afbba0",
Scheme: source.ImageScheme,
ImageMetadata: source.ImageMetadata{
UserInput: "user-image-input",

View file

@ -10,12 +10,14 @@ import (
// Source object represents the thing that was cataloged
type Source struct {
ID string `json:"id"`
Type string `json:"type"`
Target interface{} `json:"target"`
}
// sourceUnpacker is used to unmarshal Source objects
type sourceUnpacker struct {
ID string `json:"id,omitempty"`
Type string `json:"type"`
Target json.RawMessage `json:"target"`
}
@ -28,6 +30,7 @@ func (s *Source) UnmarshalJSON(b []byte) error {
}
s.Type = unpacker.Type
s.ID = unpacker.ID
switch s.Type {
case "directory", "file":

View file

@ -20,10 +20,12 @@ func TestSource_UnmarshalJSON(t *testing.T) {
{
name: "directory",
input: []byte(`{
"id": "foobar",
"type": "directory",
"target":"/var/lib/foo"
}`),
expectedSource: &Source{
ID: "foobar",
Type: "directory",
Target: "/var/lib/foo",
},
@ -32,6 +34,7 @@ func TestSource_UnmarshalJSON(t *testing.T) {
{
name: "image",
input: []byte(`{
"id": "foobar",
"type": "image",
"target": {
"userInput": "alpine:3.10",
@ -55,6 +58,7 @@ func TestSource_UnmarshalJSON(t *testing.T) {
}
}`),
expectedSource: &Source{
ID: "foobar",
Type: "image",
Target: source.ImageMetadata{
UserInput: "alpine:3.10",
@ -98,10 +102,12 @@ func TestSource_UnmarshalJSON(t *testing.T) {
{
name: "file",
input: []byte(`{
"id": "foobar",
"type": "file",
"target":"/var/lib/foo/go.mod"
}`),
expectedSource: &Source{
ID: "foobar",
Type: "file",
Target: "/var/lib/foo/go.mod",
},
@ -110,10 +116,12 @@ func TestSource_UnmarshalJSON(t *testing.T) {
{
name: "unknown source type",
input: []byte(`{
"id": "foobar",
"type": "unknown-thing",
"target":"/var/lib/foo"
}`),
expectedSource: &Source{
ID: "foobar",
Type: "unknown-thing",
},
errAssertion: assert.Error,

View file

@ -67,6 +67,7 @@
],
"artifactRelationships": [],
"source": {
"id": "eda6cf0b63f1a1d2eaf7792a2a98c832c21a18e6992bcebffe6381781cc85cbc",
"type": "directory",
"target": "/some/path"
},
@ -88,7 +89,7 @@
}
},
"schema": {
"version": "4.0.0",
"url": "https://raw.githubusercontent.com/anchore/syft/main/schema/json/schema-4.0.0.json"
"version": "4.1.0",
"url": "https://raw.githubusercontent.com/anchore/syft/main/schema/json/schema-4.1.0.json"
}
}

View file

@ -139,6 +139,7 @@
}
],
"source": {
"id": "c2b46b4eb06296933b7cf0722683964e9ecbd93265b9ef6ae9642e3952afbba0",
"type": "image",
"target": {
"userInput": "user-image-input",
@ -184,7 +185,7 @@
}
},
"schema": {
"version": "4.0.0",
"url": "https://raw.githubusercontent.com/anchore/syft/main/schema/json/schema-4.0.0.json"
"version": "4.1.0",
"url": "https://raw.githubusercontent.com/anchore/syft/main/schema/json/schema-4.1.0.json"
}
}

View file

@ -9,7 +9,7 @@
"locations": [
{
"path": "/somefile-1.txt",
"layerID": "sha256:7ef28e9c2d56471ee090b578a678bdf28c3b5a311ca7b2e28c2a4185e5bb34c0"
"layerID": "sha256:4965affaf42a7174561882c5fd87e2db6f0b07df532459ba86f98a8bd2af11de"
}
],
"licenses": [
@ -40,7 +40,7 @@
"locations": [
{
"path": "/somefile-2.txt",
"layerID": "sha256:86da8aee621161bea2efaf27a2709ddab5e7d44e30ecdfda728b02c03a28fd98"
"layerID": "sha256:460c3e27be163efe75df048c4d4cf3a22e7e363f02521fa2e82a3bd257a682d4"
}
],
"licenses": [],
@ -64,10 +64,11 @@
],
"artifactRelationships": [],
"source": {
"id": "00afea0209d754683fdfcdd47cfea94ec9f2e81286be444e297a8c776c4accbf",
"type": "image",
"target": {
"userInput": "user-image-input",
"imageID": "sha256:5dd5f5f4247e4e946f555f0de7681a631a5240b614e52717d0aed04808e8c65f",
"imageID": "sha256:6b1b476e6dc187bb688566606cf7a59d7804d81169967d8c6bb121627b0a387f",
"manifestDigest": "sha256:2731251dc34951c0e50fcc643b4c5f74922dad1a5d98f302b504cf46cd5d9368",
"mediaType": "application/vnd.docker.distribution.manifest.v2+json",
"tags": [
@ -77,17 +78,17 @@
"layers": [
{
"mediaType": "application/vnd.docker.image.rootfs.diff.tar.gzip",
"digest": "sha256:7ef28e9c2d56471ee090b578a678bdf28c3b5a311ca7b2e28c2a4185e5bb34c0",
"digest": "sha256:4965affaf42a7174561882c5fd87e2db6f0b07df532459ba86f98a8bd2af11de",
"size": 22
},
{
"mediaType": "application/vnd.docker.image.rootfs.diff.tar.gzip",
"digest": "sha256:86da8aee621161bea2efaf27a2709ddab5e7d44e30ecdfda728b02c03a28fd98",
"digest": "sha256:460c3e27be163efe75df048c4d4cf3a22e7e363f02521fa2e82a3bd257a682d4",
"size": 16
}
],
"manifest": "eyJzY2hlbWFWZXJzaW9uIjoyLCJtZWRpYVR5cGUiOiJhcHBsaWNhdGlvbi92bmQuZG9ja2VyLmRpc3RyaWJ1dGlvbi5tYW5pZmVzdC52Mitqc29uIiwiY29uZmlnIjp7Im1lZGlhVHlwZSI6ImFwcGxpY2F0aW9uL3ZuZC5kb2NrZXIuY29udGFpbmVyLmltYWdlLnYxK2pzb24iLCJzaXplIjo2NzMsImRpZ2VzdCI6InNoYTI1Njo1ZGQ1ZjVmNDI0N2U0ZTk0NmY1NTVmMGRlNzY4MWE2MzFhNTI0MGI2MTRlNTI3MTdkMGFlZDA0ODA4ZThjNjVmIn0sImxheWVycyI6W3sibWVkaWFUeXBlIjoiYXBwbGljYXRpb24vdm5kLmRvY2tlci5pbWFnZS5yb290ZnMuZGlmZi50YXIuZ3ppcCIsInNpemUiOjIwNDgsImRpZ2VzdCI6InNoYTI1Njo3ZWYyOGU5YzJkNTY0NzFlZTA5MGI1NzhhNjc4YmRmMjhjM2I1YTMxMWNhN2IyZTI4YzJhNDE4NWU1YmIzNGMwIn0seyJtZWRpYVR5cGUiOiJhcHBsaWNhdGlvbi92bmQuZG9ja2VyLmltYWdlLnJvb3Rmcy5kaWZmLnRhci5nemlwIiwic2l6ZSI6MjA0OCwiZGlnZXN0Ijoic2hhMjU2Ojg2ZGE4YWVlNjIxMTYxYmVhMmVmYWYyN2EyNzA5ZGRhYjVlN2Q0NGUzMGVjZGZkYTcyOGIwMmMwM2EyOGZkOTgifV19",
"config": "eyJhcmNoaXRlY3R1cmUiOiJhbWQ2NCIsImNvbmZpZyI6eyJFbnYiOlsiUEFUSD0vdXNyL2xvY2FsL3NiaW46L3Vzci9sb2NhbC9iaW46L3Vzci9zYmluOi91c3IvYmluOi9zYmluOi9iaW4iXSwiV29ya2luZ0RpciI6Ii8iLCJPbkJ1aWxkIjpudWxsfSwiY3JlYXRlZCI6IjIwMjItMDYtMDJUMTQ6MzQ6MzQuNzE5MTM1MTc0WiIsImhpc3RvcnkiOlt7ImNyZWF0ZWQiOiIyMDIyLTA2LTAyVDE0OjM0OjM0LjY4NjkzMzI2M1oiLCJjcmVhdGVkX2J5IjoiQUREIGZpbGUtMS50eHQgL3NvbWVmaWxlLTEudHh0ICMgYnVpbGRraXQiLCJjb21tZW50IjoiYnVpbGRraXQuZG9ja2VyZmlsZS52MCJ9LHsiY3JlYXRlZCI6IjIwMjItMDYtMDJUMTQ6MzQ6MzQuNzE5MTM1MTc0WiIsImNyZWF0ZWRfYnkiOiJBREQgZmlsZS0yLnR4dCAvc29tZWZpbGUtMi50eHQgIyBidWlsZGtpdCIsImNvbW1lbnQiOiJidWlsZGtpdC5kb2NrZXJmaWxlLnYwIn1dLCJvcyI6ImxpbnV4Iiwicm9vdGZzIjp7InR5cGUiOiJsYXllcnMiLCJkaWZmX2lkcyI6WyJzaGEyNTY6N2VmMjhlOWMyZDU2NDcxZWUwOTBiNTc4YTY3OGJkZjI4YzNiNWEzMTFjYTdiMmUyOGMyYTQxODVlNWJiMzRjMCIsInNoYTI1Njo4NmRhOGFlZTYyMTE2MWJlYTJlZmFmMjdhMjcwOWRkYWI1ZTdkNDRlMzBlY2RmZGE3MjhiMDJjMDNhMjhmZDk4Il19fQ==",
"manifest": "eyJzY2hlbWFWZXJzaW9uIjoyLCJtZWRpYVR5cGUiOiJhcHBsaWNhdGlvbi92bmQuZG9ja2VyLmRpc3RyaWJ1dGlvbi5tYW5pZmVzdC52Mitqc29uIiwiY29uZmlnIjp7Im1lZGlhVHlwZSI6ImFwcGxpY2F0aW9uL3ZuZC5kb2NrZXIuY29udGFpbmVyLmltYWdlLnYxK2pzb24iLCJzaXplIjo2NzMsImRpZ2VzdCI6InNoYTI1Njo2YjFiNDc2ZTZkYzE4N2JiNjg4NTY2NjA2Y2Y3YTU5ZDc4MDRkODExNjk5NjdkOGM2YmIxMjE2MjdiMGEzODdmIn0sImxheWVycyI6W3sibWVkaWFUeXBlIjoiYXBwbGljYXRpb24vdm5kLmRvY2tlci5pbWFnZS5yb290ZnMuZGlmZi50YXIuZ3ppcCIsInNpemUiOjIwNDgsImRpZ2VzdCI6InNoYTI1Njo0OTY1YWZmYWY0MmE3MTc0NTYxODgyYzVmZDg3ZTJkYjZmMGIwN2RmNTMyNDU5YmE4NmY5OGE4YmQyYWYxMWRlIn0seyJtZWRpYVR5cGUiOiJhcHBsaWNhdGlvbi92bmQuZG9ja2VyLmltYWdlLnJvb3Rmcy5kaWZmLnRhci5nemlwIiwic2l6ZSI6MjA0OCwiZGlnZXN0Ijoic2hhMjU2OjQ2MGMzZTI3YmUxNjNlZmU3NWRmMDQ4YzRkNGNmM2EyMmU3ZTM2M2YwMjUyMWZhMmU4MmEzYmQyNTdhNjgyZDQifV19",
"config": "eyJhcmNoaXRlY3R1cmUiOiJhbWQ2NCIsImNvbmZpZyI6eyJFbnYiOlsiUEFUSD0vdXNyL2xvY2FsL3NiaW46L3Vzci9sb2NhbC9iaW46L3Vzci9zYmluOi91c3IvYmluOi9zYmluOi9iaW4iXSwiV29ya2luZ0RpciI6Ii8iLCJPbkJ1aWxkIjpudWxsfSwiY3JlYXRlZCI6IjIwMjItMTAtMDVUMTQ6MjQ6NTguNzc0NTY2MjM2WiIsImhpc3RvcnkiOlt7ImNyZWF0ZWQiOiIyMDIyLTEwLTA1VDE0OjI0OjU4Ljc0NDY3NTEyOVoiLCJjcmVhdGVkX2J5IjoiQUREIGZpbGUtMS50eHQgL3NvbWVmaWxlLTEudHh0ICMgYnVpbGRraXQiLCJjb21tZW50IjoiYnVpbGRraXQuZG9ja2VyZmlsZS52MCJ9LHsiY3JlYXRlZCI6IjIwMjItMTAtMDVUMTQ6MjQ6NTguNzc0NTY2MjM2WiIsImNyZWF0ZWRfYnkiOiJBREQgZmlsZS0yLnR4dCAvc29tZWZpbGUtMi50eHQgIyBidWlsZGtpdCIsImNvbW1lbnQiOiJidWlsZGtpdC5kb2NrZXJmaWxlLnYwIn1dLCJvcyI6ImxpbnV4Iiwicm9vdGZzIjp7InR5cGUiOiJsYXllcnMiLCJkaWZmX2lkcyI6WyJzaGEyNTY6NDk2NWFmZmFmNDJhNzE3NDU2MTg4MmM1ZmQ4N2UyZGI2ZjBiMDdkZjUzMjQ1OWJhODZmOThhOGJkMmFmMTFkZSIsInNoYTI1Njo0NjBjM2UyN2JlMTYzZWZlNzVkZjA0OGM0ZDRjZjNhMjJlN2UzNjNmMDI1MjFmYTJlODJhM2JkMjU3YTY4MmQ0Il19fQ==",
"repoDigests": [],
"architecture": "",
"os": ""
@ -111,7 +112,7 @@
}
},
"schema": {
"version": "4.0.0",
"url": "https://raw.githubusercontent.com/anchore/syft/main/schema/json/schema-4.0.0.json"
"version": "4.1.0",
"url": "https://raw.githubusercontent.com/anchore/syft/main/schema/json/schema-4.1.0.json"
}
}

View file

@ -17,8 +17,6 @@ import (
)
// ToFormatModel transforms the sbom import a format-specific model.
// note: this is needed for anchore import functionality
// TODO: unexport this when/if anchore import functionality is removed
func ToFormatModel(s sbom.SBOM) model.Document {
src, err := toSourceModel(s.Source)
if err != nil {
@ -236,16 +234,19 @@ func toSourceModel(src source.Metadata) (model.Source, error) {
metadata.Tags = []string{}
}
return model.Source{
ID: src.ID,
Type: "image",
Target: metadata,
}, nil
case source.DirectoryScheme:
return model.Source{
ID: src.ID,
Type: "directory",
Target: src.Path,
}, nil
case source.FileScheme:
return model.Source{
ID: src.ID,
Type: "file",
Target: src.Path,
}, nil

View file

@ -26,10 +26,12 @@ func Test_toSourceModel(t *testing.T) {
{
name: "directory",
src: source.Metadata{
ID: "test-id",
Scheme: source.DirectoryScheme,
Path: "some/path",
},
expected: model.Source{
ID: "test-id",
Type: "directory",
Target: "some/path",
},
@ -37,10 +39,12 @@ func Test_toSourceModel(t *testing.T) {
{
name: "file",
src: source.Metadata{
ID: "test-id",
Scheme: source.FileScheme,
Path: "some/path",
},
expected: model.Source{
ID: "test-id",
Type: "file",
Target: "some/path",
},
@ -48,6 +52,7 @@ func Test_toSourceModel(t *testing.T) {
{
name: "image",
src: source.Metadata{
ID: "test-id",
Scheme: source.ImageScheme,
ImageMetadata: source.ImageMetadata{
UserInput: "user-input",
@ -57,6 +62,7 @@ func Test_toSourceModel(t *testing.T) {
},
},
expected: model.Source{
ID: "test-id",
Type: "image",
Target: source.ImageMetadata{
UserInput: "user-input",

View file

@ -64,6 +64,9 @@ func toSyftRelationships(doc *model.Document, catalog *pkg.Catalog, relationship
}
}
// set source metadata in identifier map
idMap[doc.Source.ID] = toSyftSource(doc.Source)
for _, f := range doc.Files {
idMap[f.ID] = f.Location
}
@ -78,6 +81,14 @@ func toSyftRelationships(doc *model.Document, catalog *pkg.Catalog, relationship
return out
}
func toSyftSource(s model.Source) *source.Source {
newSrc := &source.Source{
Metadata: *toSyftSourceData(s),
}
newSrc.SetID()
return newSrc
}
func toSyftRelationship(idMap map[string]interface{}, relationship model.Relationship, idAliases map[string]string) *artifact.Relationship {
id := func(id string) string {
aliased, ok := idAliases[id]
@ -86,16 +97,19 @@ func toSyftRelationship(idMap map[string]interface{}, relationship model.Relatio
}
return id
}
from, ok := idMap[id(relationship.Parent)].(artifact.Identifiable)
if !ok {
log.Warnf("relationship mapping from key %s is not a valid artifact.Identifiable type: %+v", relationship.Parent, idMap[relationship.Parent])
return nil
}
to, ok := idMap[id(relationship.Child)].(artifact.Identifiable)
if !ok {
log.Warnf("relationship mapping to key %s is not a valid artifact.Identifiable type: %+v", relationship.Child, idMap[relationship.Child])
return nil
}
typ := artifact.RelationshipType(relationship.Type)
switch typ {
@ -126,16 +140,19 @@ func toSyftSourceData(s model.Source) *source.Metadata {
switch s.Type {
case "directory":
return &source.Metadata{
ID: s.ID,
Scheme: source.DirectoryScheme,
Path: s.Target.(string),
}
case "file":
return &source.Metadata{
ID: s.ID,
Scheme: source.FileScheme,
Path: s.Target.(string),
}
case "image":
return &source.Metadata{
ID: s.ID,
Scheme: source.ImageScheme,
ImageMetadata: s.Target.(source.ImageMetadata),
}

View file

@ -74,9 +74,24 @@ func CatalogPackages(src *source.Source, cfg cataloger.Config) (*pkg.Catalog, []
return nil, nil, nil, err
}
relationships = append(relationships, newSourceRelationshipsFromCatalog(src, catalog)...)
return catalog, relationships, release, nil
}
func newSourceRelationshipsFromCatalog(src *source.Source, c *pkg.Catalog) []artifact.Relationship {
relationships := make([]artifact.Relationship, 0) // Should we pre-allocate this by giving catalog a Len() method?
for p := range c.Enumerate() {
relationships = append(relationships, artifact.Relationship{
From: src,
To: p,
Type: artifact.ContainsRelationship,
})
}
return relationships
}
// SetLogger sets the logger object used for all syft logging calls.
func SetLogger(logger logger.Logger) {
log.Log = logger

View file

@ -2,6 +2,7 @@ package source
// Metadata represents any static source data that helps describe "what" was cataloged.
type Metadata struct {
ID string `hash:"ignore"` // the id generated from the parent source struct
Scheme Scheme // the source data scheme type (directory or image)
ImageMetadata ImageMetadata // all image info (image only)
Path string // the root path to be cataloged (directory only)

View file

@ -27,13 +27,13 @@ import (
// Source is an object that captures the data source to be cataloged, configuration, and a specific resolver used
// in cataloging (based on the data source and configuration)
type Source struct {
id artifact.ID
Image *image.Image // the image object to be cataloged (image only)
id artifact.ID `hash:"ignore"`
Image *image.Image `hash:"ignore"` // the image object to be cataloged (image only)
Metadata Metadata
directoryResolver *directoryResolver
directoryResolver *directoryResolver `hash:"ignore"`
path string
mutex *sync.Mutex
Exclusions []string
Exclusions []string `hash:"ignore"`
}
// Input is an object that captures the detected user input regarding source location, scheme, and provider type.
@ -241,28 +241,33 @@ func generateFileSource(fs afero.Fs, location string) (*Source, func(), error) {
// NewFromDirectory creates a new source object tailored to catalog a given filesystem directory recursively.
func NewFromDirectory(path string) (Source, error) {
return Source{
s := Source{
mutex: &sync.Mutex{},
Metadata: Metadata{
Scheme: DirectoryScheme,
Path: path,
},
path: path,
}, nil
}
s.SetID()
return s, nil
}
// NewFromFile creates a new source object tailored to catalog a file.
func NewFromFile(path string) (Source, func()) {
analysisPath, cleanupFn := fileAnalysisPath(path)
return Source{
s := Source{
mutex: &sync.Mutex{},
Metadata: Metadata{
Scheme: FileScheme,
Path: path,
},
path: analysisPath,
}, cleanupFn
}
s.SetID()
return s, cleanupFn
}
// fileAnalysisPath returns the path given, or in the case the path is an archive, the location where the archive
@ -298,16 +303,18 @@ func NewFromImage(img *image.Image, userImageStr string) (Source, error) {
return Source{}, fmt.Errorf("no image given")
}
return Source{
s := Source{
Image: img,
Metadata: Metadata{
Scheme: ImageScheme,
ImageMetadata: NewImageMetadata(img, userImageStr),
},
}, nil
}
s.SetID()
return s, nil
}
func (s Source) ID() artifact.ID {
func (s *Source) ID() artifact.ID {
if s.id == "" {
s.SetID()
}
@ -333,7 +340,7 @@ func (s *Source) SetID() {
}
d = di.String()
case ImageScheme:
manifestDigest := digest.FromBytes(s.Image.Metadata.RawManifest).String()
manifestDigest := digest.FromBytes(s.Metadata.ImageMetadata.RawManifest).String()
if manifestDigest != "" {
d = manifestDigest
break
@ -341,7 +348,7 @@ func (s *Source) SetID() {
// calcuate chain ID for image sources where manifestDigest is not available
// https://github.com/opencontainers/image-spec/blob/main/config.md#layer-chainid
d = calculateChainID(s.Image)
d = calculateChainID(s.Metadata.ImageMetadata.Layers)
if d == "" {
// TODO what happens here if image has no layers?
// Is this case possible
@ -353,27 +360,28 @@ func (s *Source) SetID() {
}
s.id = artifact.ID(strings.TrimPrefix(d, "sha256:"))
s.Metadata.ID = strings.TrimPrefix(d, "sha256:")
}
func calculateChainID(img *image.Image) string {
if len(img.Layers) < 1 {
func calculateChainID(lm []LayerMetadata) string {
if len(lm) < 1 {
return ""
}
// DiffID(L0) = digest of layer 0
// https://github.com/anchore/stereoscope/blob/1b1b744a919964f38d14e1416fb3f25221b761ce/pkg/image/layer_metadata.go#L19-L32
chainID := img.Layers[0].Metadata.Digest
id := chain(chainID, img.Layers[1:])
chainID := lm[0].Digest
id := chain(chainID, lm[1:])
return id
}
func chain(chainID string, layers []*image.Layer) string {
func chain(chainID string, layers []LayerMetadata) string {
if len(layers) < 1 {
return chainID
}
chainID = digest.FromString(layers[0].Metadata.Digest + " " + chainID).String()
chainID = digest.FromString(layers[0].Digest + " " + chainID).String()
return chain(chainID, layers[1:])
}

View file

@ -120,7 +120,7 @@ func TestSetID(t *testing.T) {
Path: "test-fixtures/image-simple",
},
},
expected: artifact.ID("febd2d6148dc327d"),
expected: artifact.ID("26e8f4daad203793"),
},
}