Update portage cataloger to new generic cataloger (#1316)

* port portage (ha) cataloger to new generic cataloger pattern

Signed-off-by: Alex Goodman <alex.goodman@anchore.com>

* update JSON schema to account for removing portage fields

Signed-off-by: Alex Goodman <alex.goodman@anchore.com>

Signed-off-by: Alex Goodman <alex.goodman@anchore.com>

internal/constants.go

@@ -6,5 +6,5 @@ const (
 
 	// JSONSchemaVersion is the current schema version output by the JSON encoder
 	// This is roughly following the "SchemaVer" guidelines for versioning the JSON schema. Please see schema/json/README.md for details on how to increment.
-	JSONSchemaVersion = "4.1.0"
+	JSONSchemaVersion = "5.0.0"
 )

syft/formats/syftjson/format.go

@@ -4,7 +4,7 @@ import (
 	"github.com/anchore/syft/syft/sbom"
 )
 
-const ID sbom.FormatID = "syft-4-json"
+const ID sbom.FormatID = "syft-5-json"
 
 func Format() sbom.Format {
 	return sbom.NewFormat(

syft/formats/syftjson/test-fixtures/snapshot/TestDirectoryEncoder.golden

@@ -89,7 +89,7 @@
   }
  },
  "schema": {
-  "version": "4.1.0",
-  "url": "https://raw.githubusercontent.com/anchore/syft/main/schema/json/schema-4.1.0.json"
+  "version": "5.0.0",
+  "url": "https://raw.githubusercontent.com/anchore/syft/main/schema/json/schema-5.0.0.json"
  }
 }

syft/formats/syftjson/test-fixtures/snapshot/TestEncodeFullJSONDocument.golden

@@ -185,7 +185,7 @@
   }
  },
  "schema": {
-  "version": "4.1.0",
-  "url": "https://raw.githubusercontent.com/anchore/syft/main/schema/json/schema-4.1.0.json"
+  "version": "5.0.0",
+  "url": "https://raw.githubusercontent.com/anchore/syft/main/schema/json/schema-5.0.0.json"
  }
 }

syft/formats/syftjson/test-fixtures/snapshot/TestImageEncoder.golden

@@ -112,7 +112,7 @@
   }
  },
  "schema": {
-  "version": "4.1.0",
-  "url": "https://raw.githubusercontent.com/anchore/syft/main/schema/json/schema-4.1.0.json"
+  "version": "5.0.0",
+  "url": "https://raw.githubusercontent.com/anchore/syft/main/schema/json/schema-5.0.0.json"
  }
 }

syft/pkg/cataloger/portage/cataloger.go

@@ -18,71 +18,63 @@ import (
 	"github.com/anchore/syft/syft/artifact"
 	"github.com/anchore/syft/syft/file"
 	"github.com/anchore/syft/syft/pkg"
+	"github.com/anchore/syft/syft/pkg/cataloger/generic"
 	"github.com/anchore/syft/syft/source"
 )
 
 var (
-	cpvRe = regexp.MustCompile(`/([^/]*/[\w+][\w+-]*)-((\d+)((\.\d+)*)([a-z]?)((_(pre|p|beta|alpha|rc)\d*)*)(-r\d+)?)/CONTENTS$`)
+	cpvRe                = regexp.MustCompile(`/([^/]*/[\w+][\w+-]*)-((\d+)((\.\d+)*)([a-z]?)((_(pre|p|beta|alpha|rc)\d*)*)(-r\d+)?)/CONTENTS$`)
+	_     generic.Parser = parsePortageContents
 )
 
-type Cataloger struct{}
-
-// NewPortageCataloger returns a new Portage package cataloger object.
-func NewPortageCataloger() *Cataloger {
-	return &Cataloger{}
+func NewPortageCataloger() *generic.Cataloger {
+	return generic.NewCataloger("portage-cataloger").
+		WithParserByGlobs(parsePortageContents, "**/var/db/pkg/*/*/CONTENTS")
 }
 
-// Name returns a string that uniquely describes a cataloger
-func (c *Cataloger) Name() string {
-	return "portage-cataloger"
-}
-
-// Catalog is given an object to resolve file references and content, this function returns any discovered Packages after analyzing portage support files.
-func (c *Cataloger) Catalog(resolver source.FileResolver) ([]pkg.Package, []artifact.Relationship, error) {
-	dbFileMatches, err := resolver.FilesByGlob(pkg.PortageDBGlob)
-	if err != nil {
-		return nil, nil, fmt.Errorf("failed to find portage files by glob: %w", err)
+func parsePortageContents(resolver source.FileResolver, _ *generic.Environment, reader source.LocationReadCloser) ([]pkg.Package, []artifact.Relationship, error) {
+	cpvMatch := cpvRe.FindStringSubmatch(reader.Location.RealPath)
+	if cpvMatch == nil {
+		return nil, nil, fmt.Errorf("failed to match package and version in %s", reader.Location.RealPath)
 	}
-	var allPackages []pkg.Package
-	for _, dbLocation := range dbFileMatches {
-		cpvMatch := cpvRe.FindStringSubmatch(dbLocation.RealPath)
-		if cpvMatch == nil {
-			return nil, nil, fmt.Errorf("failed to match package and version in %s", dbLocation.RealPath)
-		}
-		entry := pkg.PortageMetadata{
+
+	name, version := cpvMatch[1], cpvMatch[2]
+	if name == "" || version == "" {
+		log.WithFields("path", reader.Location.RealPath).Warnf("failed to parse portage name and version")
+		return nil, nil, nil
+	}
+
+	p := pkg.Package{
+		Name:         name,
+		Version:      version,
+		PURL:         packageURL(name, version),
+		Locations:    source.NewLocationSet(),
+		Type:         pkg.PortagePkg,
+		MetadataType: pkg.PortageMetadataType,
+		Metadata: pkg.PortageMetadata{
 			// ensure the default value for a collection is never nil since this may be shown as JSON
-			Files:   make([]pkg.PortageFileRecord, 0),
-			Package: cpvMatch[1],
-			Version: cpvMatch[2],
-		}
-
-		err = addFiles(resolver, dbLocation, &entry)
-		if err != nil {
-			return nil, nil, err
-		}
-
-		addSize(resolver, dbLocation, &entry)
-
-		p := pkg.Package{
-			Name:         entry.Package,
-			Version:      entry.Version,
-			Type:         pkg.PortagePkg,
-			MetadataType: pkg.PortageMetadataType,
-			Metadata:     entry,
-		}
-		addLicenses(resolver, dbLocation, &p)
-		p.FoundBy = c.Name()
-		p.Locations.Add(dbLocation)
-		p.SetID()
-		allPackages = append(allPackages, p)
+			Files: make([]pkg.PortageFileRecord, 0),
+		},
 	}
-	return allPackages, nil, nil
+	addLicenses(resolver, reader.Location, &p)
+	addSize(resolver, reader.Location, &p)
+	addFiles(resolver, reader.Location, &p)
+
+	p.SetID()
+
+	return []pkg.Package{p}, nil, nil
 }
 
-func addFiles(resolver source.FileResolver, dbLocation source.Location, entry *pkg.PortageMetadata) error {
+func addFiles(resolver source.FileResolver, dbLocation source.Location, p *pkg.Package) {
 	contentsReader, err := resolver.FileContentsByLocation(dbLocation)
 	if err != nil {
-		return err
+		log.WithFields("path", dbLocation.RealPath).Warnf("failed to fetch portage contents (package=%s): %+v", p.Name, err)
+		return
+	}
+
+	entry, ok := p.Metadata.(pkg.PortageMetadata)
+	if !ok {
+		return
 	}
 
 	scanner := bufio.NewScanner(contentsReader)
@@ -101,7 +93,9 @@ func addFiles(resolver source.FileResolver, dbLocation source.Location, entry *p
 			entry.Files = append(entry.Files, record)
 		}
 	}
-	return nil
+
+	p.Metadata = entry
+	p.Locations.Add(dbLocation)
 }
 
 func addLicenses(resolver source.FileResolver, dbLocation source.Location, p *pkg.Package) {
@@ -109,43 +103,60 @@ func addLicenses(resolver source.FileResolver, dbLocation source.Location, p *pk
 
 	location := resolver.RelativeFileByPath(dbLocation, path.Join(parentPath, "LICENSE"))
 
-	if location != nil {
-		licenseReader, err := resolver.FileContentsByLocation(*location)
-		if err == nil {
-			findings := internal.NewStringSet()
-			scanner := bufio.NewScanner(licenseReader)
-			scanner.Split(bufio.ScanWords)
-			for scanner.Scan() {
-				token := scanner.Text()
-				if token != "||" && token != "(" && token != ")" {
-					findings.Add(token)
-				}
-			}
-			p.Licenses = findings.ToSlice()
+	if location == nil {
+		return
+	}
 
-			sort.Strings(p.Licenses)
+	licenseReader, err := resolver.FileContentsByLocation(*location)
+	if err != nil {
+		log.WithFields("path", dbLocation.RealPath).Warnf("failed to fetch portage LICENSE: %+v", err)
+		return
+	}
+
+	findings := internal.NewStringSet()
+	scanner := bufio.NewScanner(licenseReader)
+	scanner.Split(bufio.ScanWords)
+	for scanner.Scan() {
+		token := scanner.Text()
+		if token != "||" && token != "(" && token != ")" {
+			findings.Add(token)
 		}
 	}
+	licenses := findings.ToSlice()
+	sort.Strings(licenses)
+	p.Licenses = licenses
+	p.Locations.Add(*location)
 }
 
-func addSize(resolver source.FileResolver, dbLocation source.Location, entry *pkg.PortageMetadata) {
+func addSize(resolver source.FileResolver, dbLocation source.Location, p *pkg.Package) {
 	parentPath := filepath.Dir(dbLocation.RealPath)
 
 	location := resolver.RelativeFileByPath(dbLocation, path.Join(parentPath, "SIZE"))
 
-	if location != nil {
-		sizeReader, err := resolver.FileContentsByLocation(*location)
-		if err != nil {
-			log.Warnf("failed to fetch portage SIZE (package=%s): %+v", entry.Package, err)
-		} else {
-			scanner := bufio.NewScanner(sizeReader)
-			for scanner.Scan() {
-				line := strings.Trim(scanner.Text(), "\n")
-				size, err := strconv.Atoi(line)
-				if err == nil {
-					entry.InstalledSize = size
-				}
-			}
+	if location == nil {
+		return
+	}
+
+	entry, ok := p.Metadata.(pkg.PortageMetadata)
+	if !ok {
+		return
+	}
+
+	sizeReader, err := resolver.FileContentsByLocation(*location)
+	if err != nil {
+		log.WithFields("name", p.Name).Warnf("failed to fetch portage SIZE: %+v", err)
+		return
+	}
+
+	scanner := bufio.NewScanner(sizeReader)
+	for scanner.Scan() {
+		line := strings.Trim(scanner.Text(), "\n")
+		size, err := strconv.Atoi(line)
+		if err == nil {
+			entry.InstalledSize = size
 		}
 	}
+
+	p.Metadata = entry
+	p.Locations.Add(*location)
 }

syft/pkg/cataloger/portage/cataloger_test.go

@@ -3,62 +3,58 @@ package portage
 import (
 	"testing"
 
-	"github.com/go-test/deep"
-
-	"github.com/anchore/stereoscope/pkg/imagetest"
+	"github.com/anchore/syft/syft/artifact"
 	"github.com/anchore/syft/syft/file"
 	"github.com/anchore/syft/syft/pkg"
+	"github.com/anchore/syft/syft/pkg/cataloger/internal/pkgtest"
 	"github.com/anchore/syft/syft/source"
 )
 
 func TestPortageCataloger(t *testing.T) {
-	tests := []struct {
-		name     string
-		expected []pkg.Package
-	}{
+
+	expectedPkgs := []pkg.Package{
 		{
-			name: "go-case",
-			expected: []pkg.Package{
-				{
-					Name:         "app-containers/skopeo",
-					Version:      "1.5.1",
-					FoundBy:      "portage-cataloger",
-					Licenses:     []string{"Apache-2.0", "BSD", "BSD-2", "CC-BY-SA-4.0", "ISC", "MIT"},
-					Type:         pkg.PortagePkg,
-					MetadataType: pkg.PortageMetadataType,
-					Metadata: pkg.PortageMetadata{
-						Package:       "app-containers/skopeo",
-						Version:       "1.5.1",
-						InstalledSize: 27937835,
-						Files: []pkg.PortageFileRecord{
-							{
-								Path: "/usr/bin/skopeo",
-								Digest: &file.Digest{
-									Algorithm: "md5",
-									Value:     "376c02bd3b22804df8fdfdc895e7dbfb",
-								},
-							},
-							{
-								Path: "/etc/containers/policy.json",
-								Digest: &file.Digest{
-									Algorithm: "md5",
-									Value:     "c01eb6950f03419e09d4fc88cb42ff6f",
-								},
-							},
-							{
-								Path: "/etc/containers/registries.d/default.yaml",
-								Digest: &file.Digest{
-									Algorithm: "md5",
-									Value:     "e6e66cd3c24623e0667f26542e0e08f6",
-								},
-							},
-							{
-								Path: "/var/lib/atomic/sigstore/.keep_app-containers_skopeo-0",
-								Digest: &file.Digest{
-									Algorithm: "md5",
-									Value:     "d41d8cd98f00b204e9800998ecf8427e",
-								},
-							},
+			Name:    "app-containers/skopeo",
+			Version: "1.5.1",
+			FoundBy: "portage-cataloger",
+			PURL:    "pkg:ebuild/app-containers/skopeo@1.5.1",
+			Locations: source.NewLocationSet(
+				source.NewLocation("var/db/pkg/app-containers/skopeo-1.5.1/CONTENTS"),
+				source.NewLocation("var/db/pkg/app-containers/skopeo-1.5.1/LICENSE"),
+				source.NewLocation("var/db/pkg/app-containers/skopeo-1.5.1/SIZE"),
+			),
+			Licenses:     []string{"Apache-2.0", "BSD", "BSD-2", "CC-BY-SA-4.0", "ISC", "MIT"},
+			Type:         pkg.PortagePkg,
+			MetadataType: pkg.PortageMetadataType,
+			Metadata: pkg.PortageMetadata{
+				InstalledSize: 27937835,
+				Files: []pkg.PortageFileRecord{
+					{
+						Path: "/usr/bin/skopeo",
+						Digest: &file.Digest{
+							Algorithm: "md5",
+							Value:     "376c02bd3b22804df8fdfdc895e7dbfb",
+						},
+					},
+					{
+						Path: "/etc/containers/policy.json",
+						Digest: &file.Digest{
+							Algorithm: "md5",
+							Value:     "c01eb6950f03419e09d4fc88cb42ff6f",
+						},
+					},
+					{
+						Path: "/etc/containers/registries.d/default.yaml",
+						Digest: &file.Digest{
+							Algorithm: "md5",
+							Value:     "e6e66cd3c24623e0667f26542e0e08f6",
+						},
+					},
+					{
+						Path: "/var/lib/atomic/sigstore/.keep_app-containers_skopeo-0",
+						Digest: &file.Digest{
+							Algorithm: "md5",
+							Value:     "d41d8cd98f00b204e9800998ecf8427e",
 						},
 					},
 				},
@@ -66,41 +62,12 @@ func TestPortageCataloger(t *testing.T) {
 		},
 	}
 
-	for _, test := range tests {
-		t.Run(test.name, func(t *testing.T) {
+	// TODO: relationships are not under test yet
+	var expectedRelationships []artifact.Relationship
 
-			img := imagetest.GetFixtureImage(t, "docker-archive", "image-portage")
-
-			s, err := source.NewFromImage(img, "")
-			if err != nil {
-				t.Fatal(err)
-			}
-
-			c := NewPortageCataloger()
-
-			resolver, err := s.FileResolver(source.SquashedScope)
-			if err != nil {
-				t.Errorf("could not get resolver error: %+v", err)
-			}
-
-			actual, _, err := c.Catalog(resolver)
-			if err != nil {
-				t.Fatalf("failed to catalog: %+v", err)
-			}
-
-			if len(actual) != len(test.expected) {
-				for _, a := range actual {
-					t.Logf("   %+v", a)
-				}
-				t.Fatalf("unexpected package count: %d!=%d", len(actual), len(test.expected))
-			}
-
-			// test remaining fields...
-			for _, d := range deep.Equal(actual, test.expected) {
-				t.Errorf("diff: %+v", d)
-			}
-
-		})
-	}
+	pkgtest.NewCatalogTester().
+		FromDirectory(t, "test-fixtures/image-portage").
+		Expects(expectedPkgs, expectedRelationships).
+		TestCataloger(t, NewPortageCataloger())
 
 }

syft/pkg/cataloger/portage/purl.go

@@ -0,0 +1,18 @@
+package portage
+
+import (
+	"github.com/anchore/packageurl-go"
+)
+
+func packageURL(name, version string) string {
+	var qualifiers packageurl.Qualifiers
+
+	return packageurl.NewPackageURL(
+		"ebuild", // currently this is the proposed type for portage packages at https://github.com/package-url/purl-spec/blob/master/PURL-TYPES.rst
+		"",
+		name,
+		version,
+		qualifiers,
+		"",
+	).ToString()
+}

syft/pkg/cataloger/portage/purl_test.go

@@ -0,0 +1,28 @@
+package portage
+
+import (
+	"fmt"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func Test_packageURL(t *testing.T) {
+
+	tests := []struct {
+		name    string
+		version string
+		want    string
+	}{
+		{
+			"app-admin/eselect",
+			"1.4.15",
+			"pkg:ebuild/app-admin/eselect@1.4.15",
+		},
+	}
+	for _, tt := range tests {
+		t.Run(fmt.Sprintf("%s@%s", tt.name, tt.version), func(t *testing.T) {
+			assert.Equal(t, tt.want, packageURL(tt.name, tt.version))
+		})
+	}
+}

syft/pkg/cataloger/portage/test-fixtures/image-portage/Dockerfile

@@ -1,2 +0,0 @@
-FROM scratch
-COPY . .

syft/pkg/portage_metadata.go

@@ -4,12 +4,8 @@ import (
 	"github.com/anchore/syft/syft/file"
 )
 
-const PortageDBGlob = "**/var/db/pkg/*/*/CONTENTS"
-
 // PortageMetadata represents all captured data for a Package package DB entry.
 type PortageMetadata struct {
-	Package       string              `mapstructure:"Package" json:"package"`
-	Version       string              `mapstructure:"Version" json:"version"`
 	InstalledSize int                 `mapstructure:"InstalledSize" json:"installedSize" cyclonedx:"installedSize"`
 	Files         []PortageFileRecord `json:"files"`
 }