Add portage support for Gentoo Linux (#1076)

Co-authored-by: Christopher Phillips <christopher.phillips@anchore.com>

internal/constants.go

@@ -6,5 +6,5 @@ const (
 
 	// JSONSchemaVersion is the current schema version output by the JSON encoder
 	// This is roughly following the "SchemaVer" guidelines for versioning the JSON schema. Please see schema/json/README.md for details on how to increment.
-	JSONSchemaVersion = "3.3.0"
+	JSONSchemaVersion = "3.3.1"
 )

internal/formats/common/spdxhelpers/source_info.go

@@ -37,6 +37,8 @@ func SourceInfo(p pkg.Package) string {
 		answer = "acquired package info from PHP composer manifest"
 	case pkg.ConanPkg:
 		answer = "acquired package info from conan manifest"
+	case pkg.PortagePkg:
+		answer = "acquired package info from portage DB"
 	default:
 		answer = "acquired package info from the following paths"
 	}

internal/formats/common/spdxhelpers/source_info_test.go

@@ -158,6 +158,14 @@ func Test_SourceInfo(t *testing.T) {
 				"from conan manifest",
 			},
 		},
+		{
+			input: pkg.Package{
+				Type: pkg.PortagePkg,
+			},
+			expected: []string{
+				"from portage DB",
+			},
+		},
 	}
 	var pkgTypes []pkg.Type
 	for _, test := range tests {

internal/formats/syftjson/model/package.go

@@ -157,6 +157,12 @@ func unpackMetadata(p *Package, unpacker packageMetadataUnpacker) error {
 			return err
 		}
 		p.Metadata = payload
+	case pkg.PortageMetadataType:
+		var payload pkg.PortageMetadata
+		if err := json.Unmarshal(unpacker.Metadata, &payload); err != nil {
+			return err
+		}
+		p.Metadata = payload
 	default:
 		log.Warnf("unknown package metadata type=%q for packageID=%q", p.MetadataType, p.ID)
 	}

internal/formats/syftjson/test-fixtures/snapshot/TestDirectoryEncoder.golden

@@ -88,7 +88,7 @@
   }
  },
  "schema": {
-  "version": "3.3.0",
-  "url": "https://raw.githubusercontent.com/anchore/syft/main/schema/json/schema-3.3.0.json"
+  "version": "3.3.1",
+  "url": "https://raw.githubusercontent.com/anchore/syft/main/schema/json/schema-3.3.1.json"
  }
 }

internal/formats/syftjson/test-fixtures/snapshot/TestEncodeFullJSONDocument.golden

@@ -184,7 +184,7 @@
   }
  },
  "schema": {
-  "version": "3.3.0",
-  "url": "https://raw.githubusercontent.com/anchore/syft/main/schema/json/schema-3.3.0.json"
+  "version": "3.3.1",
+  "url": "https://raw.githubusercontent.com/anchore/syft/main/schema/json/schema-3.3.1.json"
  }
 }

internal/formats/syftjson/test-fixtures/snapshot/TestImageEncoder.golden

@@ -111,7 +111,7 @@
   }
  },
  "schema": {
-  "version": "3.3.0",
-  "url": "https://raw.githubusercontent.com/anchore/syft/main/schema/json/schema-3.3.0.json"
+  "version": "3.3.1",
+  "url": "https://raw.githubusercontent.com/anchore/syft/main/schema/json/schema-3.3.1.json"
  }
 }

schema/json/README.md

@@ -25,7 +25,7 @@ Given a version number format `MODEL.REVISION.ADDITION`:
 When adding a new `pkg.*Metadata` that is assigned to the `pkg.Package.Metadata` struct field it is important that a few things
 are done:
 
-- a new integration test case is added to `test/integration/pkg_cases_test.go` that exercises the new package type with the new metadata
+- a new integration test case is added to `test/integration/catalog_packages_cases_test.go` that exercises the new package type with the new metadata
 - the new metadata struct is added to the `artifactMetadataContainer` struct within `schema/json/generate.go`
 
 ## Generating a New Schema
@@ -36,4 +36,4 @@ Create the new schema by running `cd schema/json && go run generate.go` (note yo
 - If there is an existing schema for the given version and the new schema matches the existing schema, no action is taken
 - If there is an existing schema for the given version and the new schema **does not** match the existing schema, an error is shown indicating to increment the version appropriately (see the "Versioning" section)
 
-***Note: never delete a JSON schema and never change an existing JSON schema once it has been published in a release!*** Only add new schemas with a newly incremented version. All previous schema files must be stored in the `schema/json/` directory.
+***Note: never delete a JSON schema and never change an existing JSON schema once it has been published in a release!*** Only add new schemas with a newly incremented version. All previous schema files must be stored in the `schema/json/` directory.

schema/json/generate.go

@@ -27,19 +27,20 @@ can be extended to include specific package metadata struct shapes in the future
 // When a new package metadata definition is created it will need to be manually added here. The variable name does
 // not matter as long as it is exported.
 type artifactMetadataContainer struct {
-	Apk    pkg.ApkMetadata
-	Alpm   pkg.AlpmMetadata
-	Dpkg   pkg.DpkgMetadata
-	Gem    pkg.GemMetadata
-	Java   pkg.JavaMetadata
-	Npm    pkg.NpmPackageJSONMetadata
-	Python pkg.PythonPackageMetadata
-	Rpm    pkg.RpmdbMetadata
-	Cargo  pkg.CargoPackageMetadata
-	Go     pkg.GolangBinMetadata
-	Php    pkg.PhpComposerJSONMetadata
-	Dart   pkg.DartPubMetadata
-	Dotnet pkg.DotnetDepsMetadata
+	Apk     pkg.ApkMetadata
+	Alpm    pkg.AlpmMetadata
+	Dpkg    pkg.DpkgMetadata
+	Gem     pkg.GemMetadata
+	Java    pkg.JavaMetadata
+	Npm     pkg.NpmPackageJSONMetadata
+	Python  pkg.PythonPackageMetadata
+	Rpm     pkg.RpmdbMetadata
+	Cargo   pkg.CargoPackageMetadata
+	Go      pkg.GolangBinMetadata
+	Php     pkg.PhpComposerJSONMetadata
+	Dart    pkg.DartPubMetadata
+	Dotnet  pkg.DotnetDepsMetadata
+	Portage pkg.PortageMetadata
 }
 
 func main() {

syft/pkg/cataloger/cataloger.go

@@ -21,6 +21,7 @@ import (
 	"github.com/anchore/syft/syft/pkg/cataloger/java"
 	"github.com/anchore/syft/syft/pkg/cataloger/javascript"
 	"github.com/anchore/syft/syft/pkg/cataloger/php"
+	"github.com/anchore/syft/syft/pkg/cataloger/portage"
 	"github.com/anchore/syft/syft/pkg/cataloger/python"
 	"github.com/anchore/syft/syft/pkg/cataloger/rpmdb"
 	"github.com/anchore/syft/syft/pkg/cataloger/ruby"
@@ -54,6 +55,7 @@ func ImageCatalogers(cfg Config) []Cataloger {
 		apkdb.NewApkdbCataloger(),
 		golang.NewGoModuleBinaryCataloger(),
 		dotnet.NewDotnetDepsCataloger(),
+		portage.NewPortageCataloger(),
 	}, cfg.Catalogers)
 }
 
@@ -77,6 +79,7 @@ func DirectoryCatalogers(cfg Config) []Cataloger {
 		dart.NewPubspecLockCataloger(),
 		dotnet.NewDotnetDepsCataloger(),
 		cpp.NewConanfileCataloger(),
+		portage.NewPortageCataloger(),
 	}, cfg.Catalogers)
 }
 
@@ -103,6 +106,7 @@ func AllCatalogers(cfg Config) []Cataloger {
 		php.NewPHPComposerInstalledCataloger(),
 		php.NewPHPComposerLockCataloger(),
 		cpp.NewConanfileCataloger(),
+		portage.NewPortageCataloger(),
 	}, cfg.Catalogers)
 }
 

syft/pkg/cataloger/portage/cataloger.go

@@ -0,0 +1,151 @@
+/*
+Package portage provides a concrete Cataloger implementation for Gentoo Portage.
+*/
+package portage
+
+import (
+	"bufio"
+	"fmt"
+	"path"
+	"path/filepath"
+	"regexp"
+	"sort"
+	"strconv"
+	"strings"
+
+	"github.com/anchore/syft/internal"
+	"github.com/anchore/syft/internal/log"
+	"github.com/anchore/syft/syft/artifact"
+	"github.com/anchore/syft/syft/file"
+	"github.com/anchore/syft/syft/pkg"
+	"github.com/anchore/syft/syft/source"
+)
+
+var (
+	cpvRe = regexp.MustCompile(`/([^/]*/[\w+][\w+-]*)-((\d+)((\.\d+)*)([a-z]?)((_(pre|p|beta|alpha|rc)\d*)*)(-r\d+)?)/CONTENTS$`)
+)
+
+type Cataloger struct{}
+
+// NewPortageCataloger returns a new Portage package cataloger object.
+func NewPortageCataloger() *Cataloger {
+	return &Cataloger{}
+}
+
+// Name returns a string that uniquely describes a cataloger
+func (c *Cataloger) Name() string {
+	return "portage-cataloger"
+}
+
+// Catalog is given an object to resolve file references and content, this function returns any discovered Packages after analyzing portage support files.
+func (c *Cataloger) Catalog(resolver source.FileResolver) ([]pkg.Package, []artifact.Relationship, error) {
+	dbFileMatches, err := resolver.FilesByGlob(pkg.PortageDBGlob)
+	if err != nil {
+		return nil, nil, fmt.Errorf("failed to find portage files by glob: %w", err)
+	}
+	var allPackages []pkg.Package
+	for _, dbLocation := range dbFileMatches {
+		cpvMatch := cpvRe.FindStringSubmatch(dbLocation.RealPath)
+		if cpvMatch == nil {
+			return nil, nil, fmt.Errorf("failed to match package and version in %s", dbLocation.RealPath)
+		}
+		entry := pkg.PortageMetadata{
+			// ensure the default value for a collection is never nil since this may be shown as JSON
+			Files:   make([]pkg.PortageFileRecord, 0),
+			Package: cpvMatch[1],
+			Version: cpvMatch[2],
+		}
+
+		err = addFiles(resolver, dbLocation, &entry)
+		if err != nil {
+			return nil, nil, err
+		}
+
+		addSize(resolver, dbLocation, &entry)
+
+		p := pkg.Package{
+			Name:         entry.Package,
+			Version:      entry.Version,
+			Type:         pkg.PortagePkg,
+			MetadataType: pkg.PortageMetadataType,
+			Metadata:     entry,
+		}
+		addLicenses(resolver, dbLocation, &p)
+		p.FoundBy = c.Name()
+		p.Locations.Add(dbLocation)
+		p.SetID()
+		allPackages = append(allPackages, p)
+	}
+	return allPackages, nil, nil
+}
+
+func addFiles(resolver source.FileResolver, dbLocation source.Location, entry *pkg.PortageMetadata) error {
+	contentsReader, err := resolver.FileContentsByLocation(dbLocation)
+	if err != nil {
+		return err
+	}
+
+	scanner := bufio.NewScanner(contentsReader)
+	for scanner.Scan() {
+		line := strings.Trim(scanner.Text(), "\n")
+		fields := strings.Split(line, " ")
+
+		if fields[0] == "obj" {
+			record := pkg.PortageFileRecord{
+				Path: fields[1],
+			}
+			record.Digest = &file.Digest{
+				Algorithm: "md5",
+				Value:     fields[2],
+			}
+			entry.Files = append(entry.Files, record)
+		}
+	}
+	return nil
+}
+
+func addLicenses(resolver source.FileResolver, dbLocation source.Location, p *pkg.Package) {
+	parentPath := filepath.Dir(dbLocation.RealPath)
+
+	location := resolver.RelativeFileByPath(dbLocation, path.Join(parentPath, "LICENSE"))
+
+	if location != nil {
+		licenseReader, err := resolver.FileContentsByLocation(*location)
+		if err == nil {
+			findings := internal.NewStringSet()
+			scanner := bufio.NewScanner(licenseReader)
+			scanner.Split(bufio.ScanWords)
+			for scanner.Scan() {
+				token := scanner.Text()
+				if token != "||" && token != "(" && token != ")" {
+					findings.Add(token)
+				}
+			}
+			p.Licenses = findings.ToSlice()
+
+			sort.Strings(p.Licenses)
+		}
+	}
+}
+
+func addSize(resolver source.FileResolver, dbLocation source.Location, entry *pkg.PortageMetadata) {
+	parentPath := filepath.Dir(dbLocation.RealPath)
+
+	location := resolver.RelativeFileByPath(dbLocation, path.Join(parentPath, "SIZE"))
+
+	if location != nil {
+		sizeReader, err := resolver.FileContentsByLocation(*location)
+		if err != nil {
+			log.Warnf("failed to fetch portage SIZE (package=%s): %+v", entry.Package, err)
+		} else {
+			scanner := bufio.NewScanner(sizeReader)
+			for scanner.Scan() {
+				line := strings.Trim(scanner.Text(), "\n")
+				size, err := strconv.Atoi(line)
+				if err == nil {
+					entry.InstalledSize = size
+				}
+			}
+		}
+	}
+}

syft/pkg/cataloger/portage/cataloger_test.go

@@ -0,0 +1,106 @@
+package portage
+
+import (
+	"testing"
+
+	"github.com/anchore/syft/syft/file"
+
+	"github.com/anchore/stereoscope/pkg/imagetest"
+	"github.com/anchore/syft/syft/pkg"
+	"github.com/anchore/syft/syft/source"
+	"github.com/go-test/deep"
+)
+
+func TestPortageCataloger(t *testing.T) {
+	tests := []struct {
+		name     string
+		expected []pkg.Package
+	}{
+		{
+			name: "go-case",
+			expected: []pkg.Package{
+				{
+					Name:         "app-containers/skopeo",
+					Version:      "1.5.1",
+					FoundBy:      "portage-cataloger",
+					Licenses:     []string{"Apache-2.0", "BSD", "BSD-2", "CC-BY-SA-4.0", "ISC", "MIT"},
+					Type:         pkg.PortagePkg,
+					MetadataType: pkg.PortageMetadataType,
+					Metadata: pkg.PortageMetadata{
+						Package:       "app-containers/skopeo",
+						Version:       "1.5.1",
+						InstalledSize: 27937835,
+						Files: []pkg.PortageFileRecord{
+							{
+								Path: "/usr/bin/skopeo",
+								Digest: &file.Digest{
+									Algorithm: "md5",
+									Value:     "376c02bd3b22804df8fdfdc895e7dbfb",
+								},
+							},
+							{
+								Path: "/etc/containers/policy.json",
+								Digest: &file.Digest{
+									Algorithm: "md5",
+									Value:     "c01eb6950f03419e09d4fc88cb42ff6f",
+								},
+							},
+							{
+								Path: "/etc/containers/registries.d/default.yaml",
+								Digest: &file.Digest{
+									Algorithm: "md5",
+									Value:     "e6e66cd3c24623e0667f26542e0e08f6",
+								},
+							},
+							{
+								Path: "/var/lib/atomic/sigstore/.keep_app-containers_skopeo-0",
+								Digest: &file.Digest{
+									Algorithm: "md5",
+									Value:     "d41d8cd98f00b204e9800998ecf8427e",
+								},
+							},
+						},
+					},
+				},
+			},
+		},
+	}
+
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+
+			img := imagetest.GetFixtureImage(t, "docker-archive", "image-portage")
+
+			s, err := source.NewFromImage(img, "")
+			if err != nil {
+				t.Fatal(err)
+			}
+
+			c := NewPortageCataloger()
+
+			resolver, err := s.FileResolver(source.SquashedScope)
+			if err != nil {
+				t.Errorf("could not get resolver error: %+v", err)
+			}
+
+			actual, _, err := c.Catalog(resolver)
+			if err != nil {
+				t.Fatalf("failed to catalog: %+v", err)
+			}
+
+			if len(actual) != len(test.expected) {
+				for _, a := range actual {
+					t.Logf("   %+v", a)
+				}
+				t.Fatalf("unexpected package count: %d!=%d", len(actual), len(test.expected))
+			}
+
+			// test remaining fields...
+			for _, d := range deep.Equal(actual, test.expected) {
+				t.Errorf("diff: %+v", d)
+			}
+
+		})
+	}
+
+}

syft/pkg/cataloger/portage/test-fixtures/image-portage/Dockerfile

@@ -0,0 +1,2 @@
+FROM scratch
+COPY . .

syft/pkg/cataloger/portage/test-fixtures/image-portage/var/db/pkg/app-containers/skopeo-1.5.1/CONTENTS

@@ -0,0 +1,13 @@
+dir /usr
+dir /usr/bin
+obj /usr/bin/skopeo 376c02bd3b22804df8fdfdc895e7dbfb 1649284374
+dir /etc
+dir /etc/containers
+obj /etc/containers/policy.json c01eb6950f03419e09d4fc88cb42ff6f 1649284375
+dir /etc/containers/registries.d
+obj /etc/containers/registries.d/default.yaml e6e66cd3c24623e0667f26542e0e08f6 1649284375
+dir /var
+dir /var/lib
+dir /var/lib/atomic
+dir /var/lib/atomic/sigstore
+obj /var/lib/atomic/sigstore/.keep_app-containers_skopeo-0 d41d8cd98f00b204e9800998ecf8427e 1649284375

syft/pkg/cataloger/portage/test-fixtures/image-portage/var/db/pkg/app-containers/skopeo-1.5.1/LICENSE

@@ -0,0 +1 @@
+Apache-2.0 BSD BSD-2 CC-BY-SA-4.0 ISC MIT

syft/pkg/cataloger/portage/test-fixtures/image-portage/var/db/pkg/app-containers/skopeo-1.5.1/SIZE

@@ -0,0 +1 @@
+27937835

syft/pkg/metadata.go

@@ -26,6 +26,7 @@ const (
 	GolangBinMetadataType        MetadataType = "GolangBinMetadata"
 	PhpComposerJSONMetadataType  MetadataType = "PhpComposerJsonMetadata"
 	ConanaMetadataType           MetadataType = "ConanaMetadataType"
+	PortageMetadataType          MetadataType = "PortageMetadata"
 )
 
 var AllMetadataTypes = []MetadataType{
@@ -44,6 +45,7 @@ var AllMetadataTypes = []MetadataType{
 	GolangBinMetadataType,
 	PhpComposerJSONMetadataType,
 	ConanaMetadataType,
+	PortageMetadataType,
 }
 
 var MetadataTypeByName = map[MetadataType]reflect.Type{
@@ -62,4 +64,5 @@ var MetadataTypeByName = map[MetadataType]reflect.Type{
 	GolangBinMetadataType:        reflect.TypeOf(GolangBinMetadata{}),
 	PhpComposerJSONMetadataType:  reflect.TypeOf(PhpComposerJSONMetadata{}),
 	ConanaMetadataType:           reflect.TypeOf(ConanMetadata{}),
+	PortageMetadataType:          reflect.TypeOf(PortageMetadata{}),
 }

syft/pkg/portage_metadata.go

@@ -0,0 +1,21 @@
+package pkg
+
+import (
+	"github.com/anchore/syft/syft/file"
+)
+
+const PortageDBGlob = "**/var/db/pkg/*/*/CONTENTS"
+
+// PortageMetadata represents all captured data for a Package package DB entry.
+type PortageMetadata struct {
+	Package       string              `mapstructure:"Package" json:"package"`
+	Version       string              `mapstructure:"Version" json:"version"`
+	InstalledSize int                 `mapstructure:"InstalledSize" json:"installedSize" cyclonedx:"installedSize"`
+	Files         []PortageFileRecord `json:"files"`
+}
+
+// PortageFileRecord represents a single file attributed to a portage package.
+type PortageFileRecord struct {
+	Path   string       `json:"path"`
+	Digest *file.Digest `json:"digest,omitempty"`
+}

syft/pkg/type.go

@@ -24,6 +24,7 @@ const (
 	DartPubPkg       Type = "dart-pub"
 	DotnetPkg        Type = "dotnet"
 	ConanPkg         Type = "conan"
+	PortagePkg       Type = "portage"
 )
 
 // AllPkgs represents all supported package types
@@ -44,6 +45,7 @@ var AllPkgs = []Type{
 	DartPubPkg,
 	DotnetPkg,
 	ConanPkg,
+	PortagePkg,
 }
 
 // PackageURLType returns the PURL package type for the current package.
@@ -77,6 +79,8 @@ func (t Type) PackageURLType() string {
 		return packageurl.TypeDotnet
 	case ConanPkg:
 		return packageurl.TypeConan
+	case PortagePkg:
+		return "portage"
 	default:
 		// TODO: should this be a "generic" purl type instead?
 		return ""
@@ -122,6 +126,8 @@ func TypeByName(name string) Type {
 		return DotnetPkg
 	case packageurl.TypeConan:
 		return ConanPkg
+	case "portage":
+		return PortagePkg
 	default:
 		return UnknownPkg
 	}

syft/pkg/type_test.go

@@ -83,6 +83,7 @@ func TestTypeFromPURL(t *testing.T) {
 	// testing microsoft packages and jenkins-plugins is not valid for purl at this time
 	expectedTypes.Remove(string(KbPkg))
 	expectedTypes.Remove(string(JenkinsPluginPkg))
+	expectedTypes.Remove(string(PortagePkg))
 
 	for _, test := range tests {
 		t.Run(string(test.expected), func(t *testing.T) {

syft/pkg/url_test.go

@@ -233,6 +233,7 @@ func TestPackageURL(t *testing.T) {
 
 	// testing microsoft packages is not valid for purl at this time
 	expectedTypes.Remove(string(KbPkg))
+	expectedTypes.Remove(string(PortagePkg))
 
 	for _, test := range tests {
 		t.Run(test.name, func(t *testing.T) {

test/cli/packages_cmd_test.go

@@ -96,7 +96,7 @@ func TestPackagesCmdFlags(t *testing.T) {
 			name: "squashed-scope-flag",
 			args: []string{"packages", "-o", "json", "-s", "squashed", coverageImage},
 			assertions: []traitAssertion{
-				assertPackageCount(33),
+				assertPackageCount(34),
 				assertSuccessfulReturnCode,
 			},
 		},

test/integration/catalog_packages_cases_test.go

@@ -277,6 +277,14 @@ var commonTestCases = []testCase{
 			"netbase": "5.4",
 		},
 	},
+	{
+		name:    "find portage packages",
+		pkgType: pkg.PortagePkg,
+		pkgInfo: map[string]string{
+			"app-containers/skopeo": "1.5.1",
+		},
+	},
+
 	{
 		name:        "find jenkins plugins",
 		pkgType:     pkg.JenkinsPluginPkg,

test/integration/test-fixtures/image-pkg-coverage/pkgs/var/db/pkg/app-containers/skopeo-1.5.1/CONTENTS

@@ -0,0 +1,13 @@
+dir /usr
+dir /usr/bin
+obj /usr/bin/skopeo 376c02bd3b22804df8fdfdc895e7dbfb 1649284374
+dir /etc
+dir /etc/containers
+obj /etc/containers/policy.json c01eb6950f03419e09d4fc88cb42ff6f 1649284375
+dir /etc/containers/registries.d
+obj /etc/containers/registries.d/default.yaml e6e66cd3c24623e0667f26542e0e08f6 1649284375
+dir /var
+dir /var/lib
+dir /var/lib/atomic
+dir /var/lib/atomic/sigstore
+obj /var/lib/atomic/sigstore/.keep_app-containers_skopeo-0 d41d8cd98f00b204e9800998ecf8427e 1649284375

test/integration/test-fixtures/image-pkg-coverage/pkgs/var/db/pkg/app-containers/skopeo-1.5.1/LICENSE

@@ -0,0 +1 @@
+Apache-2.0 BSD BSD-2 CC-BY-SA-4.0 ISC MIT

test/integration/test-fixtures/image-pkg-coverage/pkgs/var/db/pkg/app-containers/skopeo-1.5.1/SIZE

@@ -0,0 +1 @@
+27937835