Mirrors/syft
2
0
Fork 0
mirror of https://github.com/anchore/syft synced 2024-11-10 06:14:16 +00:00
Code Issues Projects Releases Packages Wiki Activity

Add abstraction for adding relationships from package cataloger results (#2853)

Browse source 
* add internal dependency resolver

Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com>

* refactor dependency relationship resolution to common object

Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com>

* replace cataloger decorator with generic processor

Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com>

* refactor resolver to be a single function

Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com>

* use common dependency specifier for debian

Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com>

* use common dependency specifier for arch

Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com>

* use common dependency specifier for alpine

Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com>

* allow for generic pkg and rel assertions in testpkg helper

Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com>

* do not allow for empty results

Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com>

* move stable deduplicate comment

Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com>

* remove relationship resolver type

Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com>

---------

Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com>
This commit is contained in:
Alex Goodman 2024-05-14 09:27:36 -04:00 committed by GitHub
parent fae6f5d372
commit 4a18895545
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
32 changed files with 1207 additions and 801 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

6
internal/string_helpers.go
View file

@ -33,5 +33,9 @@ func SplitAny(s string, seps string) []string {
splitter := func(r rune) bool {
return strings.ContainsRune(seps, r)
}
return strings.FieldsFunc(s, splitter)
result := strings.FieldsFunc(s, splitter)
if len(result) == 0 {
return []string{s}
}
return result
}

2
internal/string_helpers_test.go
View file

@ -123,7 +123,7 @@ func TestSplitAny(t *testing.T) {
name: "empty",
input: "",
fields: ",",
want: []string{},
want: []string{""},
},
{
name: "multiple separators",

4
syft/pkg/cataloger/alpine/cataloger.go
View file

@ -6,10 +6,12 @@ package alpine
import (
"github.com/anchore/syft/syft/pkg"
"github.com/anchore/syft/syft/pkg/cataloger/generic"
"github.com/anchore/syft/syft/pkg/cataloger/internal/dependency"
)
// NewDBCataloger returns a new cataloger object initialized for Alpine package DB flat-file stores.
func NewDBCataloger() pkg.Cataloger {
return generic.NewCataloger("apk-db-cataloger").
WithParserByGlobs(parseApkDB, pkg.ApkDBGlob)
WithParserByGlobs(parseApkDB, pkg.ApkDBGlob).
WithProcessors(dependency.Processor(dbEntryDependencySpecifier))
}

230
syft/pkg/cataloger/alpine/cataloger_test.go
View file

@ -3,9 +3,239 @@ package alpine
import (
"testing"
"github.com/google/go-cmp/cmp"
"github.com/google/go-cmp/cmp/cmpopts"
"github.com/anchore/syft/syft/artifact"
"github.com/anchore/syft/syft/file"
"github.com/anchore/syft/syft/pkg"
"github.com/anchore/syft/syft/pkg/cataloger/internal/pkgtest"
)
func TestApkDBCataloger(t *testing.T) {
dbLocation := file.NewLocation("lib/apk/db/installed")
bashPkg := pkg.Package{
Name: "bash",
Version: "5.2.21-r0",
Type: pkg.ApkPkg,
FoundBy: "apk-db-cataloger",
Licenses: pkg.NewLicenseSet(
pkg.NewLicenseFromLocations("GPL-3.0-or-later", dbLocation),
),
Locations: file.NewLocationSet(dbLocation),
Metadata: pkg.ApkDBEntry{
Package: "bash",
OriginPackage: "bash",
Maintainer: "Natanael Copa <ncopa@alpinelinux.org>",
Version: "5.2.21-r0",
Architecture: "x86_64",
URL: "https://www.gnu.org/software/bash/bash.html",
Description: "The GNU Bourne Again shell",
Size: 448728,
InstalledSize: 1396736,
Dependencies: []string{
"/bin/sh", "so:libc.musl-x86_64.so.1", "so:libreadline.so.8",
},
Provides: []string{
"cmd:bash=5.2.21-r0",
},
// note: files not provided and not under test
},
}
busyboxBinshPkg := pkg.Package{
Name: "busybox-binsh",
Version: "1.36.1-r15",
Type: pkg.ApkPkg,
FoundBy: "apk-db-cataloger",
Licenses: pkg.NewLicenseSet(
pkg.NewLicenseFromLocations("GPL-2.0-only", dbLocation),
),
Locations: file.NewLocationSet(dbLocation),
Metadata: pkg.ApkDBEntry{
Package: "busybox-binsh",
OriginPackage: "busybox",
Maintainer: "Sören Tempel <soeren+alpine@soeren-tempel.net>",
Version: "1.36.1-r15",
Architecture: "x86_64",
URL: "https://busybox.net/",
Description: "busybox ash /bin/sh",
Size: 1543,
InstalledSize: 8192,
Dependencies: []string{
"busybox=1.36.1-r15",
},
Provides: []string{
"/bin/sh", "cmd:sh=1.36.1-r15",
},
// note: files not provided and not under test
},
}
muslPkg := pkg.Package{
Name: "musl",
Version: "1.2.4_git20230717-r4",
Type: pkg.ApkPkg,
FoundBy: "apk-db-cataloger",
Licenses: pkg.NewLicenseSet(
pkg.NewLicenseFromLocations("MIT", dbLocation),
),
Locations: file.NewLocationSet(dbLocation),
Metadata: pkg.ApkDBEntry{
Package: "musl",
OriginPackage: "musl",
Maintainer: "Timo Teräs <timo.teras@iki.fi>",
Version: "1.2.4_git20230717-r4",
Architecture: "x86_64",
URL: "https://musl.libc.org/",
Description: "the musl c library (libc) implementation",
Size: 407278,
InstalledSize: 667648,
Dependencies: []string{},
Provides: []string{
"so:libc.musl-x86_64.so.1=1",
},
// note: files not provided and not under test
},
}
readlinePkg := pkg.Package{
Name: "readline",
Version: "8.2.1-r2",
Type: pkg.ApkPkg,
FoundBy: "apk-db-cataloger",
Licenses: pkg.NewLicenseSet(
pkg.NewLicenseFromLocations("GPL-2.0-or-later", dbLocation),
),
Locations: file.NewLocationSet(dbLocation),
Metadata: pkg.ApkDBEntry{
Package: "readline",
OriginPackage: "readline",
Maintainer: "Natanael Copa <ncopa@alpinelinux.org>",
Version: "8.2.1-r2",
Architecture: "x86_64",
URL: "https://tiswww.cwru.edu/php/chet/readline/rltop.html",
Description: "GNU readline library",
Size: 119878,
InstalledSize: 303104,
Dependencies: []string{
"so:libc.musl-x86_64.so.1", "so:libncursesw.so.6",
},
Provides: []string{
"so:libreadline.so.8=8.2",
},
// note: files not provided and not under test
},
}
expectedPkgs := []pkg.Package{
bashPkg,
busyboxBinshPkg,
muslPkg,
readlinePkg,
}
// # apk info --depends bash
// bash-5.2.21-r0 depends on:
// /bin/sh
// so:libc.musl-x86_64.so.1
// so:libreadline.so.8
//
// # apk info --who-owns /bin/sh
// /bin/sh is owned by busybox-binsh-1.36.1-r15
//
// # find / | grep musl
// /lib/ld-musl-x86_64.so.1
// /lib/libc.musl-x86_64.so.1
//
// # apk info --who-owns '/lib/libc.musl-x86_64.so.1'
// /lib/libc.musl-x86_64.so.1 is owned by musl-1.2.4_git20230717-r4
//
// # find / | grep libreadline
// /usr/lib/libreadline.so.8.2
// /usr/lib/libreadline.so.8
//
// # apk info --who-owns '/usr/lib/libreadline.so.8'
// /usr/lib/libreadline.so.8 is owned by readline-8.2.1-r2
expectedRelationships := []artifact.Relationship{
{
From: busyboxBinshPkg,
To: bashPkg,
Type: artifact.DependencyOfRelationship,
},
{
From: readlinePkg,
To: bashPkg,
Type: artifact.DependencyOfRelationship,
},
{
From: muslPkg,
To: readlinePkg,
Type: artifact.DependencyOfRelationship,
},
{
From: muslPkg,
To: bashPkg,
Type: artifact.DependencyOfRelationship,
},
}
pkgtest.NewCatalogTester().
FromDirectory(t, "test-fixtures/multiple-1").
WithCompareOptions(cmpopts.IgnoreFields(pkg.ApkDBEntry{}, "Files", "GitCommit", "Checksum")).
Expects(expectedPkgs, expectedRelationships).
TestCataloger(t, NewDBCataloger())
}
func TestCatalogerDependencyTree(t *testing.T) {
assertion := func(t *testing.T, pkgs []pkg.Package, relationships []artifact.Relationship) {
expected := map[string][]string{
"alpine-baselayout": {"busybox", "alpine-baselayout-data", "musl"},
"apk-tools": {"ca-certificates-bundle", "musl", "libcrypto1.1", "libssl1.1", "zlib"},
"busybox": {"musl"},
"libc-utils": {"musl-utils"},
"libcrypto1.1": {"musl"},
"libssl1.1": {"musl", "libcrypto1.1"},
"musl-utils": {"scanelf", "musl"},
"scanelf": {"musl"},
"ssl_client": {"musl", "libcrypto1.1", "libssl1.1"},
"zlib": {"musl"},
}
pkgsByID := make(map[artifact.ID]pkg.Package)
for _, p := range pkgs {
p.SetID()
pkgsByID[p.ID()] = p
}
actualDependencies := make(map[string][]string)
for _, r := range relationships {
switch r.Type {
case artifact.DependencyOfRelationship:
to := pkgsByID[r.To.ID()]
from := pkgsByID[r.From.ID()]
actualDependencies[to.Name] = append(actualDependencies[to.Name], from.Name)
default:
t.Fatalf("unexpected relationship type: %+v", r.Type)
}
}
if d := cmp.Diff(expected, actualDependencies); d != "" {
t.Fail()
t.Log(d)
}
}
pkgtest.NewCatalogTester().
FromDirectory(t, "test-fixtures/multiple-2").
ExpectsAssertion(assertion).
TestCataloger(t, NewDBCataloger())
}
func TestCataloger_Globs(t *testing.T) {
tests := []struct {
name string

48
syft/pkg/cataloger/alpine/dependency.go Normal file
View file

@ -0,0 +1,48 @@
package alpine
import (
"strings"
"github.com/anchore/syft/internal"
"github.com/anchore/syft/internal/log"
"github.com/anchore/syft/syft/pkg"
"github.com/anchore/syft/syft/pkg/cataloger/internal/dependency"
)
var _ dependency.Specifier = dbEntryDependencySpecifier
func dbEntryDependencySpecifier(p pkg.Package) dependency.Specification {
meta, ok := p.Metadata.(pkg.ApkDBEntry)
if !ok {
log.Tracef("cataloger failed to extract apk metadata for package %+v", p.Name)
return dependency.Specification{}
}
provides := []string{p.Name}
provides = append(provides, stripVersionSpecifiers(meta.Provides)...)
return dependency.Specification{
Provides: provides,
Requires: stripVersionSpecifiers(meta.Dependencies),
}
}
func stripVersionSpecifiers(given []string) []string {
var keys []string
for _, key := range given {
key = stripVersionSpecifier(key)
if key == "" {
continue
}
keys = append(keys, key)
}
return keys
}
func stripVersionSpecifier(s string) string {
// examples:
// musl>=1 --> musl
// cmd:scanelf=1.3.4-r0 --> cmd:scanelf
return strings.TrimSpace(internal.SplitAny(s, "<>=")[0])
}

110
syft/pkg/cataloger/alpine/dependency_test.go Normal file
View file

@ -0,0 +1,110 @@
package alpine
import (
"testing"
"github.com/stretchr/testify/assert"
"github.com/anchore/syft/syft/pkg"
"github.com/anchore/syft/syft/pkg/cataloger/internal/dependency"
)
func Test_dbEntryDependencySpecifier(t *testing.T) {
tests := []struct {
name string
p pkg.Package
want dependency.Specification
}{
{
name: "keeps given values + package name",
p: pkg.Package{
Name: "package-c",
Metadata: pkg.ApkDBEntry{
Provides: []string{"a-thing"},
Dependencies: []string{"b-thing"},
},
},
want: dependency.Specification{
Provides: []string{"package-c", "a-thing"},
Requires: []string{"b-thing"},
},
},
{
name: "strip version specifiers",
p: pkg.Package{
Name: "package-a",
Metadata: pkg.ApkDBEntry{
Provides: []string{"so:libc.musl-x86_64.so.1=1"},
Dependencies: []string{"so:libc.musl-x86_64.so.2=2"},
},
},
want: dependency.Specification{
Provides: []string{"package-a", "so:libc.musl-x86_64.so.1"},
Requires: []string{"so:libc.musl-x86_64.so.2"},
},
},
{
name: "empty dependency data entries",
p: pkg.Package{
Name: "package-a",
Metadata: pkg.ApkDBEntry{
Provides: []string{""},
Dependencies: []string{""},
},
},
want: dependency.Specification{
Provides: []string{"package-a"},
Requires: nil,
},
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
assert.Equal(t, tt.want, dbEntryDependencySpecifier(tt.p))
})
}
}
func Test_stripVersionSpecifier(t *testing.T) {
tests := []struct {
name string
version string
want string
}{
{
name: "empty expression",
version: "",
want: "",
},
{
name: "no expression",
version: "cmd:foo",
want: "cmd:foo",
},
{
name: "=",
version: "cmd:scanelf=1.3.4-r0",
want: "cmd:scanelf",
},
{
name: ">=",
version: "cmd:scanelf>=1.3.4-r0",
want: "cmd:scanelf",
},
{
name: "<",
version: "cmd:scanelf<1.3.4-r0",
want: "cmd:scanelf",
},
{
name: "ignores file paths",
version: "/bin/sh",
want: "/bin/sh",
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
assert.Equal(t, tt.want, stripVersionSpecifier(tt.version))
})
}
}

56
syft/pkg/cataloger/alpine/parse_apk_db.go
View file

@ -131,7 +131,7 @@ func parseApkDB(_ context.Context, resolver file.Resolver, env *generic.Environm
pkgs = append(pkgs, newPackage(apk, r, reader.Location))
}
return pkgs, discoverPackageDependencies(pkgs), nil
return pkgs, nil, nil
}
func findReleases(resolver file.Resolver, dbPath string) []linux.Release {
@ -386,57 +386,3 @@ func processChecksum(value string) *file.Digest {
Value: value,
}
}
func discoverPackageDependencies(pkgs []pkg.Package) (relationships []artifact.Relationship) {
// map["provides" string] -> packages that provide the "p" key
lookup := make(map[string][]pkg.Package)
// read "Provides" (p) and add as keys for lookup keys as well as package names
for _, p := range pkgs {
apkg, ok := p.Metadata.(pkg.ApkDBEntry)
if !ok {
log.Warnf("cataloger failed to extract apk 'provides' metadata for package %+v", p.Name)
continue
}
lookup[p.Name] = append(lookup[p.Name], p)
for _, provides := range apkg.Provides {
k := stripVersionSpecifier(provides)
lookup[k] = append(lookup[k], p)
}
}
// read "Pull Dependencies" (D) and match with keys
for _, p := range pkgs {
apkg, ok := p.Metadata.(pkg.ApkDBEntry)
if !ok {
log.Warnf("cataloger failed to extract apk dependency metadata for package %+v", p.Name)
continue
}
for _, depSpecifier := range apkg.Dependencies {
// use the lookup to find what pkg we depend on
dep := stripVersionSpecifier(depSpecifier)
for _, depPkg := range lookup[dep] {
// this is a pkg that package "p" depends on... make a relationship
relationships = append(relationships, artifact.Relationship{
From: depPkg,
To: p,
Type: artifact.DependencyOfRelationship,
})
}
}
}
return relationships
}
func stripVersionSpecifier(s string) string {
// examples:
// musl>=1 --> musl
// cmd:scanelf=1.3.4-r0 --> cmd:scanelf
items := internal.SplitAny(s, "<>=")
if len(items) == 0 {
return s
}
return items[0]
}

410
syft/pkg/cataloger/alpine/parse_apk_db_test.go
View file

@ -9,11 +9,9 @@ import (
"testing"
"github.com/google/go-cmp/cmp"
"github.com/google/go-cmp/cmp/cmpopts"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"github.com/anchore/syft/syft/artifact"
"github.com/anchore/syft/syft/file"
"github.com/anchore/syft/syft/linux"
"github.com/anchore/syft/syft/pkg"
@ -689,144 +687,6 @@ func TestSinglePackageDetails(t *testing.T) {
}
}
func TestMultiplePackages(t *testing.T) {
fixture := "test-fixtures/multiple"
location := file.NewLocation(fixture)
fixtureLocationSet := file.NewLocationSet(location)
expectedPkgs := []pkg.Package{
{
Name: "libc-utils",
Version: "0.7.2-r0",
Licenses: pkg.NewLicenseSet(
pkg.NewLicenseFromLocations("MPL-2.0 AND MIT", location),
),
Type: pkg.ApkPkg,
PURL: "pkg:apk/alpine/libc-utils@0.7.2-r0?arch=x86_64&upstream=libc-dev&distro=alpine-3.12",
Locations: fixtureLocationSet,
Metadata: pkg.ApkDBEntry{
Package: "libc-utils",
OriginPackage: "libc-dev",
Maintainer: "Natanael Copa <ncopa@alpinelinux.org>",
Version: "0.7.2-r0",
Architecture: "x86_64",
URL: "http://alpinelinux.org",
Description: "Meta package to pull in correct libc",
Size: 1175,
InstalledSize: 4096,
Checksum: "Q1p78yvTLG094tHE1+dToJGbmYzQE=",
GitCommit: "97b1c2842faa3bfa30f5811ffbf16d5ff9f1a479",
Dependencies: []string{"musl-utils"},
Provides: []string{},
Files: []pkg.ApkFileRecord{},
},
},
{
Name: "musl-utils",
Version: "1.1.24-r2",
Type: pkg.ApkPkg,
PURL: "pkg:apk/alpine/musl-utils@1.1.24-r2?arch=x86_64&upstream=musl&distro=alpine-3.12",
Locations: fixtureLocationSet,
Licenses: pkg.NewLicenseSet(
pkg.NewLicenseFromLocations("MIT", location),
pkg.NewLicenseFromLocations("BSD", location),
pkg.NewLicenseFromLocations("GPL2+", location),
),
Metadata: pkg.ApkDBEntry{
Package: "musl-utils",
OriginPackage: "musl",
Version: "1.1.24-r2",
Description: "the musl c library (libc) implementation",
Maintainer: "Timo Teräs <timo.teras@iki.fi>",
Architecture: "x86_64",
URL: "https://musl.libc.org/",
Size: 37944,
InstalledSize: 151552,
GitCommit: "4024cc3b29ad4c65544ad068b8f59172b5494306",
Dependencies: []string{"scanelf", "so:libc.musl-x86_64.so.1"},
Provides: []string{"cmd:getconf", "cmd:getent", "cmd:iconv", "cmd:ldconfig", "cmd:ldd"},
Checksum: "Q1bTtF5526tETKfL+lnigzIDvm+2o=",
Files: []pkg.ApkFileRecord{
{
Path: "/sbin",
},
{
Path: "/sbin/ldconfig",
OwnerUID: "0",
OwnerGID: "0",
Permissions: "755",
Digest: &file.Digest{
Algorithm: "'Q1'+base64(sha1)",
Value: "Q1Kja2+POZKxEkUOZqwSjC6kmaED4=",
},
},
{
Path: "/usr",
},
{
Path: "/usr/bin",
},
{
Path: "/usr/bin/iconv",
OwnerUID: "0",
OwnerGID: "0",
Permissions: "755",
Digest: &file.Digest{
Algorithm: "'Q1'+base64(sha1)",
Value: "Q1CVmFbdY+Hv6/jAHl1gec2Kbx1EY=",
},
},
{
Path: "/usr/bin/ldd",
OwnerUID: "0",
OwnerGID: "0",
Permissions: "755",
Digest: &file.Digest{
Algorithm: "'Q1'+base64(sha1)",
Value: "Q1yFAhGggmL7ERgbIA7KQxyTzf3ks=",
},
},
{
Path: "/usr/bin/getconf",
OwnerUID: "0",
OwnerGID: "0",
Permissions: "755",
Digest: &file.Digest{
Algorithm: "'Q1'+base64(sha1)",
Value: "Q1dAdYK8M/INibRQF5B3Rw7cmNDDA=",
},
},
{
Path: "/usr/bin/getent",
OwnerUID: "0",
OwnerGID: "0",
Permissions: "755",
Digest: &file.Digest{
Algorithm: "'Q1'+base64(sha1)",
Value: "Q1eR2Dz/WylabgbWMTkd2+hGmEya4=",
},
},
},
},
},
}
expectedRelationships := []artifact.Relationship{
{
From: expectedPkgs[1], // musl-utils
To: expectedPkgs[0], // libc-utils
Type: artifact.DependencyOfRelationship,
Data: nil,
},
}
env := generic.Environment{LinuxRelease: &linux.Release{
ID: "alpine",
VersionID: "3.12",
}}
pkgtest.TestFileParserWithEnv(t, fixture, parseApkDB, &env, expectedPkgs, expectedRelationships)
}
func Test_processChecksum(t *testing.T) {
tests := []struct {
name string
@ -858,237 +718,6 @@ func Test_processChecksum(t *testing.T) {
}
}
func Test_discoverPackageDependencies(t *testing.T) {
tests := []struct {
name string
genFn func() ([]pkg.Package, []artifact.Relationship)
}{
{
name: "has no dependency",
genFn: func() ([]pkg.Package, []artifact.Relationship) {
a := pkg.Package{
Name: "package-a",
Metadata: pkg.ApkDBEntry{
Provides: []string{"a-thing"},
},
}
a.SetID()
b := pkg.Package{
Name: "package-b",
Metadata: pkg.ApkDBEntry{
Provides: []string{"b-thing"},
},
}
b.SetID()
return []pkg.Package{a, b}, nil
},
},
{
name: "has 1 dependency",
genFn: func() ([]pkg.Package, []artifact.Relationship) {
a := pkg.Package{
Name: "package-a",
Metadata: pkg.ApkDBEntry{
Dependencies: []string{"b-thing"},
},
}
a.SetID()
b := pkg.Package{
Name: "package-b",
Metadata: pkg.ApkDBEntry{
Provides: []string{"b-thing"},
},
}
b.SetID()
return []pkg.Package{a, b}, []artifact.Relationship{
{
From: b,
To: a,
Type: artifact.DependencyOfRelationship,
},
}
},
},
{
name: "strip version specifiers",
genFn: func() ([]pkg.Package, []artifact.Relationship) {
a := pkg.Package{
Name: "package-a",
Metadata: pkg.ApkDBEntry{
Dependencies: []string{"so:libc.musl-x86_64.so.1"},
},
}
a.SetID()
b := pkg.Package{
Name: "package-b",
Metadata: pkg.ApkDBEntry{
Provides: []string{"so:libc.musl-x86_64.so.1=1"},
},
}
b.SetID()
return []pkg.Package{a, b}, []artifact.Relationship{
{
From: b,
To: a,
Type: artifact.DependencyOfRelationship,
},
}
},
},
{
name: "strip version specifiers with empty provides value",
genFn: func() ([]pkg.Package, []artifact.Relationship) {
a := pkg.Package{
Name: "package-a",
Metadata: pkg.ApkDBEntry{
Dependencies: []string{"so:libc.musl-x86_64.so.1"},
},
}
a.SetID()
b := pkg.Package{
Name: "package-b",
Metadata: pkg.ApkDBEntry{
Provides: []string{""},
},
}
b.SetID()
return []pkg.Package{a, b}, nil
},
},
{
name: "depends on package name",
genFn: func() ([]pkg.Package, []artifact.Relationship) {
a := pkg.Package{
Name: "package-a",
Metadata: pkg.ApkDBEntry{
Dependencies: []string{"musl>=1.2"},
},
}
a.SetID()
b := pkg.Package{
Name: "musl",
Metadata: pkg.ApkDBEntry{
Provides: []string{"so:libc.musl-x86_64.so.1=1"},
},
}
b.SetID()
return []pkg.Package{a, b}, []artifact.Relationship{
{
From: b,
To: a,
Type: artifact.DependencyOfRelationship,
},
}
},
},
{
name: "depends on package file",
genFn: func() ([]pkg.Package, []artifact.Relationship) {
a := pkg.Package{
Name: "alpine-baselayout",
Metadata: pkg.ApkDBEntry{
Dependencies: []string{"/bin/sh"},
},
}
a.SetID()
b := pkg.Package{
Name: "busybox",
Metadata: pkg.ApkDBEntry{
Provides: []string{"/bin/sh"},
},
}
b.SetID()
return []pkg.Package{a, b}, []artifact.Relationship{
{
From: b,
To: a,
Type: artifact.DependencyOfRelationship,
},
}
},
},
}
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
pkgs, wantRelationships := test.genFn()
gotRelationships := discoverPackageDependencies(pkgs)
d := cmp.Diff(wantRelationships, gotRelationships, cmpopts.IgnoreUnexported(pkg.Package{}, file.LocationSet{}, pkg.LicenseSet{}))
if d != "" {
t.Fail()
t.Log(d)
}
})
}
}
func TestPackageDbDependenciesByParse(t *testing.T) {
tests := []struct {
fixture string
expected map[string][]string
}{
{
fixture: "test-fixtures/installed",
expected: map[string][]string{
"alpine-baselayout": {"alpine-baselayout-data", "busybox", "musl"},
"apk-tools": {"musl", "ca-certificates-bundle", "musl", "libcrypto1.1", "libssl1.1", "zlib"},
"busybox": {"musl"},
"libc-utils": {"musl-utils"},
"libcrypto1.1": {"musl"},
"libssl1.1": {"musl", "libcrypto1.1"},
"musl-utils": {"scanelf", "musl"},
"scanelf": {"musl"},
"ssl_client": {"musl", "libcrypto1.1", "libssl1.1"},
"zlib": {"musl"},
},
},
}
for _, test := range tests {
t.Run(test.fixture, func(t *testing.T) {
f, err := os.Open(test.fixture)
require.NoError(t, err)
t.Cleanup(func() { require.NoError(t, f.Close()) })
pkgs, relationships, err := parseApkDB(context.Background(), nil, nil, file.LocationReadCloser{
Location: file.NewLocation(test.fixture),
ReadCloser: f,
})
require.NoError(t, err)
pkgsByID := make(map[artifact.ID]pkg.Package)
for _, p := range pkgs {
p.SetID()
pkgsByID[p.ID()] = p
}
actualDependencies := make(map[string][]string)
for _, r := range relationships {
switch r.Type {
case artifact.DependencyOfRelationship:
to := pkgsByID[r.To.ID()]
from := pkgsByID[r.From.ID()]
actualDependencies[to.Name] = append(actualDependencies[to.Name], from.Name)
default:
t.Fatalf("unexpected relationship type: %+v", r.Type)
}
}
if d := cmp.Diff(test.expected, actualDependencies); d != "" {
t.Fail()
t.Log(d)
}
})
}
}
func Test_parseApkDB_expectedPkgNames(t *testing.T) {
tests := []struct {
fixture string
@ -1175,45 +804,6 @@ func newLocationReadCloser(t *testing.T, path string) file.LocationReadCloser {
return file.NewLocationReadCloser(file.NewLocation(path), f)
}
func Test_stripVersionSpecifier(t *testing.T) {
tests := []struct {
name string
version string
want string
}{
{
name: "empty expression",
version: "",
want: "",
},
{
name: "no expression",
version: "cmd:foo",
want: "cmd:foo",
},
{
name: "=",
version: "cmd:scanelf=1.3.4-r0",
want: "cmd:scanelf",
},
{
name: ">=",
version: "cmd:scanelf>=1.3.4-r0",
want: "cmd:scanelf",
},
{
name: "<",
version: "cmd:scanelf<1.3.4-r0",
want: "cmd:scanelf",
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
assert.Equal(t, tt.want, stripVersionSpecifier(tt.version))
})
}
}
func TestParseReleasesFromAPKRepository(t *testing.T) {
tests := []struct {
repos string

56
syft/pkg/cataloger/alpine/test-fixtures/multiple
View file

@ -1,56 +0,0 @@
C:Q1p78yvTLG094tHE1+dToJGbmYzQE=
P:libc-utils
V:0.7.2-r0
A:x86_64
S:1175
I:4096
T:Meta package to pull in correct libc
U:http://alpinelinux.org
L:MPL-2.0 AND MIT
o:libc-dev
m:Natanael Copa <ncopa@alpinelinux.org>
t:1575749004
c:97b1c2842faa3bfa30f5811ffbf16d5ff9f1a479
D:musl-utils
C:Q1bTtF5526tETKfL+lnigzIDvm+2o=
P:musl-utils
V:1.1.24-r2
A:x86_64
S:37944
I:151552
T:the musl c library (libc) implementation
U:https://musl.libc.org/
L:MIT BSD GPL2+
o:musl
m:Timo Teräs <timo.teras@iki.fi>
t:1584790550
c:4024cc3b29ad4c65544ad068b8f59172b5494306
D:scanelf so:libc.musl-x86_64.so.1
p:cmd:getconf cmd:getent cmd:iconv cmd:ldconfig cmd:ldd
r:libiconv
F:sbin
R:ldconfig
a:0:0:755
Z:Q1Kja2+POZKxEkUOZqwSjC6kmaED4=
F:usr
F:usr/bin
R:iconv
a:0:0:755
Z:Q1CVmFbdY+Hv6/jAHl1gec2Kbx1EY=
R:ldd
a:0:0:755
Z:Q1yFAhGggmL7ERgbIA7KQxyTzf3ks=
R:getconf
a:0:0:755
Z:Q1dAdYK8M/INibRQF5B3Rw7cmNDDA=
R:getent
a:0:0:755
Z:Q1eR2Dz/WylabgbWMTkd2+hGmEya4=

78
syft/pkg/cataloger/alpine/test-fixtures/multiple-1/lib/apk/db/installed Normal file
View file

@ -0,0 +1,78 @@
P:bash
V:5.2.21-r0
A:x86_64
S:448728
I:1396736
T:The GNU Bourne Again shell
U:https://www.gnu.org/software/bash/bash.html
L:GPL-3.0-or-later
o:bash
m:Natanael Copa <ncopa@alpinelinux.org>
t:1701073495
c:6a9559d98850225ba80771901ef1abda91cb29aa
D:/bin/sh so:libc.musl-x86_64.so.1 so:libreadline.so.8
p:cmd:bash=5.2.21-r0
P:busybox-binsh
V:1.36.1-r15
A:x86_64
S:1543
I:8192
T:busybox ash /bin/sh
U:https://busybox.net/
L:GPL-2.0-only
o:busybox
m:Sören Tempel <soeren+alpine@soeren-tempel.net>
t:1699383189
c:d1b6f274f29076967826e0ecf6ebcaa5d360272f
k:100
D:busybox=1.36.1-r15
p:/bin/sh cmd:sh=1.36.1-r15
r:busybox-initscripts
F:bin
R:sh
a:0:0:777
Z:Q1pcfTfDNEbNKQc2s1tia7da05M8Q=
P:musl
V:1.2.4_git20230717-r4
A:x86_64
S:407278
I:667648
T:the musl c library (libc) implementation
U:https://musl.libc.org/
L:MIT
o:musl
m:Timo Teräs <timo.teras@iki.fi>
t:1699271358
c:ca7f2ab5e88794e4e654b40776f8a92256f50639
p:so:libc.musl-x86_64.so.1=1
F:lib
R:ld-musl-x86_64.so.1
a:0:0:755
Z:Q1+zEJiG53Cxy7DkV5oZQqeWnzybY=
R:libc.musl-x86_64.so.1
a:0:0:777
Z:Q17yJ3JFNypA4mxhJJr0ou6CzsJVI=
P:readline
V:8.2.1-r2
A:x86_64
S:119878
I:303104
T:GNU readline library
U:https://tiswww.cwru.edu/php/chet/readline/rltop.html
L:GPL-2.0-or-later
o:readline
m:Natanael Copa <ncopa@alpinelinux.org>
t:1684120357
c:33283848034c9885d984c8e8697c645c57324938
D:so:libc.musl-x86_64.so.1 so:libncursesw.so.6
p:so:libreadline.so.8=8.2
F:etc
R:inputrc
Z:Q1ilcgkuEseXEH6iMo9UNjLn1pPfg=
F:usr
F:usr/lib
R:libreadline.so.8
a:0:0:777

0
syft/pkg/cataloger/alpine/test-fixtures/installed → syft/pkg/cataloger/alpine/test-fixtures/multiple-2/lib/apk/db/installed
View file

90
syft/pkg/cataloger/arch/cataloger.go
View file

@ -4,96 +4,14 @@ Package arch provides a concrete Cataloger implementations for packages relating
package arch
import (
"context"
"strings"
"github.com/anchore/syft/internal/log"
"github.com/anchore/syft/syft/artifact"
"github.com/anchore/syft/syft/file"
"github.com/anchore/syft/syft/pkg"
"github.com/anchore/syft/syft/pkg/cataloger/generic"
"github.com/anchore/syft/syft/pkg/cataloger/internal/dependency"
)
type cataloger struct {
*generic.Cataloger
}
// NewDBCataloger returns a new cataloger object initialized for arch linux pacman database flat-file stores.
func NewDBCataloger() pkg.Cataloger {
return cataloger{
Cataloger: generic.NewCataloger("alpm-db-cataloger").
WithParserByGlobs(parseAlpmDB, pkg.AlpmDBGlob),
}
}
func (c cataloger) Catalog(ctx context.Context, resolver file.Resolver) ([]pkg.Package, []artifact.Relationship, error) {
pkgs, rels, err := c.Cataloger.Catalog(ctx, resolver)
if err != nil {
return nil, nil, err
}
rels = append(rels, associateRelationships(pkgs)...)
return pkgs, rels, nil
}
// associateRelationships will create relationships between packages based on the "Depends" and "Provides"
// fields for installed packages. If there is an installed package that has a dependency that is (somehow) not installed,
// then that relationship (between the installed and uninstalled package) will NOT be created.
func associateRelationships(pkgs []pkg.Package) (relationships []artifact.Relationship) {
// map["provides" + "package"] -> packages that provide that package
lookup := make(map[string][]pkg.Package)
// read providers and add lookup keys as needed
for _, p := range pkgs {
meta, ok := p.Metadata.(pkg.AlpmDBEntry)
if !ok {
log.Warnf("cataloger failed to extract alpm 'provides' metadata for package %+v", p.Name)
continue
}
// allow for lookup by package name
lookup[p.Name] = append(lookup[p.Name], p)
for _, provides := range meta.Provides {
// allow for lookup by exact specification
lookup[provides] = append(lookup[provides], p)
// allow for lookup by library name only
k := stripVersionSpecifier(provides)
lookup[k] = append(lookup[k], p)
}
}
// read "Depends" and match with provider keys
for _, p := range pkgs {
meta, ok := p.Metadata.(pkg.AlpmDBEntry)
if !ok {
log.Warnf("cataloger failed to extract alpm 'dependency' metadata for package %+v", p.Name)
continue
}
for _, dep := range meta.Depends {
for _, depPkg := range lookup[dep] {
relationships = append(relationships, artifact.Relationship{
From: depPkg,
To: p,
Type: artifact.DependencyOfRelationship,
})
}
}
}
return relationships
}
func stripVersionSpecifier(s string) string {
// examples:
// gcc-libs --> gcc-libs
// libtree-sitter.so=0-64 --> libtree-sitter.so
items := strings.Split(s, "=")
if len(items) == 0 {
return s
}
return strings.TrimSpace(items[0])
return generic.NewCataloger("alpm-db-cataloger").
WithParserByGlobs(parseAlpmDB, pkg.AlpmDBGlob).
WithProcessors(dependency.Processor(dbEntryDependencySpecifier))
}

48
syft/pkg/cataloger/arch/dependency.go Normal file
View file

@ -0,0 +1,48 @@
package arch
import (
"strings"
"github.com/anchore/syft/internal/log"
"github.com/anchore/syft/syft/pkg"
"github.com/anchore/syft/syft/pkg/cataloger/internal/dependency"
)
var _ dependency.Specifier = dbEntryDependencySpecifier
func dbEntryDependencySpecifier(p pkg.Package) dependency.Specification {
meta, ok := p.Metadata.(pkg.AlpmDBEntry)
if !ok {
log.Tracef("cataloger failed to extract alpm metadata for package %+v", p.Name)
return dependency.Specification{}
}
provides := []string{p.Name}
for _, key := range meta.Provides {
if key == "" {
continue
}
provides = append(provides, key, stripVersionSpecifier(key))
}
var requires []string
for _, depSpecifier := range meta.Depends {
if depSpecifier == "" {
continue
}
requires = append(requires, depSpecifier)
}
return dependency.Specification{
Provides: provides,
Requires: requires,
}
}
func stripVersionSpecifier(s string) string {
// examples:
// gcc-libs --> gcc-libs
// libtree-sitter.so=0-64 --> libtree-sitter.so
return strings.TrimSpace(strings.Split(s, "=")[0])
}

100
syft/pkg/cataloger/arch/dependency_test.go Normal file
View file

@ -0,0 +1,100 @@
package arch
import (
"testing"
"github.com/stretchr/testify/assert"
"github.com/anchore/syft/syft/pkg"
"github.com/anchore/syft/syft/pkg/cataloger/internal/dependency"
)
func Test_dbEntryDependencySpecifier(t *testing.T) {
tests := []struct {
name string
p pkg.Package
want dependency.Specification
}{
{
name: "keeps given values + package name",
p: pkg.Package{
Name: "package-c",
Metadata: pkg.AlpmDBEntry{
Provides: []string{"a-thing"},
Depends: []string{"b-thing"},
},
},
want: dependency.Specification{
Provides: []string{"package-c", "a-thing", "a-thing"}, // note: gets deduplicated downstream
Requires: []string{"b-thing"},
},
},
{
name: "strip version specifiers",
p: pkg.Package{
Name: "package-a",
Metadata: pkg.AlpmDBEntry{
Provides: []string{"libtree-sitter.so.me=1-64"},
Depends: []string{"libtree-sitter.so.thing=2-64"},
},
},
want: dependency.Specification{
Provides: []string{"package-a", "libtree-sitter.so.me=1-64", "libtree-sitter.so.me"},
Requires: []string{"libtree-sitter.so.thing=2-64"},
},
},
{
name: "empty dependency data entries",
p: pkg.Package{
Name: "package-a",
Metadata: pkg.AlpmDBEntry{
Provides: []string{""},
Depends: []string{""},
},
},
want: dependency.Specification{
Provides: []string{"package-a"},
Requires: nil,
},
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
assert.Equal(t, tt.want, dbEntryDependencySpecifier(tt.p))
})
}
}
func Test_stripVersionSpecifier(t *testing.T) {
tests := []struct {
name string
version string
want string
}{
{
name: "empty expression",
version: "",
want: "",
},
{
name: "no expression",
version: "gcc-libs",
want: "gcc-libs",
},
{
name: "=",
version: "libtree-sitter.so=0-64",
want: "libtree-sitter.so",
},
{
name: "ignores file paths",
version: "/bin/sh",
want: "/bin/sh",
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
assert.Equal(t, tt.want, stripVersionSpecifier(tt.version))
})
}
}

4
syft/pkg/cataloger/debian/cataloger.go
View file

@ -6,6 +6,7 @@ package debian
import (
"github.com/anchore/syft/syft/pkg"
"github.com/anchore/syft/syft/pkg/cataloger/generic"
"github.com/anchore/syft/syft/pkg/cataloger/internal/dependency"
)
// NewDBCataloger returns a new Deb package cataloger capable of parsing DPKG status DB flat-file stores.
@ -13,5 +14,6 @@ func NewDBCataloger() pkg.Cataloger {
return generic.NewCataloger("dpkg-db-cataloger").
// note: these globs have been intentionally split up in order to improve search performance,
// please do NOT combine into: "**/var/lib/dpkg/{status,status.d/*}"
WithParserByGlobs(parseDpkgDB, "**/var/lib/dpkg/status", "**/var/lib/dpkg/status.d/*", "**/lib/opkg/info/*.control", "**/lib/opkg/status")
WithParserByGlobs(parseDpkgDB, "**/var/lib/dpkg/status", "**/var/lib/dpkg/status.d/*", "**/lib/opkg/info/*.control", "**/lib/opkg/status").
WithProcessors(dependency.Processor(dbEntryDependencySpecifier))
}

62
syft/pkg/cataloger/debian/cataloger_test.go
View file

@ -1,8 +1,12 @@
package debian
import (
"context"
"testing"
"github.com/google/go-cmp/cmp"
"github.com/stretchr/testify/require"
"github.com/anchore/syft/syft/file"
"github.com/anchore/syft/syft/pkg"
"github.com/anchore/syft/syft/pkg/cataloger/internal/pkgtest"
@ -160,6 +164,64 @@ func TestDpkgCataloger(t *testing.T) {
}
}
func Test_CatalogerRelationships(t *testing.T) {
tests := []struct {
name string
fixture string
wantRelationships map[string][]string
}{
{
name: "relationships for coreutils",
fixture: "test-fixtures/var/lib/dpkg/status.d/coreutils-relationships",
wantRelationships: map[string][]string{
"coreutils": {"libacl1", "libattr1", "libc6", "libgmp10", "libselinux1"},
"libacl1": {"libc6"},
"libattr1": {"libc6"},
"libc6": {"libgcc-s1"},
"libgcc-s1": {"gcc-12-base", "libc6"},
"libgmp10": {"libc6"},
"libpcre2-8-0": {"libc6"},
"libselinux1": {"libc6", "libpcre2-8-0"},
},
},
{
name: "relationships from dpkg example docs",
fixture: "test-fixtures/var/lib/dpkg/status.d/doc-examples",
wantRelationships: map[string][]string{
"made-up-package-1": {"gnumach-dev", "hurd-dev", "kernel-headers-2.2.10"},
"made-up-package-2": {"liblua5.1-dev", "libluajit5.1-dev"},
"made-up-package-3": {"bar", "foo"},
// note that the "made-up-package-4" depends on "made-up-package-5" but not via the direct
// package name, but through the "provides" virtual package name "virtual-package-5".
"made-up-package-4": {"made-up-package-5"},
// note that though there is a "default-mta | mail-transport-agent | not-installed"
// dependency choice we raise up the packages that are installed for every choice.
// In this case that means that "default-mta" and "mail-transport-agent".
"mutt": {"default-mta", "libc6", "mail-transport-agent"},
},
},
{
name: "relationships for libpam-runtime",
fixture: "test-fixtures/var/lib/dpkg/status.d/libpam-runtime",
wantRelationships: map[string][]string{
"libpam-runtime": {"cdebconf", "debconf-2.0", "debconf1", "debconf2", "libpam-modules"},
},
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
pkgs, relationships, err := NewDBCataloger().Catalog(context.Background(), file.NewMockResolverForPaths(tt.fixture))
require.NotEmpty(t, pkgs)
require.NotEmpty(t, relationships)
require.NoError(t, err)
if d := cmp.Diff(tt.wantRelationships, abstractRelationships(t, relationships)); d != "" {
t.Errorf("unexpected relationships (-want +got):\n%s", d)
}
})
}
}
func TestCataloger_Globs(t *testing.T) {
tests := []struct {
name string

66
syft/pkg/cataloger/debian/dependency.go Normal file
View file

@ -0,0 +1,66 @@
package debian
import (
"strings"
"github.com/anchore/syft/internal"
"github.com/anchore/syft/internal/log"
"github.com/anchore/syft/syft/pkg"
"github.com/anchore/syft/syft/pkg/cataloger/internal/dependency"
)
var _ dependency.Specifier = dbEntryDependencySpecifier
func dbEntryDependencySpecifier(p pkg.Package) dependency.Specification {
meta, ok := p.Metadata.(pkg.DpkgDBEntry)
if !ok {
log.Tracef("cataloger failed to extract dpkg metadata for package %+v", p.Name)
return dependency.Specification{}
}
provides := []string{p.Name}
for _, key := range meta.Provides {
if key == "" {
continue
}
provides = append(provides, stripVersionSpecifier(key))
}
var allDeps []string
allDeps = append(allDeps, meta.Depends...)
allDeps = append(allDeps, meta.PreDepends...)
var requires []string
for _, depSpecifier := range allDeps {
if depSpecifier == "" {
continue
}
requires = append(requires, splitPackageChoice(depSpecifier)...)
}
return dependency.Specification{
Provides: provides,
Requires: requires,
}
}
func stripVersionSpecifier(s string) string {
// examples:
// libgmp10 (>= 2:6.2.1+dfsg1) --> libgmp10
// libgmp10 --> libgmp10
// foo [i386] --> foo
// default-mta | mail-transport-agent --> default-mta | mail-transport-agent
// kernel-headers-2.2.10 [!hurd-i386] --> kernel-headers-2.2.10
return strings.TrimSpace(internal.SplitAny(s, "[(<>=")[0])
}
func splitPackageChoice(s string) (ret []string) {
fields := strings.Split(s, "|")
for _, field := range fields {
field = strings.TrimSpace(field)
if field != "" {
ret = append(ret, stripVersionSpecifier(field))
}
}
return ret
}

101
syft/pkg/cataloger/debian/dependency_test.go Normal file
View file

@ -0,0 +1,101 @@
package debian
import (
"testing"
"github.com/stretchr/testify/assert"
"github.com/anchore/syft/syft/pkg"
"github.com/anchore/syft/syft/pkg/cataloger/internal/dependency"
)
func Test_dbEntryDependencySpecifier(t *testing.T) {
tests := []struct {
name string
p pkg.Package
want dependency.Specification
}{
{
name: "keeps given values + package name",
p: pkg.Package{
Name: "package-c",
Metadata: pkg.DpkgDBEntry{
Provides: []string{"a-thing"},
Depends: []string{"b-thing"},
},
},
want: dependency.Specification{
Provides: []string{"package-c", "a-thing"},
Requires: []string{"b-thing"},
},
},
{
name: "strip version specifiers + split package deps",
p: pkg.Package{
Name: "package-a",
Metadata: pkg.DpkgDBEntry{
Provides: []string{"foo [i386]"},
Depends: []string{"libgmp10 (>= 2:6.2.1+dfsg1)", "default-mta | mail-transport-agent"},
},
},
want: dependency.Specification{
Provides: []string{"package-a", "foo"},
Requires: []string{"libgmp10", "default-mta", "mail-transport-agent"},
},
},
{
name: "empty dependency data entries",
p: pkg.Package{
Name: "package-a",
Metadata: pkg.DpkgDBEntry{
Provides: []string{""},
Depends: []string{""},
},
},
want: dependency.Specification{
Provides: []string{"package-a"},
Requires: nil,
},
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
assert.Equal(t, tt.want, dbEntryDependencySpecifier(tt.p))
})
}
}
func Test_stripVersionSpecifier(t *testing.T) {
tests := []struct {
name string
input string
want string
}{
{
name: "package name only",
input: "test",
want: "test",
},
{
name: "with version",
input: "test (1.2.3)",
want: "test",
},
{
name: "multiple packages",
input: "test | other",
want: "test | other",
},
{
name: "with architecture specifiers",
input: "test [amd64 i386]",
want: "test",
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
assert.Equal(t, tt.want, stripVersionSpecifier(tt.input))
})
}
}

78
syft/pkg/cataloger/debian/parse_dpkg_db.go
View file

@ -37,7 +37,7 @@ func parseDpkgDB(_ context.Context, resolver file.Resolver, env *generic.Environ
pkgs = append(pkgs, newDpkgPackage(m, reader.Location, resolver, env.LinuxRelease))
}
return pkgs, associateRelationships(pkgs), nil
return pkgs, nil, nil
}
// parseDpkgStatus is a parser function for Debian DB status contents, returning all Debian packages listed.
@ -238,79 +238,3 @@ func handleNewKeyValue(line string) (key string, val interface{}, err error) {
return "", nil, fmt.Errorf("cannot parse field from line: '%s'", line)
}
// associateRelationships will create relationships between packages based on the "Depends", "Pre-Depends", and "Provides"
// fields for installed packages. if there is an installed package that has a dependency that is (somehow) not installed,
// then that relationship (between the installed and uninstalled package) will NOT be created.
func associateRelationships(pkgs []pkg.Package) (relationships []artifact.Relationship) {
// map["provides" + "package"] -> packages that provide that package
lookup := make(map[string][]pkg.Package)
// read provided and add as keys for lookup keys as well as package names
for _, p := range pkgs {
meta, ok := p.Metadata.(pkg.DpkgDBEntry)
if !ok {
log.Warnf("cataloger failed to extract dpkg 'provides' metadata for package %+v", p.Name)
continue
}
lookup[p.Name] = append(lookup[p.Name], p)
for _, provides := range meta.Provides {
k := stripVersionSpecifier(provides)
lookup[k] = append(lookup[k], p)
}
}
// read "Depends" and "Pre-Depends" and match with keys
for _, p := range pkgs {
meta, ok := p.Metadata.(pkg.DpkgDBEntry)
if !ok {
log.Warnf("cataloger failed to extract dpkg 'dependency' metadata for package %+v", p.Name)
continue
}
var allDeps []string
allDeps = append(allDeps, meta.Depends...)
allDeps = append(allDeps, meta.PreDepends...)
for _, depSpecifier := range allDeps {
deps := splitPackageChoice(depSpecifier)
for _, dep := range deps {
for _, depPkg := range lookup[dep] {
relationships = append(relationships, artifact.Relationship{
From: depPkg,
To: p,
Type: artifact.DependencyOfRelationship,
})
}
}
}
}
return relationships
}
func stripVersionSpecifier(s string) string {
// examples:
// libgmp10 (>= 2:6.2.1+dfsg1) --> libgmp10
// libgmp10 --> libgmp10
// foo [i386] --> foo
// default-mta | mail-transport-agent --> default-mta | mail-transport-agent
// kernel-headers-2.2.10 [!hurd-i386] --> kernel-headers-2.2.10
items := internal.SplitAny(s, "[(<>=")
if len(items) == 0 {
return s
}
return strings.TrimSpace(items[0])
}
func splitPackageChoice(s string) (ret []string) {
fields := strings.Split(s, "|")
for _, field := range fields {
field = strings.TrimSpace(field)
if field != "" {
ret = append(ret, stripVersionSpecifier(field))
}
}
return ret
}

106
syft/pkg/cataloger/debian/parse_dpkg_db_test.go
View file

@ -2,7 +2,6 @@ package debian
import (
"bufio"
"context"
"errors"
"fmt"
"os"
@ -16,7 +15,6 @@ import (
"github.com/anchore/syft/syft/file"
"github.com/anchore/syft/syft/linux"
"github.com/anchore/syft/syft/pkg"
"github.com/anchore/syft/syft/pkg/cataloger/generic"
"github.com/anchore/syft/syft/pkg/cataloger/internal/pkgtest"
)
@ -28,7 +26,7 @@ func Test_parseDpkgStatus(t *testing.T) {
}{
{
name: "single package",
fixturePath: "test-fixtures/status/single",
fixturePath: "test-fixtures/var/lib/dpkg/status.d/single",
expected: []pkg.DpkgDBEntry{
{
Package: "apt",
@ -102,7 +100,7 @@ func Test_parseDpkgStatus(t *testing.T) {
},
{
name: "single package with installed size",
fixturePath: "test-fixtures/status/installed-size-4KB",
fixturePath: "test-fixtures/var/lib/dpkg/status.d/installed-size-4KB",
expected: []pkg.DpkgDBEntry{
{
Package: "apt",
@ -143,7 +141,7 @@ func Test_parseDpkgStatus(t *testing.T) {
},
{
name: "multiple entries",
fixturePath: "test-fixtures/status/multiple",
fixturePath: "test-fixtures/var/lib/dpkg/status.d/multiple",
expected: []pkg.DpkgDBEntry{
{
Package: "no-version",
@ -434,104 +432,6 @@ func Test_handleNewKeyValue(t *testing.T) {
}
}
func Test_stripVersionSpecifier(t *testing.T) {
tests := []struct {
name string
input string
want string
}{
{
name: "package name only",
input: "test",
want: "test",
},
{
name: "with version",
input: "test (1.2.3)",
want: "test",
},
{
name: "multiple packages",
input: "test | other",
want: "test | other",
},
{
name: "with architecture specifiers",
input: "test [amd64 i386]",
want: "test",
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
assert.Equal(t, tt.want, stripVersionSpecifier(tt.input))
})
}
}
func Test_associateRelationships(t *testing.T) {
tests := []struct {
name string
fixture string
wantRelationships map[string][]string
}{
{
name: "relationships for coreutils",
fixture: "test-fixtures/status/coreutils-relationships",
wantRelationships: map[string][]string{
"coreutils": {"libacl1", "libattr1", "libc6", "libgmp10", "libselinux1"},
"libacl1": {"libc6"},
"libattr1": {"libc6"},
"libc6": {"libgcc-s1"},
"libgcc-s1": {"gcc-12-base", "libc6"},
"libgmp10": {"libc6"},
"libpcre2-8-0": {"libc6"},
"libselinux1": {"libc6", "libpcre2-8-0"},
},
},
{
name: "relationships from dpkg example docs",
fixture: "test-fixtures/status/doc-examples",
wantRelationships: map[string][]string{
"made-up-package-1": {"kernel-headers-2.2.10", "hurd-dev", "gnumach-dev"},
"made-up-package-2": {"libluajit5.1-dev", "liblua5.1-dev"},
"made-up-package-3": {"foo", "bar"},
// note that the "made-up-package-4" depends on "made-up-package-5" but not via the direct
// package name, but through the "provides" virtual package name "virtual-package-5".
"made-up-package-4": {"made-up-package-5"},
// note that though there is a "default-mta | mail-transport-agent | not-installed"
// dependency choice we raise up the packages that are installed for every choice.
// In this case that means that "default-mta" and "mail-transport-agent".
"mutt": {"libc6", "default-mta", "mail-transport-agent"},
},
},
{
name: "relationships for libpam-runtime",
fixture: "test-fixtures/status/libpam-runtime",
wantRelationships: map[string][]string{
"libpam-runtime": {"debconf1", "debconf-2.0", "debconf2", "cdebconf", "libpam-modules"},
},
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
f, err := os.Open(tt.fixture)
require.NoError(t, err)
reader := file.NewLocationReadCloser(file.NewLocation(tt.fixture), f)
pkgs, relationships, err := parseDpkgDB(context.Background(), nil, &generic.Environment{}, reader)
require.NotEmpty(t, pkgs)
require.NotEmpty(t, relationships)
require.NoError(t, err)
if d := cmp.Diff(tt.wantRelationships, abstractRelationships(t, relationships)); d != "" {
t.Errorf("unexpected relationships (-want +got):\n%s", d)
}
})
}
}
func abstractRelationships(t testing.TB, relationships []artifact.Relationship) map[string][]string {
t.Helper()

0
syft/pkg/cataloger/debian/test-fixtures/status/coreutils-relationships → syft/pkg/cataloger/debian/test-fixtures/var/lib/dpkg/status.d/coreutils-relationships
View file

0
syft/pkg/cataloger/debian/test-fixtures/status/doc-examples → syft/pkg/cataloger/debian/test-fixtures/var/lib/dpkg/status.d/doc-examples
View file

0
syft/pkg/cataloger/debian/test-fixtures/status/empty → syft/pkg/cataloger/debian/test-fixtures/var/lib/dpkg/status.d/empty
View file

0
syft/pkg/cataloger/debian/test-fixtures/status/installed-size-4KB → syft/pkg/cataloger/debian/test-fixtures/var/lib/dpkg/status.d/installed-size-4KB
View file

0
syft/pkg/cataloger/debian/test-fixtures/status/libpam-runtime → syft/pkg/cataloger/debian/test-fixtures/var/lib/dpkg/status.d/libpam-runtime
View file

0
syft/pkg/cataloger/debian/test-fixtures/status/multiple → syft/pkg/cataloger/debian/test-fixtures/var/lib/dpkg/status.d/multiple
View file

0
syft/pkg/cataloger/debian/test-fixtures/status/single → syft/pkg/cataloger/debian/test-fixtures/var/lib/dpkg/status.d/single
View file

29
syft/pkg/cataloger/generic/cataloger.go
View file

@ -12,7 +12,9 @@ import (
"github.com/anchore/syft/syft/pkg"
)
type processor func(resolver file.Resolver, env Environment) []request
type Processor func([]pkg.Package, []artifact.Relationship, error) ([]pkg.Package, []artifact.Relationship, error)
type requester func(resolver file.Resolver, env Environment) []request
type request struct {
file.Location
@ -22,12 +24,13 @@ type request struct {
// Cataloger implements the Catalog interface and is responsible for dispatching the proper parser function for
// a given path or glob pattern. This is intended to be reusable across many package cataloger types.
type Cataloger struct {
processor []processor
processors []Processor
requesters []requester
upstreamCataloger string
}
func (c *Cataloger) WithParserByGlobs(parser Parser, globs ...string) *Cataloger {
c.processor = append(c.processor,
c.requesters = append(c.requesters,
func(resolver file.Resolver, _ Environment) []request {
var requests []request
for _, g := range globs {
@ -47,7 +50,7 @@ func (c *Cataloger) WithParserByGlobs(parser Parser, globs ...string) *Cataloger
}
func (c *Cataloger) WithParserByMimeTypes(parser Parser, types ...string) *Cataloger {
c.processor = append(c.processor,
c.requesters = append(c.requesters,
func(resolver file.Resolver, _ Environment) []request {
var requests []request
log.WithFields("mimetypes", types).Trace("searching for paths matching mimetype")
@ -64,7 +67,7 @@ func (c *Cataloger) WithParserByMimeTypes(parser Parser, types ...string) *Catal
}
func (c *Cataloger) WithParserByPath(parser Parser, paths ...string) *Cataloger {
c.processor = append(c.processor,
c.requesters = append(c.requesters,
func(resolver file.Resolver, _ Environment) []request {
var requests []request
for _, p := range paths {
@ -83,6 +86,11 @@ func (c *Cataloger) WithParserByPath(parser Parser, paths ...string) *Cataloger
return c
}
func (c *Cataloger) WithProcessors(processors ...Processor) *Cataloger {
c.processors = append(c.processors, processors...)
return c
}
func makeRequests(parser Parser, locations []file.Location) []request {
var requests []request
for _, l := range locations {
@ -135,7 +143,14 @@ func (c *Cataloger) Catalog(ctx context.Context, resolver file.Resolver) ([]pkg.
relationships = append(relationships, discoveredRelationships...)
}
return packages, relationships, nil
return c.process(packages, relationships, nil)
}
func (c *Cataloger) process(pkgs []pkg.Package, rels []artifact.Relationship, err error) ([]pkg.Package, []artifact.Relationship, error) {
for _, proc := range c.processors {
pkgs, rels, err = proc(pkgs, rels, err)
}
return pkgs, rels, err
}
func invokeParser(ctx context.Context, resolver file.Resolver, location file.Location, logger logger.Logger, parser Parser, env *Environment) ([]pkg.Package, []artifact.Relationship, error) {
@ -158,7 +173,7 @@ func invokeParser(ctx context.Context, resolver file.Resolver, location file.Loc
// selectFiles takes a set of file trees and resolves and file references of interest for future cataloging
func (c *Cataloger) selectFiles(resolver file.Resolver) []request {
var requests []request
for _, proc := range c.processor {
for _, proc := range c.requesters {
requests = append(requests, proc(resolver, Environment{})...)
}
return requests

4
syft/pkg/cataloger/generic/cataloger_test.go
View file

@ -159,7 +159,7 @@ func TestClosesFileOnParserPanic(t *testing.T) {
resolver := newSpyReturningFileResolver(&spy, "test-fixtures/another-path.txt")
ctx := context.TODO()
processors := []processor{
processors := []requester{
func(resolver file.Resolver, env Environment) []request {
return []request{
{
@ -178,7 +178,7 @@ func TestClosesFileOnParserPanic(t *testing.T) {
}
c := Cataloger{
processor: processors,
requesters: processors,
upstreamCataloger: "unit-test-cataloger",
}

95
syft/pkg/cataloger/internal/dependency/resolver.go Normal file
View file

@ -0,0 +1,95 @@
package dependency
import (
"sort"
"github.com/scylladb/go-set/strset"
"github.com/anchore/syft/syft/artifact"
"github.com/anchore/syft/syft/pkg"
"github.com/anchore/syft/syft/pkg/cataloger/generic"
)
// Specification holds strings that indicate abstract resources that a package provides for other packages and
// requires for itself. These strings can represent anything from file paths, package names, or any other concept
// that is useful for dependency resolution within that packing ecosystem.
type Specification struct {
// Provides holds a list of abstract resources that this package provides for other packages.
Provides []string
// Requires holds a list of abstract resources that this package requires from other packages.
Requires []string
}
// Specifier is a function that takes a package and extracts a Specification, describing resources
// the package provides and needs.
type Specifier func(pkg.Package) Specification
// Processor returns a generic processor that will resolve relationships between packages based on the dependency claims.
func Processor(s Specifier) generic.Processor {
return func(pkgs []pkg.Package, rels []artifact.Relationship, err error) ([]pkg.Package, []artifact.Relationship, error) {
// we can't move forward unless all package IDs have been set
for idx, p := range pkgs {
id := p.ID()
if id == "" {
p.SetID()
pkgs[idx] = p
}
}
rels = append(rels, resolve(s, pkgs)...)
return pkgs, rels, err
}
}
// resolve will create relationships between packages based on the dependency claims of each package.
func resolve(specifier Specifier, pkgs []pkg.Package) (relationships []artifact.Relationship) {
pkgsProvidingResource := make(map[string][]artifact.ID)
pkgsByID := make(map[artifact.ID]pkg.Package)
specsByPkg := make(map[artifact.ID]Specification)
for _, p := range pkgs {
id := p.ID()
pkgsByID[id] = p
specsByPkg[id] = specifier(p)
for _, resource := range deduplicate(specifier(p).Provides) {
pkgsProvidingResource[resource] = append(pkgsProvidingResource[resource], id)
}
}
seen := strset.New()
for _, dependantPkg := range pkgs {
spec := specsByPkg[dependantPkg.ID()]
for _, resource := range deduplicate(spec.Requires) {
for _, providingPkgID := range pkgsProvidingResource[resource] {
// prevent creating duplicate relationships
pairKey := string(providingPkgID) + "-" + string(dependantPkg.ID())
if seen.Has(pairKey) {
continue
}
providingPkg := pkgsByID[providingPkgID]
relationships = append(relationships,
artifact.Relationship{
From: providingPkg,
To: dependantPkg,
Type: artifact.DependencyOfRelationship,
},
)
seen.Add(pairKey)
}
}
}
return relationships
}
func deduplicate(ss []string) []string {
// note: we sort the set such that multiple invocations of this function will be deterministic
set := strset.New(ss...)
list := set.List()
sort.Strings(list)
return list
}

211
syft/pkg/cataloger/internal/dependency/resolver_test.go Normal file
View file

@ -0,0 +1,211 @@
package dependency
import (
"errors"
"testing"
"github.com/google/go-cmp/cmp"
"github.com/stretchr/testify/assert"
"github.com/anchore/syft/syft/artifact"
"github.com/anchore/syft/syft/pkg"
)
func Test_resolve(t *testing.T) {
a := pkg.Package{
Name: "a",
}
a.SetID()
b := pkg.Package{
Name: "b",
}
b.SetID()
c := pkg.Package{
Name: "c",
}
c.SetID()
subjects := []pkg.Package{a, b, c}
tests := []struct {
name string
s Specifier
want map[string][]string
}{
{
name: "find relationships between packages",
s: newSpecifierBuilder().
WithProvides(a /* provides */, "a-resource").
WithRequires(b /* requires */, "a-resource").
Specifier(),
want: map[string][]string{
"b": /* depends on */ {"a"},
},
},
{
name: "deduplicates provider keys",
s: newSpecifierBuilder().
WithProvides(a /* provides */, "a-resource", "a-resource", "a-resource").
WithRequires(b /* requires */, "a-resource", "a-resource", "a-resource").
Specifier(),
want: map[string][]string{
"b": /* depends on */ {"a"},
// note: we're NOT seeing:
// "b": /* depends on */ {"a", "a", "a"},
},
},
{
name: "deduplicates crafted relationships",
s: newSpecifierBuilder().
WithProvides(a /* provides */, "a1-resource", "a2-resource", "a3-resource").
WithRequires(b /* requires */, "a1-resource", "a2-resource").
Specifier(),
want: map[string][]string{
"b": /* depends on */ {"a"},
// note: we're NOT seeing:
// "b": /* depends on */ {"a", "a"},
},
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
relationships := resolve(tt.s, subjects)
if d := cmp.Diff(tt.want, abstractRelationships(t, relationships)); d != "" {
t.Errorf("unexpected relationships (-want +got):\n%s", d)
}
})
}
}
type specifierBuilder struct {
provides map[string][]string
requires map[string][]string
}
func newSpecifierBuilder() *specifierBuilder {
return &specifierBuilder{
provides: make(map[string][]string),
requires: make(map[string][]string),
}
}
func (m *specifierBuilder) WithProvides(p pkg.Package, provides ...string) *specifierBuilder {
m.provides[p.Name] = append(m.provides[p.Name], provides...)
return m
}
func (m *specifierBuilder) WithRequires(p pkg.Package, requires ...string) *specifierBuilder {
m.requires[p.Name] = append(m.requires[p.Name], requires...)
return m
}
func (m specifierBuilder) Specifier() Specifier {
return func(p pkg.Package) Specification {
return Specification{
Provides: m.provides[p.Name],
Requires: m.requires[p.Name],
}
}
}
func abstractRelationships(t testing.TB, relationships []artifact.Relationship) map[string][]string {
t.Helper()
abstracted := make(map[string][]string)
for _, relationship := range relationships {
fromPkg, ok := relationship.From.(pkg.Package)
if !ok {
continue
}
toPkg, ok := relationship.To.(pkg.Package)
if !ok {
continue
}
// we build this backwards since we use DependencyOfRelationship instead of DependsOn
abstracted[toPkg.Name] = append(abstracted[toPkg.Name], fromPkg.Name)
}
return abstracted
}
func Test_Processor(t *testing.T) {
a := pkg.Package{
Name: "a",
}
a.SetID()
b := pkg.Package{
Name: "b",
}
b.SetID()
c := pkg.Package{
Name: "c",
}
c.SetID()
tests := []struct {
name string
sp Specifier
pkgs []pkg.Package
rels []artifact.Relationship
err error
wantPkgCount int
wantRelCount int
wantErr assert.ErrorAssertionFunc
}{
{
name: "happy path preserves decorated values",
sp: newSpecifierBuilder().
WithProvides(b, "b-resource").
WithRequires(c, "b-resource").
Specifier(),
pkgs: []pkg.Package{a, b, c},
rels: []artifact.Relationship{
{
From: a,
To: b,
Type: artifact.DependencyOfRelationship,
},
},
wantPkgCount: 3,
wantRelCount: 2, // original + new
},
{
name: "error from cataloger is propagated",
sp: newSpecifierBuilder().
WithProvides(b, "b-resource").
WithRequires(c, "b-resource").
Specifier(),
err: errors.New("surprise!"),
pkgs: []pkg.Package{a, b, c},
rels: []artifact.Relationship{
{
From: a,
To: b,
Type: artifact.DependencyOfRelationship,
},
},
wantPkgCount: 3,
wantRelCount: 2, // original + new
wantErr: assert.Error,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
if tt.wantErr == nil {
tt.wantErr = assert.NoError
}
gotPkgs, gotRels, err := Processor(tt.sp)(tt.pkgs, tt.rels, tt.err)
tt.wantErr(t, err)
assert.Len(t, gotPkgs, tt.wantPkgCount)
assert.Len(t, gotRels, tt.wantRelCount)
})
}
}

14
syft/pkg/cataloger/internal/pkgtest/test_generic_parser.go
View file

@ -44,6 +44,7 @@ type CatalogTester struct {
compareOptions []cmp.Option
locationComparer locationComparer
licenseComparer licenseComparer
customAssertions []func(t *testing.T, pkgs []pkg.Package, relationships []artifact.Relationship)
}
func NewCatalogTester() *CatalogTester {
@ -164,6 +165,11 @@ func (p *CatalogTester) WithImageResolver(t *testing.T, fixtureName string) *Cat
return p
}
func (p *CatalogTester) ExpectsAssertion(a func(t *testing.T, pkgs []pkg.Package, relationships []artifact.Relationship)) *CatalogTester {
p.customAssertions = append(p.customAssertions, a)
return p
}
func (p *CatalogTester) IgnoreLocationLayer() *CatalogTester {
p.locationComparer = func(x, y file.Location) bool {
return cmp.Equal(x.Coordinates.RealPath, y.Coordinates.RealPath) && cmp.Equal(x.AccessPath, y.AccessPath)
@ -250,7 +256,13 @@ func (p *CatalogTester) TestCataloger(t *testing.T, cataloger pkg.Cataloger) {
if p.assertResultExpectations {
p.wantErr(t, err)
p.assertPkgs(t, pkgs, relationships)
} else {
}
for _, a := range p.customAssertions {
a(t, pkgs, relationships)
}
if !p.assertResultExpectations && len(p.customAssertions) == 0 {
resolver.PruneUnfulfilledPathResponses(p.ignoreUnfulfilledPathResponses, p.ignoreAnyUnfulfilledPaths...)
// if we aren't testing the results, we should focus on what was searched for (for glob-centric tests)