move package purl and cpes (identities) to pkg.Package

Signed-off-by: Alex Goodman <alex.goodman@anchore.com>
This commit is contained in:
Alex Goodman 2020-11-19 11:11:02 -05:00
parent 71939557e6
commit 3aaa0e5566
No known key found for this signature in database
GPG key ID: 5CB45AE22BAB7EA7
35 changed files with 580 additions and 206 deletions

View file

@ -15,7 +15,7 @@ RESET := $(shell tput -T linux sgr0)
TITLE := $(BOLD)$(PURPLE)
SUCCESS := $(BOLD)$(GREEN)
# the quality gate lower threshold for unit test total % coverage (by function statements)
COVERAGE_THRESHOLD := 72
COVERAGE_THRESHOLD := 70
## Build variables
DISTDIR=./dist

1
go.mod
View file

@ -11,6 +11,7 @@ require (
github.com/bmatcuk/doublestar v1.3.3
github.com/docker/docker v17.12.0-ce-rc1.0.20200309214505-aa6a9891b09c+incompatible
github.com/dustin/go-humanize v1.0.0
github.com/facebookincubator/nvdtools v0.1.4
github.com/go-test/deep v1.0.7
github.com/google/uuid v1.1.1
github.com/gookit/color v1.2.7

2
go.sum
View file

@ -254,6 +254,8 @@ github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.m
github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98=
github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c=
github.com/evanphx/json-patch v4.2.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk=
github.com/facebookincubator/nvdtools v0.1.4 h1:x1Ucw9+bSkMd8DJJN4jNQ1Lk4PSFlJarGOxp9D6WUMo=
github.com/facebookincubator/nvdtools v0.1.4/go.mod h1:0/FIVnSEl9YHXLq3tKBPpKaI0iUceDhdSHPlIwIX44Y=
github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4=
github.com/fatih/color v1.9.0 h1:8xPHl4/q1VyqGIPif1F+1V3Y3lSmrq01EabUW3CoW5s=
github.com/fatih/color v1.9.0/go.mod h1:eQcE1qtQxscV5RaZvpXrrb8Drkc3/DdQ+uUYCNjL+zU=

View file

@ -3,6 +3,7 @@ package cataloger
import (
"github.com/anchore/syft/internal/bus"
"github.com/anchore/syft/internal/log"
"github.com/anchore/syft/syft/distro"
"github.com/anchore/syft/syft/event"
"github.com/anchore/syft/syft/pkg"
"github.com/anchore/syft/syft/source"
@ -36,13 +37,14 @@ func newMonitor() (*progress.Manual, *progress.Manual) {
// In order to efficiently retrieve contents from a underlying container image the content fetch requests are
// done in bulk. Specifically, all files of interest are collected from each catalogers and accumulated into a single
// request.
func Catalog(resolver source.Resolver, catalogers ...Cataloger) (*pkg.Catalog, error) {
func Catalog(resolver source.Resolver, theDistro *distro.Distro, catalogers ...Cataloger) (*pkg.Catalog, error) {
catalog := pkg.NewCatalog()
filesProcessed, packagesDiscovered := newMonitor()
// perform analysis, accumulating errors for each failed analysis
var errs error
for _, theCataloger := range catalogers {
// find packages from the underlying raw data
packages, err := theCataloger.Catalog(resolver)
if err != nil {
errs = multierror.Append(errs, err)
@ -55,6 +57,13 @@ func Catalog(resolver source.Resolver, catalogers ...Cataloger) (*pkg.Catalog, e
packagesDiscovered.N += int64(catalogedPackages)
for _, p := range packages {
// generate CPEs
p.CPEs = generatePackageCPEs(p)
// generate PURL
p.PURL = generatePackageURL(p, theDistro)
// add to catalog
catalog.Add(p)
}
}

81
syft/cataloger/cpe.go Normal file
View file

@ -0,0 +1,81 @@
package cataloger
import (
"fmt"
"github.com/anchore/syft/internal"
"github.com/anchore/syft/syft/pkg"
"github.com/facebookincubator/nvdtools/wfn"
)
// this is functionally equivalent to "*" and consistent with no input given (thus easier to test)
const any = ""
// generatePackageCPEs Create a list of CPEs, trying to guess the vendor, product tuple and setting TargetSoftware if possible
func generatePackageCPEs(p pkg.Package) []pkg.CPE {
targetSws := candidateTargetSoftwareAttrs(p)
vendors := candidateVendors(p)
products := candidateProducts(p)
keys := internal.NewStringSet()
cpes := make([]pkg.CPE, 0)
for _, product := range products {
for _, vendor := range append([]string{any}, vendors...) {
for _, targetSw := range append([]string{any}, targetSws...) {
// prevent duplicate entries...
key := fmt.Sprintf("%s|%s|%s|%s", product, vendor, p.Version, targetSw)
if keys.Contains(key) {
continue
}
keys.Add(key)
// add a new entry...
candidateCpe := wfn.NewAttributesWithAny()
candidateCpe.Product = product
candidateCpe.Vendor = vendor
candidateCpe.Version = p.Version
candidateCpe.TargetSW = targetSw
cpes = append(cpes, *candidateCpe)
}
}
}
return cpes
}
func candidateTargetSoftwareAttrs(p pkg.Package) []string {
// TODO: expand with package metadata (from type assert)
// TODO: would be great to allow these to be overridden by user data/config
var targetSw []string
switch p.Language {
case pkg.Java:
targetSw = append(targetSw, "java", "maven")
case pkg.JavaScript:
targetSw = append(targetSw, "node.js", "nodejs")
case pkg.Ruby:
targetSw = append(targetSw, "ruby", "rails")
case pkg.Python:
targetSw = append(targetSw, "python")
}
if p.Type == pkg.JenkinsPluginPkg {
targetSw = append(targetSw, "jenkins", "cloudbees_jenkins")
}
return targetSw
}
func candidateVendors(p pkg.Package) []string {
// TODO: expand with package metadata (from type assert)
vendors := []string{p.Name}
if p.Language == pkg.Python {
vendors = append(vendors, fmt.Sprintf("python-%s", p.Name))
}
return vendors
}
func candidateProducts(p pkg.Package) []string {
return []string{p.Name}
}

137
syft/cataloger/cpe_test.go Normal file
View file

@ -0,0 +1,137 @@
package cataloger
import (
"testing"
"github.com/anchore/syft/syft/pkg"
)
func must(c pkg.CPE, e error) pkg.CPE {
if e != nil {
panic(e)
}
return c
}
func TestGenerate(t *testing.T) {
tests := []struct {
name string
p pkg.Package
expected []pkg.CPE
}{
{
name: "python language",
p: pkg.Package{
Name: "name",
Version: "3.2",
FoundBy: "some-analyzer",
Language: pkg.Python,
Type: pkg.DebPkg,
},
expected: []pkg.CPE{
must(pkg.NewCPE("cpe:2.3:*:*:name:3.2:*:*:*:*:*:*:*")),
must(pkg.NewCPE("cpe:2.3:*:*:name:3.2:*:*:*:*:python:*:*")),
must(pkg.NewCPE("cpe:2.3:*:name:name:3.2:*:*:*:*:*:*:*")),
must(pkg.NewCPE("cpe:2.3:*:name:name:3.2:*:*:*:*:python:*:*")),
must(pkg.NewCPE("cpe:2.3:*:python-name:name:3.2:*:*:*:*:*:*:*")),
must(pkg.NewCPE("cpe:2.3:*:python-name:name:3.2:*:*:*:*:python:*:*")),
},
},
{
name: "javascript language",
p: pkg.Package{
Name: "name",
Version: "3.2",
FoundBy: "some-analyzer",
Language: pkg.JavaScript,
Type: pkg.DebPkg,
},
expected: []pkg.CPE{
must(pkg.NewCPE("cpe:2.3:*:*:name:3.2:*:*:*:*:*:*:*")),
must(pkg.NewCPE("cpe:2.3:*:*:name:3.2:*:*:*:*:node.js:*:*")),
must(pkg.NewCPE("cpe:2.3:*:*:name:3.2:*:*:*:*:nodejs:*:*")),
must(pkg.NewCPE("cpe:2.3:*:name:name:3.2:*:*:*:*:*:*:*")),
must(pkg.NewCPE("cpe:2.3:*:name:name:3.2:*:*:*:*:node.js:*:*")),
must(pkg.NewCPE("cpe:2.3:*:name:name:3.2:*:*:*:*:nodejs:*:*")),
},
},
{
name: "ruby language",
p: pkg.Package{
Name: "name",
Version: "3.2",
FoundBy: "some-analyzer",
Language: pkg.Ruby,
Type: pkg.DebPkg,
},
expected: []pkg.CPE{
must(pkg.NewCPE("cpe:2.3:*:*:name:3.2:*:*:*:*:*:*:*")),
must(pkg.NewCPE("cpe:2.3:*:*:name:3.2:*:*:*:*:ruby:*:*")),
must(pkg.NewCPE("cpe:2.3:*:*:name:3.2:*:*:*:*:rails:*:*")),
must(pkg.NewCPE("cpe:2.3:*:name:name:3.2:*:*:*:*:*:*:*")),
must(pkg.NewCPE("cpe:2.3:*:name:name:3.2:*:*:*:*:ruby:*:*")),
must(pkg.NewCPE("cpe:2.3:*:name:name:3.2:*:*:*:*:rails:*:*")),
},
},
{
name: "java language",
p: pkg.Package{
Name: "name",
Version: "3.2",
FoundBy: "some-analyzer",
Language: pkg.Java,
Type: pkg.DebPkg,
},
expected: []pkg.CPE{
must(pkg.NewCPE("cpe:2.3:*:*:name:3.2:*:*:*:*:*:*:*")),
must(pkg.NewCPE("cpe:2.3:*:*:name:3.2:*:*:*:*:java:*:*")),
must(pkg.NewCPE("cpe:2.3:*:*:name:3.2:*:*:*:*:maven:*:*")),
must(pkg.NewCPE("cpe:2.3:*:name:name:3.2:*:*:*:*:*:*:*")),
must(pkg.NewCPE("cpe:2.3:*:name:name:3.2:*:*:*:*:java:*:*")),
must(pkg.NewCPE("cpe:2.3:*:name:name:3.2:*:*:*:*:maven:*:*")),
},
},
{
name: "jenkins package",
p: pkg.Package{
Name: "name",
Version: "3.2",
FoundBy: "some-analyzer",
Language: pkg.Java,
Type: pkg.JenkinsPluginPkg,
},
expected: []pkg.CPE{
must(pkg.NewCPE("cpe:2.3:*:*:name:3.2:*:*:*:*:*:*:*")),
must(pkg.NewCPE("cpe:2.3:*:*:name:3.2:*:*:*:*:java:*:*")),
must(pkg.NewCPE("cpe:2.3:*:*:name:3.2:*:*:*:*:maven:*:*")),
must(pkg.NewCPE("cpe:2.3:*:*:name:3.2:*:*:*:*:jenkins:*:*")),
must(pkg.NewCPE("cpe:2.3:*:*:name:3.2:*:*:*:*:cloudbees_jenkins:*:*")),
must(pkg.NewCPE("cpe:2.3:*:name:name:3.2:*:*:*:*:*:*:*")),
must(pkg.NewCPE("cpe:2.3:*:name:name:3.2:*:*:*:*:java:*:*")),
must(pkg.NewCPE("cpe:2.3:*:name:name:3.2:*:*:*:*:maven:*:*")),
must(pkg.NewCPE("cpe:2.3:*:name:name:3.2:*:*:*:*:jenkins:*:*")),
must(pkg.NewCPE("cpe:2.3:*:name:name:3.2:*:*:*:*:cloudbees_jenkins:*:*")),
},
},
}
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
actual := generatePackageCPEs(test.p)
if len(actual) != len(test.expected) {
for _, e := range actual {
t.Errorf(" unexpected entry: %+v", e.BindToFmtString())
}
t.Fatalf("unexpected number of entries: %d", len(actual))
}
for idx, a := range actual {
e := test.expected[idx]
if a.BindToFmtString() != e.BindToFmtString() {
t.Errorf("mismatched entries @ %d:\n\texpected:%+v\n\t actual:%+v\n", idx, e.BindToFmtString(), a.BindToFmtString())
}
}
})
}
}

View file

@ -0,0 +1,49 @@
package cataloger
import (
"regexp"
"strings"
"github.com/anchore/syft/syft/distro"
"github.com/anchore/syft/syft/pkg"
"github.com/package-url/packageurl-go"
)
// generatePackageURL returns a package-URL representation of the given package (see https://github.com/package-url/purl-spec)
func generatePackageURL(p pkg.Package, d *distro.Distro) string {
// default to pURLs on the metadata
if p.Metadata != nil {
if i, ok := p.Metadata.(interface{ PackageURL() string }); ok {
return i.PackageURL()
} else if i, ok := p.Metadata.(interface{ PackageURL(*distro.Distro) string }); ok {
return i.PackageURL(d)
}
}
var purlType = p.Type.PackageURLType()
var name = p.Name
var namespace = ""
switch {
case purlType == "":
// there is no purl type, don't attempt to craft a purl
// TODO: should this be a "generic" purl type instead?
return ""
case p.Type == pkg.GoModulePkg:
re := regexp.MustCompile(`(/)[^/]*$`)
fields := re.Split(p.Name, -1)
namespace = fields[0]
name = strings.TrimPrefix(p.Name, namespace+"/")
}
// generate a purl from the package data
pURL := packageurl.NewPackageURL(
purlType,
namespace,
name,
p.Version,
nil,
"")
return pURL.ToString()
}

View file

@ -1,83 +1,84 @@
package pkg
package cataloger
import (
"testing"
"github.com/anchore/syft/syft/distro"
"github.com/anchore/syft/syft/pkg"
"github.com/sergi/go-diff/diffmatchpatch"
)
func TestPackage_pURL(t *testing.T) {
func TestPackageURL(t *testing.T) {
tests := []struct {
pkg Package
distro distro.Distro
pkg pkg.Package
distro *distro.Distro
expected string
}{
{
pkg: Package{
pkg: pkg.Package{
Name: "github.com/anchore/syft",
Version: "v0.1.0",
Type: GoModulePkg,
Type: pkg.GoModulePkg,
},
expected: "pkg:golang/github.com/anchore/syft@v0.1.0",
},
{
pkg: Package{
pkg: pkg.Package{
Name: "name",
Version: "v0.1.0",
Type: PythonPkg,
Type: pkg.PythonPkg,
},
expected: "pkg:pypi/name@v0.1.0",
},
{
pkg: Package{
pkg: pkg.Package{
Name: "name",
Version: "v0.1.0",
Type: PythonPkg,
Type: pkg.PythonPkg,
},
expected: "pkg:pypi/name@v0.1.0",
},
{
pkg: Package{
pkg: pkg.Package{
Name: "name",
Version: "v0.1.0",
Type: PythonPkg,
Type: pkg.PythonPkg,
},
expected: "pkg:pypi/name@v0.1.0",
},
{
pkg: Package{
pkg: pkg.Package{
Name: "name",
Version: "v0.1.0",
Type: PythonPkg,
Type: pkg.PythonPkg,
},
expected: "pkg:pypi/name@v0.1.0",
},
{
pkg: Package{
pkg: pkg.Package{
Name: "name",
Version: "v0.1.0",
Type: GemPkg,
Type: pkg.GemPkg,
},
expected: "pkg:gem/name@v0.1.0",
},
{
pkg: Package{
pkg: pkg.Package{
Name: "name",
Version: "v0.1.0",
Type: NpmPkg,
Type: pkg.NpmPkg,
},
expected: "pkg:npm/name@v0.1.0",
},
{
distro: distro.Distro{
distro: &distro.Distro{
Type: distro.Ubuntu,
},
pkg: Package{
pkg: pkg.Package{
Name: "bad-name",
Version: "bad-v0.1.0",
Type: DebPkg,
Metadata: DpkgMetadata{
Type: pkg.DebPkg,
Metadata: pkg.DpkgMetadata{
Package: "name",
Version: "v0.1.0",
Architecture: "amd64",
@ -86,14 +87,14 @@ func TestPackage_pURL(t *testing.T) {
expected: "pkg:deb/ubuntu/name@v0.1.0?arch=amd64",
},
{
distro: distro.Distro{
distro: &distro.Distro{
Type: distro.CentOS,
},
pkg: Package{
pkg: pkg.Package{
Name: "bad-name",
Version: "bad-v0.1.0",
Type: RpmPkg,
Metadata: RpmdbMetadata{
Type: pkg.RpmPkg,
Metadata: pkg.RpmdbMetadata{
Name: "name",
Version: "v0.1.0",
Epoch: 2,
@ -104,13 +105,13 @@ func TestPackage_pURL(t *testing.T) {
expected: "pkg:rpm/centos/name@2:v0.1.0-3?arch=amd64",
},
{
distro: distro.Distro{
distro: &distro.Distro{
Type: distro.UnknownDistroType,
},
pkg: Package{
pkg: pkg.Package{
Name: "name",
Version: "v0.1.0",
Type: DebPkg,
Type: pkg.DebPkg,
},
expected: "pkg:deb/name@v0.1.0",
},
@ -118,7 +119,7 @@ func TestPackage_pURL(t *testing.T) {
for _, test := range tests {
t.Run(string(test.pkg.Type)+"|"+test.expected, func(t *testing.T) {
actual := test.pkg.PackageURL(test.distro)
actual := generatePackageURL(test.pkg, test.distro)
if actual != test.expected {
dmp := diffmatchpatch.New()
diffs := dmp.DiffMain(test.expected, actual, true)

View file

@ -14,13 +14,6 @@ type Distro struct {
IDLike string
}
// NewUnknownDistro creates a standardized Distro object for unidentifiable distros
func NewUnknownDistro() Distro {
return Distro{
Type: UnknownDistroType,
}
}
// NewDistro creates a new Distro object populated with the given values.
func NewDistro(t Type, ver, like string) (Distro, error) {
if ver == "" {

View file

@ -17,8 +17,8 @@ type parseEntry struct {
}
// Identify parses distro-specific files to determine distro metadata like version and release.
func Identify(resolver source.Resolver) Distro {
distro := NewUnknownDistro()
func Identify(resolver source.Resolver) *Distro {
var distro *Distro
identityFiles := []parseEntry{
{
@ -65,12 +65,16 @@ identifyLoop:
}
if candidateDistro := entry.fn(content); candidateDistro != nil {
distro = *candidateDistro
distro = candidateDistro
break identifyLoop
}
}
}
if distro != nil && distro.Type == UnknownDistroType {
return nil
}
return distro
}
@ -113,7 +117,7 @@ func parseOsRelease(contents string) *Distro {
return assemble(id, vers, like)
}
var busyboxVersionMatcher = regexp.MustCompile(`BusyBox v[\d\.]+`)
var busyboxVersionMatcher = regexp.MustCompile(`BusyBox v[\d.]+`)
func parseBusyBox(contents string) *Distro {
matches := busyboxVersionMatcher.FindAllString(contents, -1)

View file

@ -57,6 +57,11 @@ func TestIdentifyDistro(t *testing.T) {
Type: Ubuntu,
Version: "20.4.0",
},
{
fixture: "test-fixtures/os/oraclelinux",
Type: OracleLinux,
Version: "8.3.0",
},
{
fixture: "test-fixtures/os/empty",
Type: UnknownDistroType,
@ -90,6 +95,12 @@ func TestIdentifyDistro(t *testing.T) {
}
d := Identify(s.Resolver)
if d == nil {
if test.Type == UnknownDistroType {
return
}
t.Fatalf("expected a distro but got none")
}
observedDistros.Add(d.String())
if d.Type != test.Type {
@ -103,8 +114,8 @@ func TestIdentifyDistro(t *testing.T) {
return
}
if d.Version == nil {
t.Log("Distro doesn't have a Version")
if d.Version == nil && test.Version == "" {
// this distro does not have a version
return
}
@ -116,7 +127,13 @@ func TestIdentifyDistro(t *testing.T) {
// ensure that test cases stay in sync with the distros that can be identified
if len(observedDistros) < len(definedDistros) {
t.Errorf("distro coverage incomplete (distro=%d, coverage=%d)", len(definedDistros), len(observedDistros))
for _, d := range definedDistros.ToSlice() {
t.Logf(" defined: %s", d)
}
for _, d := range observedDistros.ToSlice() {
t.Logf(" observed: %s", d)
}
t.Errorf("distro coverage incomplete (defined=%d, coverage=%d)", len(definedDistros), len(observedDistros))
}
}

View file

@ -0,0 +1,18 @@
NAME="Oracle Linux Server"
VERSION="8.3"
ID="ol"
ID_LIKE="fedora"
VARIANT="Server"
VARIANT_ID="server"
VERSION_ID="8.3"
PLATFORM_ID="platform:el8"
PRETTY_NAME="Oracle Linux Server 8.3"
ANSI_COLOR="0;31"
CPE_NAME="cpe:/o:oracle:linux:8:3:server"
HOME_URL="https://linux.oracle.com/"
BUG_REPORT_URL="https://bugzilla.oracle.com/"
ORACLE_BUGZILLA_PRODUCT="Oracle Linux 8"
ORACLE_BUGZILLA_PRODUCT_VERSION=8.3
ORACLE_SUPPORT_PRODUCT="Oracle Linux"
ORACLE_SUPPORT_PRODUCT_VERSION=8.3

View file

@ -34,79 +34,68 @@ import (
// Catalog the given image from a particular perspective (e.g. squashed source, all-layers source). Returns the discovered
// set of packages, the identified Linux distribution, and the source object used to wrap the data source.
func Catalog(userInput string, scope source.Scope) (source.Source, *pkg.Catalog, distro.Distro, error) {
log.Info("cataloging image")
s, cleanup, err := source.New(userInput, scope)
func Catalog(userInput string, scope source.Scope) (source.Source, *pkg.Catalog, *distro.Distro, error) {
theSource, cleanup, err := source.New(userInput, scope)
defer cleanup()
if err != nil {
return source.Source{}, nil, distro.Distro{}, err
return source.Source{}, nil, nil, err
}
d := IdentifyDistro(s)
catalog, err := CatalogFromScope(s)
if err != nil {
return source.Source{}, nil, distro.Distro{}, err
}
return s, catalog, d, nil
}
// IdentifyDistro attempts to discover what the underlying Linux distribution may be from the available flat files
// provided by the given source object. If results are inconclusive a "UnknownDistro" Type is returned.
func IdentifyDistro(s source.Source) distro.Distro {
d := distro.Identify(s.Resolver)
if d.Type != distro.UnknownDistroType {
log.Infof("identified distro: %s", d.String())
// find the distro
theDistro := distro.Identify(theSource.Resolver)
if theDistro != nil {
log.Infof("identified distro: %s", theDistro.String())
} else {
log.Info("could not identify distro")
}
return d
}
// Catalog the given source, which may represent a container image or filesystem. Returns the discovered set of packages.
func CatalogFromScope(s source.Source) (*pkg.Catalog, error) {
log.Info("building the catalog")
// conditionally have two sets of catalogers
// conditionally use the correct set of loggers based on the input type (container image or directory)
var catalogers []cataloger.Cataloger
switch s.Metadata.Scheme {
switch theSource.Metadata.Scheme {
case source.ImageScheme:
log.Info("cataloging image")
catalogers = cataloger.ImageCatalogers()
case source.DirectoryScheme:
log.Info("cataloging directory")
catalogers = cataloger.DirectoryCatalogers()
default:
return nil, fmt.Errorf("unable to determine cataloger set from scheme=%+v", s.Metadata.Scheme)
return source.Source{}, nil, nil, fmt.Errorf("unable to determine cataloger set from scheme=%+v", theSource.Metadata.Scheme)
}
return cataloger.Catalog(s.Resolver, catalogers...)
catalog, err := cataloger.Catalog(theSource.Resolver, theDistro, catalogers...)
if err != nil {
return source.Source{}, nil, nil, err
}
return theSource, catalog, theDistro, nil
}
// CatalogFromJSON takes an existing syft report and generates native syft objects.
func CatalogFromJSON(reader io.Reader) (source.Metadata, *pkg.Catalog, distro.Distro, error) {
func CatalogFromJSON(reader io.Reader) (source.Metadata, *pkg.Catalog, *distro.Distro, error) {
var doc jsonPresenter.Document
var err error
decoder := json.NewDecoder(reader)
if err := decoder.Decode(&doc); err != nil {
return source.Metadata{}, nil, distro.Distro{}, err
return source.Metadata{}, nil, nil, err
}
var pkgs = make([]pkg.Package, len(doc.Artifacts))
for i, a := range doc.Artifacts {
pkgs[i] = a.ToPackage()
pkgs[i], err = a.ToPackage()
if err != nil {
return source.Metadata{}, nil, nil, err
}
}
catalog := pkg.NewCatalog(pkgs...)
var distroType distro.Type
if doc.Distro.Name == "" {
distroType = distro.UnknownDistroType
} else {
distroType = distro.Type(doc.Distro.Name)
}
theDistro, err := distro.NewDistro(distroType, doc.Distro.Version, doc.Distro.IDLike)
if err != nil {
return source.Metadata{}, nil, distro.Distro{}, err
var theDistro *distro.Distro
if doc.Distro.Name != "" {
d, err := distro.NewDistro(distro.Type(doc.Distro.Name), doc.Distro.Version, doc.Distro.IDLike)
if err != nil {
return source.Metadata{}, nil, nil, err
}
theDistro = &d
}
return doc.Source.ToSourceMetadata(), catalog, theDistro, nil

41
syft/pkg/cpe.go Normal file
View file

@ -0,0 +1,41 @@
package pkg
import (
"fmt"
"strings"
"github.com/facebookincubator/nvdtools/wfn"
)
type CPE = wfn.Attributes
func NewCPE(cpeStr string) (CPE, error) {
value, err := wfn.Parse(cpeStr)
if err != nil {
return CPE{}, fmt.Errorf("failed to parse CPE=%q: %w", cpeStr, err)
}
if value == nil {
return CPE{}, fmt.Errorf("failed to parse CPE=%q", cpeStr)
}
// we need to compare the raw data since we are constructing CPEs in other locations
value.Vendor = normalizeCpeField(value.Vendor)
value.Product = normalizeCpeField(value.Product)
value.Language = normalizeCpeField(value.Language)
value.Version = normalizeCpeField(value.Version)
value.TargetSW = normalizeCpeField(value.TargetSW)
value.Part = normalizeCpeField(value.Part)
value.Edition = normalizeCpeField(value.Edition)
value.Other = normalizeCpeField(value.Other)
value.SWEdition = normalizeCpeField(value.SWEdition)
value.TargetHW = normalizeCpeField(value.TargetHW)
value.Update = normalizeCpeField(value.Update)
return *value, nil
}
func normalizeCpeField(field string) string {
// keep dashes and forward slashes unescaped
return strings.ReplaceAll(wfn.StripSlashes(field), `\/`, "/")
}

48
syft/pkg/cpe_test.go Normal file
View file

@ -0,0 +1,48 @@
package pkg
import "testing"
func must(c CPE, e error) CPE {
if e != nil {
panic(e)
}
return c
}
func TestNewCPE(t *testing.T) {
tests := []struct {
name string
input string
expected CPE
}{
{
name: "gocase",
input: `cpe:/a:10web:form_maker:1.0.0::~~~wordpress~~`,
expected: must(NewCPE(`cpe:2.3:a:10web:form_maker:1.0.0:*:*:*:*:wordpress:*:*`)),
},
{
name: "dashes",
input: `cpe:/a:7-zip:7-zip:4.56:beta:~~~windows~~`,
expected: must(NewCPE(`cpe:2.3:a:7-zip:7-zip:4.56:beta:*:*:*:windows:*:*`)),
},
{
name: "URL escape characters",
input: `cpe:/a:%240.99_kindle_books_project:%240.99_kindle_books:6::~~~android~~`,
expected: must(NewCPE(`cpe:2.3:a:$0.99_kindle_books_project:$0.99_kindle_books:6:*:*:*:*:android:*:*`)),
},
}
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
actual, err := NewCPE(test.input)
if err != nil {
t.Fatalf("got an error while creating CPE: %+v", err)
}
if actual.BindToFmtString() != test.expected.BindToFmtString() {
t.Errorf("mismatched entries:\n\texpected:%+v\n\t actual:%+v\n", test.expected.BindToFmtString(), actual.BindToFmtString())
}
})
}
}

View file

@ -24,7 +24,10 @@ type DpkgFileRecord struct {
}
// PackageURL returns the PURL for the specific Debian package (see https://github.com/package-url/purl-spec)
func (m DpkgMetadata) PackageURL(d distro.Distro) string {
func (m DpkgMetadata) PackageURL(d *distro.Distro) string {
if d == nil {
return ""
}
pURL := packageurl.NewPackageURL(
// TODO: replace with `packageurl.TypeDebian` upon merge of https://github.com/package-url/packageurl-go/pull/21
"deb",

View file

@ -1,9 +1,10 @@
package pkg
import (
"testing"
"github.com/anchore/syft/syft/distro"
"github.com/sergi/go-diff/diffmatchpatch"
"testing"
)
func TestDpkgMetadata_pURL(t *testing.T) {
@ -40,7 +41,7 @@ func TestDpkgMetadata_pURL(t *testing.T) {
for _, test := range tests {
t.Run(test.expected, func(t *testing.T) {
actual := test.metadata.PackageURL(test.distro)
actual := test.metadata.PackageURL(&test.distro)
if actual != test.expected {
dmp := diffmatchpatch.New()
diffs := dmp.DiffMain(test.expected, actual, true)

View file

@ -5,13 +5,8 @@ package pkg
import (
"fmt"
"regexp"
"strings"
"github.com/anchore/syft/syft/source"
"github.com/anchore/syft/syft/distro"
"github.com/package-url/packageurl-go"
)
// ID represents a unique value for each package added to a package catalog.
@ -28,6 +23,8 @@ type Package struct {
Licenses []string // licenses discovered with the package metadata
Language Language // the language ecosystem this package belongs to (e.g. JavaScript, Python, etc)
Type Type // the package type (e.g. Npm, Yarn, Python, Rpm, Deb, etc)
CPEs []CPE // all possible Common Platform Enumerators
PURL string // the Package URL (see https://github.com/package-url/purl-spec)
MetadataType MetadataType // the shape of the additional data in the "metadata" field
Metadata interface{} // additional data found while parsing the package source
}
@ -41,42 +38,3 @@ func (p Package) ID() ID {
func (p Package) String() string {
return fmt.Sprintf("Pkg(type=%s, name=%s, version=%s)", p.Type, p.Name, p.Version)
}
// PackageURL returns a package-URL representation of the given package (see https://github.com/package-url/purl-spec)
func (p Package) PackageURL(d distro.Distro) string {
// default to pURLs on the metadata
if p.Metadata != nil {
if i, ok := p.Metadata.(interface{ PackageURL() string }); ok {
return i.PackageURL()
} else if i, ok := p.Metadata.(interface{ PackageURL(distro.Distro) string }); ok {
return i.PackageURL(d)
}
}
var purlType = p.Type.PackageURLType()
var name = p.Name
var namespace = ""
switch {
case purlType == "":
// there is no purl type, don't attempt to craft a purl
// TODO: should this be a "generic" purl type instead?
return ""
case p.Type == GoModulePkg:
re := regexp.MustCompile(`(\/)[^\/]*$`)
fields := re.Split(p.Name, -1)
namespace = fields[0]
name = strings.TrimPrefix(p.Name, namespace+"/")
}
// generate a purl from the package data
pURL := packageurl.NewPackageURL(
purlType,
namespace,
name,
p.Version,
nil,
"")
return pURL.ToString()
}

View file

@ -33,7 +33,11 @@ type RpmdbFileRecord struct {
type RpmdbFileMode uint16
// PackageURL returns the PURL for the specific RHEL package (see https://github.com/package-url/purl-spec)
func (m RpmdbMetadata) PackageURL(d distro.Distro) string {
func (m RpmdbMetadata) PackageURL(d *distro.Distro) string {
if d == nil {
return ""
}
pURL := packageurl.NewPackageURL(
packageurl.TypeRPM,
d.Type.String(),

View file

@ -43,7 +43,7 @@ func TestRpmMetadata_pURL(t *testing.T) {
for _, test := range tests {
t.Run(test.expected, func(t *testing.T) {
actual := test.metadata.PackageURL(test.distro)
actual := test.metadata.PackageURL(&test.distro)
if actual != test.expected {
dmp := diffmatchpatch.New()
diffs := dmp.DiffMain(test.expected, actual, true)

View file

@ -5,7 +5,6 @@ import (
"github.com/anchore/syft/internal"
"github.com/anchore/syft/internal/version"
"github.com/anchore/syft/syft/distro"
"github.com/anchore/syft/syft/pkg"
"github.com/anchore/syft/syft/source"
"github.com/google/uuid"
@ -25,7 +24,7 @@ type Document struct {
}
// NewDocumentFromCatalog returns a CycloneDX Document object populated with the catalog contents.
func NewDocument(catalog *pkg.Catalog, d distro.Distro, srcMetadata source.Metadata) Document {
func NewDocument(catalog *pkg.Catalog, srcMetadata source.Metadata) Document {
versionInfo := version.FromBuild()
doc := Document{
@ -42,7 +41,7 @@ func NewDocument(catalog *pkg.Catalog, d distro.Distro, srcMetadata source.Metad
Type: "library", // TODO: this is not accurate
Name: p.Name,
Version: p.Version,
PackageURL: p.PackageURL(d),
PackageURL: p.PURL,
}
var licenses []License
for _, licenseName := range p.Licenses {

View file

@ -7,8 +7,6 @@ import (
"encoding/xml"
"io"
"github.com/anchore/syft/syft/distro"
"github.com/anchore/syft/syft/pkg"
"github.com/anchore/syft/syft/source"
)
@ -17,21 +15,19 @@ import (
type Presenter struct {
catalog *pkg.Catalog
srcMetadata source.Metadata
distro distro.Distro
}
// NewPresenter creates a CycloneDX presenter from the given Catalog and Locations objects.
func NewPresenter(catalog *pkg.Catalog, srcMetadata source.Metadata, d distro.Distro) *Presenter {
func NewPresenter(catalog *pkg.Catalog, srcMetadata source.Metadata) *Presenter {
return &Presenter{
catalog: catalog,
srcMetadata: srcMetadata,
distro: d,
}
}
// Present writes the CycloneDX report to the given io.Writer.
func (pres *Presenter) Present(output io.Writer) error {
bom := NewDocument(pres.catalog, pres.distro, pres.srcMetadata)
bom := NewDocument(pres.catalog, pres.srcMetadata)
encoder := xml.NewEncoder(output)
encoder.Indent("", " ")

View file

@ -7,7 +7,6 @@ import (
"testing"
"github.com/anchore/stereoscope/pkg/imagetest"
"github.com/anchore/syft/syft/distro"
"github.com/anchore/go-testutils"
"github.com/anchore/syft/syft/pkg"
@ -61,12 +60,7 @@ func TestCycloneDxDirsPresenter(t *testing.T) {
t.Fatal(err)
}
d, err := distro.NewDistro(distro.Ubuntu, "20.04", "debian")
if err != nil {
t.Fatal(err)
}
pres := NewPresenter(catalog, s.Metadata, d)
pres := NewPresenter(catalog, s.Metadata)
// run presenter
err = pres.Present(&buffer)
@ -109,17 +103,7 @@ func TestCycloneDxImgsPresenter(t *testing.T) {
},
Type: pkg.RpmPkg,
FoundBy: "the-cataloger-1",
Metadata: pkg.RpmdbMetadata{
Name: "package1",
Epoch: 0,
Arch: "x86_64",
Release: "1",
Version: "1.0.1",
SourceRpm: "package1-1.0.1-1.src.rpm",
Size: 12406784,
License: "MIT",
Vendor: "",
},
PURL: "the-purl-1",
})
catalog.Add(pkg.Package{
Name: "package2",
@ -133,17 +117,7 @@ func TestCycloneDxImgsPresenter(t *testing.T) {
"MIT",
"Apache-v2",
},
Metadata: pkg.RpmdbMetadata{
Name: "package2",
Epoch: 0,
Arch: "x86_64",
Release: "1",
Version: "1.0.2",
SourceRpm: "package2-1.0.2-1.src.rpm",
Size: 12406784,
License: "MIT",
Vendor: "",
},
PURL: "the-purl-2",
})
s, err := source.NewFromImage(img, source.AllLayersScope, "user-image-input")
@ -151,11 +125,6 @@ func TestCycloneDxImgsPresenter(t *testing.T) {
t.Fatal(err)
}
d, err := distro.NewDistro(distro.RedHat, "8", "")
if err != nil {
t.Fatal(err)
}
// This accounts for the non-deterministic digest value that we end up with when
// we build a container image dynamically during testing. Ultimately, we should
// use a golden image as a test fixture in place of building this image during
@ -164,7 +133,7 @@ func TestCycloneDxImgsPresenter(t *testing.T) {
// This value is sourced from the "version" node in "./test-fixtures/snapshot/TestCycloneDxImgsPresenter.golden"
s.Metadata.ImageMetadata.Digest = "sha256:2731251dc34951c0e50fcc643b4c5f74922dad1a5d98f302b504cf46cd5d9368"
pres := NewPresenter(catalog, s.Metadata, d)
pres := NewPresenter(catalog, s.Metadata)
// run presenter
err = pres.Present(&buffer)

View file

@ -1,10 +1,9 @@
<?xml version="1.0" encoding="UTF-8"?>
<bom xmlns="http://cyclonedx.org/schema/bom/1.2" xmlns:bd="http://cyclonedx.org/schema/ext/bom-descriptor/1.0" version="1" serialNumber="urn:uuid:815fdd6b-917e-423d-8c91-1fe648141505">
<bom xmlns="http://cyclonedx.org/schema/bom/1.2" xmlns:bd="http://cyclonedx.org/schema/ext/bom-descriptor/1.0" version="1" serialNumber="urn:uuid:aa1738d1-1c6e-45e4-a530-5a04d2c2b1a7">
<components>
<component type="library">
<name>package1</name>
<version>1.0.1</version>
<purl>pkg:deb/ubuntu/package1@1.0.1?arch=amd64</purl>
</component>
<component type="library">
<name>package2</name>
@ -17,11 +16,10 @@
<name>Apache-v2</name>
</license>
</licenses>
<purl>pkg:deb/ubuntu/package2@1.0.2?arch=amd64</purl>
</component>
</components>
<bd:metadata>
<bd:timestamp>2020-11-16T08:45:54-05:00</bd:timestamp>
<bd:timestamp>2020-11-19T10:11:26-05:00</bd:timestamp>
<bd:tool>
<bd:vendor>anchore</bd:vendor>
<bd:name>syft</bd:name>

View file

@ -1,10 +1,10 @@
<?xml version="1.0" encoding="UTF-8"?>
<bom xmlns="http://cyclonedx.org/schema/bom/1.2" xmlns:bd="http://cyclonedx.org/schema/ext/bom-descriptor/1.0" version="1" serialNumber="urn:uuid:3cb10332-1645-44f6-be4a-4f8be5a60cf8">
<bom xmlns="http://cyclonedx.org/schema/bom/1.2" xmlns:bd="http://cyclonedx.org/schema/ext/bom-descriptor/1.0" version="1" serialNumber="urn:uuid:8a4c332c-365c-454d-a43c-d9344577d105">
<components>
<component type="library">
<name>package1</name>
<version>1.0.1</version>
<purl>pkg:rpm/redhat/package1@0:1.0.1-1?arch=x86_64</purl>
<purl>the-purl-1</purl>
</component>
<component type="library">
<name>package2</name>
@ -17,11 +17,11 @@
<name>Apache-v2</name>
</license>
</licenses>
<purl>pkg:rpm/redhat/package2@0:1.0.2-1?arch=x86_64</purl>
<purl>the-purl-2</purl>
</component>
</components>
<bd:metadata>
<bd:timestamp>2020-11-16T08:45:54-05:00</bd:timestamp>
<bd:timestamp>2020-11-19T10:11:26-05:00</bd:timestamp>
<bd:tool>
<bd:vendor>anchore</bd:vendor>
<bd:name>syft</bd:name>

View file

@ -10,14 +10,13 @@ type Distribution struct {
}
// NewDistribution creates a struct with the Linux distribution to be represented in JSON.
func NewDistribution(d distro.Distro) Distribution {
distroName := d.Name()
if distroName == "UnknownDistroType" {
distroName = ""
func NewDistribution(d *distro.Distro) Distribution {
if d == nil {
return Distribution{}
}
return Distribution{
Name: distroName,
Name: d.Name(),
Version: d.FullVersion(),
IDLike: d.IDLike,
}

View file

@ -17,7 +17,7 @@ type Document struct {
}
// NewDocument creates and populates a new JSON document struct from the given cataloging results.
func NewDocument(catalog *pkg.Catalog, srcMetadata source.Metadata, d distro.Distro) (Document, error) {
func NewDocument(catalog *pkg.Catalog, srcMetadata source.Metadata, d *distro.Distro) (Document, error) {
src, err := NewSource(srcMetadata)
if err != nil {
return Document{}, nil

View file

@ -23,6 +23,8 @@ type packageBasicMetadata struct {
Locations []source.Location `json:"locations"`
Licenses []string `json:"licenses"`
Language pkg.Language `json:"language"`
CPEs []string `json:"cpes"`
PURL string `json:"purl"`
}
// packageCustomMetadata contains ambiguous values (type-wise) from pkg.Package.
@ -39,6 +41,10 @@ type packageMetadataUnpacker struct {
// NewPackage crates a new Package from the given pkg.Package.
func NewPackage(p *pkg.Package) (Package, error) {
var cpes = make([]string, len(p.CPEs))
for i, c := range p.CPEs {
cpes[i] = c.BindToFmtString()
}
return Package{
packageBasicMetadata: packageBasicMetadata{
Name: p.Name,
@ -48,6 +54,8 @@ func NewPackage(p *pkg.Package) (Package, error) {
Locations: p.Locations,
Licenses: p.Licenses,
Language: p.Language,
CPEs: cpes,
PURL: p.PURL,
},
packageCustomMetadata: packageCustomMetadata{
MetadataType: p.MetadataType,
@ -57,7 +65,15 @@ func NewPackage(p *pkg.Package) (Package, error) {
}
// ToPackage generates a pkg.Package from the current Package.
func (a Package) ToPackage() pkg.Package {
func (a Package) ToPackage() (pkg.Package, error) {
var cpes = make([]pkg.CPE, len(a.CPEs))
var err error
for i, c := range a.CPEs {
cpes[i], err = pkg.NewCPE(c)
if err != nil {
return pkg.Package{}, fmt.Errorf("unable to parse CPE from JSON package: %w", err)
}
}
return pkg.Package{
// does not include found-by and locations
Name: a.Name,
@ -66,10 +82,12 @@ func (a Package) ToPackage() pkg.Package {
Licenses: a.Licenses,
Language: a.Language,
Locations: a.Locations,
CPEs: cpes,
PURL: a.PURL,
Type: a.Type,
MetadataType: a.MetadataType,
Metadata: a.Metadata,
}
}, nil
}
// UnmarshalJSON is a custom unmarshaller for handling basic values and values with ambiguous types.

View file

@ -13,11 +13,11 @@ import (
type Presenter struct {
catalog *pkg.Catalog
srcMetadata source.Metadata
distro distro.Distro
distro *distro.Distro
}
// NewPresenter creates a new JSON presenter object for the given cataloging results.
func NewPresenter(catalog *pkg.Catalog, s source.Metadata, d distro.Distro) *Presenter {
func NewPresenter(catalog *pkg.Catalog, s source.Metadata, d *distro.Distro) *Presenter {
return &Presenter{
catalog: catalog,
srcMetadata: s,

View file

@ -15,6 +15,13 @@ import (
var update = flag.Bool("update", false, "update the *.golden files for json presenters")
func must(c pkg.CPE, e error) pkg.CPE {
if e != nil {
panic(e)
}
return c
}
func TestJsonDirsPresenter(t *testing.T) {
var buffer bytes.Buffer
@ -36,6 +43,10 @@ func TestJsonDirsPresenter(t *testing.T) {
Name: "package-1",
Version: "1.0.1",
},
PURL: "a-purl-2",
CPEs: []pkg.CPE{
must(pkg.NewCPE("cpe:2.3:*:some:package:2:*:*:*:*:*:*:*")),
},
})
catalog.Add(pkg.Package{
Name: "package-2",
@ -50,8 +61,12 @@ func TestJsonDirsPresenter(t *testing.T) {
Package: "package-2",
Version: "2.0.1",
},
PURL: "a-purl-2",
CPEs: []pkg.CPE{
must(pkg.NewCPE("cpe:2.3:*:some:package:2:*:*:*:*:*:*:*")),
},
})
d := distro.NewUnknownDistro()
var d *distro.Distro
s, err := source.NewFromDirectory("/some/path")
if err != nil {
t.Fatal(err)
@ -107,6 +122,10 @@ func TestJsonImgsPresenter(t *testing.T) {
Name: "package-1",
Version: "1.0.1",
},
PURL: "a-purl-1",
CPEs: []pkg.CPE{
must(pkg.NewCPE("cpe:2.3:*:some:package:1:*:*:*:*:*:*:*")),
},
})
catalog.Add(pkg.Package{
Name: "package-2",
@ -121,10 +140,14 @@ func TestJsonImgsPresenter(t *testing.T) {
Package: "package-2",
Version: "2.0.1",
},
PURL: "a-purl-2",
CPEs: []pkg.CPE{
must(pkg.NewCPE("cpe:2.3:*:some:package:2:*:*:*:*:*:*:*")),
},
})
s, err := source.NewFromImage(img, source.AllLayersScope, "user-image-input")
d := distro.NewUnknownDistro()
var d *distro.Distro
pres := NewPresenter(catalog, s.Metadata, d)
// run presenter

View file

@ -14,6 +14,10 @@
"MIT"
],
"language": "python",
"cpes": [
"cpe:2.3:*:some:package:2:*:*:*:*:*:*:*"
],
"purl": "a-purl-2",
"metadataType": "PythonPackageMetadata",
"metadata": {
"name": "package-1",
@ -37,6 +41,10 @@
],
"licenses": null,
"language": "",
"cpes": [
"cpe:2.3:*:some:package:2:*:*:*:*:*:*:*"
],
"purl": "a-purl-2",
"metadataType": "DpkgMetadata",
"metadata": {
"package": "package-2",

View file

@ -15,6 +15,10 @@
"MIT"
],
"language": "python",
"cpes": [
"cpe:2.3:*:some:package:1:*:*:*:*:*:*:*"
],
"purl": "a-purl-1",
"metadataType": "PythonPackageMetadata",
"metadata": {
"name": "package-1",
@ -39,6 +43,10 @@
],
"licenses": null,
"language": "",
"cpes": [
"cpe:2.3:*:some:package:2:*:*:*:*:*:*:*"
],
"purl": "a-purl-2",
"metadataType": "DpkgMetadata",
"metadata": {
"package": "package-2",

View file

@ -25,7 +25,7 @@ type Presenter interface {
}
// GetPresenter returns a presenter for images or directories
func GetPresenter(option Option, srcMetadata source.Metadata, catalog *pkg.Catalog, d distro.Distro) Presenter {
func GetPresenter(option Option, srcMetadata source.Metadata, catalog *pkg.Catalog, d *distro.Distro) Presenter {
switch option {
case JSONPresenter:
return json.NewPresenter(catalog, srcMetadata, d)
@ -34,7 +34,7 @@ func GetPresenter(option Option, srcMetadata source.Metadata, catalog *pkg.Catal
case TablePresenter:
return table.NewPresenter(catalog)
case CycloneDxPresenter:
return cyclonedx.NewPresenter(catalog, srcMetadata, d)
return cyclonedx.NewPresenter(catalog, srcMetadata)
default:
return nil
}

View file

@ -26,7 +26,7 @@ func TestDistroImage(t *testing.T) {
t.Fatalf("could not create distro: %+v", err)
}
for _, d := range deep.Equal(actualDistro, expected) {
for _, d := range deep.Equal(actualDistro, &expected) {
t.Errorf("found distro difference: %+v", d)
}

View file

@ -67,7 +67,7 @@ func testJsonSchema(t *testing.T, catalog *pkg.Catalog, theScope source.Source,
t.Fatalf("bad distro: %+v", err)
}
p := presenter.GetPresenter(presenter.JSONPresenter, theScope.Metadata, catalog, d)
p := presenter.GetPresenter(presenter.JSONPresenter, theScope.Metadata, catalog, &d)
if p == nil {
t.Fatal("unable to get presenter")
}