port dotnet cataloger to new generic cataloger pattern (#1286)

Signed-off-by: Alex Goodman <alex.goodman@anchore.com>

Signed-off-by: Alex Goodman <alex.goodman@anchore.com>
This commit is contained in:
Alex Goodman 2022-10-24 17:17:27 -04:00 committed by GitHub
parent fbdde6d4f4
commit c7a653060d
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
8 changed files with 230 additions and 136 deletions

View file

@ -1,14 +1,13 @@
package dotnet
import (
"github.com/anchore/syft/syft/pkg/cataloger/common"
"github.com/anchore/syft/syft/pkg/cataloger/generic"
)
// NewDotnetDepsCataloger returns a new Dotnet cataloger object base on deps json files.
func NewDotnetDepsCataloger() *common.GenericCataloger {
globParsers := map[string]common.ParserFn{
"**/*.deps.json": parseDotnetDeps,
}
const catalogerName = "dotnet-deps-cataloger"
return common.NewGenericCataloger(nil, globParsers, "dotnet-deps-cataloger")
// NewDotnetDepsCataloger returns a new Dotnet cataloger object base on deps json files.
func NewDotnetDepsCataloger() *generic.Cataloger {
return generic.NewCataloger(catalogerName).
WithParserByGlobs(parseDotnetDeps, "**/*.deps.json")
}

View file

@ -0,0 +1,55 @@
package dotnet
import (
"strings"
"github.com/anchore/packageurl-go"
"github.com/anchore/syft/syft/pkg"
"github.com/anchore/syft/syft/source"
)
func newDotnetDepsPackage(nameVersion string, lib dotnetDepsLibrary, locations ...source.Location) *pkg.Package {
if lib.Type != "package" {
return nil
}
fields := strings.Split(nameVersion, "/")
name := fields[0]
version := fields[1]
m := pkg.DotnetDepsMetadata{
Name: name,
Version: version,
Path: lib.Path,
Sha512: lib.Sha512,
HashPath: lib.HashPath,
}
p := &pkg.Package{
Name: name,
Version: version,
Locations: source.NewLocationSet(locations...),
PURL: packageURL(m),
Language: pkg.Dotnet,
Type: pkg.DotnetPkg,
MetadataType: pkg.DotnetDepsMetadataType,
Metadata: m,
}
p.SetID()
return p
}
func packageURL(m pkg.DotnetDepsMetadata) string {
var qualifiers packageurl.Qualifiers
return packageurl.NewPackageURL(
packageurl.TypeDotnet,
"",
m.Name,
m.Version,
qualifiers,
"",
).ToString()
}

View file

@ -3,16 +3,15 @@ package dotnet
import (
"encoding/json"
"fmt"
"io"
"strings"
"sort"
"github.com/anchore/syft/syft/artifact"
"github.com/anchore/syft/syft/pkg"
"github.com/anchore/syft/syft/pkg/cataloger/common"
"github.com/anchore/syft/syft/pkg/cataloger/generic"
"github.com/anchore/syft/syft/source"
)
// integrity check
var _ common.ParserFn = parseDotnetDeps
var _ generic.Parser = parseDotnetDeps
type dotnetDeps struct {
Libraries map[string]dotnetDepsLibrary `json:"libraries"`
@ -25,8 +24,8 @@ type dotnetDepsLibrary struct {
HashPath string `json:"hashPath"`
}
func parseDotnetDeps(path string, reader io.Reader) ([]*pkg.Package, []artifact.Relationship, error) {
var packages []*pkg.Package
func parseDotnetDeps(_ source.FileResolver, _ *generic.Environment, reader source.LocationReadCloser) ([]pkg.Package, []artifact.Relationship, error) {
var pkgs []pkg.Package
dec := json.NewDecoder(reader)
@ -35,38 +34,23 @@ func parseDotnetDeps(path string, reader io.Reader) ([]*pkg.Package, []artifact.
return nil, nil, fmt.Errorf("failed to parse deps.json file: %w", err)
}
for nameVersion, lib := range p.Libraries {
dotnetPkg := newDotnetDepsPackage(nameVersion, lib)
var names []string
for nameVersion := range p.Libraries {
names = append(names, nameVersion)
}
// sort the names so that the order of the packages is deterministic
sort.Strings(names)
for _, nameVersion := range names {
lib := p.Libraries[nameVersion]
dotnetPkg := newDotnetDepsPackage(nameVersion, lib, reader.Location)
if dotnetPkg != nil {
packages = append(packages, dotnetPkg)
pkgs = append(pkgs, *dotnetPkg)
}
}
return packages, nil, nil
}
func newDotnetDepsPackage(nameVersion string, lib dotnetDepsLibrary) *pkg.Package {
if lib.Type != "package" {
return nil
}
splitted := strings.Split(nameVersion, "/")
name := splitted[0]
version := splitted[1]
return &pkg.Package{
Name: name,
Version: version,
Language: pkg.Dotnet,
Type: pkg.DotnetPkg,
MetadataType: pkg.DotnetDepsMetadataType,
Metadata: &pkg.DotnetDepsMetadata{
Name: name,
Version: version,
Path: lib.Path,
Sha512: lib.Sha512,
HashPath: lib.HashPath,
},
}
return pkgs, nil, nil
}

View file

@ -1,23 +1,23 @@
package dotnet
import (
"os"
"testing"
"github.com/stretchr/testify/assert"
"github.com/anchore/syft/syft/artifact"
"github.com/anchore/syft/syft/pkg"
"github.com/anchore/syft/syft/pkg/cataloger/internal/pkgtest"
"github.com/anchore/syft/syft/source"
)
func assertPackagesEqual(t *testing.T, actual []*pkg.Package, expected map[string]*pkg.Package) {
assert.Len(t, actual, len(expected))
}
func TestParseDotnetDeps(t *testing.T) {
expected := map[string]*pkg.Package{
"AWSSDK.Core": {
fixture := "test-fixtures/TestLibrary.deps.json"
fixtureLocationSet := source.NewLocationSet(source.NewLocation(fixture))
expected := []pkg.Package{
{
Name: "AWSSDK.Core",
Version: "3.7.10.6",
PURL: "pkg:dotnet/AWSSDK.Core@3.7.10.6",
Locations: fixtureLocationSet,
Language: pkg.Dotnet,
Type: pkg.DotnetPkg,
MetadataType: pkg.DotnetDepsMetadataType,
@ -29,9 +29,27 @@ func TestParseDotnetDeps(t *testing.T) {
HashPath: "awssdk.core.3.7.10.6.nupkg.sha512",
},
},
"Microsoft.Extensions.DependencyInjection": {
{
Name: "Microsoft.Extensions.DependencyInjection.Abstractions",
Version: "6.0.0",
PURL: "pkg:dotnet/Microsoft.Extensions.DependencyInjection.Abstractions@6.0.0",
Locations: fixtureLocationSet,
Language: pkg.Dotnet,
Type: pkg.DotnetPkg,
MetadataType: pkg.DotnetDepsMetadataType,
Metadata: pkg.DotnetDepsMetadata{
Name: "Microsoft.Extensions.DependencyInjection.Abstractions",
Version: "6.0.0",
Sha512: "sha512-xlzi2IYREJH3/m6+lUrQlujzX8wDitm4QGnUu6kUXTQAWPuZY8i+ticFJbzfqaetLA6KR/rO6Ew/HuYD+bxifg==",
Path: "microsoft.extensions.dependencyinjection.abstractions/6.0.0",
HashPath: "microsoft.extensions.dependencyinjection.abstractions.6.0.0.nupkg.sha512",
},
},
{
Name: "Microsoft.Extensions.DependencyInjection",
Version: "6.0.0",
PURL: "pkg:dotnet/Microsoft.Extensions.DependencyInjection@6.0.0",
Locations: fixtureLocationSet,
Language: pkg.Dotnet,
Type: pkg.DotnetPkg,
MetadataType: pkg.DotnetDepsMetadataType,
@ -43,23 +61,27 @@ func TestParseDotnetDeps(t *testing.T) {
HashPath: "microsoft.extensions.dependencyinjection.6.0.0.nupkg.sha512",
},
},
"Microsoft.Extensions.DependencyInjection.Abstractions": {
Name: "Microsoft.Extensions.DependencyInjection.Abstractions",
{
Name: "Microsoft.Extensions.Logging.Abstractions",
Version: "6.0.0",
PURL: "pkg:dotnet/Microsoft.Extensions.Logging.Abstractions@6.0.0",
Locations: fixtureLocationSet,
Language: pkg.Dotnet,
Type: pkg.DotnetPkg,
MetadataType: pkg.DotnetDepsMetadataType,
Metadata: pkg.DotnetDepsMetadata{
Name: "Microsoft.Extensions.DependencyInjection",
Name: "Microsoft.Extensions.Logging.Abstractions",
Version: "6.0.0",
Sha512: "sha512-xlzi2IYREJH3/m6+lUrQlujzX8wDitm4QGnUu6kUXTQAWPuZY8i+ticFJbzfqaetLA6KR/rO6Ew/HuYD+bxifg==",
Path: "microsoft.extensions.dependencyinjection.abstractions/6.0.0",
HashPath: "microsoft.extensions.dependencyinjection.abstractions.6.0.0.nupkg.sha512",
Sha512: "sha512-/HggWBbTwy8TgebGSX5DBZ24ndhzi93sHUBDvP1IxbZD7FDokYzdAr6+vbWGjw2XAfR2EJ1sfKUotpjHnFWPxA==",
Path: "microsoft.extensions.logging.abstractions/6.0.0",
HashPath: "microsoft.extensions.logging.abstractions.6.0.0.nupkg.sha512",
},
},
"Microsoft.Extensions.Logging": {
{
Name: "Microsoft.Extensions.Logging",
Version: "6.0.0",
PURL: "pkg:dotnet/Microsoft.Extensions.Logging@6.0.0",
Locations: fixtureLocationSet,
Language: pkg.Dotnet,
Type: pkg.DotnetPkg,
MetadataType: pkg.DotnetDepsMetadataType,
@ -71,23 +93,12 @@ func TestParseDotnetDeps(t *testing.T) {
HashPath: "microsoft.extensions.logging.6.0.0.nupkg.sha512",
},
},
"Microsoft.Extensions.Logging.Abstractions": {
Name: "Microsoft.Extensions.Logging.Abstractions",
Version: "6.0.0",
Language: pkg.Dotnet,
Type: pkg.DotnetPkg,
MetadataType: pkg.DotnetDepsMetadataType,
Metadata: pkg.DotnetDepsMetadata{
Name: "Microsoft.Extensions.Logging",
Version: "6.0.0",
Sha512: "sha512-/HggWBbTwy8TgebGSX5DBZ24ndhzi93sHUBDvP1IxbZD7FDokYzdAr6+vbWGjw2XAfR2EJ1sfKUotpjHnFWPxA==",
Path: "microsoft.extensions.logging.abstractions/6.0.0",
HashPath: "microsoft.extensions.logging.abstractions.6.0.0.nupkg.sha512",
},
},
"Microsoft.Extensions.Options": {
{
Name: "Microsoft.Extensions.Options",
Version: "6.0.0",
PURL: "pkg:dotnet/Microsoft.Extensions.Options@6.0.0",
Locations: fixtureLocationSet,
Language: pkg.Dotnet,
Type: pkg.DotnetPkg,
MetadataType: pkg.DotnetDepsMetadataType,
@ -99,9 +110,11 @@ func TestParseDotnetDeps(t *testing.T) {
HashPath: "microsoft.extensions.options.6.0.0.nupkg.sha512",
},
},
"Microsoft.Extensions.Primitives": {
{
Name: "Microsoft.Extensions.Primitives",
Version: "6.0.0",
PURL: "pkg:dotnet/Microsoft.Extensions.Primitives@6.0.0",
Locations: fixtureLocationSet,
Language: pkg.Dotnet,
Type: pkg.DotnetPkg,
MetadataType: pkg.DotnetDepsMetadataType,
@ -113,9 +126,11 @@ func TestParseDotnetDeps(t *testing.T) {
HashPath: "microsoft.extensions.primitives.6.0.0.nupkg.sha512",
},
},
"Newtonsoft.Json": {
{
Name: "Newtonsoft.Json",
Version: "13.0.1",
PURL: "pkg:dotnet/Newtonsoft.Json@13.0.1",
Locations: fixtureLocationSet,
Language: pkg.Dotnet,
Type: pkg.DotnetPkg,
MetadataType: pkg.DotnetDepsMetadataType,
@ -127,23 +142,11 @@ func TestParseDotnetDeps(t *testing.T) {
HashPath: "newtonsoft.json.13.0.1.nupkg.sha512",
},
},
"Serilog": {
Name: "Serilog",
Version: "2.10.0",
Language: pkg.Dotnet,
Type: pkg.DotnetPkg,
MetadataType: pkg.DotnetDepsMetadataType,
Metadata: pkg.DotnetDepsMetadata{
Name: "Serilog",
Version: "2.10.0",
Sha512: "sha512-+QX0hmf37a0/OZLxM3wL7V6/ADvC1XihXN4Kq/p6d8lCPfgkRdiuhbWlMaFjR9Av0dy5F0+MBeDmDdRZN/YwQA==",
Path: "serilog/2.10.0",
HashPath: "serilog.2.10.0.nupkg.sha512",
},
},
"Serilog.Sinks.Console": {
{
Name: "Serilog.Sinks.Console",
Version: "4.0.1",
PURL: "pkg:dotnet/Serilog.Sinks.Console@4.0.1",
Locations: fixtureLocationSet,
Language: pkg.Dotnet,
Type: pkg.DotnetPkg,
MetadataType: pkg.DotnetDepsMetadataType,
@ -155,9 +158,27 @@ func TestParseDotnetDeps(t *testing.T) {
HashPath: "serilog.sinks.console.4.0.1.nupkg.sha512",
},
},
"System.Diagnostics.DiagnosticSource": {
{
Name: "Serilog",
Version: "2.10.0",
PURL: "pkg:dotnet/Serilog@2.10.0",
Locations: fixtureLocationSet,
Language: pkg.Dotnet,
Type: pkg.DotnetPkg,
MetadataType: pkg.DotnetDepsMetadataType,
Metadata: pkg.DotnetDepsMetadata{
Name: "Serilog",
Version: "2.10.0",
Sha512: "sha512-+QX0hmf37a0/OZLxM3wL7V6/ADvC1XihXN4Kq/p6d8lCPfgkRdiuhbWlMaFjR9Av0dy5F0+MBeDmDdRZN/YwQA==",
Path: "serilog/2.10.0",
HashPath: "serilog.2.10.0.nupkg.sha512",
},
},
{
Name: "System.Diagnostics.DiagnosticSource",
Version: "6.0.0",
PURL: "pkg:dotnet/System.Diagnostics.DiagnosticSource@6.0.0",
Locations: fixtureLocationSet,
Language: pkg.Dotnet,
Type: pkg.DotnetPkg,
MetadataType: pkg.DotnetDepsMetadataType,
@ -169,9 +190,11 @@ func TestParseDotnetDeps(t *testing.T) {
HashPath: "system.diagnostics.diagnosticsource.6.0.0.nupkg.sha512",
},
},
"System.Runtime.CompilerServices.Unsafe": {
{
Name: "System.Runtime.CompilerServices.Unsafe",
Version: "6.0.0",
PURL: "pkg:dotnet/System.Runtime.CompilerServices.Unsafe@6.0.0",
Locations: fixtureLocationSet,
Language: pkg.Dotnet,
Type: pkg.DotnetPkg,
MetadataType: pkg.DotnetDepsMetadataType,
@ -185,15 +208,6 @@ func TestParseDotnetDeps(t *testing.T) {
},
}
fixture, err := os.Open("test-fixtures/TestLibrary.deps.json")
if err != nil {
t.Fatalf("failed to open fixture: %+v", err)
}
actual, _, err := parseDotnetDeps(fixture.Name(), fixture)
if err != nil {
t.Fatalf("failed to parse deps.json: %+v", err)
}
assertPackagesEqual(t, actual, expected)
var expectedRelationships []artifact.Relationship
pkgtest.TestGenericParser(t, fixture, parseDotnetDeps, expected, expectedRelationships)
}

View file

@ -0,0 +1,37 @@
package pkgtest
import (
"testing"
"github.com/google/go-cmp/cmp"
"github.com/google/go-cmp/cmp/cmpopts"
"github.com/anchore/syft/syft/pkg"
"github.com/anchore/syft/syft/source"
)
func AssertPackagesEqual(t testing.TB, expected, actual []pkg.Package) {
if diff := cmp.Diff(expected, actual,
cmpopts.IgnoreFields(pkg.Package{}, "id"), // note: ID is not deterministic for test purposes
cmp.Comparer(
func(x, y source.LocationSet) bool {
xs := x.ToSlice()
ys := y.ToSlice()
if len(xs) != len(ys) {
return false
}
for i, xe := range xs {
ye := ys[i]
if !(cmp.Equal(xe.Coordinates, ye.Coordinates) && cmp.Equal(xe.VirtualPath, ye.VirtualPath)) {
return false
}
}
return true
},
),
); diff != "" {
t.Errorf("unexpected packages from parsing (-expected +actual)\n%s", diff)
}
}

View file

@ -0,0 +1,35 @@
package pkgtest
import (
"os"
"testing"
"github.com/google/go-cmp/cmp"
"github.com/stretchr/testify/require"
"github.com/anchore/syft/syft/artifact"
"github.com/anchore/syft/syft/pkg"
"github.com/anchore/syft/syft/pkg/cataloger/generic"
"github.com/anchore/syft/syft/source"
)
func TestGenericParser(t *testing.T, fixturePath string, parser generic.Parser, expectedPkgs []pkg.Package, expectedRelationships []artifact.Relationship) {
TestGenericParserWithEnv(t, fixturePath, parser, nil, expectedPkgs, expectedRelationships)
}
func TestGenericParserWithEnv(t *testing.T, fixturePath string, parser generic.Parser, env *generic.Environment, expectedPkgs []pkg.Package, expectedRelationships []artifact.Relationship) {
fixture, err := os.Open(fixturePath)
require.NoError(t, err)
actualPkgs, actualRelationships, err := parser(nil, env, source.LocationReadCloser{
Location: source.NewLocation(fixture.Name()),
ReadCloser: fixture,
})
require.NoError(t, err)
AssertPackagesEqual(t, expectedPkgs, actualPkgs)
if diff := cmp.Diff(expectedRelationships, actualRelationships); diff != "" {
t.Errorf("unexpected relationships from parsing (-expected +actual)\n%s", diff)
}
}

View file

@ -1,10 +1,5 @@
package pkg
import (
"github.com/anchore/packageurl-go"
"github.com/anchore/syft/syft/linux"
)
type DotnetDepsMetadata struct {
Name string `mapstructure:"name" json:"name"`
Version string `mapstructure:"version" json:"version"`
@ -12,16 +7,3 @@ type DotnetDepsMetadata struct {
Sha512 string `mapstructure:"sha512" json:"sha512"`
HashPath string `mapstructure:"hashPath" json:"hashPath"`
}
func (m DotnetDepsMetadata) PackageURL(_ *linux.Release) string {
var qualifiers packageurl.Qualifiers
return packageurl.NewPackageURL(
packageurl.TypeDotnet,
"",
m.Name,
m.Version,
qualifiers,
"",
).ToString()
}

View file

@ -35,19 +35,6 @@ func TestPackageURL(t *testing.T) {
},
expected: "pkg:golang/go.opencensus.io@v0.23.0",
},
{
name: "dotnet",
pkg: Package{
Name: "Microsoft.CodeAnalysis.Razor",
Version: "2.2.0",
Type: DotnetPkg,
Metadata: DotnetDepsMetadata{
Name: "Microsoft.CodeAnalysis.Razor",
Version: "2.2.0",
},
},
expected: "pkg:dotnet/Microsoft.CodeAnalysis.Razor@2.2.0",
},
{
name: "python",
pkg: Package{
@ -211,6 +198,7 @@ func TestPackageURL(t *testing.T) {
expectedTypes.Remove(string(ApkPkg))
expectedTypes.Remove(string(ConanPkg))
expectedTypes.Remove(string(DartPubPkg))
expectedTypes.Remove(string(DotnetPkg))
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {