From 21eb77206020c04c9ca5316562e32179c73926a9 Mon Sep 17 00:00:00 2001 From: Keith Zantow Date: Tue, 16 Aug 2022 14:14:02 -0400 Subject: [PATCH] Associate node package licenses from node_modules (#1152) --- internal/spdxlicense/license.go | 2 +- internal/spdxlicense/license_list.go | 23 +++- .../pkg/cataloger/common/generic_cataloger.go | 16 ++- syft/pkg/cataloger/javascript/cataloger.go | 58 +++++++++- .../cataloger/javascript/cataloger_test.go | 102 ++++++++++++++++++ .../javascript/parse_package_lock_test.go | 6 ++ .../node_modules/@actions/core/package.json | 44 ++++++++ .../pkg-lock/node_modules/cowsay/package.json | 59 ++++++++++ .../test-fixtures/pkg-lock/package-lock.json | 8 ++ 9 files changed, 309 insertions(+), 9 deletions(-) create mode 100644 syft/pkg/cataloger/javascript/cataloger_test.go create mode 100644 syft/pkg/cataloger/javascript/test-fixtures/pkg-lock/node_modules/@actions/core/package.json create mode 100644 syft/pkg/cataloger/javascript/test-fixtures/pkg-lock/node_modules/cowsay/package.json diff --git a/internal/spdxlicense/license.go b/internal/spdxlicense/license.go index c0f7f9350..a3029d283 100644 --- a/internal/spdxlicense/license.go +++ b/internal/spdxlicense/license.go @@ -12,7 +12,7 @@ import ( // EX: gpl-2.0.0-only ---> GPL-2.0-only // See the debian link for more details on the spdx license differences -//go:generate go run generate/generate_license_list.go +//go:generate go run ./generate func ID(id string) (string, bool) { value, exists := licenseIDs[strings.ToLower(id)] diff --git a/internal/spdxlicense/license_list.go b/internal/spdxlicense/license_list.go index 695f89d55..65c47f438 100644 --- a/internal/spdxlicense/license_list.go +++ b/internal/spdxlicense/license_list.go @@ -1,9 +1,9 @@ // Code generated by go generate; DO NOT EDIT. -// This file was generated by robots at 2022-08-02 12:46:45.90647 -0400 EDT m=+0.327870639 +// This file was generated by robots at 2022-08-16 13:37:43.053262 -0400 EDT m=+0.183888850 // using data from https://spdx.org/licenses/licenses.json package spdxlicense -const Version = "3.17" +const Version = "3.18" var licenseIDs = map[string]string{ "0bsd": "0BSD", @@ -201,16 +201,19 @@ var licenseIDs = map[string]string{ "cc-by-3": "CC-BY-3.0", "cc-by-3-at": "CC-BY-3.0-AT", "cc-by-3-de": "CC-BY-3.0-DE", + "cc-by-3-igo": "CC-BY-3.0-IGO", "cc-by-3-nl": "CC-BY-3.0-NL", "cc-by-3-us": "CC-BY-3.0-US", "cc-by-3.0": "CC-BY-3.0", "cc-by-3.0-at": "CC-BY-3.0-AT", "cc-by-3.0-de": "CC-BY-3.0-DE", + "cc-by-3.0-igo": "CC-BY-3.0-IGO", "cc-by-3.0-nl": "CC-BY-3.0-NL", "cc-by-3.0-us": "CC-BY-3.0-US", "cc-by-3.0.0": "CC-BY-3.0", "cc-by-3.0.0-at": "CC-BY-3.0-AT", "cc-by-3.0.0-de": "CC-BY-3.0-DE", + "cc-by-3.0.0-igo": "CC-BY-3.0-IGO", "cc-by-3.0.0-nl": "CC-BY-3.0-NL", "cc-by-3.0.0-us": "CC-BY-3.0-US", "cc-by-4": "CC-BY-4.0", @@ -614,7 +617,6 @@ var licenseIDs = map[string]string{ "jasper-2.0.0": "JasPer-2.0", "jpnic": "JPNIC", "json": "JSON", - "kicad-libraries-exception": "KiCad-libraries-exception", "lal-1": "LAL-1.2", "lal-1.2": "LAL-1.2", "lal-1.2.0": "LAL-1.2", @@ -622,7 +624,7 @@ var licenseIDs = map[string]string{ "lal-1.3.0": "LAL-1.3", "latex2e": "Latex2e", "leptonica": "Leptonica", - "lgpl-2": "LGPL-2.1-only", + "lgpl-2": "LGPL-2.0-only", "lgpl-2+": "LGPL-2.0-or-later", "lgpl-2-only": "LGPL-2.0-only", "lgpl-2-or-later": "LGPL-2.0-or-later", @@ -692,7 +694,14 @@ var licenseIDs = map[string]string{ "lppl-1.3c": "LPPL-1.3c", "lppl-1a": "LPPL-1.3a", "lppl-1c": "LPPL-1.3c", + "lzma-sdk-9": "LZMA-SDK-9.22", + "lzma-sdk-9-to-9.20": "LZMA-SDK-9.11-to-9.20", + "lzma-sdk-9.11-to-9.20": "LZMA-SDK-9.11-to-9.20", + "lzma-sdk-9.11.0-to-9.20": "LZMA-SDK-9.11-to-9.20", + "lzma-sdk-9.22": "LZMA-SDK-9.22", + "lzma-sdk-9.22.0": "LZMA-SDK-9.22", "makeindex": "MakeIndex", + "minpack": "Minpack", "miros": "MirOS", "mit": "MIT", "mit-0": "MIT-0", @@ -704,6 +713,7 @@ var licenseIDs = map[string]string{ "mit-open-group": "MIT-open-group", "mitnfa": "MITNFA", "motosoto": "Motosoto", + "mpi-permissive": "mpi-permissive", "mpich2": "mpich2", "mpl-1": "MPL-1.0", "mpl-1.0": "MPL-1.0", @@ -717,6 +727,7 @@ var licenseIDs = map[string]string{ "mpl-2.0.0": "MPL-2.0", "mpl-2.0.0-no-copyleft-exception": "MPL-2.0-no-copyleft-exception", "mplus": "mplus", + "ms-lpl": "MS-LPL", "ms-pl": "MS-PL", "ms-rl": "MS-RL", "mtll": "MTLL", @@ -746,6 +757,9 @@ var licenseIDs = map[string]string{ "netcdf": "NetCDF", "newsletr": "Newsletr", "ngpl": "NGPL", + "nicta-1": "NICTA-1.0", + "nicta-1.0": "NICTA-1.0", + "nicta-1.0.0": "NICTA-1.0", "nist-pd": "NIST-PD", "nist-pd-fallback": "NIST-PD-fallback", "nlod-1": "NLOD-1.0", @@ -902,6 +916,7 @@ var licenseIDs = map[string]string{ "python-2": "Python-2.0", "python-2.0": "Python-2.0", "python-2.0.0": "Python-2.0", + "python-2.0.1": "Python-2.0.1", "qhull": "Qhull", "qpl-1": "QPL-1.0", "qpl-1.0": "QPL-1.0", diff --git a/syft/pkg/cataloger/common/generic_cataloger.go b/syft/pkg/cataloger/common/generic_cataloger.go index 5aa068114..5de3666bf 100644 --- a/syft/pkg/cataloger/common/generic_cataloger.go +++ b/syft/pkg/cataloger/common/generic_cataloger.go @@ -6,10 +6,9 @@ package common import ( "fmt" - "github.com/anchore/syft/syft/artifact" - "github.com/anchore/syft/internal" "github.com/anchore/syft/internal/log" + "github.com/anchore/syft/syft/artifact" "github.com/anchore/syft/syft/pkg" "github.com/anchore/syft/syft/source" ) @@ -19,14 +18,18 @@ import ( type GenericCataloger struct { globParsers map[string]ParserFn pathParsers map[string]ParserFn + postProcessors []PostProcessFunc upstreamCataloger string } +type PostProcessFunc func(resolver source.FileResolver, location source.Location, p *pkg.Package) error + // NewGenericCataloger if provided path-to-parser-function and glob-to-parser-function lookups creates a GenericCataloger -func NewGenericCataloger(pathParsers map[string]ParserFn, globParsers map[string]ParserFn, upstreamCataloger string) *GenericCataloger { +func NewGenericCataloger(pathParsers map[string]ParserFn, globParsers map[string]ParserFn, upstreamCataloger string, postProcessors ...PostProcessFunc) *GenericCataloger { return &GenericCataloger{ globParsers: globParsers, pathParsers: pathParsers, + postProcessors: postProcessors, upstreamCataloger: upstreamCataloger, } } @@ -69,6 +72,13 @@ func (c *GenericCataloger) Catalog(resolver source.FileResolver) ([]pkg.Package, continue } + for _, postProcess := range c.postProcessors { + err = postProcess(resolver, location, p) + if err != nil { + return nil, nil, err + } + } + packages = append(packages, *p) } diff --git a/syft/pkg/cataloger/javascript/cataloger.go b/syft/pkg/cataloger/javascript/cataloger.go index 791554a5d..80b4d8077 100644 --- a/syft/pkg/cataloger/javascript/cataloger.go +++ b/syft/pkg/cataloger/javascript/cataloger.go @@ -4,7 +4,15 @@ Package javascript provides a concrete Cataloger implementation for JavaScript e package javascript import ( + "encoding/json" + "io" + "path" + "strings" + + "github.com/anchore/syft/internal/log" + "github.com/anchore/syft/syft/pkg" "github.com/anchore/syft/syft/pkg/cataloger/common" + "github.com/anchore/syft/syft/source" ) // NewJavascriptPackageCataloger returns a new JavaScript cataloger object based on detection of npm based packages. @@ -23,5 +31,53 @@ func NewJavascriptLockCataloger() *common.GenericCataloger { "**/yarn.lock": parseYarnLock, } - return common.NewGenericCataloger(nil, globParsers, "javascript-lock-cataloger") + return common.NewGenericCataloger(nil, globParsers, "javascript-lock-cataloger", addLicenses) +} + +func addLicenses(resolver source.FileResolver, location source.Location, p *pkg.Package) error { + dir := path.Dir(location.RealPath) + pkgPath := []string{dir, "node_modules"} + pkgPath = append(pkgPath, strings.Split(p.Name, "/")...) + pkgPath = append(pkgPath, "package.json") + pkgFile := path.Join(pkgPath...) + locations, err := resolver.FilesByPath(pkgFile) + if err != nil { + log.Debugf("an error occurred attempting to read: %s - %+v", pkgFile, err) + return nil + } + + if len(locations) == 0 { + return nil + } + + for _, location := range locations { + contentReader, err := resolver.FileContentsByLocation(location) + if err != nil { + log.Debugf("error getting file content reader for %s: %v", pkgFile, err) + return nil + } + + contents, err := io.ReadAll(contentReader) + if err != nil { + log.Debugf("error reading file contents for %s: %v", pkgFile, err) + return nil + } + + var pkgJSON packageJSON + err = json.Unmarshal(contents, &pkgJSON) + if err != nil { + log.Debugf("error parsing %s: %v", pkgFile, err) + return nil + } + + licenses, err := pkgJSON.licensesFromJSON() + if err != nil { + log.Debugf("error getting licenses from %s: %v", pkgFile, err) + return nil + } + + p.Licenses = licenses + } + + return nil } diff --git a/syft/pkg/cataloger/javascript/cataloger_test.go b/syft/pkg/cataloger/javascript/cataloger_test.go new file mode 100644 index 000000000..325d0b65f --- /dev/null +++ b/syft/pkg/cataloger/javascript/cataloger_test.go @@ -0,0 +1,102 @@ +package javascript + +import ( + "testing" + + "github.com/stretchr/testify/require" + + "github.com/anchore/syft/syft/pkg" + "github.com/anchore/syft/syft/source" +) + +func Test_JavascriptCataloger(t *testing.T) { + expected := map[string]pkg.Package{ + "@actions/core": { + Name: "@actions/core", + Version: "1.6.0", + Language: pkg.JavaScript, + Type: pkg.NpmPkg, + Licenses: []string{"MIT"}, + }, + "wordwrap": { + Name: "wordwrap", + Version: "0.0.3", + Language: pkg.JavaScript, + Type: pkg.NpmPkg, + }, + "get-stdin": { + Name: "get-stdin", + Version: "5.0.1", + Language: pkg.JavaScript, + Type: pkg.NpmPkg, + }, + "minimist": { + Name: "minimist", + Version: "0.0.10", + Language: pkg.JavaScript, + Type: pkg.NpmPkg, + }, + "optimist": { + Name: "optimist", + Version: "0.6.1", + Language: pkg.JavaScript, + Type: pkg.NpmPkg, + }, + "string-width": { + Name: "string-width", + Version: "2.1.1", + Language: pkg.JavaScript, + Type: pkg.NpmPkg, + }, + "strip-ansi": { + Name: "strip-ansi", + Version: "4.0.0", + Language: pkg.JavaScript, + Type: pkg.NpmPkg, + }, + "strip-eof": { + Name: "wordwrap", + Version: "1.0.0", + Language: pkg.JavaScript, + Type: pkg.NpmPkg, + }, + "ansi-regex": { + Name: "ansi-regex", + Version: "3.0.0", + Language: pkg.JavaScript, + Type: pkg.NpmPkg, + }, + "is-fullwidth-code-point": { + Name: "is-fullwidth-code-point", + Version: "2.0.0", + Language: pkg.JavaScript, + Type: pkg.NpmPkg, + }, + "cowsay": { + Name: "cowsay", + Version: "1.4.0", + Language: pkg.JavaScript, + Type: pkg.NpmPkg, + Licenses: []string{"MIT"}, + }, + } + + s, err := source.NewFromDirectory("test-fixtures/pkg-lock") + require.NoError(t, err) + + resolver, err := s.FileResolver(source.AllLayersScope) + require.NoError(t, err) + + actual, _, err := NewJavascriptLockCataloger().Catalog(resolver) + if err != nil { + t.Fatalf("failed to parse package-lock.json: %+v", err) + } + + var pkgs []*pkg.Package + for _, p := range actual { + p2 := p + pkgs = append(pkgs, &p2) + } + + assertPkgsEqual(t, pkgs, expected) +} diff --git a/syft/pkg/cataloger/javascript/parse_package_lock_test.go b/syft/pkg/cataloger/javascript/parse_package_lock_test.go index ab88e2f16..a809441d9 100644 --- a/syft/pkg/cataloger/javascript/parse_package_lock_test.go +++ b/syft/pkg/cataloger/javascript/parse_package_lock_test.go @@ -30,6 +30,12 @@ func assertPkgsEqual(t *testing.T, actual []*pkg.Package, expected map[string]pk func TestParsePackageLock(t *testing.T) { expected := map[string]pkg.Package{ + "@actions/core": { + Name: "@actions/core", + Version: "1.6.0", + Language: pkg.JavaScript, + Type: pkg.NpmPkg, + }, "wordwrap": { Name: "wordwrap", Version: "0.0.3", diff --git a/syft/pkg/cataloger/javascript/test-fixtures/pkg-lock/node_modules/@actions/core/package.json b/syft/pkg/cataloger/javascript/test-fixtures/pkg-lock/node_modules/@actions/core/package.json new file mode 100644 index 000000000..8d7a39974 --- /dev/null +++ b/syft/pkg/cataloger/javascript/test-fixtures/pkg-lock/node_modules/@actions/core/package.json @@ -0,0 +1,44 @@ +{ + "name": "@actions/core", + "version": "1.6.0", + "description": "Actions core lib", + "keywords": [ + "github", + "actions", + "core" + ], + "homepage": "https://github.com/actions/toolkit/tree/main/packages/core", + "license": "MIT", + "main": "lib/core.js", + "types": "lib/core.d.ts", + "directories": { + "lib": "lib", + "test": "__tests__" + }, + "files": [ + "lib", + "!.DS_Store" + ], + "publishConfig": { + "access": "public" + }, + "repository": { + "type": "git", + "url": "git+https://github.com/actions/toolkit.git", + "directory": "packages/core" + }, + "scripts": { + "audit-moderate": "npm install && npm audit --json --audit-level=moderate > audit.json", + "test": "echo \"Error: run tests from root\" && exit 1", + "tsc": "tsc" + }, + "bugs": { + "url": "https://github.com/actions/toolkit/issues" + }, + "dependencies": { + "@actions/http-client": "^1.0.11" + }, + "devDependencies": { + "@types/node": "^12.0.2" + } +} diff --git a/syft/pkg/cataloger/javascript/test-fixtures/pkg-lock/node_modules/cowsay/package.json b/syft/pkg/cataloger/javascript/test-fixtures/pkg-lock/node_modules/cowsay/package.json new file mode 100644 index 000000000..7492c95f1 --- /dev/null +++ b/syft/pkg/cataloger/javascript/test-fixtures/pkg-lock/node_modules/cowsay/package.json @@ -0,0 +1,59 @@ +{ + "name": "cowsay", + "version": "1.4.0", + "description": "cowsay is a configurable talking cow", + "keywords": [ + "cow", + "cowsay", + "cowthink", + "figlet", + "talking", + "ASCII" + ], + "homepage": "https://github.com/piuccio/cowsay", + "author": { + "name": "Fabio Crisci", + "email": "piuccio@gmail.com", + "url": "https://github.com/piuccio/" + }, + "license": "MIT", + "main": "./index", + "module": "./build/cowsay.es.js", + "browser": "./build/cowsay.umd.js", + "bin": { + "cowsay": "./cli.js", + "cowthink": "./cli.js" + }, + "files": [ + "index.js", + "cli.js", + "build/", + "cows/", + "lib/" + ], + "repository": { + "type": "git", + "url": "https://github.com/piuccio/cowsay.git" + }, + "scripts": { + "prepublish": "rollup -c", + "test": "node test.js" + }, + "dependencies": { + "get-stdin": "^5.0.1", + "optimist": "~0.6.1", + "string-width": "~2.1.1", + "strip-eof": "^1.0.0" + }, + "devDependencies": { + "nodeunit": "~0.11.1", + "rollup": "^0.48.2", + "rollup-plugin-commonjs": "^8.2.0", + "rollup-plugin-node-resolve": "^3.0.0", + "rollup-plugin-string": "^2.0.2" + }, + "preferGlobal": true, + "engines": { + "node": ">= 4" + } +} diff --git a/syft/pkg/cataloger/javascript/test-fixtures/pkg-lock/package-lock.json b/syft/pkg/cataloger/javascript/test-fixtures/pkg-lock/package-lock.json index 4056cac02..7a14a9e21 100644 --- a/syft/pkg/cataloger/javascript/test-fixtures/pkg-lock/package-lock.json +++ b/syft/pkg/cataloger/javascript/test-fixtures/pkg-lock/package-lock.json @@ -2,6 +2,14 @@ "requires": true, "lockfileVersion": 1, "dependencies": { + "@actions/core": { + "version": "1.6.0", + "resolved": "https://registry.npmjs.org/@actions/core/-/core-1.6.0.tgz", + "integrity": "sha512-NB1UAZomZlCV/LmJqkLhNTqtKfFXJZAUPcfl/zqG7EfsQdeUJtaWO98SGbuQ3pydJ3fHl2CvI/51OKYlCYYcaw==", + "requires": { + "@actions/http-client": "^1.0.11" + } + }, "ansi-regex": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-3.0.0.tgz",