finalize the json output (no schema yet) (#102)

Signed-off-by: Alex Goodman <alex.goodman@anchore.com>
This commit is contained in:
Alex Goodman 2020-08-07 13:05:58 -04:00 committed by GitHub
parent 76ff9737db
commit 6de7e4030d
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
21 changed files with 383 additions and 81 deletions

View file

@ -118,7 +118,9 @@ func startWorker(userInput string) <-chan error {
}
var provider vulnerability.Provider
var metadataProvider vulnerability.MetadataProvider
var catalog *pkg.Catalog
var theScope *scope.Scope
var theDistro *distro.Distro
var err error
var wg = &sync.WaitGroup{}
@ -127,7 +129,7 @@ func startWorker(userInput string) <-chan error {
go func() {
defer wg.Done()
provider, err = grype.LoadVulnerabilityDb(appConfig.Db.ToCuratorConfig(), appConfig.Db.AutoUpdate)
provider, metadataProvider, err = grype.LoadVulnerabilityDb(appConfig.Db.ToCuratorConfig(), appConfig.Db.AutoUpdate)
if err != nil {
errs <- fmt.Errorf("failed to load vulnerability db: %w", err)
}
@ -135,7 +137,7 @@ func startWorker(userInput string) <-chan error {
go func() {
defer wg.Done()
catalog, _, theDistro, err = syft.Catalog(userInput, appConfig.ScopeOpt)
catalog, theScope, theDistro, err = syft.Catalog(userInput, appConfig.ScopeOpt)
if err != nil {
errs <- fmt.Errorf("failed to catalog: %w", err)
}
@ -150,7 +152,7 @@ func startWorker(userInput string) <-chan error {
bus.Publish(partybus.Event{
Type: event.VulnerabilityScanningFinished,
Value: presenter.GetPresenter(appConfig.PresenterOpt, results, catalog),
Value: presenter.GetPresenter(appConfig.PresenterOpt, results, catalog, *theScope, metadataProvider),
})
}()
return errs

4
go.mod
View file

@ -6,9 +6,9 @@ require (
github.com/adrg/xdg v0.2.1
github.com/anchore/go-testutils v0.0.0-20200624184116-66aa578126db
github.com/anchore/go-version v1.2.2-0.20200701162849-18adb9c92b9b
github.com/anchore/grype-db v0.0.0-20200806124046-3929ab5bb802
github.com/anchore/grype-db v0.0.0-20200807151757-5aee0401bf56
github.com/anchore/stereoscope v0.0.0-20200803190343-146f38f8cc19
github.com/anchore/syft v0.1.0-beta.2.0.20200804222243-70e673204c0c
github.com/anchore/syft v0.1.0-beta.2.0.20200807140516-817ce610368c
github.com/dustin/go-humanize v1.0.0
github.com/facebookincubator/nvdtools v0.1.4-0.20200622182922-aed862a62ae6
github.com/go-test/deep v1.0.7

8
go.sum
View file

@ -115,14 +115,14 @@ github.com/anchore/go-testutils v0.0.0-20200624184116-66aa578126db h1:LWKezJnFTF
github.com/anchore/go-testutils v0.0.0-20200624184116-66aa578126db/go.mod h1:D3rc2L/q4Hcp9eeX6AIJH4Q+kPjOtJCFhG9za90j+nU=
github.com/anchore/go-version v1.2.2-0.20200701162849-18adb9c92b9b h1:e1bmaoJfZVsCYMrIZBpFxwV26CbsuoEh5muXD5I1Ods=
github.com/anchore/go-version v1.2.2-0.20200701162849-18adb9c92b9b/go.mod h1:Bkc+JYWjMCF8OyZ340IMSIi2Ebf3uwByOk6ho4wne1E=
github.com/anchore/grype-db v0.0.0-20200806124046-3929ab5bb802 h1:9Ngp0fOUcj7u/ps/Gte8mizVFHbmlTqn+pr1my1Tnkw=
github.com/anchore/grype-db v0.0.0-20200806124046-3929ab5bb802/go.mod h1:LINmipRzG88vnJEWvgMMDVCFH1qZsj7+bjmpERlSyaA=
github.com/anchore/grype-db v0.0.0-20200807151757-5aee0401bf56 h1:Hf1i3Imipp+2dmf70U+l7+aYIkzfd3myoUG0t+dBw5w=
github.com/anchore/grype-db v0.0.0-20200807151757-5aee0401bf56/go.mod h1:LINmipRzG88vnJEWvgMMDVCFH1qZsj7+bjmpERlSyaA=
github.com/anchore/stereoscope v0.0.0-20200520221116-025e07f1c93e h1:QBwtrM0MXi0z+GcHk3RoSyzaQ+CLgas0bC/uOd1P+PQ=
github.com/anchore/stereoscope v0.0.0-20200520221116-025e07f1c93e/go.mod h1:bkyLl5VITnrmgErv4S1vDfVz/TGAZ5il6161IQo7w2g=
github.com/anchore/stereoscope v0.0.0-20200803190343-146f38f8cc19 h1:iJ6du/cA9KJ0ctP905u2zCcpJubsy6kxLZBvG4XG+uY=
github.com/anchore/stereoscope v0.0.0-20200803190343-146f38f8cc19/go.mod h1:WntReQTI/I27FOQ87UgLVVzWgku6+ZsqfOTLxpIZFCs=
github.com/anchore/syft v0.1.0-beta.2.0.20200804222243-70e673204c0c h1:kaJt+it1ZQCnnBApiCGhhEgTsva/abJ6QRbywTE5TZs=
github.com/anchore/syft v0.1.0-beta.2.0.20200804222243-70e673204c0c/go.mod h1:v/x/mLoNlq5cIjlLmTcdLahHnbHfi+w1VrM6Jvcf7Y4=
github.com/anchore/syft v0.1.0-beta.2.0.20200807140516-817ce610368c h1:nxWfD5olJGXCABqvj8ulecDFCTdWTnrXygybWs6LH1c=
github.com/anchore/syft v0.1.0-beta.2.0.20200807140516-817ce610368c/go.mod h1:v/x/mLoNlq5cIjlLmTcdLahHnbHfi+w1VrM6Jvcf7Y4=
github.com/andreyvit/diff v0.0.0-20170406064948-c7f18ee00883 h1:bvNMNQO63//z+xNgfBlViaCIJKLlCJ6/fmUseuG0wVQ=
github.com/andreyvit/diff v0.0.0-20170406064948-c7f18ee00883/go.mod h1:rCTlJbsFo29Kk6CurOXKm700vrz8f0KW0JNfpkRJY/8=
github.com/andybalholm/cascadia v1.1.0/go.mod h1:GsXiBklL0woXo1j/WYWtSYYC4ouU9PqHO0sqidkEA4Y=

View file

@ -43,7 +43,7 @@ func NewCurator(cfg Config) Curator {
}
}
func (c *Curator) GetStore() (v1.VulnerabilityStoreReader, error) {
func (c *Curator) GetStore() (v1.StoreReader, error) {
// ensure the DB is ok
err := c.Validate()
if err != nil {

View file

@ -39,22 +39,22 @@ func FindVulnerabilitiesForPackage(provider vulnerability.Provider, d distro.Dis
return matcher.FindMatches(provider, d, packages...)
}
func LoadVulnerabilityDb(cfg db.Config, update bool) (vulnerability.Provider, error) {
func LoadVulnerabilityDb(cfg db.Config, update bool) (vulnerability.Provider, vulnerability.MetadataProvider, error) {
dbCurator := db.NewCurator(cfg)
if update {
_, err := dbCurator.Update()
if err != nil {
return nil, err
return nil, nil, err
}
}
store, err := dbCurator.GetStore()
if err != nil {
return nil, err
return nil, nil, err
}
return vulnerability.NewProviderFromStore(store), nil
return vulnerability.NewProviderFromStore(store), vulnerability.NewMetadataStoreProvider(store), nil
}
func SetLogger(logger logger.Logger) {

View file

@ -2,61 +2,73 @@ package json
import (
"encoding/json"
"fmt"
"io"
"github.com/anchore/grype/grype/result"
"github.com/anchore/grype/grype/vulnerability"
"github.com/anchore/syft/syft/pkg"
syftJson "github.com/anchore/syft/syft/presenter/json"
"github.com/anchore/syft/syft/scope"
)
// Presenter is a generic struct for holding fields needed for reporting
type Presenter struct {
results result.Result
catalog *pkg.Catalog
results result.Result
catalog *pkg.Catalog
scope scope.Scope
metadataProvider vulnerability.MetadataProvider
}
// NewPresenter is a *Presenter constructor
func NewPresenter(results result.Result, catalog *pkg.Catalog) *Presenter {
func NewPresenter(results result.Result, catalog *pkg.Catalog, theScope scope.Scope, metadataProvider vulnerability.MetadataProvider) *Presenter {
return &Presenter{
results: results,
catalog: catalog,
results: results,
catalog: catalog,
metadataProvider: metadataProvider,
scope: theScope,
}
}
// ResultObj is a single item for the JSON array reported
type ResultObj struct {
Cve string `json:"cve"`
FoundBy FoundBy `json:"found-by"`
Package Package `json:"package"`
// Finding is a single item for the JSON array reported
type Finding struct {
Vulnerability Vulnerability `json:"vulnerability"`
MatchDetails MatchDetails `json:"matched-by"`
Artifact syftJson.Artifact `json:"artifact"`
}
// FoundBy contains all data that indicates how the result match was found
type FoundBy struct {
// MatchDetails contains all data that indicates how the result match was found
type MatchDetails struct {
Matcher string `json:"matcher"`
SearchKey string `json:"search-key"`
}
// Package is a nested JSON object from ResultObj
type Package struct {
Name string `json:"name"`
Version string `json:"version"`
Type string `json:"type"`
}
// Present creates a JSON-based reporting
func (pres *Presenter) Present(output io.Writer) error {
doc := make([]ResultObj, 0)
doc := make([]Finding, 0)
for m := range pres.results.Enumerate() {
p := pres.catalog.Package(m.Package.ID())
art, err := syftJson.NewArtifact(p, pres.scope)
if err != nil {
return err
}
metadata, err := pres.metadataProvider.GetMetadata(m.Vulnerability.ID, m.Vulnerability.RecordSource)
if err != nil {
return fmt.Errorf("unable to fetch vuln=%q metadata: %+v", m.Vulnerability.ID, err)
}
for match := range pres.results.Enumerate() {
p := pres.catalog.Package(match.Package.ID())
doc = append(
doc,
ResultObj{
Cve: match.Vulnerability.ID,
FoundBy: FoundBy{
Matcher: match.Matcher.String(),
SearchKey: match.SearchKey,
Finding{
Vulnerability: NewVulnerability(m, metadata),
Artifact: art,
MatchDetails: MatchDetails{
Matcher: m.Matcher.String(),
SearchKey: m.SearchKey,
},
Package: Package{Name: p.Name, Version: p.Version, Type: string(p.Type)},
},
)
}

View file

@ -3,6 +3,8 @@ package json
import (
"bytes"
"flag"
"github.com/anchore/stereoscope/pkg/file"
"github.com/anchore/syft/syft/scope"
"testing"
"github.com/anchore/go-testutils"
@ -15,53 +17,122 @@ import (
var update = flag.Bool("update", false, "update the *.golden files for json presenters")
func TestJsonPresenter(t *testing.T) {
type metadataMock struct {
store map[string]map[string]vulnerability.Metadata
}
func newMetadataMock() *metadataMock {
return &metadataMock{
store: map[string]map[string]vulnerability.Metadata{
"CVE-1999-0001": {
"source-1": {
Description: "1999-01 description",
CvssV3: &vulnerability.Cvss{
BaseScore: 4,
Vector: "another vector",
},
},
},
"CVE-1999-0002": {
"source-2": {
Description: "1999-02 description",
CvssV2: &vulnerability.Cvss{
BaseScore: 1,
ExploitabilityScore: 2,
ImpactScore: 3,
Vector: "vector",
},
},
},
"CVE-1999-0003": {
"source-1": {
Description: "1999-03 description",
},
},
},
}
}
func (m *metadataMock) GetMetadata(id, recordSource string) (*vulnerability.Metadata, error) {
value := m.store[id][recordSource]
return &value, nil
}
func TestJsonPresenter(t *testing.T) {
var buffer bytes.Buffer
var testImage = "image-simple"
if *update {
testutils.UpdateGoldenFixtureImage(t, testImage)
}
img := testutils.GetGoldenFixtureImage(t, testImage)
var pkg1 = pkg.Package{
Name: "package-1",
Version: "1.0.1",
Type: pkg.DebPkg,
Source: []file.Reference{
*img.SquashedTree().File("/somefile-1.txt"),
},
FoundBy: "the-cataloger-1",
}
var pkg2 = pkg.Package{
Name: "package-2",
Version: "2.0.1",
Type: pkg.DebPkg,
Source: []file.Reference{
*img.SquashedTree().File("/somefile-2.txt"),
},
FoundBy: "the-cataloger-2",
}
var match1 = match.Match{
Type: match.ExactDirectMatch,
Vulnerability: vulnerability.Vulnerability{ID: "CVE-1999-0001"},
Package: &pkg1,
Matcher: match.DpkgMatcher,
Type: match.ExactDirectMatch,
Vulnerability: vulnerability.Vulnerability{
ID: "CVE-1999-0001",
RecordSource: "source-1",
},
Package: &pkg1,
Matcher: match.DpkgMatcher,
}
var match2 = match.Match{
Type: match.ExactIndirectMatch,
Vulnerability: vulnerability.Vulnerability{ID: "CVE-1999-0002"},
Package: &pkg1,
Matcher: match.DpkgMatcher,
Type: match.ExactIndirectMatch,
Vulnerability: vulnerability.Vulnerability{
ID: "CVE-1999-0002",
RecordSource: "source-2",
},
Package: &pkg1,
Matcher: match.DpkgMatcher,
}
var match3 = match.Match{
Type: match.ExactIndirectMatch,
Vulnerability: vulnerability.Vulnerability{
ID: "CVE-1999-0003",
RecordSource: "source-1",
},
Package: &pkg1,
Matcher: match.DpkgMatcher,
}
results := result.NewResult()
results.Add(&pkg1, match1, match2)
results.Add(&pkg1, match1, match2, match3)
catalog := pkg.NewCatalog()
// populate catalog with test data
catalog.Add(pkg1)
catalog.Add(pkg2)
pres := NewPresenter(results, catalog)
theScope, err := scope.NewScopeFromImage(img, scope.AllLayersScope)
pres := NewPresenter(results, catalog, theScope, newMetadataMock())
// TODO: add a constructor for a match.Match when the data is better shaped
// run presenter
err := pres.Present(&buffer)
if err != nil {
if err = pres.Present(&buffer); err != nil {
t.Fatal(err)
}
actual := buffer.Bytes()
@ -89,7 +160,7 @@ func TestEmptyJsonPresenter(t *testing.T) {
catalog := pkg.NewCatalog()
pres := NewPresenter(results, catalog)
pres := NewPresenter(results, catalog, scope.Scope{}, nil)
// run presenter
err := pres.Present(&buffer)

View file

@ -0,0 +1 @@
this file has contents

View file

@ -0,0 +1 @@
file-2 contents!

View file

@ -0,0 +1,2 @@
another file!
with lines...

View file

@ -1,26 +1,84 @@
[
{
"cve": "CVE-1999-0001",
"found-by": {
"vulnerability": {
"id": "CVE-1999-0001",
"description": "1999-01 description",
"cvss-v3": {
"base-score": 4,
"vector": "another vector"
}
},
"matched-by": {
"matcher": "dpkg-matcher",
"search-key": ""
},
"package": {
"artifact": {
"name": "package-1",
"version": "1.0.1",
"type": "deb"
"type": "deb",
"found-by": [
"the-cataloger-1"
],
"locations": [
{
"path": "/somefile-1.txt",
"layer-index": 0
}
]
}
},
{
"cve": "CVE-1999-0002",
"found-by": {
"vulnerability": {
"id": "CVE-1999-0002",
"description": "1999-02 description",
"cvss-v2": {
"base-score": 1,
"exploitability-score": 2,
"impact-score": 3,
"vector": "vector"
}
},
"matched-by": {
"matcher": "dpkg-matcher",
"search-key": ""
},
"package": {
"artifact": {
"name": "package-1",
"version": "1.0.1",
"type": "deb"
"type": "deb",
"found-by": [
"the-cataloger-1"
],
"locations": [
{
"path": "/somefile-1.txt",
"layer-index": 0
}
]
}
},
{
"vulnerability": {
"id": "CVE-1999-0003",
"description": "1999-03 description"
},
"matched-by": {
"matcher": "dpkg-matcher",
"search-key": ""
},
"artifact": {
"name": "package-1",
"version": "1.0.1",
"type": "deb",
"found-by": [
"the-cataloger-1"
],
"locations": [
{
"path": "/somefile-1.txt",
"layer-index": 0
}
]
}
}
]

View file

@ -0,0 +1,73 @@
package json
import (
"github.com/anchore/grype/grype/match"
"github.com/anchore/grype/grype/vulnerability"
)
type Cvss struct {
BaseScore float64 `json:"base-score"`
ExploitabilityScore *float64 `json:"exploitability-score,omitempty"`
ImpactScore *float64 `json:"impact-score,omitempty"`
Vector string `json:"vector"`
}
type Vulnerability struct {
ID string `json:"id"`
Severity string `json:"severity,omitempty"`
Links []string `json:"links,omitempty"`
Description string `json:"description,omitempty"`
CvssV2 *Cvss `json:"cvss-v2,omitempty"`
CvssV3 *Cvss `json:"cvss-v3,omitempty"`
}
func NewVulnerability(m match.Match, metadata *vulnerability.Metadata) Vulnerability {
if metadata == nil {
return Vulnerability{
ID: m.Vulnerability.ID,
}
}
var cvssV2 *Cvss
if metadata.CvssV2 != nil {
var exploitability, impact *float64
if metadata.CvssV2.ExploitabilityScore > 0 {
exploitability = &metadata.CvssV2.ExploitabilityScore
}
if metadata.CvssV2.ImpactScore > 0 {
impact = &metadata.CvssV2.ImpactScore
}
cvssV2 = &Cvss{
BaseScore: metadata.CvssV2.BaseScore,
ExploitabilityScore: exploitability,
ImpactScore: impact,
Vector: metadata.CvssV2.Vector,
}
}
var cvssV3 *Cvss
if metadata.CvssV3 != nil {
var exploitability, impact *float64
if metadata.CvssV3.ExploitabilityScore > 0 {
exploitability = &metadata.CvssV3.ExploitabilityScore
}
if metadata.CvssV3.ImpactScore > 0 {
impact = &metadata.CvssV3.ImpactScore
}
cvssV3 = &Cvss{
BaseScore: metadata.CvssV3.BaseScore,
ExploitabilityScore: exploitability,
ImpactScore: impact,
Vector: metadata.CvssV3.Vector,
}
}
return Vulnerability{
ID: m.Vulnerability.ID,
Severity: metadata.Severity,
Links: metadata.Links,
Description: metadata.Description,
CvssV2: cvssV2,
CvssV3: cvssV3,
}
}

View file

@ -3,6 +3,9 @@ package presenter
import (
"io"
"github.com/anchore/grype/grype/vulnerability"
"github.com/anchore/syft/syft/scope"
"github.com/anchore/grype/grype/presenter/json"
"github.com/anchore/grype/grype/presenter/table"
"github.com/anchore/grype/grype/result"
@ -15,10 +18,10 @@ type Presenter interface {
}
// GetPresenter retrieves a Presenter that matches a CLI option
func GetPresenter(option Option, results result.Result, catalog *pkg.Catalog) Presenter {
func GetPresenter(option Option, results result.Result, catalog *pkg.Catalog, theScope scope.Scope, metadataProvider vulnerability.MetadataProvider) Presenter {
switch option {
case JSONPresenter:
return json.NewPresenter(results, catalog)
return json.NewPresenter(results, catalog, theScope, metadataProvider)
case TablePresenter:
return table.NewPresenter(results, catalog)
default:

View file

@ -0,0 +1,47 @@
package vulnerability
import (
v1 "github.com/anchore/grype-db/pkg/db/v1"
)
type Metadata struct {
ID string
Severity string
Links []string
Description string
CvssV2 *Cvss
CvssV3 *Cvss
}
type Cvss struct {
BaseScore float64
ExploitabilityScore float64
ImpactScore float64
Vector string
}
func NewMetadata(m *v1.VulnerabilityMetadata) (*Metadata, error) {
if m == nil {
return nil, nil
}
return &Metadata{
ID: m.ID,
Severity: m.Severity,
Links: m.Links,
Description: m.Description,
CvssV2: NewCvss(m.CvssV2),
CvssV3: NewCvss(m.CvssV3),
}, nil
}
func NewCvss(m *v1.Cvss) *Cvss {
if m == nil {
return nil
}
return &Cvss{
BaseScore: m.BaseScore,
ExploitabilityScore: m.ExploitabilityScore,
ImpactScore: m.ImpactScore,
Vector: m.Vector,
}
}

View file

@ -0,0 +1,26 @@
package vulnerability
import (
"fmt"
v1 "github.com/anchore/grype-db/pkg/db/v1"
)
type MetadataStoreAdapter struct {
store v1.VulnerabilityMetadataStoreReader
}
func NewMetadataStoreProvider(store v1.VulnerabilityMetadataStoreReader) *MetadataStoreAdapter {
return &MetadataStoreAdapter{
store: store,
}
}
func (pr *MetadataStoreAdapter) GetMetadata(id, recordSource string) (*Metadata, error) {
metadata, err := pr.store.GetVulnerabilityMetadata(id, recordSource)
if err != nil {
return nil, fmt.Errorf("metadata provider failed to fetch id='%s' recordsource='%s': %w", id, recordSource, err)
}
return NewMetadata(metadata)
}

View file

@ -23,3 +23,7 @@ type ProviderByLanguage interface {
type ProviderByCPE interface {
GetByCPE(cpe.CPE) ([]*Vulnerability, error)
}
type MetadataProvider interface {
GetMetadata(id, recordSource string) (*Metadata, error)
}

View file

@ -10,17 +10,17 @@ import (
"github.com/facebookincubator/nvdtools/wfn"
)
type StoreProvider struct {
type StoreAdapter struct {
store v1.VulnerabilityStoreReader
}
func NewProviderFromStore(store v1.VulnerabilityStoreReader) *StoreProvider {
return &StoreProvider{
func NewProviderFromStore(store v1.VulnerabilityStoreReader) *StoreAdapter {
return &StoreAdapter{
store: store,
}
}
func (pr *StoreProvider) GetByDistro(d distro.Distro, p *pkg.Package) ([]*Vulnerability, error) {
func (pr *StoreAdapter) GetByDistro(d distro.Distro, p *pkg.Package) ([]*Vulnerability, error) {
vulns := make([]*Vulnerability, 0)
namespace := distroNamespace(d)
@ -41,7 +41,7 @@ func (pr *StoreProvider) GetByDistro(d distro.Distro, p *pkg.Package) ([]*Vulner
return vulns, nil
}
func (pr *StoreProvider) GetByLanguage(l pkg.Language, p *pkg.Package) ([]*Vulnerability, error) {
func (pr *StoreAdapter) GetByLanguage(l pkg.Language, p *pkg.Package) ([]*Vulnerability, error) {
vulns := make([]*Vulnerability, 0)
namespaces := languageNamespaces(l)
@ -70,7 +70,7 @@ func (pr *StoreProvider) GetByLanguage(l pkg.Language, p *pkg.Package) ([]*Vulne
return vulns, nil
}
func (pr *StoreProvider) GetByCPE(requestCPE cpe.CPE) ([]*Vulnerability, error) {
func (pr *StoreAdapter) GetByCPE(requestCPE cpe.CPE) ([]*Vulnerability, error) {
vulns := make([]*Vulnerability, 0)
namespaces := cpeNamespaces()

View file

@ -9,9 +9,10 @@ import (
)
type Vulnerability struct {
Constraint version.Constraint
CPEs []cpe.CPE
ID string
Constraint version.Constraint
CPEs []cpe.CPE
ID string
RecordSource string
}
func NewVulnerability(vuln v1.Vulnerability) (*Vulnerability, error) {
@ -23,9 +24,10 @@ func NewVulnerability(vuln v1.Vulnerability) (*Vulnerability, error) {
}
return &Vulnerability{
Constraint: constraint,
ID: vuln.ID,
CPEs: make([]cpe.CPE, 0),
Constraint: constraint,
ID: vuln.ID,
CPEs: make([]cpe.CPE, 0),
RecordSource: vuln.RecordSource,
}, nil
}