mirror of
https://github.com/anchore/syft
synced 2024-11-10 06:14:16 +00:00
Add inline-comparison as acceptance test (#130)
* add inline-compare as acceptance test Signed-off-by: Alex Goodman <alex.goodman@anchore.com> * add additional RPM metadata Signed-off-by: Alex Goodman <alex.goodman@anchore.com> * add comments and doc strings to the compare-* make targets Signed-off-by: Alex Goodman <alex.goodman@anchore.com>
This commit is contained in:
Alex Goodman
GitHub
parent
4c7784da62
commit
2d452bf59e
17 changed files with 274 additions and 112 deletions
29
.github/workflows/acceptance-test.yaml
vendored
29
.github/workflows/acceptance-test.yaml
vendored
|
|
@ -1,5 +1,6 @@
|
|||
name: 'Acceptance'
|
||||
on:
|
||||
workflow_dispatch:
|
||||
push:
|
||||
# ... only act on pushes to main
|
||||
branches:
|
||||
|
|
@ -7,8 +8,10 @@ on:
|
|||
# ... do not act on release tags
|
||||
tags-ignore:
|
||||
- v*
|
||||
|
||||
env:
|
||||
GO_VERSION: "1.14.x"
|
||||
|
||||
jobs:
|
||||
Build-Snapshot-Artifacts:
|
||||
runs-on: ubuntu-latest
|
||||
|
|
@ -98,3 +101,29 @@ jobs:
|
|||
|
||||
- name: Run Acceptance Tests (Mac)
|
||||
run: make acceptance-mac
|
||||
|
||||
# Note: changing this job name requires making the same update in the .github/workflows/release.yaml pipeline
|
||||
Inline-Compare:
|
||||
needs: [ Build-Snapshot-Artifacts ]
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
|
||||
- uses: actions/checkout@v2
|
||||
|
||||
- name: Fingerprint inline-compare sources
|
||||
run: make compare-fingerprint
|
||||
|
||||
- name: Restore inline reports cache
|
||||
id: cache
|
||||
uses: actions/cache@v2
|
||||
with:
|
||||
path: ${{ github.workspace }}/test/inline-compare/inline-reports
|
||||
key: inline-reports-${{ hashFiles('**/inline-compare.fingerprint') }}
|
||||
|
||||
- uses: actions/download-artifact@v2
|
||||
with:
|
||||
name: artifacts
|
||||
path: snapshot
|
||||
|
||||
- name: Compare Anchore inline-scan results against snapshot build output
|
||||
run: make compare-snapshot
|
||||
14
.github/workflows/release.yaml
vendored
14
.github/workflows/release.yaml
vendored
|
|
@ -7,8 +7,10 @@ on:
|
|||
# ... only act on release tags
|
||||
tags:
|
||||
- 'v*'
|
||||
|
||||
env:
|
||||
GO_VERSION: "1.14.x"
|
||||
|
||||
jobs:
|
||||
wait-for-checks:
|
||||
runs-on: ubuntu-latest
|
||||
|
|
@ -50,12 +52,22 @@ jobs:
|
|||
checkName: "Acceptance-Mac"
|
||||
ref: ${{ github.event.pull_request.head.sha || github.sha }}
|
||||
|
||||
- name: Check inline comparison test results
|
||||
uses: fountainhead/action-wait-for-check@v1.0.0
|
||||
id: inline-compare
|
||||
with:
|
||||
token: ${{ secrets.GITHUB_TOKEN }}
|
||||
# This check name is defined as the github action job name (in .github/workflows/acceptance-test.yaml)
|
||||
checkName: "Inline-Compare"
|
||||
ref: ${{ github.event.pull_request.head.sha || github.sha }}
|
||||
|
||||
- name: Quality gate
|
||||
if: steps.sa-unit-int.outputs.conclusion != 'success' || steps.acceptance-linux.outputs.conclusion != 'success' || steps.acceptance-mac.outputs.conclusion != 'success'
|
||||
if: steps.sa-unit-int.outputs.conclusion != 'success' || steps.inline-compare.outputs.conclusion != 'success' || steps.acceptance-linux.outputs.conclusion != 'success' || steps.acceptance-mac.outputs.conclusion != 'success'
|
||||
run: |
|
||||
echo "Static/Unit/Integration Status: ${{ steps.sa-unit-int.outputs.conclusion }}"
|
||||
echo "Acceptance Test (Linux) Status: ${{ steps.acceptance-linux.outputs.conclusion }}"
|
||||
echo "Acceptance Test (Mac) Status: ${{ steps.acceptance-mac.outputs.conclusion }}"
|
||||
echo "Inline Compare Status: ${{ steps.inline-compare.outputs.conclusion }}"
|
||||
false
|
||||
|
||||
release:
|
||||
|
|
|
|||
1
.gitignore
vendored
1
.gitignore
vendored
|
|
@ -2,6 +2,7 @@
|
|||
/snapshot
|
||||
.server/
|
||||
.vscode/
|
||||
*.fingerprint
|
||||
*.tar
|
||||
*.jar
|
||||
*.war
|
||||
|
|
|
|||
22
Makefile
22
Makefile
|
|
@ -21,6 +21,7 @@ COVERAGE_THRESHOLD := 72
|
|||
DISTDIR=./dist
|
||||
SNAPSHOTDIR=./snapshot
|
||||
GITTREESTATE=$(if $(shell git status --porcelain),dirty,clean)
|
||||
SNAPSHOT_CMD=$(shell realpath $(shell pwd)/$(SNAPSHOTDIR)/syft_linux_amd64/syft)
|
||||
|
||||
ifeq "$(strip $(VERSION))" ""
|
||||
override VERSION = $(shell git describe --always --tags --dirty)
|
||||
|
|
@ -58,10 +59,6 @@ endef
|
|||
all: clean static-analysis test ## Run all linux-based checks (linting, license check, unit, integration, and linux acceptance tests)
|
||||
@printf '$(SUCCESS)All checks pass!$(RESET)\n'
|
||||
|
||||
.PHONY: compare
|
||||
compare:
|
||||
@cd test/inline-compare && make
|
||||
|
||||
.PHONY: test
|
||||
test: unit integration acceptance-linux ## Run all tests (currently unit, integration, and linux acceptance tests)
|
||||
|
||||
|
|
@ -127,7 +124,8 @@ integration: ## Run integration tests
|
|||
$(call title,Running integration tests)
|
||||
go test -v -tags=integration ./test/integration
|
||||
|
||||
test/integration/test-fixtures/tar-cache.key, integration-fingerprint:
|
||||
# note: this is used by CI to determine if the integration test fixture cache (docker image tars) should be busted
|
||||
integration-fingerprint:
|
||||
find test/integration/test-fixtures/image-* -type f -exec md5sum {} + | awk '{print $1}' | sort | md5sum | tee test/integration/test-fixtures/tar-cache.fingerprint
|
||||
|
||||
.PHONY: java-packages-fingerprint
|
||||
|
|
@ -192,6 +190,20 @@ acceptance-mac: $(SNAPSHOTDIR) ## Run acceptance tests on build snapshot binarie
|
|||
.PHONY: acceptance-linux
|
||||
acceptance-linux: acceptance-test-deb-package-install acceptance-test-rpm-package-install ## Run acceptance tests on build snapshot binaries and packages (Linux)
|
||||
|
||||
# note: this is used by CI to determine if the inline-scan report cache should be busted for the inline-compare tests
|
||||
.PHONY: compare-fingerprint
|
||||
compare-fingerprint:
|
||||
find test/inline-compare/* -type f -exec md5sum {} + | grep -v '\-reports' | grep -v 'fingerprint' | awk '{print $1}' | sort | md5sum | tee test/inline-compare/inline-compare.fingerprint
|
||||
|
||||
.PHONY: compare-snapshot
|
||||
compare-snapshot: $(SNAPSHOTDIR) ## Compare the reports of a run of a snapshot build of syft against inline-scan
|
||||
chmod 755 $(SNAPSHOT_CMD)
|
||||
@cd test/inline-compare && SYFT_CMD=$(SNAPSHOT_CMD) make
|
||||
|
||||
.PHONY: compare
|
||||
compare: ## Compare the reports of a run of a main-branch build of syft against inline-scan
|
||||
@cd test/inline-compare && make
|
||||
|
||||
.PHONY: acceptance-test-deb-package-install
|
||||
acceptance-test-deb-package-install: $(SNAPSHOTDIR)
|
||||
$(call title,Running acceptance test: DEB install)
|
||||
|
|
|
|||
2
go.mod
2
go.mod
|
|
@ -27,7 +27,7 @@ require (
|
|||
github.com/spf13/viper v1.7.0
|
||||
github.com/wagoodman/go-partybus v0.0.0-20200526224238-eb215533f07d
|
||||
github.com/wagoodman/go-progress v0.0.0-20200731105512-1020f39e6240
|
||||
github.com/wagoodman/go-rpmdb v0.0.0-20200719223757-ce54a4b0607b
|
||||
github.com/wagoodman/go-rpmdb v0.0.0-20200810111121-8136676cb95c
|
||||
github.com/wagoodman/jotframe v0.0.0-20200730190914-3517092dd163
|
||||
github.com/x-cray/logrus-prefixed-formatter v0.5.2
|
||||
github.com/xeipuuv/gojsonschema v1.2.0
|
||||
|
|
|
|||
4
go.sum
4
go.sum
|
|
@ -832,8 +832,8 @@ github.com/wagoodman/go-progress v0.0.0-20200621122631-1a2120f0695a h1:lV3ioFpbq
|
|||
github.com/wagoodman/go-progress v0.0.0-20200621122631-1a2120f0695a/go.mod h1:jLXFoL31zFaHKAAyZUh+sxiTDFe1L1ZHrcK2T1itVKA=
|
||||
github.com/wagoodman/go-progress v0.0.0-20200731105512-1020f39e6240 h1:r6BlIP7CVZtMlxUQhT40h1IE1TzEgKVqwmsVGuscvdk=
|
||||
github.com/wagoodman/go-progress v0.0.0-20200731105512-1020f39e6240/go.mod h1:jLXFoL31zFaHKAAyZUh+sxiTDFe1L1ZHrcK2T1itVKA=
|
||||
github.com/wagoodman/go-rpmdb v0.0.0-20200719223757-ce54a4b0607b h1:elYGLFZPymeTWJ6qA3tIzFet3LQ9D/Jl6HLWNyFjdQc=
|
||||
github.com/wagoodman/go-rpmdb v0.0.0-20200719223757-ce54a4b0607b/go.mod h1:MjoIZzKmbYfcpbC6ARWMcHijAjtLBViDaHcayXKWQWI=
|
||||
github.com/wagoodman/go-rpmdb v0.0.0-20200810111121-8136676cb95c h1:eEWc4HjIq0gSno1apdb5MjRn2995xNrNmRTiJyjUJd8=
|
||||
github.com/wagoodman/go-rpmdb v0.0.0-20200810111121-8136676cb95c/go.mod h1:MjoIZzKmbYfcpbC6ARWMcHijAjtLBViDaHcayXKWQWI=
|
||||
github.com/wagoodman/jotframe v0.0.0-20200730190914-3517092dd163 h1:qoZwR+bHbFFNirY4Yt7lqbOXnFAMnlFfR89w0TXwjrc=
|
||||
github.com/wagoodman/jotframe v0.0.0-20200730190914-3517092dd163/go.mod h1:DzXZ1wfRedNhC3KQTick8Gf3CEPMFHsP5k4R/ldjKtw=
|
||||
github.com/x-cray/logrus-prefixed-formatter v0.5.2 h1:00txxvfBM9muc0jiLIEAkAcIMJzfthRT6usrui8uGmg=
|
||||
|
|
|
|||
|
|
@ -45,12 +45,18 @@ func parseRpmDB(_ string, reader io.Reader) ([]pkg.Package, error) {
|
|||
for _, entry := range pkgList {
|
||||
p := pkg.Package{
|
||||
Name: entry.Name,
|
||||
Version: entry.Version,
|
||||
Type: pkg.RpmPkg,
|
||||
Version: fmt.Sprintf("%s-%s", entry.Version, entry.Release), // this is what engine does
|
||||
//Version: fmt.Sprintf("%d:%s-%s.%s", entry.Epoch, entry.Version, entry.Release, entry.Arch),
|
||||
Type: pkg.RpmPkg,
|
||||
Metadata: pkg.RpmMetadata{
|
||||
Epoch: entry.Epoch,
|
||||
Arch: entry.Arch,
|
||||
Release: entry.Release,
|
||||
Version: entry.Version,
|
||||
Epoch: entry.Epoch,
|
||||
Arch: entry.Arch,
|
||||
Release: entry.Release,
|
||||
SourceRpm: entry.SourceRpm,
|
||||
Vendor: entry.Vendor,
|
||||
License: entry.License,
|
||||
Size: entry.Size,
|
||||
},
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -11,12 +11,17 @@ func TestParseRpmDB(t *testing.T) {
|
|||
expected := map[string]pkg.Package{
|
||||
"dive": {
|
||||
Name: "dive",
|
||||
Version: "0.9.2",
|
||||
Version: "0.9.2-1",
|
||||
Type: pkg.RpmPkg,
|
||||
Metadata: pkg.RpmMetadata{
|
||||
Epoch: 0,
|
||||
Arch: "x86_64",
|
||||
Release: "1",
|
||||
Epoch: 0,
|
||||
Arch: "x86_64",
|
||||
Release: "1",
|
||||
Version: "0.9.2",
|
||||
SourceRpm: "dive-0.9.2-1.src.rpm",
|
||||
Size: 12406784,
|
||||
License: "MIT",
|
||||
Vendor: "",
|
||||
},
|
||||
},
|
||||
}
|
||||
|
|
@ -31,11 +36,11 @@ func TestParseRpmDB(t *testing.T) {
|
|||
t.Fatalf("failed to parse rpmdb: %+v", err)
|
||||
}
|
||||
|
||||
if len(actual) != 1 {
|
||||
if len(actual) != len(expected) {
|
||||
for _, a := range actual {
|
||||
t.Log(" ", a)
|
||||
}
|
||||
t.Fatalf("unexpected package count: %d!=%d", len(actual), 1)
|
||||
t.Fatalf("unexpected package count: %d!=%d", len(actual), len(expected))
|
||||
}
|
||||
|
||||
for _, a := range actual {
|
||||
|
|
|
|||
|
|
@ -1,7 +1,7 @@
|
|||
#!/usr/bin/env bash
|
||||
set -eux
|
||||
|
||||
docker create --name generate-rpmdb-fixture centos:latest sh -c 'tail -f /dev/null'
|
||||
docker create --name generate-rpmdb-fixture centos:8 sh -c 'tail -f /dev/null'
|
||||
|
||||
function cleanup {
|
||||
docker kill generate-rpmdb-fixture
|
||||
|
|
|
|||
|
|
@ -10,10 +10,14 @@ type DpkgMetadata struct {
|
|||
}
|
||||
|
||||
type RpmMetadata struct {
|
||||
Epoch int `mapstructure:"Epoch" json:"epoch"`
|
||||
Arch string `mapstructure:"Arch" json:"architecture"`
|
||||
Release string `mapstructure:"Release" json:"release"`
|
||||
// TODO: consider keeping the remaining values as an embedded map
|
||||
Version string `mapstructure:"Version" json:"version"`
|
||||
Epoch int `mapstructure:"Epoch" json:"epoch"`
|
||||
Arch string `mapstructure:"Arch" json:"architecture"`
|
||||
Release string `mapstructure:"Release" json:"release"`
|
||||
SourceRpm string `mapstructure:"SourceRpm" json:"source-rpm"`
|
||||
Size int `mapstructure:"Size" json:"size"`
|
||||
License string `mapstructure:"License" json:"license"`
|
||||
Vendor string `mapstructure:"Vendor" json:"vendor"`
|
||||
}
|
||||
|
||||
type JavaManifest struct {
|
||||
|
|
|
|||
|
|
@ -5,10 +5,9 @@ import collections
|
|||
|
||||
Metadata = collections.namedtuple("Metadata", "metadata sources")
|
||||
Package = collections.namedtuple("Package", "name type version")
|
||||
Vulnerability = collections.namedtuple("Vulnerability", "cve package")
|
||||
|
||||
|
||||
class syft:
|
||||
class Syft:
|
||||
def __init__(self, report_path):
|
||||
self.report_path = report_path
|
||||
|
||||
|
|
@ -35,10 +34,10 @@ class syft:
|
|||
|
||||
|
||||
def main(baseline_report, new_report):
|
||||
report1_obj = syft(report_path=baseline_report)
|
||||
report1_obj = Syft(report_path=baseline_report)
|
||||
report1_packages, report1_metadata = report1_obj.packages()
|
||||
|
||||
report2_obj = syft(report_path=new_report)
|
||||
report2_obj = Syft(report_path=new_report)
|
||||
report2_packages, report2_metadata = report2_obj.packages()
|
||||
|
||||
if len(report2_packages) == 0 and len(report1_packages) == 0:
|
||||
|
|
@ -102,9 +101,9 @@ def main(baseline_report, new_report):
|
|||
|
||||
|
||||
if __name__ == "__main__":
|
||||
print("\nComparing two syft reports...\n")
|
||||
print("\nComparing two Syft reports...\n")
|
||||
if len(sys.argv) != 3:
|
||||
sys.exit("please provide two syft json files")
|
||||
sys.exit("please provide two Syft json files")
|
||||
|
||||
rc = main(sys.argv[1], sys.argv[2])
|
||||
sys.exit(rc)
|
||||
|
|
|
|||
|
|
@ -1,27 +0,0 @@
|
|||
#!/usr/bin/env bash
|
||||
set -eu
|
||||
|
||||
BOLD="$(tput -T linux bold)"
|
||||
RED="$(tput -T linux setaf 1)"
|
||||
RESET="$(tput -T linux sgr0)"
|
||||
FAIL="${BOLD}${RED}"
|
||||
SUCCESS="${BOLD}"
|
||||
JQ_ARGS="-S .artifacts"
|
||||
|
||||
if ! command -v jq &> /dev/null ;then
|
||||
JQ_IMAGE="imega/jq:latest"
|
||||
JQ_CMD="docker run --rm -i ${JQ_IMAGE} ${JQ_ARGS}"
|
||||
docker pull "${JQ_IMAGE}"
|
||||
else
|
||||
JQ_CMD="jq ${JQ_ARGS}"
|
||||
fi
|
||||
|
||||
if [[ $(cat $1 | ${JQ_CMD}) ]]; then
|
||||
set -x
|
||||
# compare the output of both results
|
||||
diff <(cat $1 | ${JQ_CMD}) <(cat $2 | ${JQ_CMD})
|
||||
set +x
|
||||
echo "${SUCCESS}Comparison passed!${RESET}"
|
||||
else
|
||||
exit "${FAIL}Failing since one of the test files is empty ($1)${RESET}"
|
||||
fi
|
||||
|
|
@ -1,7 +0,0 @@
|
|||
FROM python:3
|
||||
|
||||
WORKDIR /
|
||||
COPY syft-reports /syft-reports
|
||||
COPY inline-reports /inline-reports
|
||||
COPY compare.py .
|
||||
ENTRYPOINT ["/compare.py"]
|
||||
|
|
@ -1,12 +1,15 @@
|
|||
IMAGE = "centos:8"
|
||||
IMAGE_CLEAN = $(shell echo $(IMAGE) | tr ":" "_")
|
||||
syft_DIR = syft-reports
|
||||
syft_REPORT = $(syft_DIR)/$(IMAGE_CLEAN).json
|
||||
ifndef SYFT_CMD
|
||||
SYFT_CMD = go run ../../main.go
|
||||
endif
|
||||
|
||||
IMAGE_CLEAN = $(shell echo $(COMPARE_IMAGE) | tr ":" "_")
|
||||
SYFT_DIR = syft-reports
|
||||
SYFT_REPORT = $(SYFT_DIR)/$(IMAGE_CLEAN).json
|
||||
INLINE_DIR = inline-reports
|
||||
INLINE_REPORT = $(INLINE_DIR)/$(IMAGE_CLEAN)-content-os.json
|
||||
|
||||
ifndef syft_DIR
|
||||
$(error syft_DIR is not set)
|
||||
ifndef SYFT_DIR
|
||||
$(error SYFT_DIR is not set)
|
||||
endif
|
||||
|
||||
ifndef INLINE_DIR
|
||||
|
|
@ -14,26 +17,33 @@ ifndef INLINE_DIR
|
|||
endif
|
||||
|
||||
.PHONY: all
|
||||
all: compare
|
||||
.DEFAULT_GOAL :=
|
||||
all: clean-syft
|
||||
./compare-all.sh
|
||||
|
||||
.PHONY: compare
|
||||
compare: $(INLINE_REPORT) $(syft_REPORT)
|
||||
docker build -t compare-syft:latest .
|
||||
docker run compare-syft:latest $(IMAGE)
|
||||
.PHONY: compare-image
|
||||
compare-image: $(SYFT_REPORT) $(INLINE_REPORT)
|
||||
./compare.py $(COMPARE_IMAGE)
|
||||
|
||||
.PHONY: gather-iamge
|
||||
gather-image: $(SYFT_REPORT) $(INLINE_REPORT)
|
||||
|
||||
$(INLINE_REPORT):
|
||||
echo "Creating $(INLINE_REPORT)..."
|
||||
mkdir -p $(INLINE_DIR)
|
||||
curl -s https://ci-tools.anchore.io/inline_scan-v0.7.0 | bash -s -- -p -r $(IMAGE)
|
||||
curl -s https://ci-tools.anchore.io/inline_scan-v0.7.0 | bash -s -- -p -r $(COMPARE_IMAGE)
|
||||
mv anchore-reports/* $(INLINE_DIR)/
|
||||
rmdir anchore-reports
|
||||
|
||||
$(syft_REPORT):
|
||||
echo "Creating $(syft_REPORT)..."
|
||||
mkdir -p $(syft_DIR)
|
||||
docker pull $(IMAGE)
|
||||
go run ../../main.go $(IMAGE) -o json > $(syft_REPORT)
|
||||
$(SYFT_REPORT):
|
||||
echo "Creating $(SYFT_REPORT)..."
|
||||
mkdir -p $(SYFT_DIR)
|
||||
$(SYFT_CMD) $(COMPARE_IMAGE) -o json > $(SYFT_REPORT)
|
||||
|
||||
.PHONY: clean
|
||||
clean:
|
||||
rm -f $(INLINE_DIR)/* $(syft_DIR)/*
|
||||
clean: clean-syft
|
||||
rm -f $(INLINE_DIR)/*
|
||||
|
||||
.PHONY: clean-syft
|
||||
clean-syft:
|
||||
rm -f $(SYFT_DIR)/*
|
||||
17
test/inline-compare/compare-all.sh
Executable file
17
test/inline-compare/compare-all.sh
Executable file
|
|
@ -0,0 +1,17 @@
|
|||
#!/usr/bin/env bash
|
||||
set -eu
|
||||
|
||||
# TODO: add "alpine:3.12.0" back
|
||||
images=("debian:10.5" "centos:8.2.2004" )
|
||||
|
||||
# gather all image analyses
|
||||
for img in "${images[@]}"; do
|
||||
echo "Gathering facts for $img"
|
||||
COMPARE_IMAGE=${img} make gather-image
|
||||
done
|
||||
|
||||
# compare all results
|
||||
for img in "${images[@]}"; do
|
||||
echo "Comparing results for $img"
|
||||
COMPARE_IMAGE=${img} make compare-image
|
||||
done
|
||||
|
|
@ -2,14 +2,25 @@
|
|||
import os
|
||||
import sys
|
||||
import json
|
||||
import functools
|
||||
import collections
|
||||
|
||||
QUALITY_GATE_THRESHOLD = 0.9
|
||||
QUALITY_GATE_THRESHOLD = 0.95
|
||||
INDENT = " "
|
||||
IMAGE_QUALITY_GATE = collections.defaultdict(lambda: QUALITY_GATE_THRESHOLD, **{
|
||||
|
||||
})
|
||||
|
||||
# We additionally fail if an image is above a particular threshold. Why? We expect the lower threshold to be 90%,
|
||||
# however additional functionality in grype is still being implemented, so this threshold may not be able to be met.
|
||||
# In these cases the IMAGE_QUALITY_GATE is set to a lower value to allow the test to pass for known issues. Once these
|
||||
# issues/enhancements are done we want to ensure that the lower threshold is bumped up to catch regression. The only way
|
||||
# to do this is to select an upper threshold for images with known threshold values, so we have a failure that
|
||||
# loudly indicates the lower threshold should be bumped.
|
||||
IMAGE_UPPER_THRESHOLD = collections.defaultdict(lambda: 1, **{
|
||||
|
||||
})
|
||||
Metadata = collections.namedtuple("Metadata", "version")
|
||||
Package = collections.namedtuple("Package", "name type")
|
||||
Vulnerability = collections.namedtuple("Vulnerability", "cve package")
|
||||
|
||||
|
||||
class InlineScan:
|
||||
|
|
@ -33,12 +44,17 @@ class InlineScan:
|
|||
|
||||
def _enumerate_section(self, report, section):
|
||||
report_path = self._report_path(report=report)
|
||||
os_report_path = self._report_path(report="content-os")
|
||||
|
||||
if os.path.exists(os_report_path) and not os.path.exists(report_path):
|
||||
# if the OS report is there but the target report is not, that is engine's way of saying "no findings"
|
||||
return
|
||||
|
||||
with open(report_path) as json_file:
|
||||
data = json.load(json_file)
|
||||
for entry in data[section]:
|
||||
yield entry
|
||||
|
||||
@functools.lru_cache
|
||||
def _python_packages(self):
|
||||
packages = set()
|
||||
metadata = collections.defaultdict(dict)
|
||||
|
|
@ -51,7 +67,6 @@ class InlineScan:
|
|||
|
||||
return packages, metadata
|
||||
|
||||
@functools.lru_cache
|
||||
def _os_packages(self):
|
||||
packages = set()
|
||||
metadata = collections.defaultdict(dict)
|
||||
|
|
@ -63,7 +78,7 @@ class InlineScan:
|
|||
return packages, metadata
|
||||
|
||||
|
||||
class syft:
|
||||
class Syft:
|
||||
|
||||
report_tmpl = "{image}.json"
|
||||
|
||||
|
|
@ -78,31 +93,54 @@ class syft:
|
|||
for entry in data[section]:
|
||||
yield entry
|
||||
|
||||
@functools.lru_cache
|
||||
def packages(self):
|
||||
packages = set()
|
||||
metadata = collections.defaultdict(dict)
|
||||
for entry in self._enumerate_section(section="artifacts"):
|
||||
|
||||
# normalize to inline
|
||||
pType = entry["type"].lower()
|
||||
if pType in ("wheel", "egg"):
|
||||
pType = "python"
|
||||
pkg_type = entry["type"].lower()
|
||||
if pkg_type in ("wheel", "egg"):
|
||||
pkg_type = "python"
|
||||
elif pkg_type in ("deb",):
|
||||
pkg_type = "dpkg"
|
||||
elif pkg_type in ("java-archive",):
|
||||
pkg_type = "java"
|
||||
elif pkg_type in ("apk",):
|
||||
pkg_type = "apkg"
|
||||
|
||||
package = Package(name=entry["name"], type=pType,)
|
||||
package = Package(name=entry["name"], type=pkg_type,)
|
||||
|
||||
packages.add(package)
|
||||
metadata[package.type][package] = Metadata(version=entry["version"])
|
||||
return packages, metadata
|
||||
|
||||
|
||||
def print_rows(rows):
|
||||
if not rows:
|
||||
return
|
||||
widths = []
|
||||
for col, _ in enumerate(rows[0]):
|
||||
width = max(len(row[col]) for row in rows) + 2 # padding
|
||||
widths.append(width)
|
||||
for row in rows:
|
||||
print("".join(word.ljust(widths[col_idx]) for col_idx, word in enumerate(row)))
|
||||
|
||||
|
||||
def main(image):
|
||||
print(colors.bold+"Image:", image, colors.reset)
|
||||
|
||||
inline = InlineScan(image=image, report_dir="inline-reports")
|
||||
inline_packages, inline_metadata = inline.packages()
|
||||
|
||||
syft = syft(image=image, report_dir="syft-reports")
|
||||
syft = Syft(image=image, report_dir="syft-reports")
|
||||
syft_packages, syft_metadata = syft.packages()
|
||||
|
||||
if len(inline_packages) == 0:
|
||||
# we are purposefully selecting test images that are guaranteed to have packages, so this should never happen
|
||||
print(colors.bold + colors.fg.red + "inline found no packages!", colors.reset)
|
||||
return 1
|
||||
|
||||
if len(syft_packages) == 0 and len(inline_packages) == 0:
|
||||
print("nobody found any packages")
|
||||
return 0
|
||||
|
|
@ -120,51 +158,114 @@ def main(image):
|
|||
metadata = inline_metadata[package.type][package]
|
||||
inline_metadata_set.add((package, metadata))
|
||||
|
||||
syft_metadata_set = set()
|
||||
syft_overlap_metadata_set = set()
|
||||
for package in syft_packages:
|
||||
metadata = syft_metadata[package.type][package]
|
||||
syft_metadata_set.add((package, metadata))
|
||||
# we only want to really count mismatched metadata for packages that are at least found by inline
|
||||
if package in inline_metadata[package.type]:
|
||||
syft_overlap_metadata_set.add((package, metadata))
|
||||
|
||||
same_metadata = syft_metadata_set & inline_metadata_set
|
||||
same_metadata = syft_overlap_metadata_set & inline_metadata_set
|
||||
percent_overlap_metadata = (
|
||||
float(len(same_metadata)) / float(len(inline_metadata_set))
|
||||
) * 100.0
|
||||
missing_metadata = inline_metadata_set - same_metadata
|
||||
|
||||
if len(bonus_packages) > 0:
|
||||
print("syft Bonus packages:")
|
||||
rows = []
|
||||
print(colors.bold + "Syft found extra packages:", colors.reset)
|
||||
for package in sorted(list(bonus_packages)):
|
||||
print(" " + repr(package))
|
||||
rows.append([INDENT, repr(package)])
|
||||
print_rows(rows)
|
||||
print()
|
||||
|
||||
if len(missing_pacakges) > 0:
|
||||
print("syft Missing packages:")
|
||||
rows = []
|
||||
print(colors.bold + "Syft missed packages:", colors.reset)
|
||||
for package in sorted(list(missing_pacakges)):
|
||||
print(" " + repr(package))
|
||||
rows.append([INDENT, repr(package)])
|
||||
print_rows(rows)
|
||||
print()
|
||||
|
||||
print("Inline Packages: %d" % len(inline_packages))
|
||||
print("syft Packages: %d" % len(syft_packages))
|
||||
print()
|
||||
if len(missing_metadata) > 0:
|
||||
rows = []
|
||||
print(colors.bold + "Syft mismatched metadata:", colors.reset)
|
||||
for inline_metadata_pair in sorted(list(missing_metadata)):
|
||||
pkg, metadata = inline_metadata_pair
|
||||
if pkg in syft_metadata[pkg.type]:
|
||||
syft_metadata_item = syft_metadata[pkg.type][pkg]
|
||||
else:
|
||||
syft_metadata_item = "--- MISSING ---"
|
||||
rows.append([INDENT, "for:", repr(pkg), ":", repr(syft_metadata_item), "!=", repr(metadata)])
|
||||
print_rows(rows)
|
||||
print()
|
||||
|
||||
print(colors.bold+"Summary:", colors.reset)
|
||||
print(" Image: %s" % image)
|
||||
print(" Inline Packages: %d" % len(inline_packages))
|
||||
print(" Syft Packages: %d" % len(syft_packages))
|
||||
print(
|
||||
"Baseline Packages Matched: %2.3f %% (%d/%d packages)"
|
||||
" Baseline Packages Matched: %2.3f %% (%d/%d packages)"
|
||||
% (percent_overlap_packages, len(same_packages), len(inline_packages))
|
||||
)
|
||||
print(
|
||||
"Baseline Metadata Matched: %2.3f %% (%d/%d metadata)"
|
||||
" Baseline Metadata Matched: %2.3f %% (%d/%d metadata)"
|
||||
% (percent_overlap_metadata, len(same_metadata), len(inline_metadata_set))
|
||||
)
|
||||
|
||||
overall_score = (percent_overlap_packages + percent_overlap_metadata) / 2.0
|
||||
|
||||
print("Overall Score: %2.3f %%" % overall_score)
|
||||
print(colors.bold + " Overall Score: %2.1f %%" % overall_score, colors.reset)
|
||||
|
||||
if overall_score < (QUALITY_GATE_THRESHOLD * 100):
|
||||
print("failed quality gate (>= %d %%)" % (QUALITY_GATE_THRESHOLD * 100))
|
||||
upper_gate_value = IMAGE_UPPER_THRESHOLD[image] * 100
|
||||
lower_gate_value = IMAGE_QUALITY_GATE[image] * 100
|
||||
if overall_score < lower_gate_value:
|
||||
print(colors.bold + " Quality Gate: " + colors.fg.red + "FAILED (is not >= %d %%)\n" % lower_gate_value, colors.reset)
|
||||
return 1
|
||||
elif overall_score > upper_gate_value:
|
||||
print(colors.bold + " Quality Gate: " + colors.fg.orange + "FAILED (lower threshold is artificially low and should be updated)\n", colors.reset)
|
||||
return 1
|
||||
else:
|
||||
print(colors.bold + " Quality Gate: " + colors.fg.green + "pass (>= %d %%)\n" % lower_gate_value, colors.reset)
|
||||
|
||||
return 0
|
||||
|
||||
|
||||
class colors:
|
||||
reset='\033[0m'
|
||||
bold='\033[01m'
|
||||
disable='\033[02m'
|
||||
underline='\033[04m'
|
||||
reverse='\033[07m'
|
||||
strikethrough='\033[09m'
|
||||
invisible='\033[08m'
|
||||
class fg:
|
||||
black='\033[30m'
|
||||
red='\033[31m'
|
||||
green='\033[32m'
|
||||
orange='\033[33m'
|
||||
blue='\033[34m'
|
||||
purple='\033[35m'
|
||||
cyan='\033[36m'
|
||||
lightgrey='\033[37m'
|
||||
darkgrey='\033[90m'
|
||||
lightred='\033[91m'
|
||||
lightgreen='\033[92m'
|
||||
yellow='\033[93m'
|
||||
lightblue='\033[94m'
|
||||
pink='\033[95m'
|
||||
lightcyan='\033[96m'
|
||||
class bg:
|
||||
black='\033[40m'
|
||||
red='\033[41m'
|
||||
green='\033[42m'
|
||||
orange='\033[43m'
|
||||
blue='\033[44m'
|
||||
purple='\033[45m'
|
||||
cyan='\033[46m'
|
||||
lightgrey='\033[47m'
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
if len(sys.argv) != 2:
|
||||
sys.exit("provide an image")
|
||||
|
|
|
|||
|
|
@ -14,7 +14,7 @@ var cases = []struct {
|
|||
name: "find rpmdb packages",
|
||||
pkgType: pkg.RpmPkg,
|
||||
pkgInfo: map[string]string{
|
||||
"dive": "0.9.2",
|
||||
"dive": "0.9.2-1",
|
||||
},
|
||||
},
|
||||
{
|
||||
|
|
|
|||
Loading…
Reference in a new issue