Merge pull request #107 from anchore/fix-acceptance-compare

Fix acceptance test compare script & persist results

Makefile

@@ -164,8 +164,9 @@ acceptance-mac: $(SNAPSHOTDIR) ## Run acceptance tests on build snapshot binarie
 	$(call title,Running acceptance test: Run on Mac)
 	$(ACC_DIR)/mac.sh \
 			$(SNAPSHOTDIR) \
-			$(ACC_DIR)\
-			$(ACC_TEST_IMAGE)
+			$(ACC_DIR) \
+			$(ACC_TEST_IMAGE) \
+			$(RESULTSDIR)
 
 .PHONY: acceptance-linux
 acceptance-linux: acceptance-test-deb-package-install acceptance-test-rpm-package-install ## Run acceptance tests on build snapshot binaries and packages (Linux)
@@ -175,16 +176,18 @@ acceptance-test-deb-package-install: $(SNAPSHOTDIR)
 	$(call title,Running acceptance test: DEB install)
 	$(ACC_DIR)/deb.sh \
 			$(SNAPSHOTDIR) \
-			$(ACC_DIR)\
-			$(ACC_TEST_IMAGE)
+			$(ACC_DIR) \
+			$(ACC_TEST_IMAGE) \
+			$(RESULTSDIR)
 
 .PHONY: acceptance-test-rpm-package-install
 acceptance-test-rpm-package-install: $(SNAPSHOTDIR)
 	$(call title,Running acceptance test: RPM install)
 	$(ACC_DIR)/rpm.sh \
 			$(SNAPSHOTDIR) \
-			$(ACC_DIR)\
-			$(ACC_TEST_IMAGE)
+			$(ACC_DIR) \
+			$(ACC_TEST_IMAGE) \
+			$(RESULTSDIR)
 
 # TODO: this is not releasing yet
 .PHONY: release
@@ -203,11 +206,11 @@ release: clean-dist ## Build and publish final binaries and packages
 	# TODO: add upload to bucket
 
 .PHONY: clean
-clean: clean-dist clean-shapshot  ## Remove previous builds and result reports
+clean: clean-dist clean-snapshot  ## Remove previous builds and result reports
 	rm -rf $(RESULTSDIR)/*
 
-.PHONY: clean-shapshot
-clean-shapshot:
+.PHONY: clean-snapshot
+clean-snapshot:
 	rm -rf $(SNAPSHOTDIR) $(TEMPDIR)/goreleaser.yaml
 
 .PHONY: clean-dist

test/acceptance/compare.py

@@ -28,7 +28,8 @@ class syft:
 
             packages.add(package)
             metadata[package.type][package] = Metadata(
-                metadata=repr(entry["metadata"]), sources=repr(entry["sources"])
+                # note: the metadata entry is optional
+                metadata=repr(entry.get("metadata")), sources=repr(entry["sources"])
             )
         return packages, metadata
 
@@ -50,7 +51,7 @@ def main(baseline_report, new_report):
     ) * 100.0
 
     extra_packages = report2_packages - report1_packages
-    missing_pacakges = report1_packages - report2_packages
+    missing_packages = report1_packages - report2_packages
 
     report1_metadata_set = set()
     for package in report1_packages:
@@ -75,9 +76,9 @@ def main(baseline_report, new_report):
             print("    " + repr(package))
         print()
 
-    if len(missing_pacakges) > 0:
+    if len(missing_packages) > 0:
         print("Missing packages:")
-        for package in sorted(list(missing_pacakges)):
+        for package in sorted(list(missing_packages)):
             print("    " + repr(package))
         print()
 

test/acceptance/deb.sh

@@ -4,6 +4,7 @@ set -eux
 DISTDIR=$1
 ACC_DIR=$2
 TEST_IMAGE=$3
+RESULTSDIR=$4
 
 TEST_TYPE=deb
 WORK_DIR=`mktemp -d -t "syft-acceptance-test-${TEST_TYPE}-XXXXXX"`
@@ -40,7 +41,10 @@ docker run --rm \
             cat ${REPORT} \
         "
 
+# keep the generated report around
+cp ${REPORT} ${RESULTSDIR}
+
 # compare the results to a known good output
 ${ACC_DIR}/compare.py \
     ${GOLDEN_REPORT} \
-    ${REPORT}
+    ${REPORT} | tee ${RESULTSDIR}/acceptance-${TEST_TYPE}.txt

test/acceptance/mac.sh

@@ -4,6 +4,7 @@ set -eux
 DISTDIR=$1
 ACC_DIR=$2
 TEST_IMAGE=$3
+RESULTSDIR=$4
 
 TEST_IMAGE_TAR=/tmp/image.tar
 TEST_TYPE=mac
@@ -37,7 +38,10 @@ ${DISTDIR}/syft_darwin_amd64/syft version -v
 ${DISTDIR}/syft_darwin_amd64/syft docker-archive://${TEST_IMAGE_TAR} -vv -o json > ${REPORT}
 cat ${REPORT}
 
+# keep the generated report around
+cp ${REPORT} ${RESULTSDIR}
+
 # compare the results to a known good output
 ${ACC_DIR}/compare.py \
     ${GOLDEN_REPORT} \
-    ${REPORT}
+    ${REPORT} | tee ${RESULTSDIR}/acceptance-${TEST_TYPE}.txt

test/acceptance/rpm.sh

@@ -4,6 +4,7 @@ set -eux
 DISTDIR=$1
 ACC_DIR=$2
 TEST_IMAGE=$3
+RESULTSDIR=$4
 
 TEST_TYPE=rpm
 WORK_DIR=`mktemp -d -t "syft-acceptance-test-${TEST_TYPE}-XXXXXX"`
@@ -40,7 +41,10 @@ docker run --rm \
             cat ${REPORT} \
         "
 
+# keep the generated report around
+cp ${REPORT} ${RESULTSDIR}
+
 # compare the results to a known good output
 ${ACC_DIR}/compare.py \
     ${GOLDEN_REPORT} \
-    ${REPORT}
+    ${REPORT} | tee ${RESULTSDIR}/acceptance-${TEST_TYPE}.txt