update pipeline with new levels of testing

Signed-off-by: Alex Goodman <alex.goodman@anchore.com>
2024-09-20 06:01:53 +00:00 · 2021-03-18 09:01:07 -04:00 · 2021-03-18 09:01:07 -04:00 · 1d87f07da1
commit 1d87f07da1
parent cf516add95
7 changed files with 470 additions and 144 deletions
--- a/.github/workflows/release.yaml
+++ b/.github/workflows/release.yaml
@ -9,7 +9,7 @@ on:
      - "v*"

 env:
-  GO_VERSION: "1.14.x"
+  GO_VERSION: "1.16.x"

 jobs:
  wait-for-checks:
@ -29,17 +29,26 @@ jobs:
        id: static-analysis
        with:
          token: ${{ secrets.GITHUB_TOKEN }}
-          # This check name is defined as the circle-ci workflow name (in .circleci/config.yaml)
-          checkName: "Static-Analysis (1.x, ubuntu-latest)"
+          # This check name is defined as the github action job name (in .github/workflows/testing.yaml)
+          checkName: "Static-Analysis"
          ref: ${{ github.event.pull_request.head.sha || github.sha }}

-      - name: Check unit + integration results (latest go version)
+      - name: Check unit test results
        uses: fountainhead/action-wait-for-check@v1.0.0
-        id: unit-integration
+        id: unit
        with:
          token: ${{ secrets.GITHUB_TOKEN }}
-          # This check name is defined as the circle-ci workflow name (in .circleci/config.yaml)
-          checkName: "Tests (1.x, ubuntu-latest)"
+          # This check name is defined as the github action job name (in .github/workflows/testing.yaml)
+          checkName: "Unit-Test"
+          ref: ${{ github.event.pull_request.head.sha || github.sha }}
+
+      - name: Check integration test results
+        uses: fountainhead/action-wait-for-check@v1.0.0
+        id: integration
+        with:
+          token: ${{ secrets.GITHUB_TOKEN }}
+          # This check name is defined as the github action job name (in .github/workflows/testing.yaml)
+          checkName: "Integration-Test"
          ref: ${{ github.event.pull_request.head.sha || github.sha }}

      - name: Check acceptance test results (linux)
@ -47,7 +56,7 @@ jobs:
        id: acceptance-linux
        with:
          token: ${{ secrets.GITHUB_TOKEN }}
-          # This check name is defined as the github action job name (in .github/workflows/acceptance-test.yaml)
+          # This check name is defined as the github action job name (in .github/workflows/testing.yaml)
          checkName: "Acceptance-Linux"
          ref: ${{ github.event.pull_request.head.sha || github.sha }}

@ -56,27 +65,28 @@ jobs:
        id: acceptance-mac
        with:
          token: ${{ secrets.GITHUB_TOKEN }}
-          # This check name is defined as the github action job name (in .github/workflows/acceptance-test.yaml)
+          # This check name is defined as the github action job name (in .github/workflows/testing.yaml)
          checkName: "Acceptance-Mac"
          ref: ${{ github.event.pull_request.head.sha || github.sha }}

-      - name: Check inline comparison test results
+      - name: Check cli test results (linux)
        uses: fountainhead/action-wait-for-check@v1.0.0
-        id: inline-compare
+        id: cli-linux
        with:
          token: ${{ secrets.GITHUB_TOKEN }}
-          # This check name is defined as the github action job name (in .github/workflows/acceptance-test.yaml)
-          checkName: "Inline-Compare"
+          # This check name is defined as the github action job name (in .github/workflows/testing.yaml)
+          checkName: "Cli-Linux"
          ref: ${{ github.event.pull_request.head.sha || github.sha }}

      - name: Quality gate
-        if: steps.static-analysis.outputs.conclusion != 'success' || steps.unit-integration.outputs.conclusion != 'success' || steps.inline-compare.outputs.conclusion != 'success' || steps.acceptance-linux.outputs.conclusion != 'success' || steps.acceptance-mac.outputs.conclusion != 'success'
+        if: steps.static-analysis.outputs.conclusion != 'success' || steps.unit.outputs.conclusion != 'success' || steps.integration.outputs.conclusion != 'success' || steps.cli-linux.outputs.conclusion != 'success' || steps.acceptance-linux.outputs.conclusion != 'success' || steps.acceptance-mac.outputs.conclusion != 'success'
        run: |
          echo "Static Analysis Status: ${{ steps.static-analysis.conclusion }}"
-          echo "Unit & Integration Test Status: ${{ steps.unit-integration.outputs.conclusion }}"
+          echo "Unit Test Status: ${{ steps.unit.outputs.conclusion }}"
+          echo "Integration Test Status: ${{ steps.integration.outputs.conclusion }}"
          echo "Acceptance Test (Linux) Status: ${{ steps.acceptance-linux.outputs.conclusion }}"
          echo "Acceptance Test (Mac) Status: ${{ steps.acceptance-mac.outputs.conclusion }}"
-          echo "Inline Compare Status: ${{ steps.inline-compare.outputs.conclusion }}"
+          echo "CLI Test (Linux) Status: ${{ steps.cli-linux.outputs.conclusion }}"
          false

  release:
--- a/.github/workflows/static-unit-integration.yaml
+++ b/.github/workflows/static-unit-integration.yaml
@ -1,101 +0,0 @@
-name: "Static Analysis + Unit + Integration"
-on:
-  workflow_dispatch:
-  push:
-  pull_request:
-jobs:
-  Static-Analysis:
-    strategy:
-      matrix:
-        go-version: [1.x]
-        platform: [ubuntu-latest]
-    runs-on: ${{ matrix.platform }}
-    steps:
-      - uses: actions/setup-go@v2
-        with:
-          go-version: ${{ matrix.go-version }}
-
-      - uses: actions/checkout@v2
-
-      - name: Restore bootstrap cache
-        id: bootstrap-cache
-        uses: actions/cache@v2.1.3
-        with:
-          path: |
-            ~/go/pkg/mod
-            ${{ github.workspace }}/.tmp
-          key: ${{ runner.os }}-go-${{ matrix.go-version }}-${{ hashFiles('**/go.sum') }}-${{ hashFiles('Makefile') }}
-          restore-keys: |
-            ${{ runner.os }}-go-${{ matrix.go-version }}-${{ hashFiles('**/go.sum') }}-
-            ${{ runner.os }}-go-${{ matrix.go-version }}-
-
-      - name: Bootstrap project dependencies
-        if: steps.bootstrap-cache.outputs.cache-hit != 'true'
-        run: make bootstrap
-
-      - name: Bootstrap CI dependencies
-        run: make ci-bootstrap
-
-      - name: Run static analysis
-        run: make static-analysis
-
-  Tests:
-    strategy:
-      matrix:
-        # test the lower bounds of support, and the latest available
-        go-version: [1.13.x, 1.x]
-        platform: [ubuntu-latest]
-    runs-on: ${{ matrix.platform }}
-    steps:
-      - uses: actions/setup-go@v2
-        with:
-          go-version: ${{ matrix.go-version }}
-
-      - uses: actions/checkout@v2
-
-      - name: Restore bootstrap cache
-        id: bootstrap-cache
-        uses: actions/cache@v2.1.3
-        with:
-          path: |
-            ~/go/pkg/mod
-            ${{ github.workspace }}/.tmp
-          key: ${{ runner.os }}-go-${{ matrix.go-version }}-${{ hashFiles('**/go.sum') }}-${{ hashFiles('Makefile') }}
-          restore-keys: |
-            ${{ runner.os }}-go-${{ matrix.go-version }}-${{ hashFiles('**/go.sum') }}-
-            ${{ runner.os }}-go-${{ matrix.go-version }}-
-
-      - name: Bootstrap project dependencies
-        if: steps.bootstrap-cache.outputs.cache-hit != 'true'
-        run: make bootstrap
-
-      - name: Bootstrap CI dependencies
-        run: make ci-bootstrap
-
-      - name: Build cache key for java test-fixture blobs (for unit tests)
-        run: make java-packages-fingerprint
-
-      - name: Restore Java test-fixture cache
-        id: unit-java-cache
-        uses: actions/cache@v2.1.3
-        with:
-          path: syft/cataloger/java/test-fixtures/java-builds/packages
-          key: ${{ runner.os }}-unit-java-cache-${{ hashFiles( 'syft/cataloger/java/test-fixtures/java-builds/packages.fingerprint' ) }}
-
-      - name: Run unit tests
-        run: make unit
-
-      - name: Validate syft output against the CycloneDX schema
-        run: make validate-cyclonedx-schema
-
-      - name: Build key for tar cache
-        run: make integration-fingerprint
-
-      - name: Restore integration test cache
-        uses: actions/cache@v2.1.3
-        with:
-          path: ${{ github.workspace }}/integration/test-fixtures/cache
-          key: ${{ runner.os }}-integration-test-cache-${{ hashFiles('integration/test-fixtures/cache.fingerprint') }}
-
-      - name: Run integration tests
-        run: make integration
--- a/.github/workflows/validations.yaml
+++ b/.github/workflows/validations.yaml
@ -0,0 +1,368 @@
+name: "Validations"
+on:
+  workflow_dispatch:
+  push:
+  pull_request:
+
+env:
+  GO_VERSION: "1.16.x"
+
+jobs:
+
+  # Note: changing this job name requires making the same update in the .github/workflows/release.yaml pipeline
+  Static-Analysis:
+    name: "Static analysis"
+    runs-on: ubuntu-20.04
+    # run only on push event (internal PRs) or on a pull_request event that is from a fork (external PR)
+    # skip if this is a pull_request event on an internal PR (which is already covered by push events)
+    if: github.event_name == 'push' || github.event.pull_request.head.repo.full_name != github.repository
+    steps:
+      - uses: actions/setup-go@v2
+        with:
+          go-version: ${{ env.GO_VERSION }}
+
+      - uses: actions/checkout@v2
+
+      - name: Restore tool cache
+        id: tool-cache
+        uses: actions/cache@v2.1.3
+        with:
+          path: ${{ github.workspace }}/.tmp
+          key: ${{ runner.os }}-tool-${{ hashFiles('Makefile') }}
+
+      - name: Restore go cache
+        id: go-cache
+        uses: actions/cache@v2.1.3
+        with:
+          path: ~/go/pkg/mod
+          key: ${{ runner.os }}-go-${{ env.GO_VERSION }}-${{ hashFiles('**/go.sum') }}
+          restore-keys: |
+            ${{ runner.os }}-go-${{ env.GO_VERSION }}-
+
+      - name: (cache-miss) Bootstrap all project dependencies
+        if: steps.tool-cache.outputs.cache-hit != 'true' || steps.go-cache.outputs.cache-hit != 'true'
+        run: make bootstrap
+
+      - name: Bootstrap CI environment dependencies
+        run: make ci-bootstrap
+
+      - name: Run static analysis
+        run: make static-analysis
+
+  # Note: changing this job name requires making the same update in the .github/workflows/release.yaml pipeline
+  Unit-Test:
+    name: "Unit tests"
+    runs-on: ubuntu-20.04
+    # run only on push event (internal PRs) or on a pull_request event that is from a fork (external PR)
+    # skip if this is a pull_request event on an internal PR (which is already covered by push events)
+    if: github.event_name == 'push' || github.event.pull_request.head.repo.full_name != github.repository
+    steps:
+      - uses: actions/setup-go@v2
+        with:
+          go-version: ${{ env.GO_VERSION }}
+
+      - uses: actions/checkout@v2
+
+      - name: Restore docker cache
+        uses: satackey/action-docker-layer-caching@v0.0.11
+        continue-on-error: true
+
+      - name: Restore tool cache
+        id: tool-cache
+        uses: actions/cache@v2.1.3
+        with:
+          path: ${{ github.workspace }}/.tmp
+          key: ${{ runner.os }}-tool-${{ hashFiles('Makefile') }}
+
+      - name: Restore go cache
+        id: go-cache
+        uses: actions/cache@v2.1.3
+        with:
+          path: ~/go/pkg/mod
+          key: ${{ runner.os }}-go-${{ env.GO_VERSION }}-${{ hashFiles('**/go.sum') }}
+          restore-keys: |
+            ${{ runner.os }}-go-${{ env.GO_VERSION }}-
+
+      - name: (cache-miss) Bootstrap all project dependencies
+        if: steps.tool-cache.outputs.cache-hit != 'true' || steps.go-cache.outputs.cache-hit != 'true'
+        run: make bootstrap
+
+      - name: Bootstrap CI environment dependencies
+        run: make ci-bootstrap
+
+      - name: Build cache key for java test-fixture blobs (for unit tests)
+        run: make java-packages-fingerprint
+
+      - name: Restore Java test-fixture cache
+        id: unit-java-cache
+        uses: actions/cache@v2.1.3
+        with:
+          path: syft/pkg/cataloger/java/test-fixtures/java-builds/packages
+          key: ${{ runner.os }}-unit-java-cache-${{ hashFiles( 'syft/pkg/cataloger/java/test-fixtures/java-builds/packages.fingerprint' ) }}
+
+      - name: Run unit tests
+        run: make unit
+
+      - uses: actions/upload-artifact@v2
+        with:
+          name: unit-test-results
+          path: test/results/**/*
+
+  # Note: changing this job name requires making the same update in the .github/workflows/release.yaml pipeline
+  Integration-Test:
+    name: "Integration tests"
+    runs-on: ubuntu-20.04
+    # run only on push event (internal PRs) or on a pull_request event that is from a fork (external PR)
+    # skip if this is a pull_request event on an internal PR (which is already covered by push events)
+    if: github.event_name == 'push' || github.event.pull_request.head.repo.full_name != github.repository
+    steps:
+      - uses: actions/setup-go@v2
+        with:
+          go-version: ${{ env.GO_VERSION }}
+
+      - uses: actions/checkout@v2
+
+      - name: Restore docker cache
+        uses: satackey/action-docker-layer-caching@v0.0.11
+        continue-on-error: true
+
+      - name: Restore tool cache
+        id: tool-cache
+        uses: actions/cache@v2.1.3
+        with:
+          path: ${{ github.workspace }}/.tmp
+          key: ${{ runner.os }}-tool-${{ hashFiles('Makefile') }}
+
+      - name: Restore go cache
+        id: go-cache
+        uses: actions/cache@v2.1.3
+        with:
+          path: ~/go/pkg/mod
+          key: ${{ runner.os }}-go-${{ env.GO_VERSION }}-${{ hashFiles('**/go.sum') }}
+          restore-keys: |
+            ${{ runner.os }}-go-${{ env.GO_VERSION }}-
+
+      - name: (cache-miss) Bootstrap all project dependencies
+        if: steps.tool-cache.outputs.cache-hit != 'true' || steps.go-cache.outputs.cache-hit != 'true'
+        run: make bootstrap
+
+      - name: Bootstrap CI environment dependencies
+        run: make ci-bootstrap
+
+      - name: Validate syft output against the CycloneDX schema
+        run: make validate-cyclonedx-schema
+
+      - name: Build key for tar cache
+        run: make integration-fingerprint
+
+      - name: Restore integration test cache
+        uses: actions/cache@v2.1.3
+        with:
+          path: ${{ github.workspace }}/test/integration/test-fixtures/cache
+          key: ${{ runner.os }}-integration-test-cache-${{ hashFiles('test/integration/test-fixtures/cache.fingerprint') }}
+
+      - name: Run integration tests
+        run: make integration
+
+  Benchmark-Test:
+    name: "Benchmark tests"
+    runs-on: ubuntu-20.04
+    # run only on push event (internal PRs) or on a pull_request event that is from a fork (external PR)
+    # skip if this is a pull_request event on an internal PR (which is already covered by push events)
+    if: github.event_name == 'push' || github.event.pull_request.head.repo.full_name != github.repository
+    steps:
+      - uses: actions/setup-go@v2
+        with:
+          go-version: ${{ env.GO_VERSION }}
+
+      - uses: actions/checkout@v2
+
+      - name: Restore docker cache
+        uses: satackey/action-docker-layer-caching@v0.0.11
+        continue-on-error: true
+
+      - name: Restore tool cache
+        id: tool-cache
+        uses: actions/cache@v2.1.3
+        with:
+          path: ${{ github.workspace }}/.tmp
+          key: ${{ runner.os }}-tool-${{ hashFiles('Makefile') }}
+
+      - name: Restore go cache
+        id: go-cache
+        uses: actions/cache@v2.1.3
+        with:
+          path: ~/go/pkg/mod
+          key: ${{ runner.os }}-go-${{ env.GO_VERSION }}-${{ hashFiles('**/go.sum') }}
+          restore-keys: |
+            ${{ runner.os }}-go-${{ env.GO_VERSION }}-
+
+      - name: (cache-miss) Bootstrap all project dependencies
+        if: steps.tool-cache.outputs.cache-hit != 'true' || steps.go-cache.outputs.cache-hit != 'true'
+        run: make bootstrap
+
+      - name: Bootstrap CI environment dependencies
+        run: make ci-bootstrap
+
+      - name: Restore base benchmark result
+        uses: actions/cache@v2
+        with:
+          path: test/results/benchmark-main.txt
+          # use base sha for PR or new commit hash for main push in benchmark result key
+          key: ${{ runner.os }}-bench-${{ (github.event.pull_request.base.sha != github.event.after) && github.event.pull_request.base.sha || github.event.after }}
+
+      - name: Run benchmark tests
+        id: benchmark
+        run: |
+          REF_NAME=${GITHUB_REF##*/} make benchmark
+          OUTPUT=$(make show-benchstat)
+          OUTPUT="${OUTPUT//'%'/'%25'}"
+          OUTPUT="${OUTPUT//$'\n'/'%0A'}"
+          OUTPUT="${OUTPUT//$'\r'/'%0D'}"
+          echo "::set-output name=result::$OUTPUT"
+
+      - uses: actions/upload-artifact@v2
+        with:
+          name: branchmark-test-results
+          path: test/results/**/*
+
+      - name: Update PR benchmark results comment
+        uses: marocchino/sticky-pull-request-comment@v2
+        with:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          header: benchmark
+          message: |
+            ### Benchmark Test Results
+
+            <details>
+              <summary>Benchmark results from the latest changes vs base branch</summary>
+
+            ```
+            ${{ steps.benchmark.outputs.result }}
+            ```
+
+            </details>
+
+  Build-Snapshot-Artifacts:
+    name: "Build snapshot artifacts"
+    runs-on: macos-latest # We're creating these snapshot builds on macOS to be consistent with our release workflow's build process, which also takes place on macOS (due to code signing requirements).
+    # run only on push event (internal PRs) or on a pull_request event that is from a fork (external PR)
+    # skip if this is a pull_request event on an internal PR (which is already covered by push events)
+    if: github.event_name == 'push' || github.event.pull_request.head.repo.full_name != github.repository
+    steps:
+      - uses: actions/setup-go@v2
+        with:
+          go-version: ${{ env.GO_VERSION }}
+
+      - uses: actions/checkout@v2
+
+      - name: Restore tool cache
+        id: tool-cache
+        uses: actions/cache@v2.1.3
+        with:
+          path: ${{ github.workspace }}/.tmp
+          key: ${{ runner.os }}-tool-${{ hashFiles('Makefile') }}
+
+      - name: Restore go cache
+        id: go-cache
+        uses: actions/cache@v2.1.3
+        with:
+          path: ~/go/pkg/mod
+          key: ${{ runner.os }}-go-${{ env.GO_VERSION }}-${{ hashFiles('**/go.sum') }}
+          restore-keys: |
+            ${{ runner.os }}-go-${{ env.GO_VERSION }}-
+
+      - name: (cache-miss) Bootstrap all project dependencies
+        if: steps.tool-cache.outputs.cache-hit != 'true' || steps.go-cache.outputs.cache-hit != 'true'
+        run: make bootstrap
+
+      - name: Build snapshot artifacts
+        run: make snapshot
+
+      - uses: actions/upload-artifact@v2
+        with:
+          name: artifacts
+          path: snapshot/**/*
+
+  # Note: changing this job name requires making the same update in the .github/workflows/release.yaml pipeline
+  Acceptance-Linux:
+    name: "Acceptance tests (Linux)"
+    needs: [Build-Snapshot-Artifacts]
+    runs-on: ubuntu-20.04
+    # run only on push event (internal PRs) or on a pull_request event that is from a fork (external PR)
+    # skip if this is a pull_request event on an internal PR (which is already covered by push events)
+    if: github.event_name == 'push' || github.event.pull_request.head.repo.full_name != github.repository
+    steps:
+      - uses: actions/checkout@v2
+
+      - uses: actions/download-artifact@v2
+        with:
+          name: artifacts
+          path: snapshot
+
+      - name: Run Acceptance Tests (Linux)
+        run: make acceptance-linux
+
+  # Note: changing this job name requires making the same update in the .github/workflows/release.yaml pipeline
+  Acceptance-Mac:
+    name: "Acceptance tests (Mac)"
+    needs: [Build-Snapshot-Artifacts]
+    runs-on: macos-latest
+    # run only on push event (internal PRs) or on a pull_request event that is from a fork (external PR)
+    # skip if this is a pull_request event on an internal PR (which is already covered by push events)
+    if: github.event_name == 'push' || github.event.pull_request.head.repo.full_name != github.repository
+    steps:
+      - uses: actions/checkout@v2
+
+      - uses: actions/download-artifact@v2
+        with:
+          name: artifacts
+          path: snapshot
+
+      - name: Run Acceptance Tests (Mac)
+        run: make acceptance-mac
+
+  # Note: changing this job name requires making the same update in the .github/workflows/release.yaml pipeline
+  Cli-Linux:
+    name: "CLI tests (Linux)"
+    needs: [Build-Snapshot-Artifacts]
+    runs-on: ubuntu-20.04
+    # run only on push event (internal PRs) or on a pull_request event that is from a fork (external PR)
+    # skip if this is a pull_request event on an internal PR (which is already covered by push events)
+    if: github.event_name == 'push' || github.event.pull_request.head.repo.full_name != github.repository
+    steps:
+      - uses: actions/checkout@v2
+
+      - name: Restore docker cache
+        uses: satackey/action-docker-layer-caching@v0.0.11
+        continue-on-error: true
+
+      - name: Restore go cache
+        id: go-cache
+        uses: actions/cache@v2.1.3
+        with:
+          path: ~/go/pkg/mod
+          key: ${{ runner.os }}-go-${{ env.GO_VERSION }}-${{ hashFiles('**/go.sum') }}
+          restore-keys: |
+            ${{ runner.os }}-go-${{ env.GO_VERSION }}-
+
+      - name: (cache-miss) Bootstrap go dependencies
+        if: steps.go-cache.outputs.cache-hit != 'true'
+        run: make bootstrap-go
+
+      - name: Build key for tar cache
+        run: make cli-fingerprint
+
+      - name: Restore CLI test cache
+        uses: actions/cache@v2.1.3
+        with:
+          path: ${{ github.workspace }}/test/cli/test-fixtures/cache
+          key: ${{ runner.os }}-cli-test-cache-${{ hashFiles('test/cli/test-fixtures/cache.fingerprint') }}
+
+      - uses: actions/download-artifact@v2
+        with:
+          name: artifacts
+          path: snapshot
+
+      - name: Run CLI Tests (Linux)
+        run: make cli-linux
--- a/.gitignore
+++ b/.gitignore
@ -1,4 +1,5 @@
 CHANGELOG.md
+/test/results
 /dist
 /snapshot
 .server/
--- a/85
+++ b/85
@ -1,8 +1,8 @@
 BIN = syft
 TEMPDIR = ./.tmp
-RESULTSDIR = $(TEMPDIR)/results
-COVER_REPORT = $(RESULTSDIR)/cover.report
-COVER_TOTAL = $(RESULTSDIR)/cover.total
+RESULTSDIR = test/results
+COVER_REPORT = $(RESULTSDIR)/unit-coverage-details.txt
+COVER_TOTAL = $(RESULTSDIR)/unit-coverage-summary.txt
 LINTCMD = $(TEMPDIR)/golangci-lint run --tests=false --config .golangci.yaml
 ACC_TEST_IMAGE = centos:8.2.2004
 ACC_DIR = ./test/acceptance
@ -19,6 +19,7 @@ COVERAGE_THRESHOLD := 68
 # CI cache busting values; change these if you want CI to not use previous stored cache
 COMPARE_CACHE_BUSTER="f7e689d76a9"
 INTEGRATION_CACHE_BUSTER="789bacdf"
+CLI_CACHE_BUSTER="789bacdf"
 BOOTSTRAP_CACHE="789bacdf"

 ## Build variables
@ -57,6 +58,10 @@ ifndef SNAPSHOTDIR
 	$(error SNAPSHOTDIR is not set)
 endif

+ifndef REF_NAME
+	REF_NAME = $(VERSION)
+endif
+
 define title
    @printf '$(TITLE)$(1)$(RESET)\n'
 endef
@ -68,7 +73,7 @@ all: clean static-analysis test ## Run all linux-based checks (linting, license
 	@printf '$(SUCCESS)All checks pass!$(RESET)\n'

 .PHONY: test
-test: unit validate-cyclonedx-schema integration acceptance-linux ## Run all tests (currently unit, integration, and linux acceptance tests)
+test: unit validate-cyclonedx-schema integration benchmark acceptance-linux ## Run all tests (currently unit, integration, and linux acceptance tests)

 .PHONY: help
 help:
@ -78,20 +83,31 @@ help:
 ci-bootstrap:
 	DEBIAN_FRONTEND=noninteractive sudo apt update && sudo -E apt install -y bc jq libxml2-utils

-.PHONY: bootstrap
-bootstrap: ## Download and install all go dependencies (+ prep tooling in the ./tmp dir)
-	$(call title,Bootstrapping dependencies)
-	@pwd
-	# prep temp dirs
-	mkdir -p $(TEMPDIR)
+.PHONY:
+ci-bootstrap-mac:
+	github_changelog_generator --version || sudo gem install github_changelog_generator
+
+$(RESULTSDIR):
 	mkdir -p $(RESULTSDIR)
-	# install go dependencies
-	go mod download
-	# install utilities
+
+$(TEMPDIR):
+	mkdir -p $(TEMPDIR)
+
+.PHONY: bootstrap-tools
+bootstrap-tools: $(TEMPDIR)
+	[ -f "$(TEMPDIR)/benchstat" ] || GO111MODULE=off GOBIN=$(shell realpath $(TEMPDIR)) go get -u golang.org/x/perf/cmd/benchstat
 	[ -f "$(TEMPDIR)/golangci" ] || curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b $(TEMPDIR)/ v1.26.0
 	[ -f "$(TEMPDIR)/bouncer" ] || curl -sSfL https://raw.githubusercontent.com/wagoodman/go-bouncer/master/bouncer.sh | sh -s -- -b $(TEMPDIR)/ v0.2.0
 	[ -f "$(TEMPDIR)/goreleaser" ] || curl -sfL https://install.goreleaser.com/github.com/goreleaser/goreleaser.sh | sh -s -- -b $(TEMPDIR)/ v0.140.0

+.PHONY: bootstrap-go
+bootstrap-go:
+	go mod download
+
+.PHONY: bootstrap
+bootstrap: $(RESULTSDIR) bootstrap-go bootstrap-tools ## Download and install all go dependencies (+ prep tooling in the ./tmp dir)
+	$(call title,Bootstrapping dependencies)
+
 .PHONY: static-analysis
 static-analysis: lint check-licenses

@ -124,33 +140,45 @@ validate-cyclonedx-schema:
 	cd schema/cyclonedx && make

 .PHONY: unit
-unit: fixtures ## Run unit tests (with coverage)
+unit: $(RESULTSDIR) fixtures ## Run unit tests (with coverage)
 	$(call title,Running unit tests)
 	go test -coverprofile $(COVER_REPORT) $(shell go list ./... | grep -v anchore/syft/test)
 	@go tool cover -func $(COVER_REPORT) | grep total |  awk '{print substr($$3, 1, length($$3)-1)}' > $(COVER_TOTAL)
 	@echo "Coverage: $$(cat $(COVER_TOTAL))"
 	@if [ $$(echo "$$(cat $(COVER_TOTAL)) >= $(COVERAGE_THRESHOLD)" | bc -l) -ne 1 ]; then echo "$(RED)$(BOLD)Failed coverage quality gate (> $(COVERAGE_THRESHOLD)%)$(RESET)" && false; fi

+.PHONY: benchmark
+benchmark: $(RESULTSDIR) ## Run benchmark tests and compare against the baseline (if available)
+	$(call title,Running benchmark tests)
+	go test -p 1 -run=^Benchmark -bench=. -count=5 -benchmem ./... | tee $(RESULTSDIR)/benchmark-$(REF_NAME).txt
+	(test -s $(RESULTSDIR)/benchmark-main.txt && \
+		$(TEMPDIR)/benchstat $(RESULTSDIR)/benchmark-main.txt $(RESULTSDIR)/benchmark-$(REF_NAME).txt || \
+		$(TEMPDIR)/benchstat $(RESULTSDIR)/benchmark-$(REF_NAME).txt) \
+			| tee $(RESULTSDIR)/benchstat.txt
+
+.PHONY: show-benchstat
+show-benchstat:
+	@cat $(RESULTSDIR)/benchstat.txt
+
 .PHONY: integration
 integration: ## Run integration tests
 	$(call title,Running integration tests)

 	go test -v ./test/integration

-
 # note: this is used by CI to determine if the integration test fixture cache (docker image tars) should be busted
 integration-fingerprint:
 	find test/integration/test-fixtures/image-* -type f -exec md5sum {} + | awk '{print $1}' | sort | md5sum | tee test/integration/test-fixtures/cache.fingerprint && echo "$(INTEGRATION_CACHE_BUSTER)" >> test/integration/test-fixtures/cache.fingerprint

 .PHONY: java-packages-fingerprint
 java-packages-fingerprint:
-	@cd syft/cataloger/java/test-fixtures/java-builds && \
+	@cd syft/pkg/cataloger/java/test-fixtures/java-builds && \
 	make packages.fingerprint

 .PHONY: fixtures
 fixtures:
 	$(call title,Generating test fixtures)
-	cd syft/cataloger/java/test-fixtures/java-builds && make
+	cd syft/pkg/cataloger/java/test-fixtures/java-builds && make

 .PHONY: generate-json-schema
 generate-json-schema:  ## Generate a new json schema
@ -175,7 +203,7 @@ $(SNAPSHOTDIR): ## Build snapshot release binaries and packages

 # note: we cannot clean the snapshot directory since the pipeline builds the snapshot separately
 .PHONY: acceptance-mac
-acceptance-mac: $(SNAPSHOTDIR) ## Run acceptance tests on build snapshot binaries and packages (Mac)
+acceptance-mac: $(RESULTSDIR) $(SNAPSHOTDIR) ## Run acceptance tests on build snapshot binaries and packages (Mac)
 	$(call title,Running acceptance test: Run on Mac)
 	$(ACC_DIR)/mac.sh \
 			$(SNAPSHOTDIR) \
@ -202,7 +230,7 @@ compare:  ## Compare the reports of a run of a main-branch build of syft against
 	@cd test/inline-compare && make

 .PHONY: acceptance-test-deb-package-install
-acceptance-test-deb-package-install: $(SNAPSHOTDIR)
+acceptance-test-deb-package-install: $(RESULTSDIR) $(SNAPSHOTDIR)
 	$(call title,Running acceptance test: DEB install)
 	$(ACC_DIR)/deb.sh \
 			$(SNAPSHOTDIR) \
@ -211,7 +239,7 @@ acceptance-test-deb-package-install: $(SNAPSHOTDIR)
 			$(RESULTSDIR)

 .PHONY: acceptance-test-rpm-package-install
-acceptance-test-rpm-package-install: $(SNAPSHOTDIR)
+acceptance-test-rpm-package-install: $(RESULTSDIR) $(SNAPSHOTDIR)
 	$(call title,Running acceptance test: RPM install)
 	$(ACC_DIR)/rpm.sh \
 			$(SNAPSHOTDIR) \
@ -219,6 +247,23 @@ acceptance-test-rpm-package-install: $(SNAPSHOTDIR)
 			$(ACC_TEST_IMAGE) \
 			$(RESULTSDIR)

+# note: this is used by CI to determine if the integration test fixture cache (docker image tars) should be busted
+cli-fingerprint:
+	find test/cli/test-fixtures/image-* -type f -exec md5sum {} + | awk '{print $1}' | sort | md5sum | tee test/cli/test-fixtures/cache.fingerprint && echo "$(CLI_CACHE_BUSTER)" >> test/cli/test-fixtures/cache.fingerprint
+
+.PHONY: cli-linux
+cli-linux: $(SNAPSHOTDIR) ## Run CLI tests for Linux executable
+	chmod 755 "$(SNAPSHOTDIR)/$(BIN)_linux_amd64/$(BIN)"
+	$(SNAPSHOTDIR)/$(BIN)_linux_amd64/$(BIN) version
+	SYFT_BINARY_LOCATION='$(SNAPSHOTDIR)/$(BIN)_linux_amd64/$(BIN)' \
+		go test -count=1 -v ./test/cli
+
+.PHONY: cli-macos
+cli-macos: $(SNAPSHOTDIR) ## Run CLI tests for macOS executable
+	$(SNAPSHOTDIR)/$(BIN)_linux_amd64/$(BIN) version
+	SYFT_BINARY_LOCATION='$(SNAPSHOTDIR)/$(BIN)-macos_darwin_amd64/$(BIN)' \
+		go test -count=1 -v ./test/cli
+
 .PHONY: changlog-release
 changelog-release:
 	@echo "Last tag: $(SECOND_TO_LAST_TAG)"
--- a/go.mod
+++ b/go.mod
@ -3,13 +3,14 @@ module github.com/anchore/syft
 go 1.14

 require (
+	github.com/acarl005/stripansi v0.0.0-20180116102854-5a71ef0e047d
 	github.com/adrg/xdg v0.2.1
-	github.com/alecthomas/jsonschema v0.0.0-20200530073317-71f438968921
+	github.com/alecthomas/jsonschema v0.0.0-20210301060011-54c507b6f074
 	github.com/anchore/client-go v0.0.0-20210222170800-9c70f9b80bcf
 	github.com/anchore/go-rpmdb v0.0.0-20201106153645-0043963c2e12
 	github.com/anchore/go-testutils v0.0.0-20200925183923-d5f45b0d3c04
 	github.com/anchore/go-version v1.2.2-0.20200701162849-18adb9c92b9b
-	github.com/anchore/stereoscope v0.0.0-20210201165248-e94c52b4052d
+	github.com/anchore/stereoscope v0.0.0-20210317203852-f77bbcbede40
 	github.com/antihax/optional v1.0.0
 	github.com/bmatcuk/doublestar/v2 v2.0.4
 	github.com/docker/docker v17.12.0-ce-rc1.0.20200309214505-aa6a9891b09c+incompatible
@ -31,6 +32,7 @@ require (
 	github.com/sirupsen/logrus v1.6.0
 	github.com/spf13/afero v1.2.2
 	github.com/spf13/cobra v1.0.1-0.20200909172742-8a63648dd905
+	github.com/spf13/pflag v1.0.5
 	github.com/spf13/viper v1.7.0
 	github.com/wagoodman/go-partybus v0.0.0-20200526224238-eb215533f07d
 	github.com/wagoodman/go-progress v0.0.0-20200731105512-1020f39e6240
--- a/go.sum
+++ b/go.sum
@ -96,10 +96,12 @@ github.com/PuerkitoBio/purell v1.1.1/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbt
 github.com/PuerkitoBio/urlesc v0.0.0-20160726150825-5bd2802263f2/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE=
 github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE=
 github.com/StackExchange/wmi v0.0.0-20180116203802-5d049714c4a6/go.mod h1:3eOhrUMpNV+6aFIbp5/iudMxNCF27Vw2OZgy4xEx0Fg=
+github.com/acarl005/stripansi v0.0.0-20180116102854-5a71ef0e047d h1:licZJFw2RwpHMqeKTCYkitsPqHNxTmd4SNR5r94FGM8=
+github.com/acarl005/stripansi v0.0.0-20180116102854-5a71ef0e047d/go.mod h1:asat636LX7Bqt5lYEZ27JNDcqxfjdBQuJ/MM4CN/Lzo=
 github.com/adrg/xdg v0.2.1 h1:VSVdnH7cQ7V+B33qSJHTCRlNgra1607Q8PzEmnvb2Ic=
 github.com/adrg/xdg v0.2.1/go.mod h1:ZuOshBmzV4Ta+s23hdfFZnBsdzmoR3US0d7ErpqSbTQ=
-github.com/alecthomas/jsonschema v0.0.0-20200530073317-71f438968921 h1:T3+cD5fYvuH36h7EZq+TDpm+d8a6FSD4pQsbmuGGQ8o=
-github.com/alecthomas/jsonschema v0.0.0-20200530073317-71f438968921/go.mod h1:/n6+1/DWPltRLWL/VKyUxg6tzsl5kHUCcraimt4vr60=
+github.com/alecthomas/jsonschema v0.0.0-20210301060011-54c507b6f074 h1:Lw9q+WyJLFOR+AULchS5/2GKfM+6gOh4szzizdfH3MU=
+github.com/alecthomas/jsonschema v0.0.0-20210301060011-54c507b6f074/go.mod h1:/n6+1/DWPltRLWL/VKyUxg6tzsl5kHUCcraimt4vr60=
 github.com/alecthomas/kingpin v2.2.6+incompatible/go.mod h1:59OFYbFVLKQKq+mqrL6Rw5bR0c3ACQaawgXx0QYndlE=
 github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
 github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
@ -113,8 +115,8 @@ github.com/anchore/go-testutils v0.0.0-20200925183923-d5f45b0d3c04 h1:VzprUTpc0v
 github.com/anchore/go-testutils v0.0.0-20200925183923-d5f45b0d3c04/go.mod h1:6dK64g27Qi1qGQZ67gFmBFvEHScy0/C8qhQhNe5B5pQ=
 github.com/anchore/go-version v1.2.2-0.20200701162849-18adb9c92b9b h1:e1bmaoJfZVsCYMrIZBpFxwV26CbsuoEh5muXD5I1Ods=
 github.com/anchore/go-version v1.2.2-0.20200701162849-18adb9c92b9b/go.mod h1:Bkc+JYWjMCF8OyZ340IMSIi2Ebf3uwByOk6ho4wne1E=
-github.com/anchore/stereoscope v0.0.0-20210201165248-e94c52b4052d h1:2hv5NOZ0fD8tPk1UdGiW9PHxmjBmBLL+sFlhLXjjKgo=
-github.com/anchore/stereoscope v0.0.0-20210201165248-e94c52b4052d/go.mod h1:lhSEYyGLXTXMIFHAz7Ls/MNQ5EjYd5ziLxovKZp1xOs=
+github.com/anchore/stereoscope v0.0.0-20210317203852-f77bbcbede40 h1:k3/JigkYl7NjMad9eDBBcsg9qiXJFreW6rKNgE0aMUI=
+github.com/anchore/stereoscope v0.0.0-20210317203852-f77bbcbede40/go.mod h1:T/OUHXgngXFlo2vknQsDx+n/jErCLPt5o0H1ZXFBpig=
 github.com/andreyvit/diff v0.0.0-20170406064948-c7f18ee00883/go.mod h1:rCTlJbsFo29Kk6CurOXKm700vrz8f0KW0JNfpkRJY/8=
 github.com/antihax/optional v1.0.0 h1:xK2lYat7ZLaVVcIuj82J8kIro4V6kDe0AUDFboUCwcg=
 github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY=
@ -737,7 +739,6 @@ github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyC
 github.com/valyala/fasthttp v1.2.0/go.mod h1:4vX61m6KN+xDduDNwXrhIAVZaZaZiQ1luJk8LWSxF3s=
 github.com/valyala/quicktemplate v1.2.0/go.mod h1:EH+4AkTd43SvgIbQHYu59/cJyxDoOVRUAfrukLPuGJ4=
 github.com/valyala/tcplisten v0.0.0-20161114210144-ceec8f93295a/go.mod h1:v3UYOV9WzVtRmSR+PDvWpU/qWl4Wa5LApYYX4ZtKbio=
-github.com/vbatts/tar-split v0.11.1/go.mod h1:LEuURwDEiWjRjwu46yU3KVGuUdVv/dcnpcEPSzR8z6g=
 github.com/vdemeester/k8s-pkg-credentialprovider v1.17.4/go.mod h1:inCTmtUdr5KJbreVojo06krnTgaeAz/Z7lynpPk/Q2c=
 github.com/vmware/govmomi v0.20.3/go.mod h1:URlwyTFZX72RmxtxuaFL2Uj3fD1JTvZdx59bHWk6aFU=
 github.com/wagoodman/go-partybus v0.0.0-20200526224238-eb215533f07d h1:KOxOL6qpmqwoPloNwi+CEgc1ayjHNOFNrvoOmeDOjDg=