diff --git a/.github/actions/bootstrap/action.yaml b/.github/actions/bootstrap/action.yaml new file mode 100644 index 000000000..cd3d1c6c7 --- /dev/null +++ b/.github/actions/bootstrap/action.yaml @@ -0,0 +1,81 @@ +name: "Bootstrap" +description: "Bootstrap all tools and dependencies" +inputs: + go-version: + description: "Go version to install" + required: true + default: "1.18.x" + use-go-cache: + description: "Restore go cache" + required: true + default: "true" + cache-key-prefix: + description: "Prefix all cache keys with this value" + required: true + default: "831180ac25" + build-cache-key-prefix: + description: "Prefix build cache key with this value" + required: true + default: "f8b6d31dea" + bootstrap-apt-packages: + description: "Space delimited list of tools to install via apt" + default: "libxml2-utils" + +runs: + using: "composite" + steps: + - uses: actions/setup-go@v3 + with: + go-version: ${{ inputs.go-version }} + + - name: Restore tool cache + id: tool-cache + uses: actions/cache@v3 + with: + path: ${{ github.workspace }}/.tmp + key: ${{ inputs.cache-key-prefix }}-${{ runner.os }}-tool-${{ hashFiles('Makefile') }} + + # note: we need to keep restoring the go mod cache before bootstrapping tools since `go install` is used in + # some installations of project tools. + - name: Restore go module cache + id: go-mod-cache + if: inputs.use-go-cache == 'true' + uses: actions/cache@v3 + with: + path: | + ~/go/pkg/mod + key: ${{ inputs.cache-key-prefix }}-${{ runner.os }}-go-${{ inputs.go-version }}-${{ hashFiles('**/go.sum') }} + restore-keys: | + ${{ inputs.cache-key-prefix }}-${{ runner.os }}-go-${{ inputs.go-version }}- + + - name: (cache-miss) Bootstrap project tools + shell: bash + if: steps.tool-cache.outputs.cache-hit != 'true' + run: make bootstrap-tools + + - name: Restore go build cache + id: go-cache + if: inputs.use-go-cache == 'true' + uses: actions/cache@v3 + with: + path: | + ~/.cache/go-build + key: ${{ inputs.cache-key-prefix }}-${{ inputs.build-cache-key-prefix }}-${{ runner.os }}-go-${{ inputs.go-version }}-${{ hashFiles('**/go.sum') }} + restore-keys: | + ${{ inputs.cache-key-prefix }}-${{ inputs.build-cache-key-prefix }}-${{ runner.os }}-go-${{ inputs.go-version }}- + + - name: (cache-miss) Bootstrap go dependencies + shell: bash + if: steps.go-mod-cache.outputs.cache-hit != 'true' && inputs.use-go-cache == 'true' + run: make bootstrap-go + + - name: Install apt packages + if: inputs.bootstrap-apt-packages != '' + shell: bash + run: | + DEBIAN_FRONTEND=noninteractive sudo apt update && sudo -E apt install -y ${{ inputs.bootstrap-apt-packages }} + + - name: Create all cache fingerprints + shell: bash + run: make fingerprints + diff --git a/.github/scripts/ci-check.sh b/.github/scripts/ci-check.sh new file mode 100755 index 000000000..b507c327f --- /dev/null +++ b/.github/scripts/ci-check.sh @@ -0,0 +1,11 @@ +#!/usr/bin/env bash + +red=$(tput setaf 1) +bold=$(tput bold) +normal=$(tput sgr0) + +# assert we are running in CI (or die!) +if [[ -z "$CI" ]]; then + echo "${bold}${red}This script should ONLY be run in CI. Exiting...${normal}" + exit 1 +fi diff --git a/.github/scripts/coverage.py b/.github/scripts/coverage.py new file mode 100755 index 000000000..db14135cd --- /dev/null +++ b/.github/scripts/coverage.py @@ -0,0 +1,36 @@ +#!/usr/bin/env python3 +import subprocess +import sys +import shlex + + +class bcolors: + HEADER = '\033[95m' + OKBLUE = '\033[94m' + OKCYAN = '\033[96m' + OKGREEN = '\033[92m' + WARNING = '\033[93m' + FAIL = '\033[91m' + ENDC = '\033[0m' + BOLD = '\033[1m' + UNDERLINE = '\033[4m' + + +if len(sys.argv) < 3: + print("Usage: coverage.py [threshold] [go-coverage-report]") + sys.exit(1) + + +threshold = float(sys.argv[1]) +report = sys.argv[2] + + +args = shlex.split(f"go tool cover -func {report}") +p = subprocess.run(args, capture_output=True, text=True) + +percent_coverage = float(p.stdout.splitlines()[-1].split()[-1].replace("%", "")) +print(f"{bcolors.BOLD}Coverage: {percent_coverage}%{bcolors.ENDC}") + +if percent_coverage < threshold: + print(f"{bcolors.BOLD}{bcolors.FAIL}Coverage below threshold of {threshold}%{bcolors.ENDC}") + sys.exit(1) diff --git a/.github/scripts/go-mod-tidy-check.sh b/.github/scripts/go-mod-tidy-check.sh index 41bc63910..6ba8333e8 100755 --- a/.github/scripts/go-mod-tidy-check.sh +++ b/.github/scripts/go-mod-tidy-check.sh @@ -4,19 +4,18 @@ set -eu ORIGINAL_STATE_DIR=$(mktemp -d "TEMP-original-state-XXXXXXXXX") TIDY_STATE_DIR=$(mktemp -d "TEMP-tidy-state-XXXXXXXXX") -trap "cp -v ${ORIGINAL_STATE_DIR}/* ./ && rm -fR ${ORIGINAL_STATE_DIR} ${TIDY_STATE_DIR}" EXIT +trap "cp ${ORIGINAL_STATE_DIR}/* ./ && rm -fR ${ORIGINAL_STATE_DIR} ${TIDY_STATE_DIR}" EXIT -echo "Capturing original state of files..." -cp -v go.mod go.sum "${ORIGINAL_STATE_DIR}" +# capturing original state of files... +cp go.mod go.sum "${ORIGINAL_STATE_DIR}" -echo "Capturing state of go.mod and go.sum after running go mod tidy..." +# capturing state of go.mod and go.sum after running go mod tidy... go mod tidy -cp -v go.mod go.sum "${TIDY_STATE_DIR}" -echo "" +cp go.mod go.sum "${TIDY_STATE_DIR}" set +e -# Detect difference between the git HEAD state and the go mod tidy state +# detect difference between the git HEAD state and the go mod tidy state DIFF_MOD=$(diff -u "${ORIGINAL_STATE_DIR}/go.mod" "${TIDY_STATE_DIR}/go.mod") DIFF_SUM=$(diff -u "${ORIGINAL_STATE_DIR}/go.sum" "${TIDY_STATE_DIR}/go.sum") diff --git a/.github/workflows/benchmark-testing.yaml b/.github/workflows/benchmark-testing.yaml new file mode 100644 index 000000000..4cd87594b --- /dev/null +++ b/.github/workflows/benchmark-testing.yaml @@ -0,0 +1,58 @@ +name: "Benchmark testing" + +on: + workflow_dispatch: + pull_request: + +jobs: + + Benchmark-Test: + name: "Benchmark tests" + runs-on: ubuntu-20.04 + # note: we want benchmarks to run on pull_request events in order to publish results to a sticky comment, and + # we also want to run on push such that merges to main are recorded to the cache. For this reason we don't filter + # the job by event. + steps: + - uses: actions/checkout@v3 + + - name: Bootstrap environment + uses: ./.github/actions/bootstrap + + - name: Restore base benchmark result + uses: actions/cache@v3 + with: + path: test/results/benchmark-main.txt + # use base sha for PR or new commit hash for main push in benchmark result key + key: ${{ runner.os }}-bench-${{ (github.event.pull_request.base.sha != github.event.after) && github.event.pull_request.base.sha || github.event.after }} + + - name: Run benchmark tests + id: benchmark + run: | + REF_NAME=${GITHUB_REF##*/} make benchmark + OUTPUT=$(make show-benchstat) + OUTPUT="${OUTPUT//'%'/'%25'}" # URL encode all '%' characters + OUTPUT="${OUTPUT//$'\n'/'%0A'}" # URL encode all '\n' characters + OUTPUT="${OUTPUT//$'\r'/'%0D'}" # URL encode all '\r' characters + echo "::set-output name=result::$OUTPUT" + + - uses: actions/upload-artifact@v3 + with: + name: benchmark-test-results + path: test/results/**/* + + - name: Update PR benchmark results comment + uses: marocchino/sticky-pull-request-comment@v2 + continue-on-error: true + with: + header: benchmark + message: | + ### Benchmark Test Results + +
+ Benchmark results from the latest changes vs base branch + + ``` + ${{ steps.benchmark.outputs.result }} + ``` + +
diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index 4199a8504..239faf683 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -16,7 +16,7 @@ jobs: environment: release runs-on: ubuntu-20.04 steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 # we don't want to release commits that have been pushed and tagged, but not necessarily merged onto main - name: Ensure tagged commit is on main @@ -26,7 +26,7 @@ jobs: git merge-base --is-ancestor ${GITHUB_REF##*/} origin/main && echo "${GITHUB_REF##*/} is a commit on main!" - name: Check static analysis results - uses: fountainhead/action-wait-for-check@v1.0.0 + uses: fountainhead/action-wait-for-check@v1.1.0 id: static-analysis with: token: ${{ secrets.GITHUB_TOKEN }} @@ -35,7 +35,7 @@ jobs: ref: ${{ github.event.pull_request.head.sha || github.sha }} - name: Check unit test results - uses: fountainhead/action-wait-for-check@v1.0.0 + uses: fountainhead/action-wait-for-check@v1.1.0 id: unit with: token: ${{ secrets.GITHUB_TOKEN }} @@ -44,7 +44,7 @@ jobs: ref: ${{ github.event.pull_request.head.sha || github.sha }} - name: Check integration test results - uses: fountainhead/action-wait-for-check@v1.0.0 + uses: fountainhead/action-wait-for-check@v1.1.0 id: integration with: token: ${{ secrets.GITHUB_TOKEN }} @@ -53,7 +53,7 @@ jobs: ref: ${{ github.event.pull_request.head.sha || github.sha }} - name: Check acceptance test results (linux) - uses: fountainhead/action-wait-for-check@v1.0.0 + uses: fountainhead/action-wait-for-check@v1.1.0 id: acceptance-linux with: token: ${{ secrets.GITHUB_TOKEN }} @@ -62,7 +62,7 @@ jobs: ref: ${{ github.event.pull_request.head.sha || github.sha }} - name: Check acceptance test results (mac) - uses: fountainhead/action-wait-for-check@v1.0.0 + uses: fountainhead/action-wait-for-check@v1.1.0 id: acceptance-mac with: token: ${{ secrets.GITHUB_TOKEN }} @@ -71,7 +71,7 @@ jobs: ref: ${{ github.event.pull_request.head.sha || github.sha }} - name: Check cli test results (linux) - uses: fountainhead/action-wait-for-check@v1.0.0 + uses: fountainhead/action-wait-for-check@v1.1.0 id: cli-linux with: token: ${{ secrets.GITHUB_TOKEN }} @@ -97,33 +97,13 @@ jobs: contents: write packages: write steps: - - uses: actions/setup-go@v2 - with: - go-version: ${{ env.GO_VERSION }} + - uses: actions/checkout@v3 - - uses: actions/checkout@v2 + - name: Bootstrap environment + uses: ./.github/actions/bootstrap with: - fetch-depth: 0 - - - name: Restore tool cache - id: tool-cache - uses: actions/cache@v2.1.3 - with: - path: ${{ github.workspace }}/.tmp - key: ${{ runner.os }}-tool-${{ hashFiles('Makefile') }} - - - name: Restore go cache - id: go-cache - uses: actions/cache@v2.1.3 - with: - path: ~/go/pkg/mod - key: ${{ runner.os }}-go-${{ env.GO_VERSION }}-${{ hashFiles('**/go.sum') }} - restore-keys: | - ${{ runner.os }}-go-${{ env.GO_VERSION }}- - - - name: (cache-miss) Bootstrap all project dependencies - if: steps.tool-cache.outputs.cache-hit != 'true' || steps.go-cache.outputs.cache-hit != 'true' - run: make bootstrap + # use the same cache we used for building snapshots + build-cache-key-prefix: "snapshot" - name: Login to Docker Hub uses: docker/login-action@v2 diff --git a/.github/workflows/validations.yaml b/.github/workflows/validations.yaml index b01a2d829..dbb376e75 100644 --- a/.github/workflows/validations.yaml +++ b/.github/workflows/validations.yaml @@ -1,14 +1,11 @@ name: "Validations" + on: workflow_dispatch: + pull_request: push: branches: - main - pull_request: - -env: - GO_VERSION: "1.18.x" - GO_STABLE_VERSION: true jobs: @@ -17,100 +14,42 @@ jobs: name: "Static analysis" runs-on: ubuntu-20.04 steps: - - uses: actions/setup-go@v2 - with: - go-version: ${{ env.GO_VERSION }} - stable: ${{ env.GO_STABLE_VERSION }} + - uses: actions/checkout@v3 - - uses: actions/checkout@v2 - - - name: Restore tool cache - id: tool-cache - uses: actions/cache@v2.1.3 - with: - path: ${{ github.workspace }}/.tmp - key: ${{ runner.os }}-tool-${{ hashFiles('Makefile') }} - - - name: Restore go cache - id: go-cache - uses: actions/cache@v2.1.3 - with: - path: ~/go/pkg/mod - key: ${{ runner.os }}-go-${{ env.GO_VERSION }}-${{ hashFiles('**/go.sum') }} - restore-keys: | - ${{ runner.os }}-go-${{ env.GO_VERSION }}- - - - name: (cache-miss) Bootstrap all project dependencies - if: steps.tool-cache.outputs.cache-hit != 'true' || steps.go-cache.outputs.cache-hit != 'true' - run: make bootstrap - - - name: Bootstrap CI environment dependencies - run: make ci-bootstrap + - name: Bootstrap environment + uses: ./.github/actions/bootstrap - name: Run static analysis run: make static-analysis + Unit-Test: # Note: changing this job name requires making the same update in the .github/workflows/release.yaml pipeline name: "Unit tests" runs-on: ubuntu-20.04 steps: - - uses: actions/setup-go@v2 - with: - go-version: ${{ env.GO_VERSION }} - stable: ${{ env.GO_STABLE_VERSION }} + - uses: actions/checkout@v3 - - uses: actions/checkout@v2 - - - name: Restore tool cache - id: tool-cache - uses: actions/cache@v2.1.3 - with: - path: ${{ github.workspace }}/.tmp - key: ${{ runner.os }}-tool-${{ hashFiles('Makefile') }} - - - name: Restore go cache - id: go-cache - uses: actions/cache@v2.1.3 - with: - path: ~/go/pkg/mod - key: ${{ runner.os }}-go-${{ env.GO_VERSION }}-${{ hashFiles('**/go.sum') }} - restore-keys: | - ${{ runner.os }}-go-${{ env.GO_VERSION }}- - - - name: (cache-miss) Bootstrap all project dependencies - if: steps.tool-cache.outputs.cache-hit != 'true' || steps.go-cache.outputs.cache-hit != 'true' - run: make bootstrap - - - name: Bootstrap CI environment dependencies - run: make ci-bootstrap - - - name: Build cache key for java test-fixture blobs (for unit tests) - run: make java-packages-fingerprint + - name: Bootstrap environment + uses: ./.github/actions/bootstrap - name: Restore Java test-fixture cache id: unit-java-cache - uses: actions/cache@v2.1.3 + uses: actions/cache@v3 with: path: syft/pkg/cataloger/java/test-fixtures/java-builds/packages key: ${{ runner.os }}-unit-java-cache-${{ hashFiles( 'syft/pkg/cataloger/java/test-fixtures/java-builds/packages.fingerprint' ) }} - - name: Build cache key for rpm test-fixture blobs (for unit tests) - run: make rpm-binaries-fingerprint - - name: Restore RPM test-fixture cache id: unit-rpm-cache - uses: actions/cache@v2.1.3 + uses: actions/cache@v3 with: path: syft/pkg/cataloger/rpm/test-fixtures/rpms key: ${{ runner.os }}-unit-rpm-cache-${{ hashFiles( 'syft/pkg/cataloger/rpm/test-fixtures/rpms.fingerprint' ) }} - - name: Build cache key for go binary test-fixture blobs (for unit tests) - run: make go-binaries-fingerprint - - - name: Restore Go binary test-fixture cache + - name: Restore go binary test-fixture cache id: unit-go-binary-cache - uses: actions/cache@v2.1.3 + uses: actions/cache@v3 with: path: syft/pkg/cataloger/golang/test-fixtures/archs/binaries key: ${{ runner.os }}-unit-go-binaries-cache-${{ hashFiles( 'syft/pkg/cataloger/golang/test-fixtures/archs/binaries.fingerprint' ) }} @@ -118,54 +57,22 @@ jobs: - name: Run unit tests run: make unit - - uses: actions/upload-artifact@v2 - with: - name: unit-test-results - path: test/results/**/* Integration-Test: # Note: changing this job name requires making the same update in the .github/workflows/release.yaml pipeline name: "Integration tests" runs-on: ubuntu-20.04 steps: - - uses: actions/setup-go@v2 - with: - go-version: ${{ env.GO_VERSION }} - stable: ${{ env.GO_STABLE_VERSION }} + - uses: actions/checkout@v3 - - uses: actions/checkout@v2 - - - name: Restore tool cache - id: tool-cache - uses: actions/cache@v2.1.3 - with: - path: ${{ github.workspace }}/.tmp - key: ${{ runner.os }}-tool-${{ hashFiles('Makefile') }} - - - name: Restore go cache - id: go-cache - uses: actions/cache@v2.1.3 - with: - path: ~/go/pkg/mod - key: ${{ runner.os }}-go-${{ env.GO_VERSION }}-${{ hashFiles('**/go.sum') }} - restore-keys: | - ${{ runner.os }}-go-${{ env.GO_VERSION }}- - - - name: (cache-miss) Bootstrap all project dependencies - if: steps.tool-cache.outputs.cache-hit != 'true' || steps.go-cache.outputs.cache-hit != 'true' - run: make bootstrap - - - name: Bootstrap CI environment dependencies - run: make ci-bootstrap + - name: Bootstrap environment + uses: ./.github/actions/bootstrap - name: Validate syft output against the CycloneDX schema run: make validate-cyclonedx-schema - - name: Build key for tar cache - run: make integration-fingerprint - - name: Restore integration test cache - uses: actions/cache@v2.1.3 + uses: actions/cache@v3 with: path: ${{ github.workspace }}/test/integration/test-fixtures/cache key: ${{ runner.os }}-integration-test-cache-${{ hashFiles('test/integration/test-fixtures/cache.fingerprint') }} @@ -173,123 +80,35 @@ jobs: - name: Run integration tests run: make integration - Benchmark-Test: - name: "Benchmark tests" - runs-on: ubuntu-20.04 - # note: we want benchmarks to run on pull_request events in order to publish results to a sticky comment, and - # we also want to run on push such that merges to main are recorded to the cache. For this reason we don't filter - # the job by event. - steps: - - uses: actions/setup-go@v2 - with: - go-version: ${{ env.GO_VERSION }} - stable: ${{ env.GO_STABLE_VERSION }} - - - uses: actions/checkout@v2 - - - name: Restore tool cache - id: tool-cache - uses: actions/cache@v2.1.3 - with: - path: ${{ github.workspace }}/.tmp - key: ${{ runner.os }}-tool-${{ hashFiles('Makefile') }} - - - name: Restore go cache - id: go-cache - uses: actions/cache@v2.1.3 - with: - path: ~/go/pkg/mod - key: ${{ runner.os }}-go-${{ env.GO_VERSION }}-${{ hashFiles('**/go.sum') }} - restore-keys: | - ${{ runner.os }}-go-${{ env.GO_VERSION }}- - - - name: (cache-miss) Bootstrap all project dependencies - if: steps.tool-cache.outputs.cache-hit != 'true' || steps.go-cache.outputs.cache-hit != 'true' - run: make bootstrap - - - name: Bootstrap CI environment dependencies - run: make ci-bootstrap - - - name: Restore base benchmark result - uses: actions/cache@v2 - with: - path: test/results/benchmark-main.txt - # use base sha for PR or new commit hash for main push in benchmark result key - key: ${{ runner.os }}-bench-${{ (github.event.pull_request.base.sha != github.event.after) && github.event.pull_request.base.sha || github.event.after }} - - - name: Run benchmark tests - id: benchmark - run: | - REF_NAME=${GITHUB_REF##*/} make benchmark - OUTPUT=$(make show-benchstat) - OUTPUT="${OUTPUT//'%'/'%25'}" # URL encode all '%' characters - OUTPUT="${OUTPUT//$'\n'/'%0A'}" # URL encode all '\n' characters - OUTPUT="${OUTPUT//$'\r'/'%0D'}" # URL encode all '\r' characters - echo "::set-output name=result::$OUTPUT" - - - uses: actions/upload-artifact@v2 - with: - name: benchmark-test-results - path: test/results/**/* - - - name: Update PR benchmark results comment - uses: marocchino/sticky-pull-request-comment@v2 - continue-on-error: true - with: - header: benchmark - message: | - ### Benchmark Test Results - -
- Benchmark results from the latest changes vs base branch - - ``` - ${{ steps.benchmark.outputs.result }} - ``` - -
Build-Snapshot-Artifacts: name: "Build snapshot artifacts" runs-on: ubuntu-20.04 steps: - - uses: actions/setup-go@v2 + - uses: actions/checkout@v3 + + - name: Bootstrap environment + uses: ./.github/actions/bootstrap with: - go-version: ${{ env.GO_VERSION }} - stable: ${{ env.GO_STABLE_VERSION }} - - - uses: actions/checkout@v2 - - - name: Set up QEMU - uses: docker/setup-qemu-action@v1 - - - name: Restore tool cache - id: tool-cache - uses: actions/cache@v2.1.3 - with: - path: ${{ github.workspace }}/.tmp - key: ${{ runner.os }}-tool-${{ hashFiles('Makefile') }} - - - name: Restore go cache - id: go-cache - uses: actions/cache@v2.1.3 - with: - path: ~/go/pkg/mod - key: ${{ runner.os }}-go-${{ env.GO_VERSION }}-${{ hashFiles('**/go.sum') }} - restore-keys: | - ${{ runner.os }}-go-${{ env.GO_VERSION }}- - - - name: (cache-miss) Bootstrap all project dependencies - if: steps.tool-cache.outputs.cache-hit != 'true' || steps.go-cache.outputs.cache-hit != 'true' - run: make bootstrap + # why have another build cache key? We don't want unit/integration/etc test build caches to replace + # the snapshot build cache, which includes builds for all OSs and architectures. As long as this key is + # unique from the build-cache-key-prefix in other CI jobs, we should be fine. + # + # note: ideally this value should match what is used in release (just to help with build times). + build-cache-key-prefix: "snapshot" + bootstrap-apt-packages: "" - name: Build snapshot artifacts run: make snapshot - - uses: actions/upload-artifact@v2 + # why not use actions/upload-artifact? It is very slow (3 minutes to upload ~600MB of data, vs 10 seconds with this approach). + # see https://github.com/actions/upload-artifact/issues/199 for more info + - name: Upload snapshot artifacts + uses: actions/cache/save@v3 with: - name: artifacts - path: snapshot/**/* + path: snapshot + key: snapshot-build-${{ github.run_id }} + Acceptance-Linux: # Note: changing this job name requires making the same update in the .github/workflows/release.yaml pipeline @@ -297,22 +116,20 @@ jobs: needs: [Build-Snapshot-Artifacts] runs-on: ubuntu-20.04 steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 - - uses: actions/download-artifact@v2 + - name: Download snapshot build + uses: actions/cache/restore@v3 with: - name: artifacts path: snapshot + key: snapshot-build-${{ github.run_id }} - name: Run comparison tests (Linux) run: make compare-linux - - name: Build key for image cache - run: make install-fingerprint - - name: Restore install.sh test image cache id: install-test-image-cache - uses: actions/cache@v2.1.3 + uses: actions/cache@v3 with: path: ${{ github.workspace }}/test/install/cache key: ${{ runner.os }}-install-test-image-cache-${{ hashFiles('test/install/cache.fingerprint') }} @@ -335,16 +152,17 @@ jobs: needs: [Build-Snapshot-Artifacts] runs-on: macos-latest steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 - - uses: actions/download-artifact@v2 + - name: Download snapshot build + uses: actions/cache/restore@v3 with: - name: artifacts path: snapshot + key: snapshot-build-${{ github.run_id }} - name: Restore docker image cache for compare testing id: mac-compare-testing-cache - uses: actions/cache@v2.1.3 + uses: actions/cache@v3 with: path: image.tar key: ${{ runner.os }}-${{ hashFiles('test/compare/mac.sh') }} @@ -355,52 +173,29 @@ jobs: - name: Run install.sh tests (Mac) run: make install-test-ci-mac + Cli-Linux: # Note: changing this job name requires making the same update in the .github/workflows/release.yaml pipeline name: "CLI tests (Linux)" needs: [Build-Snapshot-Artifacts] runs-on: ubuntu-20.04 steps: - - uses: actions/setup-go@v2 - with: - go-version: ${{ env.GO_VERSION }} - stable: ${{ env.GO_STABLE_VERSION }} + - uses: actions/checkout@v3 - - uses: actions/checkout@v2 + - name: Bootstrap environment + uses: ./.github/actions/bootstrap - - name: Restore go cache - id: go-cache - uses: actions/cache@v2.1.3 - with: - path: ~/go/pkg/mod - key: ${{ runner.os }}-go-${{ env.GO_VERSION }}-${{ hashFiles('**/go.sum') }} - restore-keys: | - ${{ runner.os }}-go-${{ env.GO_VERSION }}- - - - name: Restore tool cache - id: tool-cache - uses: actions/cache@v2.1.3 - with: - path: ${{ github.workspace }}/.tmp - key: ${{ runner.os }}-tool-${{ hashFiles('Makefile') }} - - - name: (cache-miss) Bootstrap all project dependencies - if: steps.tool-cache.outputs.cache-hit != 'true' || steps.go-cache.outputs.cache-hit != 'true' - run: make bootstrap - - - name: Build key for tar cache - run: make cli-fingerprint - - - name: Restore CLI test cache - uses: actions/cache@v2.1.3 + - name: Restore CLI test-fixture cache + uses: actions/cache@v3 with: path: ${{ github.workspace }}/test/cli/test-fixtures/cache key: ${{ runner.os }}-cli-test-cache-${{ hashFiles('test/cli/test-fixtures/cache.fingerprint') }} - - uses: actions/download-artifact@v2 + - name: Download snapshot build + uses: actions/cache/restore@v3 with: - name: artifacts path: snapshot + key: snapshot-build-${{ github.run_id }} - name: Run CLI Tests (Linux) run: make cli diff --git a/.goreleaser.yaml b/.goreleaser.yaml index b22088fa5..d4e05af8e 100644 --- a/.goreleaser.yaml +++ b/.goreleaser.yaml @@ -5,6 +5,7 @@ release: env: # required to support multi architecture docker builds - DOCKER_CLI_EXPERIMENTAL=enabled + - CGO_ENABLED=0 builds: - id: linux-build @@ -19,8 +20,6 @@ builds: - s390x # set the modified timestamp on the output binary to the git timestamp to ensure a reproducible build mod_timestamp: &build-timestamp '{{ .CommitTimestamp }}' - env: &build-env - - CGO_ENABLED=0 ldflags: &build-ldflags | -w -s @@ -39,7 +38,6 @@ builds: - amd64 - arm64 mod_timestamp: *build-timestamp - env: *build-env ldflags: *build-ldflags hooks: post: @@ -55,7 +53,6 @@ builds: goarch: - amd64 mod_timestamp: *build-timestamp - env: *build-env ldflags: *build-ldflags archives: diff --git a/DEVELOPING.md b/DEVELOPING.md index 35f958f6a..e5242647d 100644 --- a/DEVELOPING.md +++ b/DEVELOPING.md @@ -15,37 +15,16 @@ After cloning the following step can help you get setup: - this command `go run cmd/syft/main.go alpine:latest` will compile and run syft against `alpine:latest` 5. view the README or syft help output for more output options -#### Make output The main make tasks for common static analysis and testing are `lint`, `lint-fix`, `unit`, `integration`, and `cli`. -``` -all Run all linux-based checks (linting, license check, unit, integration, and linux compare tests) -benchmark Run benchmark tests and compare against the baseline (if available) -bootstrap Download and install all tooling dependencies (+ prep tooling in the ./tmp dir) -build Build release snapshot binaries and packages -check-licenses Ensure transitive dependencies are compliant with the current license policy -clean-test-image-cache Clean test image cache -clean Remove previous builds, result reports, and test cache -cli Run CLI tests -compare-linux Run compare tests on build snapshot binaries and packages (Linux) -compare-mac Run compare tests on build snapshot binaries and packages (Mac) -generate-json-schema Generate a new json schema -generate-license-list Generate an updated spdx license list -help Display this help -integration Run integration tests -lint-fix Auto-format all source code + run golangci lint fixers -lint Run gofmt + golangci lint checks -show-test-image-cache Show all docker and image tar cache -show-test-snapshots Show all test snapshots -snapshot-with-signing Build snapshot release binaries and packages (with dummy signing) -test Run all tests (currently unit, integration, linux compare, and cli tests) -unit Run unit tests (with coverage) -``` + +See `make help` for all the current make tasks. ## Architecture Syft is used to generate a Software Bill of Materials (SBOM) from different kinds of input. ### Code organization for the cmd package + Syft's entrypoint can be found in the `cmd` package at `cmd/syft/main.go`. `main.go` builds a new syft `cli` via `cli.New()` and then executes the `cli` via `cli.Execute()`. The `cli` package is responsible for parsing command line arguments, setting up the application context and configuration, and executing the application. Each of syft's commands @@ -55,23 +34,24 @@ They are registered in `syft/cli/commands/go`. . └── syft/ ├── cli/ - │   ├── attest/ - │   ├── attest.go - │   ├── commands.go - │   ├── completion.go - │   ├── convert/ - │   ├── convert.go - │   ├── eventloop/ - │   ├── options/ - │   ├── packages/ - │   ├── packages.go - │   ├── poweruser/ - │   ├── poweruser.go - │   └── version.go + │ ├── attest/ + │ ├── attest.go + │ ├── commands.go + │ ├── completion.go + │ ├── convert/ + │ ├── convert.go + │ ├── eventloop/ + │ ├── options/ + │ ├── packages/ + │ ├── packages.go + │ ├── poweruser/ + │ ├── poweruser.go + │ └── version.go └── main.go ``` #### Execution flow + ```mermaid sequenceDiagram participant main as cmd/syft/main @@ -100,19 +80,19 @@ sequenceDiagram Syft's core library (see, exported) functionality is implemented in the `syft` package. The `syft` package is responsible for organizing the core SBOM data model, it's translated output formats, and the core SBOM generation logic. -#### Organization and design notes for the syft library - analysis creates a static SBOM which can be encoded and decoded - format objects, should strive to not add or enrich data in encoding that could otherwise be done during analysis - package catalogers and their organization can be viewed/added to the `syft/pkg/cataloger` package - file catalogers and their organization can be viewed/added to the `syft/file` package - The source package provides an abstraction to allow a user to loosely define a data source that can be cataloged -- Logging Abstraction ... #### Code example of syft as a library -Here is a gist of using syft as a library to generate a SBOM from a docker image: [link](https://gist.github.com/wagoodman/57ed59a6d57600c23913071b8470175b). + +Here is a gist of using syft as a library to generate a SBOM for a docker image: [link](https://gist.github.com/wagoodman/57ed59a6d57600c23913071b8470175b). The execution flow for the example is detailed below. #### Execution flow examples for the syft library + ```mermaid sequenceDiagram participant source as source.New(ubuntu:latest) @@ -135,12 +115,14 @@ sequenceDiagram ### Syft Catalogers ##### Summary + Catalogers are the way in which syft is able to identify and construct packages given some amount of source metadata. For example, Syft can locate and process `package-lock.json` files when performing filesystem scans. See: [how to specify file globs](https://github.com/anchore/syft/blob/main/syft/pkg/cataloger/javascript/cataloger.go#L16-L21) and an implementation of the [package-lock.json parser](https://github.com/anchore/syft/blob/main/syft/pkg/cataloger/javascript/cataloger.go#L16-L21) fora quick review. #### Building a new Cataloger + Catalogers must fulfill the interface [found here](https://github.com/anchore/syft/blob/main/syft/pkg/cataloger.go). This means that when building a new cataloger, the new struct must implement both method signatures of `Catalog` and `Name`. diff --git a/Makefile b/Makefile index 40716e502..3750e8678 100644 --- a/Makefile +++ b/Makefile @@ -1,24 +1,23 @@ -BIN = syft -VERSION=$(shell git describe --dirty --always --tags) -TEMPDIR = ./.tmp +BIN := syft +TEMP_DIR := ./.tmp -# commands and versions -LINTCMD = $(TEMPDIR)/golangci-lint run --tests=false -GOIMPORTS_CMD = $(TEMPDIR)/gosimports -local github.com/anchore -RELEASE_CMD=$(TEMPDIR)/goreleaser release --rm-dist -SNAPSHOT_CMD=$(RELEASE_CMD) --skip-publish --snapshot +# Command templates ################################# +LINT_CMD := $(TEMP_DIR)/golangci-lint run --tests=false +GOIMPORTS_CMD := $(TEMP_DIR)/gosimports -local github.com/anchore +RELEASE_CMD := $(TEMP_DIR)/goreleaser release --rm-dist +SNAPSHOT_CMD := $(RELEASE_CMD) --skip-publish --skip-sign --snapshot -# tool versions -GOLANGCILINT_VERSION = v1.50.1 -GOSIMPORTS_VERSION = v0.3.5 -BOUNCER_VERSION = v0.4.0 -CHRONICLE_VERSION = v0.4.2 -GORELEASER_VERSION = v1.14.1 -YAJSV_VERSION = v1.4.1 -COSIGN_VERSION = v1.13.1 -QUILL_VERSION = v0.2.0 +# Tool versions ################################# +GOLANGCILINT_VERSION := v1.50.1 +GOSIMPORTS_VERSION := v0.3.5 +BOUNCER_VERSION := v0.4.0 +CHRONICLE_VERSION := v0.4.2 +GORELEASER_VERSION := v1.14.1 +YAJSV_VERSION := v1.4.1 +COSIGN_VERSION := v1.13.1 +QUILL_VERSION := v0.2.0 -# formatting variables +# Formatting variables ################################# BOLD := $(shell tput -T linux bold) PURPLE := $(shell tput -T linux setaf 5) GREEN := $(shell tput -T linux setaf 2) @@ -28,55 +27,23 @@ RESET := $(shell tput -T linux sgr0) TITLE := $(BOLD)$(PURPLE) SUCCESS := $(BOLD)$(GREEN) -# test variables -RESULTSDIR = test/results -COMPARE_DIR = ./test/compare -COMPARE_TEST_IMAGE = centos:8.2.2004 -COVER_REPORT = $(RESULTSDIR)/unit-coverage-details.txt -COVER_TOTAL = $(RESULTSDIR)/unit-coverage-summary.txt -# the quality gate lower threshold for unit test total % coverage (by function statements) -COVERAGE_THRESHOLD := 62 +# Test variables ################################# +COMPARE_DIR := ./test/compare +COMPARE_TEST_IMAGE := centos:8.2.2004 +COVERAGE_THRESHOLD := 62 # the quality gate lower threshold for unit test total % coverage (by function statements) -# CI cache busting values; change these if you want CI to not use previous stored cache -INTEGRATION_CACHE_BUSTER="894d8ca" -CLI_CACHE_BUSTER="e5cdfd8" -BOOTSTRAP_CACHE="c7afb99ad" - -## Build variables -DISTDIR=./dist -SNAPSHOTDIR=./snapshot -OS=$(shell uname | tr '[:upper:]' '[:lower:]') -SNAPSHOT_BIN=$(realpath $(shell pwd)/$(SNAPSHOTDIR)/$(OS)-build_$(OS)_amd64_v1/$(BIN)) - -## Variable assertions -ifndef TEMPDIR - $(error TEMPDIR is not set) -endif - -ifndef RESULTSDIR - $(error RESULTSDIR is not set) -endif - -ifndef COMPARE_DIR - $(error COMPARE_DIR is not set) -endif - -ifndef DISTDIR - $(error DISTDIR is not set) -endif - -ifndef SNAPSHOTDIR - $(error SNAPSHOTDIR is not set) -endif +## Build variables ################################# +VERSION := $(shell git describe --dirty --always --tags) +DIST_DIR := ./dist +SNAPSHOT_DIR := ./snapshot +CHANGELOG := CHANGELOG.md +OS := $(shell uname | tr '[:upper:]' '[:lower:]') +SNAPSHOT_BIN := $(realpath $(shell pwd)/$(SNAPSHOT_DIR)/$(OS)-build_$(OS)_amd64_v1/$(BIN)) ifndef VERSION $(error VERSION is not set) endif -ifndef REF_NAME - REF_NAME = $(VERSION) -endif - define title @printf '$(TITLE)$(1)$(RESET)\n' endef @@ -89,65 +56,58 @@ define safe_rm_rf_children bash -c 'test -z "$(1)" && false || rm -rf $(1)/*' endef -## Default Task .DEFAULT_GOAL:=help -## Tasks .PHONY: all -all: clean static-analysis test ## Run all linux-based checks (linting, license check, unit, integration, and linux compare tests) +all: static-analysis test ## Run all linux-based checks (linting, license check, unit, integration, and linux compare tests) @printf '$(SUCCESS)All checks pass!$(RESET)\n' +.PHONY: static-analysis +static-analysis: check-go-mod-tidy lint check-licenses check-json-schema-drift ## Run all static analysis checks + .PHONY: test -test: unit validate-cyclonedx-schema integration benchmark compare-linux cli ## Run all tests (currently unit, integration, linux compare, and cli tests) +test: unit integration validate-cyclonedx-schema benchmark cli ## Run all tests (currently unit, integration, linux compare, and cli tests) -.PHONY: ci-bootstrap -ci-bootstrap: - DEBIAN_FRONTEND=noninteractive sudo apt update && sudo -E apt install -y bc jq libxml2-utils -.PHONY: -ci-bootstrap-mac: - github_changelog_generator --version || sudo gem install github_changelog_generator +## Bootstrapping targets ################################# -$(RESULTSDIR): - mkdir -p $(RESULTSDIR) - -$(TEMPDIR): - mkdir -p $(TEMPDIR) +.PHONY: bootstrap +bootstrap: $(TEMP_DIR) bootstrap-go bootstrap-tools ## Download and install all tooling dependencies (+ prep tooling in the ./tmp dir) + $(call title,Bootstrapping dependencies) .PHONY: bootstrap-tools -bootstrap-tools: $(TEMPDIR) - curl -sSfL https://raw.githubusercontent.com/anchore/quill/main/install.sh | sh -s -- -b $(TEMPDIR)/ $(QUILL_VERSION) - GO111MODULE=off GOBIN=$(realpath $(TEMPDIR)) go get -u golang.org/x/perf/cmd/benchstat - curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b $(TEMPDIR)/ $(GOLANGCILINT_VERSION) - curl -sSfL https://raw.githubusercontent.com/wagoodman/go-bouncer/master/bouncer.sh | sh -s -- -b $(TEMPDIR)/ $(BOUNCER_VERSION) - curl -sSfL https://raw.githubusercontent.com/anchore/chronicle/main/install.sh | sh -s -- -b $(TEMPDIR)/ $(CHRONICLE_VERSION) - .github/scripts/goreleaser-install.sh -d -b $(TEMPDIR)/ $(GORELEASER_VERSION) +bootstrap-tools: $(TEMP_DIR) + curl -sSfL https://raw.githubusercontent.com/anchore/quill/main/install.sh | sh -s -- -b $(TEMP_DIR)/ $(QUILL_VERSION) + GO111MODULE=off GOBIN=$(realpath $(TEMP_DIR)) go get -u golang.org/x/perf/cmd/benchstat + curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b $(TEMP_DIR)/ $(GOLANGCILINT_VERSION) + curl -sSfL https://raw.githubusercontent.com/wagoodman/go-bouncer/master/bouncer.sh | sh -s -- -b $(TEMP_DIR)/ $(BOUNCER_VERSION) + curl -sSfL https://raw.githubusercontent.com/anchore/chronicle/main/install.sh | sh -s -- -b $(TEMP_DIR)/ $(CHRONICLE_VERSION) + .github/scripts/goreleaser-install.sh -d -b $(TEMP_DIR)/ $(GORELEASER_VERSION) # the only difference between goimports and gosimports is that gosimports removes extra whitespace between import blocks (see https://github.com/golang/go/issues/20818) - GOBIN="$(realpath $(TEMPDIR))" go install github.com/rinchsan/gosimports/cmd/gosimports@$(GOSIMPORTS_VERSION) - GOBIN="$(realpath $(TEMPDIR))" go install github.com/neilpa/yajsv@$(YAJSV_VERSION) - GOBIN="$(realpath $(TEMPDIR))" go install github.com/sigstore/cosign/cmd/cosign@$(COSIGN_VERSION) + GOBIN="$(realpath $(TEMP_DIR))" go install github.com/rinchsan/gosimports/cmd/gosimports@$(GOSIMPORTS_VERSION) + GOBIN="$(realpath $(TEMP_DIR))" go install github.com/neilpa/yajsv@$(YAJSV_VERSION) + GOBIN="$(realpath $(TEMP_DIR))" go install github.com/sigstore/cosign/cmd/cosign@$(COSIGN_VERSION) .PHONY: bootstrap-go bootstrap-go: go mod download -.PHONY: bootstrap -bootstrap: $(RESULTSDIR) bootstrap-go bootstrap-tools ## Download and install all tooling dependencies (+ prep tooling in the ./tmp dir) - $(call title,Bootstrapping dependencies) +$(TEMP_DIR): + mkdir -p $(TEMP_DIR) -.PHONY: static-analysis -static-analysis: check-go-mod-tidy check-licenses lint + +## Static analysis targets ################################# .PHONY: lint -lint: ## Run gofmt + golangci lint checks +lint: ## Run gofmt + golangci lint checks $(call title,Running linters) # ensure there are no go fmt differences @printf "files with gofmt issues: [$(shell gofmt -l -s .)]\n" @test -z "$(shell gofmt -l -s .)" # run all golangci-lint rules - $(LINTCMD) + $(LINT_CMD) @[ -z "$(shell $(GOIMPORTS_CMD) -d .)" ] || (echo "goimports needs to be fixed" && false) # go tooling does not play well with certain filename characters, ensure the common cases don't result in future "go get" failures @@ -155,243 +115,107 @@ lint: ## Run gofmt + golangci lint checks @bash -c "[[ '$(MALFORMED_FILENAMES)' == '' ]] || (printf '\nfound unsupported filename characters:\n$(MALFORMED_FILENAMES)\n\n' && false)" .PHONY: lint-fix -lint-fix: ## Auto-format all source code + run golangci lint fixers +lint-fix: ## Auto-format all source code + run golangci lint fixers $(call title,Running lint fixers) gofmt -w -s . $(GOIMPORTS_CMD) -w . - $(LINTCMD) --fix + $(LINT_CMD) --fix go mod tidy .PHONY: check-licenses -check-licenses: ## Ensure transitive dependencies are compliant with the current license policy - $(TEMPDIR)/bouncer check ./... +check-licenses: ## Ensure transitive dependencies are compliant with the current license policy + $(call title,Checking for license compliance) + $(TEMP_DIR)/bouncer check ./... check-go-mod-tidy: @ .github/scripts/go-mod-tidy-check.sh && echo "go.mod and go.sum are tidy!" +check-json-schema-drift: + $(call title,Ensure there is no drift between the JSON schema and the code) + @git diff-index --quiet HEAD -- || (echo "there are uncommitted changes, please commit them before running this check" && false) + @make generate-json-schema || (echo "$(RED)$(BOLD)JSON schema drift detected!$(RESET)" && false) + @git diff-index --quiet HEAD -- || (echo "$(RED)$(BOLD)JSON schema drift detected!$(RESET)" && false) + + +## Testing targets ################################# + +.PHONY: unit +unit: $(TEMP_DIR) fixtures ## Run unit tests (with coverage) + $(call title,Running unit tests) + go test -coverprofile $(TEMP_DIR)/unit-coverage-details.txt $(shell go list ./... | grep -v anchore/syft/test) + @.github/scripts/coverage.py $(COVERAGE_THRESHOLD) $(TEMP_DIR)/unit-coverage-details.txt + +.PHONY: integration +integration: ## Run integration tests + $(call title,Running integration tests) + go test -v ./test/integration + .PHONY: validate-cyclonedx-schema validate-cyclonedx-schema: cd schema/cyclonedx && make -.PHONY: unit -unit: $(RESULTSDIR) fixtures ## Run unit tests (with coverage) - $(call title,Running unit tests) - go test -coverprofile $(COVER_REPORT) $(shell go list ./... | grep -v anchore/syft/test) - @go tool cover -func $(COVER_REPORT) | grep total | awk '{print substr($$3, 1, length($$3)-1)}' > $(COVER_TOTAL) - @echo "Coverage: $$(cat $(COVER_TOTAL))" - @if [ $$(echo "$$(cat $(COVER_TOTAL)) >= $(COVERAGE_THRESHOLD)" | bc -l) -ne 1 ]; then echo "$(RED)$(BOLD)Failed coverage quality gate (> $(COVERAGE_THRESHOLD)%)$(RESET)" && false; fi +.PHONY: cli +cli: $(SNAPSHOT_DIR) ## Run CLI tests + chmod 755 "$(SNAPSHOT_BIN)" + $(SNAPSHOT_BIN) version + SYFT_BINARY_LOCATION='$(SNAPSHOT_BIN)' \ + go test -count=1 -timeout=15m -v ./test/cli + + +## Benchmark test targets ################################# .PHONY: benchmark -benchmark: $(RESULTSDIR) ## Run benchmark tests and compare against the baseline (if available) +benchmark: $(TEMP_DIR) ## Run benchmark tests and compare against the baseline (if available) $(call title,Running benchmark tests) - go test -p 1 -run=^Benchmark -bench=. -count=5 -benchmem ./... | tee $(RESULTSDIR)/benchmark-$(REF_NAME).txt - (test -s $(RESULTSDIR)/benchmark-main.txt && \ - $(TEMPDIR)/benchstat $(RESULTSDIR)/benchmark-main.txt $(RESULTSDIR)/benchmark-$(REF_NAME).txt || \ - $(TEMPDIR)/benchstat $(RESULTSDIR)/benchmark-$(REF_NAME).txt) \ - | tee $(RESULTSDIR)/benchstat.txt + go test -p 1 -run=^Benchmark -bench=. -count=7 -benchmem ./... | tee $(TEMP_DIR)/benchmark-$(VERSION).txt + (test -s $(TEMP_DIR)/benchmark-main.txt && \ + $(TEMP_DIR)/benchstat $(TEMP_DIR)/benchmark-main.txt $(TEMP_DIR)/benchmark-$(VERSION).txt || \ + $(TEMP_DIR)/benchstat $(TEMP_DIR)/benchmark-$(VERSION).txt) \ + | tee $(TEMP_DIR)/benchstat.txt .PHONY: show-benchstat show-benchstat: - @cat $(RESULTSDIR)/benchstat.txt + @cat $(TEMP_DIR)/benchstat.txt -# note: this is used by CI to determine if the install test fixture cache (docker image tars) should be busted -install-fingerprint: - cd test/install && \ + +## Test-fixture-related targets ################################# + +# note: this is used by CI to determine if various test fixture cache should be restored or recreated +fingerprints: + $(call title,Creating all test cache input fingerprints) + + # for IMAGE integration test fixtures + cd test/integration/test-fixtures && \ make cache.fingerprint -install-test: $(SNAPSHOTDIR) - cd test/install && \ - make - -install-test-cache-save: $(SNAPSHOTDIR) - cd test/install && \ - make save - -install-test-cache-load: $(SNAPSHOTDIR) - cd test/install && \ - make load - -install-test-ci-mac: $(SNAPSHOTDIR) - cd test/install && \ - make ci-test-mac - -.PHONY: integration -integration: ## Run integration tests - $(call title,Running integration tests) - go test -v ./test/integration - -# note: this is used by CI to determine if the integration test fixture cache (docker image tars) should be busted -integration-fingerprint: - $(call title,Integration test fixture fingerprint) - find test/integration/test-fixtures/image-* -type f -exec md5sum {} + | awk '{print $1}' | sort | tee /dev/stderr | md5sum | tee test/integration/test-fixtures/cache.fingerprint && echo "$(INTEGRATION_CACHE_BUSTER)" >> test/integration/test-fixtures/cache.fingerprint - -.PHONY: java-packages-fingerprint -java-packages-fingerprint: - $(call title,Java test fixture fingerprint) + # for JAVA BUILD test fixtures cd syft/pkg/cataloger/java/test-fixtures/java-builds && \ make packages.fingerprint -.PHONY: go-binaries-fingerprint -go-binaries-fingerprint: - $(call title,Go binaries test fixture fingerprint) + # for GO BINARY test fixtures cd syft/pkg/cataloger/golang/test-fixtures/archs && \ make binaries.fingerprint -.PHONY: rpm-binaries-fingerprint -rpm-binaries-fingerprint: - $(call title,RPM binary test fixture fingerprint) + # for RPM test fixtures cd syft/pkg/cataloger/rpm/test-fixtures && \ make rpms.fingerprint + # for INSTALL integration test fixtures + cd test/install && \ + make cache.fingerprint + + # for CLI test fixtures + cd test/cli/test-fixtures && \ + make cache.fingerprint + .PHONY: fixtures fixtures: $(call title,Generating test fixtures) cd syft/pkg/cataloger/java/test-fixtures/java-builds && make cd syft/pkg/cataloger/rpm/test-fixtures && make -.PHONY: generate-json-schema -generate-json-schema: ## Generate a new json schema - cd schema/json && go run generate.go - -.PHONY: generate-license-list -generate-license-list: ## Generate an updated spdx license list - go generate ./internal/spdxlicense/... - gofmt -s -w ./internal/spdxlicense - -.PHONY: build -build: $(SNAPSHOTDIR) ## Build release snapshot binaries and packages - -$(SNAPSHOTDIR): ## Build snapshot release binaries and packages - $(call title,Building snapshot artifacts) - - # create a config with the dist dir overridden - echo "dist: $(SNAPSHOTDIR)" > $(TEMPDIR)/goreleaser.yaml - cat .goreleaser.yaml >> $(TEMPDIR)/goreleaser.yaml - - # build release snapshots - bash -c "SKIP_SIGNING=true $(SNAPSHOT_CMD) --skip-sign --config $(TEMPDIR)/goreleaser.yaml" - -.PHONY: snapshot-with-signing -snapshot-with-signing: ## Build snapshot release binaries and packages (with dummy signing) - $(call title,Building snapshot artifacts (+ signing)) - - # create a config with the dist dir overridden - echo "dist: $(SNAPSHOTDIR)" > $(TEMPDIR)/goreleaser.yaml - cat .goreleaser.yaml >> $(TEMPDIR)/goreleaser.yaml - - # build release snapshots - bash -c "\ - $(SNAPSHOT_CMD) \ - --config $(TEMPDIR)/goreleaser.yaml \ - || (cat /tmp/quill-*.log && false)" - -# note: we cannot clean the snapshot directory since the pipeline builds the snapshot separately -.PHONY: compare-mac -compare-mac: $(RESULTSDIR) $(SNAPSHOTDIR) ## Run compare tests on build snapshot binaries and packages (Mac) - $(call title,Running compare test: Run on Mac) - $(COMPARE_DIR)/mac.sh \ - $(SNAPSHOTDIR) \ - $(COMPARE_DIR) \ - $(COMPARE_TEST_IMAGE) \ - $(RESULTSDIR) - -# note: we cannot clean the snapshot directory since the pipeline builds the snapshot separately -.PHONY: compare-linux -compare-linux: compare-test-deb-package-install compare-test-rpm-package-install ## Run compare tests on build snapshot binaries and packages (Linux) - -.PHONY: compare-test-deb-package-install -compare-test-deb-package-install: $(RESULTSDIR) $(SNAPSHOTDIR) - $(call title,Running compare test: DEB install) - $(COMPARE_DIR)/deb.sh \ - $(SNAPSHOTDIR) \ - $(COMPARE_DIR) \ - $(COMPARE_TEST_IMAGE) \ - $(RESULTSDIR) - -.PHONY: compare-test-rpm-package-install -compare-test-rpm-package-install: $(RESULTSDIR) $(SNAPSHOTDIR) - $(call title,Running compare test: RPM install) - $(COMPARE_DIR)/rpm.sh \ - $(SNAPSHOTDIR) \ - $(COMPARE_DIR) \ - $(COMPARE_TEST_IMAGE) \ - $(RESULTSDIR) - -# note: this is used by CI to determine if the integration test fixture cache (docker image tars) should be busted -cli-fingerprint: - $(call title,CLI test fixture fingerprint) - find test/cli/test-fixtures/image-* -type f -exec md5sum {} + | awk '{print $1}' | sort | md5sum | tee test/cli/test-fixtures/cache.fingerprint && echo "$(CLI_CACHE_BUSTER)" >> test/cli/test-fixtures/cache.fingerprint - -.PHONY: cli -cli: $(SNAPSHOTDIR) ## Run CLI tests - chmod 755 "$(SNAPSHOT_BIN)" - $(SNAPSHOT_BIN) version - SYFT_BINARY_LOCATION='$(SNAPSHOT_BIN)' \ - go test -count=1 -timeout=15m -v ./test/cli - -.PHONY: changelog -changelog: clean-changelog CHANGELOG.md - @docker run -it --rm \ - -v $(shell pwd)/CHANGELOG.md:/CHANGELOG.md \ - rawkode/mdv \ - -t 748.5989 \ - /CHANGELOG.md - -CHANGELOG.md: - $(TEMPDIR)/chronicle -vv > CHANGELOG.md - -.PHONY: release -release: clean-dist CHANGELOG.md - $(call title,Publishing release artifacts) - - # create a config with the dist dir overridden - echo "dist: $(DISTDIR)" > $(TEMPDIR)/goreleaser.yaml - cat .goreleaser.yaml >> $(TEMPDIR)/goreleaser.yaml - - # note: notarization cannot be done in parallel, thus --parallelism 1 - bash -c "\ - $(RELEASE_CMD) \ - --config $(TEMPDIR)/goreleaser.yaml \ - --parallelism 1 \ - --release-notes <(cat CHANGELOG.md) \ - || (cat /tmp/quill-*.log && false)" - - # TODO: turn this into a post-release hook - # upload the version file that supports the application version update check (excluding pre-releases) - .github/scripts/update-version-file.sh "$(DISTDIR)" "$(VERSION)" - -.PHONY: clean -clean: clean-dist clean-snapshot clean-test-image-cache ## Remove previous builds, result reports, and test cache - $(call safe_rm_rf_children,$(RESULTSDIR)) - -.PHONY: clean-snapshot -clean-snapshot: - $(call safe_rm_rf,$(SNAPSHOTDIR)) - rm -f $(TEMPDIR)/goreleaser.yaml - -.PHONY: clean-dist -clean-dist: clean-changelog - $(call safe_rm_rf,$(DISTDIR)) - rm -f $(TEMPDIR)/goreleaser.yaml - -.PHONY: clean-changelog -clean-changelog: - rm -f CHANGELOG.md - -clean-test-image-cache: clean-test-image-tar-cache clean-test-image-docker-cache ## Clean test image cache - -.PHONY: clear-test-image-tar-cache -clean-test-image-tar-cache: - ## Delete all test cache (built docker image tars) - find . -type f -wholename "**/test-fixtures/cache/stereoscope-fixture-*.tar" -delete - -.PHONY: clear-test-image-docker-cache -clean-test-image-docker-cache: - ## Purge all test docker images - docker images --format '{{.ID}} {{.Repository}}' | grep stereoscope-fixture- | awk '{print $$1}' | uniq | xargs -r docker rmi --force - .PHONY: show-test-image-cache -show-test-image-cache: ## Show all docker and image tar cache +show-test-image-cache: ## Show all docker and image tar cache $(call title,Docker daemon cache) @docker images --format '{{.ID}} {{.Repository}}:{{.Tag}}' | grep stereoscope-fixture- | sort @@ -399,10 +223,150 @@ show-test-image-cache: ## Show all docker and image tar cache @find . -type f -wholename "**/test-fixtures/cache/stereoscope-fixture-*.tar" | sort .PHONY: show-test-snapshots -show-test-snapshots: ## Show all test snapshots +show-test-snapshots: ## Show all test snapshots $(call title,Test snapshots) @find . -type f -wholename "**/test-fixtures/snapshot/*" | sort + +## install.sh testing targets ################################# + +install-test: $(SNAPSHOT_DIR) + cd test/install && \ + make + +install-test-cache-save: $(SNAPSHOT_DIR) + cd test/install && \ + make save + +install-test-cache-load: $(SNAPSHOT_DIR) + cd test/install && \ + make load + +install-test-ci-mac: $(SNAPSHOT_DIR) + cd test/install && \ + make ci-test-mac + +# note: we cannot clean the snapshot directory since the pipeline builds the snapshot separately +.PHONY: compare-mac +compare-mac: $(TEMP_DIR) $(SNAPSHOT_DIR) ## Run compare tests on build snapshot binaries and packages (Mac) + $(call title,Running compare test: Run on Mac) + $(COMPARE_DIR)/mac.sh \ + $(SNAPSHOT_DIR) \ + $(COMPARE_DIR) \ + $(COMPARE_TEST_IMAGE) \ + $(TEMP_DIR) + +# note: we cannot clean the snapshot directory since the pipeline builds the snapshot separately +.PHONY: compare-linux +compare-linux: compare-test-deb-package-install compare-test-rpm-package-install ## Run compare tests on build snapshot binaries and packages (Linux) + +.PHONY: compare-test-deb-package-install +compare-test-deb-package-install: $(TEMP_DIR) $(SNAPSHOT_DIR) + $(call title,Running compare test: DEB install) + $(COMPARE_DIR)/deb.sh \ + $(SNAPSHOT_DIR) \ + $(COMPARE_DIR) \ + $(COMPARE_TEST_IMAGE) \ + $(TEMP_DIR) + +.PHONY: compare-test-rpm-package-install +compare-test-rpm-package-install: $(TEMP_DIR) $(SNAPSHOT_DIR) + $(call title,Running compare test: RPM install) + $(COMPARE_DIR)/rpm.sh \ + $(SNAPSHOT_DIR) \ + $(COMPARE_DIR) \ + $(COMPARE_TEST_IMAGE) \ + $(TEMP_DIR) + + +## Code generation targets ################################# + +.PHONY: generate-json-schema +generate-json-schema: ## Generate a new json schema + cd schema/json && go run generate.go + +.PHONY: generate-license-list +generate-license-list: ## Generate an updated spdx license list + go generate ./internal/spdxlicense/... + gofmt -s -w ./internal/spdxlicense + + +## Build-related targets ################################# + +.PHONY: build +build: $(SNAPSHOT_DIR) ## Build release snapshot binaries and packages + +$(SNAPSHOT_DIR): ## Build snapshot release binaries and packages + $(call title,Building snapshot artifacts) + + # create a config with the dist dir overridden + echo "dist: $(SNAPSHOT_DIR)" > $(TEMP_DIR)/goreleaser.yaml + cat .goreleaser.yaml >> $(TEMP_DIR)/goreleaser.yaml + + # build release snapshots + $(SNAPSHOT_CMD) --config $(TEMP_DIR)/goreleaser.yaml + +.PHONY: changelog +changelog: clean-changelog $(CHANGELOG) ## Generate and show the changelog for the current unreleased version + @docker run -it --rm \ + -v $(shell pwd)/$(CHANGELOG):/$(CHANGELOG) \ + rawkode/mdv \ + -t 748.5989 \ + /$(CHANGELOG) + +$(CHANGELOG): + $(TEMP_DIR)/chronicle -vv > $(CHANGELOG) + +.PHONY: release +release: clean-dist $(CHANGELOG) + $(call title,Publishing release artifacts) + @.github/scripts/ci-check.sh + + # create a config with the dist dir overridden + echo "dist: $(DIST_DIR)" > $(TEMP_DIR)/goreleaser.yaml + cat .goreleaser.yaml >> $(TEMP_DIR)/goreleaser.yaml + + bash -c "\ + $(RELEASE_CMD) \ + --config $(TEMP_DIR)/goreleaser.yaml \ + --release-notes <(cat $(CHANGELOG)) \ + || (cat /tmp/quill-*.log && false)" + + # upload the version file that supports the application version update check (excluding pre-releases) + .github/scripts/update-version-file.sh "$(DIST_DIR)" "$(VERSION)" + +## Cleanup targets ################################# + +.PHONY: clean +clean: clean-dist clean-snapshot clean-test-image-cache ## Remove previous builds, result reports, and test cache + $(call safe_rm_rf_children,$(TEMP_DIR)) + +.PHONY: clean-snapshot +clean-snapshot: + $(call safe_rm_rf,$(SNAPSHOT_DIR)) + rm -f $(TEMP_DIR)/goreleaser.yaml + +.PHONY: clean-dist +clean-dist: clean-changelog + $(call safe_rm_rf,$(DIST_DIR)) + rm -f $(TEMP_DIR)/goreleaser.yaml + +.PHONY: clean-changelog +clean-changelog: + rm -f $(CHANGELOG) + +clean-test-image-cache: clean-test-image-tar-cache clean-test-image-docker-cache ## Clean test image cache + +.PHONY: clear-test-image-tar-cache +clean-test-image-tar-cache: ## Delete all test cache (built docker image tars) + find . -type f -wholename "**/test-fixtures/cache/stereoscope-fixture-*.tar" -delete + +.PHONY: clear-test-image-docker-cache +clean-test-image-docker-cache: ## Purge all test docker images + docker images --format '{{.ID}} {{.Repository}}' | grep stereoscope-fixture- | awk '{print $$1}' | uniq | xargs -r docker rmi --force + +## Halp! ################################# + .PHONY: help help: ## Display this help @grep -E '^[a-zA-Z_-]+:.*?## .*$$' $(MAKEFILE_LIST) | sort | awk 'BEGIN {FS = ":.*?## "}; {printf "$(BOLD)$(CYAN)%-25s$(RESET)%s\n", $$1, $$2}' diff --git a/syft/pkg/cataloger/java/test-fixtures/java-builds/Makefile b/syft/pkg/cataloger/java/test-fixtures/java-builds/Makefile index 1f13986c2..fcc61446b 100644 --- a/syft/pkg/cataloger/java/test-fixtures/java-builds/Makefile +++ b/syft/pkg/cataloger/java/test-fixtures/java-builds/Makefile @@ -71,5 +71,5 @@ $(PKGSDIR)/example-java-app: $(PKGSDIR)/example-java-app-maven-0.1.0.jar # we need a way to determine if CI should bust the test cache based on the source material $(PKGSDIR).fingerprint: clean-examples mkdir -p $(PKGSDIR) - find example-* -type f -exec sha256sum {} \; | sort | tee /dev/stderr | tee $(PKGSDIR).fingerprint + find example-* build-* Makefile -type f -exec sha256sum {} \; | sort | tee /dev/stderr | tee $(PKGSDIR).fingerprint sha256sum $(PKGSDIR).fingerprint diff --git a/test/cli/test-fixtures/Makefile b/test/cli/test-fixtures/Makefile new file mode 100644 index 000000000..5042a5aad --- /dev/null +++ b/test/cli/test-fixtures/Makefile @@ -0,0 +1,6 @@ +# change these if you want CI to not use previous stored cache +CLI_CACHE_BUSTER := "e5cdfd8" + +.PHONY: cache.fingerprint +cache.fingerprint: + find image-* -type f -exec md5sum {} + | awk '{print $1}' | sort | md5sum | tee cache.fingerprint && echo "$(CLI_CACHE_BUSTER)" >> cache.fingerprint diff --git a/test/integration/test-fixtures/Makefile b/test/integration/test-fixtures/Makefile new file mode 100644 index 000000000..2a75aa436 --- /dev/null +++ b/test/integration/test-fixtures/Makefile @@ -0,0 +1,6 @@ +# change these if you want CI to not use previous stored cache +INTEGRATION_CACHE_BUSTER := "894d8ca" + +.PHONY: cache.fingerprint +cache.fingerprint: + find image-* -type f -exec md5sum {} + | awk '{print $1}' | sort | tee /dev/stderr | md5sum | tee cache.fingerprint && echo "$(INTEGRATION_CACHE_BUSTER)" >> cache.fingerprint