mirror of
https://github.com/anchore/syft
synced 2024-09-20 06:01:53 +00:00
refactor test cache workflow
Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com>
This commit is contained in:
parent
c1b189b948
commit
9e51842185
46 changed files with 851 additions and 272 deletions
16
.binny.yaml
16
.binny.yaml
|
@ -115,3 +115,19 @@ tools:
|
|||
method: github-release
|
||||
with:
|
||||
repo: cli/cli
|
||||
|
||||
# used to upload test fixture cache
|
||||
- name: oras
|
||||
version:
|
||||
want: v1.2.0
|
||||
method: github-release
|
||||
with:
|
||||
repo: oras-project/oras
|
||||
|
||||
# used to upload test fixture cache
|
||||
- name: yq
|
||||
version:
|
||||
want: v4.44.3
|
||||
method: github-release
|
||||
with:
|
||||
repo: mikefarah/yq
|
23
.github/actions/bootstrap/action.yaml
vendored
23
.github/actions/bootstrap/action.yaml
vendored
|
@ -13,16 +13,15 @@ inputs:
|
|||
cache-key-prefix:
|
||||
description: "Prefix all cache keys with this value"
|
||||
required: true
|
||||
default: "1ac8281053"
|
||||
compute-fingerprints:
|
||||
description: "Compute test fixture fingerprints"
|
||||
default: "181053ac82"
|
||||
download-test-fixture-cache:
|
||||
description: "Download test fixture cache from OCI and github actions"
|
||||
required: true
|
||||
default: "true"
|
||||
default: "false"
|
||||
bootstrap-apt-packages:
|
||||
description: "Space delimited list of tools to install via apt"
|
||||
default: "libxml2-utils"
|
||||
|
||||
|
||||
runs:
|
||||
using: "composite"
|
||||
steps:
|
||||
|
@ -54,8 +53,14 @@ runs:
|
|||
run: |
|
||||
DEBIAN_FRONTEND=noninteractive sudo apt update && sudo -E apt install -y ${{ inputs.bootstrap-apt-packages }}
|
||||
|
||||
- name: Create all cache fingerprints
|
||||
if: inputs.compute-fingerprints == 'true'
|
||||
shell: bash
|
||||
run: make fingerprints
|
||||
- name: Restore ORAS cache from github actions
|
||||
if: inputs.download-test-fixture-cache == 'true'
|
||||
uses: actions/cache@704facf57e6136b1bc63b828d79edcd491f0ee84 # v3.3.2
|
||||
with:
|
||||
path: ${{ github.workspace }}/.tmp/oras-cache
|
||||
key: ${{ inputs.cache-key-prefix }}-${{ runner.os }}-oras-cache
|
||||
|
||||
- name: Download test fixture cache
|
||||
if: inputs.download-test-fixture-cache == 'true'
|
||||
shell: bash
|
||||
run: make download-test-fixture-cache
|
||||
|
|
11
.github/scripts/ci-check.sh
vendored
11
.github/scripts/ci-check.sh
vendored
|
@ -1,11 +0,0 @@
|
|||
#!/usr/bin/env bash
|
||||
|
||||
red=$(tput setaf 1)
|
||||
bold=$(tput bold)
|
||||
normal=$(tput sgr0)
|
||||
|
||||
# assert we are running in CI (or die!)
|
||||
if [[ -z "$CI" ]]; then
|
||||
echo "${bold}${red}This step should ONLY be run in CI. Exiting...${normal}"
|
||||
exit 1
|
||||
fi
|
135
.github/scripts/find_cache_paths.py
vendored
Executable file
135
.github/scripts/find_cache_paths.py
vendored
Executable file
|
@ -0,0 +1,135 @@
|
|||
#!/usr/bin/env python3
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import glob
|
||||
import sys
|
||||
import json
|
||||
import hashlib
|
||||
|
||||
|
||||
IGNORED_PREFIXES = []
|
||||
|
||||
|
||||
def find_fingerprints_and_check_dirs(base_dir):
|
||||
all_fingerprints = set(glob.glob(os.path.join(base_dir, '**', 'test*', '**', '*.fingerprint'), recursive=True))
|
||||
|
||||
all_fingerprints = {os.path.relpath(fp) for fp in all_fingerprints
|
||||
if not any(fp.startswith(prefix) for prefix in IGNORED_PREFIXES)}
|
||||
|
||||
if not all_fingerprints:
|
||||
show("No .fingerprint files or cache directories found.")
|
||||
exit(1)
|
||||
|
||||
missing_content = []
|
||||
valid_paths = set()
|
||||
fingerprint_contents = []
|
||||
|
||||
for fingerprint in all_fingerprints:
|
||||
path = fingerprint.replace('.fingerprint', '')
|
||||
|
||||
if not os.path.exists(path):
|
||||
missing_content.append(path)
|
||||
continue
|
||||
|
||||
if not os.path.isdir(path):
|
||||
valid_paths.add(path)
|
||||
continue
|
||||
|
||||
if os.listdir(path):
|
||||
valid_paths.add(path)
|
||||
else:
|
||||
missing_content.append(path)
|
||||
|
||||
with open(fingerprint, 'r') as f:
|
||||
content = f.read().strip()
|
||||
fingerprint_contents.append((fingerprint, content))
|
||||
|
||||
return sorted(valid_paths), missing_content, fingerprint_contents
|
||||
|
||||
|
||||
def parse_fingerprint_contents(fingerprint_content):
|
||||
input_map = {}
|
||||
for line in fingerprint_content.splitlines():
|
||||
digest, path = line.split()
|
||||
input_map[path] = digest
|
||||
return input_map
|
||||
|
||||
|
||||
def calculate_sha256(fingerprint_contents):
|
||||
sorted_fingerprint_contents = sorted(fingerprint_contents, key=lambda x: x[0])
|
||||
|
||||
concatenated_contents = ''.join(content for _, content in sorted_fingerprint_contents)
|
||||
|
||||
sha256_hash = hashlib.sha256(concatenated_contents.encode()).hexdigest()
|
||||
|
||||
return sha256_hash
|
||||
|
||||
|
||||
def calculate_file_sha256(file_path):
|
||||
sha256_hash = hashlib.sha256()
|
||||
with open(file_path, 'rb') as f:
|
||||
for byte_block in iter(lambda: f.read(4096), b""):
|
||||
sha256_hash.update(byte_block)
|
||||
return sha256_hash.hexdigest()
|
||||
|
||||
|
||||
def show(*s: str):
|
||||
print(*s, file=sys.stderr)
|
||||
|
||||
|
||||
def main(file_path: str | None):
|
||||
base_dir = '.'
|
||||
valid_paths, missing_content, fingerprint_contents = find_fingerprints_and_check_dirs(base_dir)
|
||||
|
||||
if missing_content:
|
||||
show("The following paths are missing or have no content, but have corresponding .fingerprint files:")
|
||||
for path in sorted(missing_content):
|
||||
show(f"- {path}")
|
||||
show("Please ensure these paths exist and have content if they are directories.")
|
||||
exit(1)
|
||||
|
||||
sha256_hash = calculate_sha256(fingerprint_contents)
|
||||
|
||||
paths_with_digests = []
|
||||
for path in sorted(valid_paths):
|
||||
fingerprint_file = f"{path}.fingerprint"
|
||||
try:
|
||||
if os.path.exists(fingerprint_file):
|
||||
file_digest = calculate_file_sha256(fingerprint_file)
|
||||
|
||||
# Parse the fingerprint file to get the digest/path tuples
|
||||
with open(fingerprint_file, 'r') as f:
|
||||
fingerprint_content = f.read().strip()
|
||||
input_map = parse_fingerprint_contents(fingerprint_content)
|
||||
|
||||
paths_with_digests.append({
|
||||
"path": path,
|
||||
"digest": file_digest,
|
||||
"input": input_map
|
||||
})
|
||||
|
||||
except Exception as e:
|
||||
show(f"Error processing {fingerprint_file}: {e}")
|
||||
raise e
|
||||
|
||||
|
||||
output = {
|
||||
"digest": sha256_hash,
|
||||
"paths": paths_with_digests
|
||||
}
|
||||
|
||||
content = json.dumps(output, indent=2, sort_keys=True)
|
||||
|
||||
if file_path:
|
||||
with open(file_path, 'w') as f:
|
||||
f.write(content)
|
||||
|
||||
print(content)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
file_path = None
|
||||
if len(sys.argv) > 1:
|
||||
file_path = sys.argv[1]
|
||||
main(file_path)
|
70
.github/scripts/fingerprint_docker_fixtures.py
vendored
Executable file
70
.github/scripts/fingerprint_docker_fixtures.py
vendored
Executable file
|
@ -0,0 +1,70 @@
|
|||
#!/usr/bin/env python3
|
||||
|
||||
import os
|
||||
import subprocess
|
||||
import hashlib
|
||||
|
||||
BOLD = '\033[1m'
|
||||
YELLOW = '\033[0;33m'
|
||||
RESET = '\033[0m'
|
||||
|
||||
|
||||
def print_message(message):
|
||||
print(f"{YELLOW}{message}{RESET}")
|
||||
|
||||
|
||||
def sha256sum(filepath):
|
||||
h = hashlib.sha256()
|
||||
with open(filepath, 'rb') as f:
|
||||
for chunk in iter(lambda: f.read(4096), b""):
|
||||
h.update(chunk)
|
||||
return h.hexdigest()
|
||||
|
||||
|
||||
def is_git_tracked_or_untracked(directory):
|
||||
"""Returns a sorted list of files in the directory that are tracked or not ignored by Git."""
|
||||
result = subprocess.run(
|
||||
["git", "ls-files", "--cached", "--others", "--exclude-standard"],
|
||||
cwd=directory,
|
||||
stdout=subprocess.PIPE,
|
||||
text=True
|
||||
)
|
||||
return sorted(result.stdout.strip().splitlines())
|
||||
|
||||
|
||||
def find_test_fixture_dirs_with_images(base_dir):
|
||||
"""Find directories that contain 'test-fixtures' and at least one 'image-*' directory."""
|
||||
for root, dirs, files in os.walk(base_dir):
|
||||
if 'test-fixtures' in root:
|
||||
image_dirs = [d for d in dirs if d.startswith('image-')]
|
||||
if image_dirs:
|
||||
yield os.path.realpath(root)
|
||||
|
||||
|
||||
def generate_fingerprints():
|
||||
print_message("creating fingerprint files for docker fixtures...")
|
||||
|
||||
for test_fixture_dir in find_test_fixture_dirs_with_images('.'):
|
||||
cache_fingerprint_path = os.path.join(test_fixture_dir, 'cache.fingerprint')
|
||||
|
||||
with open(cache_fingerprint_path, 'w') as fingerprint_file:
|
||||
for image_dir in find_image_dirs(test_fixture_dir):
|
||||
for file in is_git_tracked_or_untracked(image_dir):
|
||||
file_path = os.path.join(image_dir, file)
|
||||
checksum = sha256sum(file_path)
|
||||
path_from_fixture_dir = os.path.relpath(file_path, test_fixture_dir)
|
||||
fingerprint_file.write(f"{checksum} {path_from_fixture_dir}\n")
|
||||
|
||||
|
||||
def find_image_dirs(test_fixture_dir):
|
||||
"""Find all 'image-*' directories inside a given test-fixture directory."""
|
||||
result = []
|
||||
for root, dirs, files in os.walk(test_fixture_dir):
|
||||
for dir_name in dirs:
|
||||
if dir_name.startswith('image-'):
|
||||
result.append(os.path.join(root, dir_name))
|
||||
return sorted(result)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
generate_fingerprints()
|
2
.github/scripts/labeler.py
vendored
Normal file → Executable file
2
.github/scripts/labeler.py
vendored
Normal file → Executable file
|
@ -1,3 +1,5 @@
|
|||
#!/usr/bin/env python3
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import sys
|
||||
|
|
2
.github/scripts/labeler_test.py
vendored
Normal file → Executable file
2
.github/scripts/labeler_test.py
vendored
Normal file → Executable file
|
@ -1,3 +1,5 @@
|
|||
#!/usr/bin/env python3
|
||||
|
||||
import unittest
|
||||
from unittest.mock import patch
|
||||
import subprocess
|
||||
|
|
2
.github/workflows/release-version-file.yaml
vendored
2
.github/workflows/release-version-file.yaml
vendored
|
@ -1,4 +1,4 @@
|
|||
name: "Release"
|
||||
name: "Release: version file"
|
||||
|
||||
on:
|
||||
|
||||
|
|
41
.github/workflows/test-fixture-cache-publish.yaml
vendored
Normal file
41
.github/workflows/test-fixture-cache-publish.yaml
vendored
Normal file
|
@ -0,0 +1,41 @@
|
|||
name: "Test fixture cache: publish"
|
||||
|
||||
on:
|
||||
workflow_dispatch:
|
||||
# TODO: remove me, just here for testing
|
||||
push:
|
||||
schedule:
|
||||
# run nightly at 4AM UTC
|
||||
- cron: "0 4 * * *"
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
jobs:
|
||||
|
||||
Publish:
|
||||
name: "Publish test fixture image cache"
|
||||
# we use this runner to get enough storage space for docker images and fixture cache
|
||||
runs-on: ubuntu-22.04-4core-16gb
|
||||
permissions:
|
||||
packages: write
|
||||
steps:
|
||||
- uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 #v4.1.7
|
||||
|
||||
- name: Bootstrap environment
|
||||
uses: ./.github/actions/bootstrap
|
||||
with:
|
||||
# we want to rebuild the cache with no previous state
|
||||
download-test-fixture-cache: false
|
||||
|
||||
- name: Run all tests
|
||||
run: make test
|
||||
env:
|
||||
# we want to rebuild the cache with no previous state
|
||||
DOWNLOAD_TEST_FIXTURE_CACHE: "false"
|
||||
|
||||
- name: Login to GitHub Container Registry (ORAS)
|
||||
run: echo "${{ secrets.GITHUB_TOKEN }}" | .tool/oras login ghcr.io -u ${{ github.actor }} --password-stdin
|
||||
|
||||
- name: Publish test fixture cache
|
||||
run: make upload-test-fixture-cache
|
1
.github/workflows/update-bootstrap-tools.yml
vendored
1
.github/workflows/update-bootstrap-tools.yml
vendored
|
@ -19,7 +19,6 @@ jobs:
|
|||
uses: ./.github/actions/bootstrap
|
||||
with:
|
||||
bootstrap-apt-packages: ""
|
||||
compute-fingerprints: "false"
|
||||
go-dependencies: false
|
||||
|
||||
- name: "Update tool versions"
|
||||
|
|
74
.github/workflows/validations.yaml
vendored
74
.github/workflows/validations.yaml
vendored
|
@ -35,48 +35,8 @@ jobs:
|
|||
|
||||
- name: Bootstrap environment
|
||||
uses: ./.github/actions/bootstrap
|
||||
|
||||
- name: Restore file executable test-fixture cache
|
||||
uses: actions/cache@0c45773b623bea8c8e75f6c82b208c3cf94ea4f9 #v4.0.2
|
||||
with:
|
||||
path: syft/file/cataloger/executable/test-fixtures/elf/bin
|
||||
key: ${{ runner.os }}-unit-file-executable-elf-cache-${{ hashFiles( 'syft/file/cataloger/executable/test-fixtures/elf/cache.fingerprint' ) }}
|
||||
|
||||
- name: Restore file executable shared-info test-fixture cache
|
||||
uses: actions/cache@0c45773b623bea8c8e75f6c82b208c3cf94ea4f9 #v4.0.2
|
||||
with:
|
||||
path: syft/file/cataloger/executable/test-fixtures/shared-info/bin
|
||||
key: ${{ runner.os }}-unit-file-executable-shared-info-cache-${{ hashFiles( 'syft/file/cataloger/executable/test-fixtures/shared-info/cache.fingerprint' ) }}
|
||||
|
||||
- name: Restore Java test-fixture cache
|
||||
uses: actions/cache@0c45773b623bea8c8e75f6c82b208c3cf94ea4f9 #v4.0.2
|
||||
with:
|
||||
path: syft/pkg/cataloger/java/test-fixtures/java-builds/packages
|
||||
key: ${{ runner.os }}-unit-java-cache-${{ hashFiles( 'syft/pkg/cataloger/java/test-fixtures/java-builds/cache.fingerprint' ) }}
|
||||
|
||||
- name: Restore RPM test-fixture cache
|
||||
uses: actions/cache@0c45773b623bea8c8e75f6c82b208c3cf94ea4f9 #v4.0.2
|
||||
with:
|
||||
path: syft/pkg/cataloger/redhat/test-fixtures/rpms
|
||||
key: ${{ runner.os }}-unit-rpm-cache-${{ hashFiles( 'syft/pkg/cataloger/redhat/test-fixtures/rpms.fingerprint' ) }}
|
||||
|
||||
- name: Restore go binary test-fixture cache
|
||||
uses: actions/cache@0c45773b623bea8c8e75f6c82b208c3cf94ea4f9 #v4.0.2
|
||||
with:
|
||||
path: syft/pkg/cataloger/golang/test-fixtures/archs/binaries
|
||||
key: ${{ runner.os }}-unit-go-binaries-cache-${{ hashFiles( 'syft/pkg/cataloger/golang/test-fixtures/archs/binaries.fingerprint' ) }}
|
||||
|
||||
- name: Restore binary cataloger test-fixture cache
|
||||
uses: actions/cache@0c45773b623bea8c8e75f6c82b208c3cf94ea4f9 #v4.0.2
|
||||
with:
|
||||
path: syft/pkg/cataloger/binary/test-fixtures/classifiers/bin
|
||||
key: ${{ runner.os }}-unit-binary-cataloger-cache-${{ hashFiles( 'syft/pkg/cataloger/binary/test-fixtures/cache.fingerprint' ) }}
|
||||
|
||||
- name: Restore Kernel test-fixture cache
|
||||
uses: actions/cache@0c45773b623bea8c8e75f6c82b208c3cf94ea4f9 #v4.0.2
|
||||
with:
|
||||
path: syft/pkg/cataloger/kernel/test-fixtures/cache
|
||||
key: ${{ runner.os }}-unit-kernel-cache-${{ hashFiles( 'syft/pkg/cataloger/kernel/test-fixtures/cache.fingerprint' ) }}
|
||||
download-test-fixture-cache: true
|
||||
|
||||
- name: Run unit tests
|
||||
run: make unit
|
||||
|
@ -91,16 +51,12 @@ jobs:
|
|||
|
||||
- name: Bootstrap environment
|
||||
uses: ./.github/actions/bootstrap
|
||||
with:
|
||||
download-test-fixture-cache: true
|
||||
|
||||
- name: Validate syft output against the CycloneDX schema
|
||||
run: make validate-cyclonedx-schema
|
||||
|
||||
- name: Restore integration test cache
|
||||
uses: actions/cache@0c45773b623bea8c8e75f6c82b208c3cf94ea4f9 #v4.0.2
|
||||
with:
|
||||
path: ${{ github.workspace }}/cmd/syft/internal/test/integration/test-fixtures/cache
|
||||
key: ${{ runner.os }}-integration-test-cache-${{ hashFiles('/cmd/syft/internal/test/integration/test-fixtures/cache.fingerprint') }}
|
||||
|
||||
- name: Run integration tests
|
||||
run: make integration
|
||||
|
||||
|
@ -143,6 +99,8 @@ jobs:
|
|||
|
||||
- name: Bootstrap environment
|
||||
uses: ./.github/actions/bootstrap
|
||||
with:
|
||||
download-test-fixture-cache: true
|
||||
|
||||
- name: Download snapshot build
|
||||
id: snapshot-cache
|
||||
|
@ -162,13 +120,6 @@ jobs:
|
|||
- name: Run comparison tests (Linux)
|
||||
run: make compare-linux
|
||||
|
||||
- name: Restore install.sh test image cache
|
||||
id: install-test-image-cache
|
||||
uses: actions/cache@0c45773b623bea8c8e75f6c82b208c3cf94ea4f9 #v4.0.2
|
||||
with:
|
||||
path: ${{ github.workspace }}/test/install/cache
|
||||
key: ${{ runner.os }}-install-test-image-cache-${{ hashFiles('test/install/cache.fingerprint') }}
|
||||
|
||||
- name: Load test image cache
|
||||
if: steps.install-test-image-cache.outputs.cache-hit == 'true'
|
||||
run: make install-test-cache-load
|
||||
|
@ -196,8 +147,8 @@ jobs:
|
|||
uses: ./.github/actions/bootstrap
|
||||
with:
|
||||
bootstrap-apt-packages: ""
|
||||
compute-fingerprints: "false"
|
||||
go-dependencies: false
|
||||
download-test-fixture-cache: true
|
||||
|
||||
- name: Download snapshot build
|
||||
id: snapshot-cache
|
||||
|
@ -214,13 +165,6 @@ jobs:
|
|||
if: steps.snapshot-cache.outputs.cache-hit != 'true'
|
||||
run: echo "unable to download snapshots from previous job" && false
|
||||
|
||||
- name: Restore docker image cache for compare testing
|
||||
id: mac-compare-testing-cache
|
||||
uses: actions/cache@0c45773b623bea8c8e75f6c82b208c3cf94ea4f9 #v4.0.2
|
||||
with:
|
||||
path: image.tar
|
||||
key: ${{ runner.os }}-${{ hashFiles('test/compare/mac.sh') }}
|
||||
|
||||
- name: Run comparison tests (Mac)
|
||||
run: make compare-mac
|
||||
|
||||
|
@ -238,12 +182,8 @@ jobs:
|
|||
|
||||
- name: Bootstrap environment
|
||||
uses: ./.github/actions/bootstrap
|
||||
|
||||
- name: Restore CLI test-fixture cache
|
||||
uses: actions/cache@0c45773b623bea8c8e75f6c82b208c3cf94ea4f9 #v4.0.2
|
||||
with:
|
||||
path: ${{ github.workspace }}/test/cli/test-fixtures/cache
|
||||
key: ${{ runner.os }}-cli-test-cache-${{ hashFiles('test/cli/test-fixtures/cache.fingerprint') }}
|
||||
download-test-fixture-cache: true
|
||||
|
||||
- name: Download snapshot build
|
||||
id: snapshot-cache
|
||||
|
|
4
Makefile
4
Makefile
|
@ -25,8 +25,8 @@ ci-bootstrap-go:
|
|||
|
||||
# this is a bootstrapping catch-all, where if the target doesn't exist, we'll ensure the tools are installed and then try again
|
||||
%:
|
||||
make $(TASK)
|
||||
$(TASK) $@
|
||||
@make --silent $(TASK)
|
||||
@$(TASK) $@
|
||||
|
||||
## Shim targets #################################
|
||||
|
||||
|
|
259
Taskfile.yaml
259
Taskfile.yaml
|
@ -4,9 +4,19 @@ vars:
|
|||
OWNER: anchore
|
||||
PROJECT: syft
|
||||
|
||||
CACHE_IMAGE: ghcr.io/{{ .OWNER }}/{{ .PROJECT }}/test-fixture-cache:latest
|
||||
|
||||
# static file dirs
|
||||
TOOL_DIR: .tool
|
||||
TMP_DIR: .tmp
|
||||
ORAS_CACHE: "{{ .TMP_DIR }}/oras-cache"
|
||||
CACHE_PATHS_FILE: "{{ .TMP_DIR }}/cache_paths.json"
|
||||
LAST_CACHE_PULL_FILE: "{{ .TMP_DIR }}/last_cache_paths.json"
|
||||
|
||||
# TOOLS
|
||||
ORAS: "{{ .TOOL_DIR }}/oras"
|
||||
YQ: "{{ .TOOL_DIR }}/yq"
|
||||
TASK: "{{ .TOOL_DIR }}/task"
|
||||
|
||||
# used for changelog generation
|
||||
CHANGELOG: CHANGELOG.md
|
||||
|
@ -33,6 +43,9 @@ vars:
|
|||
COMPARE_DIR: ./test/compare
|
||||
COMPARE_TEST_IMAGE: centos:8.2.2004
|
||||
|
||||
env:
|
||||
GNUMAKEFLAGS: '--no-print-directory'
|
||||
|
||||
tasks:
|
||||
|
||||
## High-level tasks #################################
|
||||
|
@ -213,10 +226,6 @@ tasks:
|
|||
# that the cache being restored with the correct binary will be rebuilt since the timestamps
|
||||
# and local checksums will not line up.
|
||||
deps: [tools, snapshot]
|
||||
sources:
|
||||
- "{{ .SNAPSHOT_BIN }}"
|
||||
- ./test/cli/**
|
||||
- ./**/*.go
|
||||
cmds:
|
||||
- cmd: "echo 'testing binary: {{ .SNAPSHOT_BIN }}'"
|
||||
silent: true
|
||||
|
@ -230,18 +239,14 @@ tasks:
|
|||
|
||||
test-utils:
|
||||
desc: Run tests for pipeline utils
|
||||
sources:
|
||||
- .github/scripts/labeler*.py
|
||||
cmds:
|
||||
- cmd: python .github/scripts/labeler_test.py
|
||||
- cmd: .github/scripts/labeler_test.py
|
||||
|
||||
|
||||
## Benchmark test targets #################################
|
||||
|
||||
benchmark:
|
||||
deps: [tmpdir]
|
||||
sources:
|
||||
- ./**/*.go
|
||||
generates:
|
||||
- "{{ .TMP_DIR }}/benchmark-main.txt"
|
||||
cmds:
|
||||
|
@ -254,8 +259,6 @@ tasks:
|
|||
|
||||
show-benchstat:
|
||||
deps: [benchmark, tmpdir]
|
||||
sources:
|
||||
- "{{ .TMP_DIR }}/benchstat.txt"
|
||||
cmds:
|
||||
- cmd: "cat {{ .TMP_DIR }}/benchstat.txt"
|
||||
silent: true
|
||||
|
@ -264,55 +267,171 @@ tasks:
|
|||
## Test-fixture-related targets #################################
|
||||
|
||||
fingerprints:
|
||||
desc: Generate test fixture fingerprints
|
||||
desc: Generate fingerprints for all non-docker test fixture
|
||||
silent: true
|
||||
# this will look for `test-fixtures/Makefile` and invoke the `fingerprint` target to calculate all cache input fingerprint files
|
||||
generates:
|
||||
- cmd/syft/internal/test/integration/test-fixtures/cache.fingerprint
|
||||
- syft/file/cataloger/executable/test-fixtures/elf/cache.fingerprint
|
||||
- syft/file/cataloger/executable/test-fixtures/shared-info/cache.fingerprint
|
||||
- syft/pkg/cataloger/binary/test-fixtures/cache.fingerprint
|
||||
- syft/pkg/cataloger/java/test-fixtures/java-builds/cache.fingerprint
|
||||
- syft/pkg/cataloger/golang/test-fixtures/archs/binaries.fingerprint
|
||||
- syft/pkg/cataloger/redhat/test-fixtures/rpms.fingerprint
|
||||
- syft/pkg/cataloger/kernel/test-fixtures/cache.fingerprint
|
||||
- '**/test-fixtures/**/*.fingerprint'
|
||||
- test/install/cache.fingerprint
|
||||
- test/cli/test-fixtures/cache.fingerprint
|
||||
cmds:
|
||||
# for EXECUTABLE unit test fixtures
|
||||
- "cd syft/file/cataloger/executable/test-fixtures/elf && make cache.fingerprint"
|
||||
- "cd syft/file/cataloger/executable/test-fixtures/shared-info && make cache.fingerprint"
|
||||
# for IMAGE integration test fixtures
|
||||
- "cd cmd/syft/internal/test/integration/test-fixtures && make cache.fingerprint"
|
||||
# for BINARY unit test fixtures
|
||||
- "cd syft/pkg/cataloger/binary/test-fixtures && make cache.fingerprint"
|
||||
# for JAVA BUILD unit test fixtures
|
||||
- "cd syft/pkg/cataloger/java/test-fixtures/java-builds && make cache.fingerprint"
|
||||
# for GO BINARY unit test fixtures
|
||||
- "cd syft/pkg/cataloger/golang/test-fixtures/archs && make binaries.fingerprint"
|
||||
# for RPM unit test fixtures
|
||||
- "cd syft/pkg/cataloger/redhat/test-fixtures && make rpms.fingerprint"
|
||||
# for Kernel unit test fixtures
|
||||
- "cd syft/pkg/cataloger/kernel/test-fixtures && make cache.fingerprint"
|
||||
# for INSTALL test fixtures
|
||||
- "cd test/install && make cache.fingerprint"
|
||||
# for CLI test fixtures
|
||||
- "cd test/cli/test-fixtures && make cache.fingerprint"
|
||||
- |
|
||||
BOLD='\033[1m'
|
||||
YELLOW='\033[0;33m'
|
||||
RESET='\033[0m'
|
||||
|
||||
fixtures:
|
||||
desc: Generate test fixtures
|
||||
echo -e "${YELLOW}creating fingerprint files for non-docker fixtures...${RESET}"
|
||||
for dir in $(find . -type d -name 'test-fixtures'); do
|
||||
if [ -f "$dir/Makefile" ]; then
|
||||
# for debugging...
|
||||
#echo -e "${YELLOW}• calculating fingerprints in $dir... ${RESET}"
|
||||
|
||||
(make -C "$dir" fingerprint)
|
||||
fi
|
||||
done
|
||||
|
||||
# for debugging...
|
||||
# echo -e "generated all fixture fingerprints"
|
||||
|
||||
- .github/scripts/fingerprint_docker_fixtures.py
|
||||
- |
|
||||
# if DOWNLOAD_TEST_FIXTURE_CACHE is set to 'false', then we don't need to calculate the fingerprint for the cache
|
||||
if [ "$DOWNLOAD_TEST_FIXTURE_CACHE" = "false" ]; then
|
||||
exit 0
|
||||
fi
|
||||
.github/scripts/find_cache_paths.py {{ .CACHE_PATHS_FILE }} > /dev/null
|
||||
|
||||
|
||||
refresh-fixtures:
|
||||
desc: Clear and fetch all test fixture cache
|
||||
aliases:
|
||||
- fixtures
|
||||
deps:
|
||||
- fingerprints
|
||||
silent: true
|
||||
cmd: |
|
||||
BOLD='\033[1m'
|
||||
PURPLE='\033[0;35m'
|
||||
RESET='\033[0m'
|
||||
|
||||
# if DOWNLOAD_TEST_FIXTURE_CACHE is set to 'false', then skip the cache download and always build
|
||||
if [ "$DOWNLOAD_TEST_FIXTURE_CACHE" = "false" ]; then
|
||||
echo -e "${BOLD}${PURPLE}skipping cache download, rebuilding cache...${RESET}"
|
||||
{{ .TASK }} build-fixtures
|
||||
exit 0
|
||||
fi
|
||||
|
||||
LATEST_FINGERPRINT=$(docker manifest inspect {{ .CACHE_IMAGE }} | {{ .YQ }} -r '.annotations.fingerprint')
|
||||
WANT_FINGERPRINT=$(cat {{ .CACHE_PATHS_FILE }} | {{ .YQ }} -r '.digest')
|
||||
|
||||
echo "latest cache: $LATEST_FINGERPRINT"
|
||||
echo "desired cache: $WANT_FINGERPRINT"
|
||||
|
||||
if [ -f {{ .LAST_CACHE_PULL_FILE }} ]; then
|
||||
LAST_PULL_FINGERPRINT=$(cat {{ .LAST_CACHE_PULL_FILE }} | {{ .YQ }} -r '.digest')
|
||||
else
|
||||
echo -e "${BOLD}${PURPLE}empty cache, downloading cache...${RESET}"
|
||||
{{ .TASK }} download-test-fixture-cache
|
||||
exit 0
|
||||
fi
|
||||
|
||||
echo "last pulled cache: $LAST_PULL_FINGERPRINT"
|
||||
|
||||
# if we already have the latest cache, skip the refresh
|
||||
if [ "$LAST_PULL_FINGERPRINT" = "$WANT_FINGERPRINT" ]; then
|
||||
echo -e "${BOLD}${PURPLE}already have the latest cache (skipping cache download)${RESET}"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
# at this point we only refresh the cache if we want the same cache that is currently available.
|
||||
# we don't by default refresh the cache if the cache if it is simply different from what we have,
|
||||
# because we may be working on a code change that doesn't require a cache refresh (but could trigger one,
|
||||
# which would be annoying to deal with in a development workflow).
|
||||
|
||||
if [ "$LATEST_FINGERPRINT" = "$WANT_FINGERPRINT" ]; then
|
||||
echo -e "${BOLD}${PURPLE}found newer cache! downloading cache...${RESET}"
|
||||
{{ .TASK }} download-test-fixture-cache
|
||||
else
|
||||
echo -e "${BOLD}${PURPLE}found different cache, but isn't clear if it's newer (skipping cache download and manually building)${RESET}"
|
||||
|
||||
{{ .YQ }} eval '.paths[] | "\(.digest) \(.path)"' {{ .LAST_CACHE_PULL_FILE }} > .tmp/last_cache_lines
|
||||
{{ .YQ }} eval '.paths[] | "\(.digest) \(.path)"' {{ .CACHE_PATHS_FILE }} > .tmp/cache_lines
|
||||
diff .tmp/last_cache_lines .tmp/cache_lines || true
|
||||
|
||||
echo -e "${BOLD}${PURPLE}diff with more context...${RESET}"
|
||||
|
||||
diff -U10000 {{ .LAST_CACHE_PULL_FILE }} {{ .CACHE_PATHS_FILE }} || true
|
||||
|
||||
echo -e "${BOLD}${PURPLE}detected changes to input material, manually building fixtures...${RESET}"
|
||||
|
||||
{{ .TASK }} build-fixtures
|
||||
fi
|
||||
|
||||
build-fixtures:
|
||||
desc: Generate all non-docker test fixtures
|
||||
silent: true
|
||||
# this will look for `test-fixtures/Makefile` and invoke the `fixtures` target to generate any and all test fixtures
|
||||
cmds:
|
||||
- "cd syft/file/cataloger/executable/test-fixtures/elf && make"
|
||||
- "cd syft/file/cataloger/executable/test-fixtures/shared-info && make"
|
||||
- "cd syft/pkg/cataloger/java/test-fixtures/java-builds && make"
|
||||
- "cd syft/pkg/cataloger/redhat/test-fixtures && make"
|
||||
- "cd syft/pkg/cataloger/binary/test-fixtures && make"
|
||||
- |
|
||||
BOLD='\033[1m'
|
||||
YELLOW='\033[0;33m'
|
||||
RESET='\033[0m'
|
||||
|
||||
# Use a for loop with command substitution to avoid subshell issues
|
||||
for dir in $(find . -type d -name 'test-fixtures'); do
|
||||
if [ -f "$dir/Makefile" ]; then
|
||||
echo -e "${YELLOW}${BOLD}generating fixtures in $dir${RESET}"
|
||||
(make -C "$dir" fixtures)
|
||||
fi
|
||||
done
|
||||
echo -e "${BOLD}generated all fixtures${RESET}"
|
||||
|
||||
download-test-fixture-cache:
|
||||
desc: Download test fixture cache from ghcr.io
|
||||
deps: [tools, clean-cache]
|
||||
vars:
|
||||
CACHE_DIGEST:
|
||||
sh: docker manifest inspect {{ .CACHE_IMAGE }} | {{ .YQ }} -r '.annotations.fingerprint'
|
||||
cmds:
|
||||
- silent: true
|
||||
cmd: |
|
||||
# if oras cache is > 4 GB, delete it
|
||||
if [ -d {{ .ORAS_CACHE }} ]; then
|
||||
total_size=$(du -c {{ .ORAS_CACHE }} | grep total | awk '{print $1}')
|
||||
if [ "$total_size" -gt 4194304 ]; then
|
||||
echo 'deleting oras cache'
|
||||
rm -rf {{ .ORAS_CACHE }}
|
||||
fi
|
||||
fi
|
||||
- "ORAS_CACHE={{ .ORAS_CACHE }} {{ .ORAS }} pull {{ .CACHE_IMAGE }}"
|
||||
- "cp {{ .CACHE_PATHS_FILE }} {{ .LAST_CACHE_PULL_FILE }}"
|
||||
|
||||
upload-test-fixture-cache:
|
||||
desc: Upload the test fixture cache to ghcr.io
|
||||
deps: [tools, fingerprints]
|
||||
silent: true
|
||||
cmd: |
|
||||
set -eu
|
||||
oras_command="{{ .ORAS }} push {{ .CACHE_IMAGE }}"
|
||||
|
||||
paths=$(cat {{ .CACHE_PATHS_FILE }} | {{ .YQ }} -r '.paths[].path')
|
||||
for path in $paths; do
|
||||
oras_command+=" $path"
|
||||
done
|
||||
oras_command+=" {{ .CACHE_PATHS_FILE }}"
|
||||
|
||||
oras_command+=" --annotation org.opencontainers.image.source=https://github.com/{{ .OWNER }}/{{ .PROJECT }}"
|
||||
oras_command+=" --annotation fingerprint=$(cat {{ .CACHE_PATHS_FILE }} | {{ .YQ }} -r '.digest')"
|
||||
|
||||
echo "Executing: $oras_command"
|
||||
eval $oras_command
|
||||
|
||||
show-test-image-cache:
|
||||
silent: true
|
||||
cmds:
|
||||
- "echo '\nDocker daemon cache:'"
|
||||
- "echo 'Docker daemon cache:'"
|
||||
- "docker images --format '{{`{{.ID}}`}} {{`{{.Repository}}`}}:{{`{{.Tag}}`}}' | grep stereoscope-fixture- | sort"
|
||||
- "echo '\nTar cache:'"
|
||||
- 'find . -type f -wholename "**/test-fixtures/snapshot/*" | sort'
|
||||
- 'find . -type f -wholename "**/test-fixtures/cache/stereoscope-fixture-*.tar" | sort'
|
||||
|
||||
check-docker-cache:
|
||||
desc: Ensure docker caches aren't using too much disk space
|
||||
|
@ -470,7 +589,16 @@ tasks:
|
|||
ci-check:
|
||||
# desc: "[CI only] Are you in CI?"
|
||||
cmds:
|
||||
- cmd: .github/scripts/ci-check.sh
|
||||
- cmd: |
|
||||
red=$(tput setaf 1)
|
||||
bold=$(tput bold)
|
||||
normal=$(tput sgr0)
|
||||
|
||||
# assert we are running in CI (or die!)
|
||||
if [[ -z "$CI" ]]; then
|
||||
echo "${bold}${red}This step should ONLY be run in CI. Exiting...${normal}"
|
||||
exit 1
|
||||
fi
|
||||
silent: true
|
||||
|
||||
ci-release:
|
||||
|
@ -502,8 +630,31 @@ tasks:
|
|||
- "rm -rf {{ .SNAPSHOT_DIR }}"
|
||||
- "rm -rf {{ .TMP_DIR }}/goreleaser.yaml"
|
||||
|
||||
clean-cache:
|
||||
desc: Remove all docker cache and local image tar cache
|
||||
clean-docker-cache:
|
||||
desc: Remove all docker cache tars and images from the daemon
|
||||
cmds:
|
||||
- 'find . -type f -wholename "**/test-fixtures/cache/stereoscope-fixture-*.tar" -delete'
|
||||
- "docker images --format '{{`{{.ID}}`}} {{`{{.Repository}}`}}' | grep stereoscope-fixture- | awk '{print $1}' | uniq | xargs -r docker rmi --force"
|
||||
- find . -type d -wholename "**/test-fixtures/cache" | xargs rm -rf
|
||||
- docker images --format '{{`{{.ID}}`}} {{`{{.Repository}}`}}' | grep stereoscope-fixture- | awk '{print $1}' | uniq | xargs -r docker rmi --force
|
||||
|
||||
clean-oras-cache:
|
||||
desc: Remove all cache for oras commands
|
||||
cmd: rm -rf {{ .ORAS_CACHE }}
|
||||
|
||||
clean-cache:
|
||||
desc: Remove all image docker tar cache, images from the docker daemon, and ephemeral test fixtures
|
||||
cmds:
|
||||
- task: clean-docker-cache
|
||||
- |
|
||||
BOLD='\033[1m'
|
||||
YELLOW='\033[0;33m'
|
||||
RESET='\033[0m'
|
||||
|
||||
# Use a for loop with command substitution to avoid subshell issues
|
||||
for dir in $(find . -type d -name 'test-fixtures'); do
|
||||
if [ -f "$dir/Makefile" ]; then
|
||||
echo -e "${YELLOW}${BOLD}deleting ephemeral test fixtures in $dir${RESET}"
|
||||
(make -C "$dir" clean)
|
||||
fi
|
||||
done
|
||||
echo -e "${BOLD}Deleted all ephemeral test fixtures${RESET}"
|
||||
- rm -f {{ .LAST_CACHE_PULL_FILE }} {{ .CACHE_PATHS_FILE }}
|
||||
|
|
|
@ -1,6 +1,21 @@
|
|||
# change these if you want CI to not use previous stored cache
|
||||
INTEGRATION_CACHE_BUSTER := "894d8ca"
|
||||
FINGERPRINT_FILE := cache.fingerprint
|
||||
|
||||
.PHONY: cache.fingerprint
|
||||
cache.fingerprint:
|
||||
find image-* -type f -exec md5sum {} + | awk '{print $1}' | sort | tee /dev/stderr | md5sum | tee cache.fingerprint && echo "$(INTEGRATION_CACHE_BUSTER)" >> cache.fingerprint
|
||||
.DEFAULT_GOAL := fixtures
|
||||
|
||||
# requirement 1: 'fixtures' goal to generate any and all test fixtures
|
||||
fixtures:
|
||||
@echo "nothing to do"
|
||||
|
||||
# requirement 2: 'fingerprint' goal to determine if the fixture input that indicates any existing cache should be busted
|
||||
fingerprint: $(FINGERPRINT_FILE)
|
||||
|
||||
# requirement 3: we always need to recalculate the fingerprint based on source regardless of any existing fingerprint
|
||||
.PHONY: $(FINGERPRINT_FILE)
|
||||
$(FINGERPRINT_FILE):
|
||||
@find image-* -type f -exec sha256sum {} \; | sort -k2 > $(FINGERPRINT_FILE)
|
||||
@#cat $(FINGERPRINT_FILE) | sha256sum | awk '{print $$1}'
|
||||
|
||||
# requirement 4: 'clean' goal to remove all generated test fixtures
|
||||
.PHONY: clean
|
||||
clean:
|
||||
rm -f $(FINGERPRINT_FILE)
|
||||
|
|
1
go.mod
1
go.mod
|
@ -88,6 +88,7 @@ require google.golang.org/genproto v0.0.0-20231106174013-bbf56f31fb17 // indirec
|
|||
|
||||
require (
|
||||
github.com/BurntSushi/toml v1.4.0
|
||||
github.com/OneOfOne/xxhash v1.2.8
|
||||
github.com/adrg/xdg v0.5.0
|
||||
github.com/magiconair/properties v1.8.7
|
||||
golang.org/x/exp v0.0.0-20231108232855-2478ac86f678
|
||||
|
|
2
go.sum
2
go.sum
|
@ -79,6 +79,8 @@ github.com/Microsoft/go-winio v0.6.1/go.mod h1:LRdKpFKfdobln8UmuiYcKPot9D2v6svN5
|
|||
github.com/Microsoft/hcsshim v0.11.4 h1:68vKo2VN8DE9AdN4tnkWnmdhqdbpUFM8OF3Airm7fz8=
|
||||
github.com/Microsoft/hcsshim v0.11.4/go.mod h1:smjE4dvqPX9Zldna+t5FG3rnoHhaB7QYxPRqGcpAD9w=
|
||||
github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU=
|
||||
github.com/OneOfOne/xxhash v1.2.8 h1:31czK/TI9sNkxIKfaUfGlU47BAxQ0ztGgd9vPyqimf8=
|
||||
github.com/OneOfOne/xxhash v1.2.8/go.mod h1:eZbhyaAYD41SGSSsnmcpxVoRiQ/MPUTjUdIIOT9Um7Q=
|
||||
github.com/ProtonMail/go-crypto v1.0.0 h1:LRuvITjQWX+WIfr930YHG2HNfjR1uOfyf5vE0kC2U78=
|
||||
github.com/ProtonMail/go-crypto v1.0.0/go.mod h1:EjAoLdwvbIOoOQr3ihjnSoLZRtE8azugULFRteWMNc0=
|
||||
github.com/acarl005/stripansi v0.0.0-20180116102854-5a71ef0e047d h1:licZJFw2RwpHMqeKTCYkitsPqHNxTmd4SNR5r94FGM8=
|
||||
|
|
15
syft/file/cataloger/executable/test-fixtures/Makefile
Normal file
15
syft/file/cataloger/executable/test-fixtures/Makefile
Normal file
|
@ -0,0 +1,15 @@
|
|||
.DEFAULT_GOAL := default
|
||||
|
||||
default:
|
||||
@for dir in $(shell find . -mindepth 1 -maxdepth 1 -type d); do \
|
||||
if [ -f "$$dir/Makefile" ]; then \
|
||||
$(MAKE) -C $$dir; \
|
||||
fi; \
|
||||
done
|
||||
|
||||
%:
|
||||
@for dir in $(shell find . -mindepth 1 -maxdepth 1 -type d); do \
|
||||
if [ -f "$$dir/Makefile" ]; then \
|
||||
$(MAKE) -C $$dir $@; \
|
||||
fi; \
|
||||
done
|
|
@ -1,8 +1,19 @@
|
|||
BIN=./bin
|
||||
TOOL_IMAGE=localhost/syft-bin-build-tools:latest
|
||||
VERIFY_FILE=actual_verify
|
||||
FINGERPRINT_FILE=$(BIN).fingerprint
|
||||
|
||||
all: build verify
|
||||
ifndef BIN
|
||||
$(error BIN is not set)
|
||||
endif
|
||||
|
||||
.DEFAULT_GOAL := fixtures
|
||||
|
||||
# requirement 1: 'fixtures' goal to generate any and all test fixtures
|
||||
fixtures: build verify
|
||||
|
||||
# requirement 2: 'fingerprint' goal to determine if the fixture input that indicates any existing cache should be busted
|
||||
fingerprint: $(FINGERPRINT_FILE)
|
||||
|
||||
tools-check:
|
||||
@sha256sum -c Dockerfile.sha256 || (echo "Tools Dockerfile has changed" && exit 1)
|
||||
|
@ -25,10 +36,14 @@ verify: tools
|
|||
debug:
|
||||
docker run -i --rm -v $(shell pwd):/mount -w /mount/project $(TOOL_IMAGE) bash
|
||||
|
||||
cache.fingerprint:
|
||||
@find project Dockerfile Makefile -type f -exec md5sum {} + | awk '{print $1}' | sort | tee cache.fingerprint
|
||||
# requirement 3: we always need to recalculate the fingerprint based on source regardless of any existing fingerprint
|
||||
.PHONY: $(FINGERPRINT_FILE)
|
||||
$(FINGERPRINT_FILE):
|
||||
@find project Dockerfile Makefile -type f -exec sha256sum {} \; | sort -k2 > $(FINGERPRINT_FILE)
|
||||
@#cat $(FINGERPRINT_FILE) | sha256sum | awk '{print $$1}'
|
||||
|
||||
# requirement 4: 'clean' goal to remove all generated test fixtures
|
||||
clean:
|
||||
rm -f $(BIN)/*
|
||||
rm -rf $(BIN) Dockerfile.sha256 $(VERIFY_FILE) $(FINGERPRINT_FILE)
|
||||
|
||||
.PHONY: build verify debug build-image build-bins clean dockerfile-check cache.fingerprint
|
||||
.PHONY: tools tools-check build verify debug clean
|
|
@ -1,8 +1,20 @@
|
|||
BIN=./bin
|
||||
TOOL_IMAGE=localhost/syft-shared-info-build-tools:latest
|
||||
VERIFY_FILE=actual_verify
|
||||
FINGERPRINT_FILE=$(BIN).fingerprint
|
||||
|
||||
ifndef BIN
|
||||
$(error BIN is not set)
|
||||
endif
|
||||
|
||||
.DEFAULT_GOAL := fixtures
|
||||
|
||||
# requirement 1: 'fixtures' goal to generate any and all test fixtures
|
||||
fixtures: build
|
||||
|
||||
# requirement 2: 'fingerprint' goal to determine if the fixture input that indicates any existing cache should be busted
|
||||
fingerprint: $(FINGERPRINT_FILE)
|
||||
|
||||
all: build
|
||||
tools-check:
|
||||
@sha256sum -c Dockerfile.sha256 || (echo "Tools Dockerfile has changed" && exit 1)
|
||||
|
||||
|
@ -10,16 +22,20 @@ tools:
|
|||
@(docker inspect $(TOOL_IMAGE) > /dev/null && make tools-check) || (docker build -t $(TOOL_IMAGE) . && sha256sum Dockerfile > Dockerfile.sha256)
|
||||
|
||||
build: tools
|
||||
mkdir -p $(BIN)
|
||||
@mkdir -p $(BIN)
|
||||
docker run --platform linux/amd64 -i -v $(shell pwd):/mount -w /mount/project $(TOOL_IMAGE) make
|
||||
|
||||
debug:
|
||||
docker run --platform linux/amd64 -i --rm -v $(shell pwd):/mount -w /mount/project $(TOOL_IMAGE) bash
|
||||
|
||||
cache.fingerprint:
|
||||
@find project Dockerfile Makefile -type f -exec md5sum {} + | awk '{print $1}' | sort | tee cache.fingerprint
|
||||
# requirement 3: we always need to recalculate the fingerprint based on source regardless of any existing fingerprint
|
||||
.PHONY: $(FINGERPRINT_FILE)
|
||||
$(FINGERPRINT_FILE):
|
||||
@find project Dockerfile Makefile -type f -exec sha256sum {} \; | sort -k2 > $(FINGERPRINT_FILE)
|
||||
@#cat $(FINGERPRINT_FILE) | sha256sum | awk '{print $$1}'
|
||||
|
||||
# requirement 4: 'clean' goal to remove all generated test fixtures
|
||||
clean:
|
||||
rm -f $(BIN)/*
|
||||
rm -rf $(BIN) Dockerfile.sha256 $(VERIFY_FILE) $(FINGERPRINT_FILE)
|
||||
|
||||
.PHONY: build verify debug build-image build-bins clean dockerfile-check cache.fingerprint
|
||||
.PHONY: tools tools-check build debug clean
|
||||
|
|
|
@ -1,4 +0,0 @@
|
|||
# Note: changes to this file will result in updating several test values. Consider making a new image fixture instead of editing this one.
|
||||
FROM scratch
|
||||
ADD file-1.txt /somefile-1.txt
|
||||
ADD file-2.txt /somefile-2.txt
|
|
@ -1 +0,0 @@
|
|||
this file has contents
|
|
@ -1 +0,0 @@
|
|||
file-2 contents!
|
|
@ -1,11 +1,11 @@
|
|||
[Image]
|
||||
Layer: 0
|
||||
Digest: sha256:fb6beecb75b39f4bb813dbf177e501edd5ddb3e69bb45cedeb78c676ee1b7a59
|
||||
Digest: sha256:100d5a55f9032faead28b7427fa3e650e4f0158f86ea89d06e1489df00cb8c6f
|
||||
Size: 22
|
||||
MediaType: application/vnd.docker.image.rootfs.diff.tar.gzip
|
||||
|
||||
Layer: 1
|
||||
Digest: sha256:319b588ce64253a87b533c8ed01cf0025e0eac98e7b516e12532957e1244fdec
|
||||
Digest: sha256:000fb9200890d3a19138478b20023023c0dce1c54352007c2863716780f049eb
|
||||
Size: 16
|
||||
MediaType: application/vnd.docker.image.rootfs.diff.tar.gzip
|
||||
|
||||
|
|
|
@ -1,6 +1,5 @@
|
|||
classifiers/dynamic
|
||||
classifiers/bin
|
||||
cache.fingerprint
|
||||
|
||||
# allow for lb patterns (rust, pytho, php and more)
|
||||
!lib*.so
|
||||
|
|
|
@ -1,8 +1,14 @@
|
|||
.PHONY: default list download download-all cache.fingerprint
|
||||
BIN=classifiers/bin
|
||||
FINGERPRINT_FILE=$(BIN).fingerprint
|
||||
|
||||
.DEFAULT_GOAL := default
|
||||
|
||||
default: download
|
||||
.DEFAULT_GOAL := fixtures
|
||||
|
||||
# requirement 1: 'fixtures' goal to generate any and all test fixtures
|
||||
fixtures: download
|
||||
|
||||
# requirement 2: 'fingerprint' goal to determine if the fixture input that indicates any existing cache should be busted
|
||||
fingerprint: clean-fingerprint $(FINGERPRINT_FILE)
|
||||
|
||||
list: ## list all managed binaries and snippets
|
||||
go run ./manager list
|
||||
|
@ -16,14 +22,23 @@ download-all: ## download all managed binaries
|
|||
add-snippet: ## add a new snippet from an existing binary
|
||||
go run ./manager add-snippet
|
||||
|
||||
cache.fingerprint: ## prints the sha256sum of the any input to the download command (to determine if there is a cache miss)
|
||||
@cat ./config.yaml | sha256sum | awk '{print $$1}' | tee cache.fingerprint
|
||||
# requirement 3: we always need to recalculate the fingerprint based on source regardless of any existing fingerprint
|
||||
.PHONY: $(FINGERPRINT_FILE)
|
||||
$(FINGERPRINT_FILE): ## prints the sha256sum of the any input to the download command (to determine if there is a cache miss)
|
||||
@sha256sum ./config.yaml > $(FINGERPRINT_FILE)
|
||||
|
||||
# requirement 4: 'clean' goal to remove all generated test fixtures
|
||||
clean: ## clean up all downloaded binaries
|
||||
rm -rf ./classifiers/bin
|
||||
rm -rf $(BIN)
|
||||
|
||||
clean-fingerprint: ## clean up all legacy fingerprint files
|
||||
@find $(BIN) -name '*.fingerprint' -delete
|
||||
|
||||
|
||||
## Halp! #################################
|
||||
|
||||
.PHONY: help
|
||||
help:
|
||||
@grep -E '^[a-zA-Z_-]+:.*?## .*$$' $(MAKEFILE_LIST) | sort | awk 'BEGIN {FS = ":.*?## "}; {printf "$(BOLD)$(CYAN)%-25s$(RESET)%s\n", $$1, $$2}'
|
||||
|
||||
.PHONY: default list download download-all clean clean-fingerprint add-snippet fingerprint
|
Binary file not shown.
|
@ -85,6 +85,7 @@ from-images:
|
|||
paths:
|
||||
- /usr/local/go/bin/go
|
||||
|
||||
# TODO: this is no longer available from dockerhub! (the snippet is vital)
|
||||
- version: 1.5.14
|
||||
images:
|
||||
- ref: haproxy:1.5.14@sha256:3d57e3921cc84e860f764e863ce729dd0765e3d28d444775127bc42d68f98e10
|
||||
|
|
|
@ -1,11 +1,11 @@
|
|||
package config
|
||||
|
||||
import (
|
||||
"crypto/sha256"
|
||||
"fmt"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
|
||||
"github.com/OneOfOne/xxhash"
|
||||
"gopkg.in/yaml.v3"
|
||||
)
|
||||
|
||||
|
@ -68,13 +68,13 @@ func PlatformAsValue(platform string) string {
|
|||
return strings.ReplaceAll(platform, "/", "-")
|
||||
}
|
||||
|
||||
func (c BinaryFromImage) Fingerprint() string {
|
||||
func (c BinaryFromImage) Digest() string {
|
||||
by, err := yaml.Marshal(c)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
|
||||
hasher := sha256.New()
|
||||
hasher.Write(by)
|
||||
hasher := xxhash.New64()
|
||||
_, _ = hasher.Write(by)
|
||||
return fmt.Sprintf("%x", hasher.Sum(nil))
|
||||
}
|
||||
|
|
|
@ -158,7 +158,7 @@ func TestPlatformAsValue(t *testing.T) {
|
|||
}
|
||||
}
|
||||
|
||||
func TestFingerprint(t *testing.T) {
|
||||
func TestDigest(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
binary BinaryFromImage
|
||||
|
@ -179,13 +179,13 @@ func TestFingerprint(t *testing.T) {
|
|||
"path/to/test",
|
||||
},
|
||||
},
|
||||
expected: "54ed081c07e4eba031afed4c04315cf96047822196473971be98d0769a0e3645",
|
||||
expected: "fc25c48e3d2f01e3",
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
assert.Equal(t, tt.expected, tt.binary.Fingerprint())
|
||||
assert.Equal(t, tt.expected, tt.binary.Digest())
|
||||
})
|
||||
}
|
||||
}
|
||||
|
|
|
@ -2,6 +2,7 @@ package internal
|
|||
|
||||
import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"os"
|
||||
"os/exec"
|
||||
|
@ -14,6 +15,8 @@ import (
|
|||
"github.com/anchore/syft/syft/pkg/cataloger/binary/test-fixtures/manager/internal/ui"
|
||||
)
|
||||
|
||||
const digestFileSuffix = ".xxh64"
|
||||
|
||||
func DownloadFromImage(dest string, config config.BinaryFromImage) error {
|
||||
t := ui.Title{Name: config.Name(), Version: config.Version}
|
||||
t.Start()
|
||||
|
@ -39,22 +42,22 @@ func DownloadFromImage(dest string, config config.BinaryFromImage) error {
|
|||
}
|
||||
|
||||
func isDownloadStale(config config.BinaryFromImage, binaryPaths []string) bool {
|
||||
currentFingerprint := config.Fingerprint()
|
||||
currentDigest := config.Digest()
|
||||
|
||||
for _, path := range binaryPaths {
|
||||
fingerprintPath := path + ".fingerprint"
|
||||
if _, err := os.Stat(fingerprintPath); err != nil {
|
||||
digestPath := path + digestFileSuffix
|
||||
if _, err := os.Stat(digestPath); err != nil {
|
||||
// missing a fingerprint file means the download is stale
|
||||
return true
|
||||
}
|
||||
|
||||
writtenFingerprint, err := os.ReadFile(fingerprintPath)
|
||||
writtenDigest, err := os.ReadFile(digestPath)
|
||||
if err != nil {
|
||||
// missing a fingerprint file means the download is stale
|
||||
return true
|
||||
}
|
||||
|
||||
if string(writtenFingerprint) != currentFingerprint {
|
||||
if string(writtenDigest) != currentDigest {
|
||||
// the fingerprint file does not match the current fingerprint, so the download is stale
|
||||
return true
|
||||
}
|
||||
|
@ -103,6 +106,12 @@ func pullDockerImage(imageReference, platform string) error {
|
|||
cmd := exec.Command("docker", "pull", "--platform", platform, imageReference)
|
||||
err := cmd.Run()
|
||||
if err != nil {
|
||||
// attach stderr to output message
|
||||
var exitErr *exec.ExitError
|
||||
if errors.As(err, &exitErr) && len(exitErr.Stderr) > 0 {
|
||||
err = fmt.Errorf("pull failed: %w:\n%s", err, exitErr.Stderr)
|
||||
}
|
||||
|
||||
a.Done(err)
|
||||
return err
|
||||
}
|
||||
|
@ -152,6 +161,12 @@ func copyBinariesFromDockerImage(config config.BinaryFromImage, destination stri
|
|||
|
||||
cmd := exec.Command("docker", "create", "--name", containerName, image.Reference)
|
||||
if err = cmd.Run(); err != nil {
|
||||
// attach stderr to output message
|
||||
var exitErr *exec.ExitError
|
||||
if errors.As(err, &exitErr) && len(exitErr.Stderr) > 0 {
|
||||
err = fmt.Errorf("%w:\n%s", err, exitErr.Stderr)
|
||||
}
|
||||
|
||||
return err
|
||||
}
|
||||
|
||||
|
@ -162,7 +177,7 @@ func copyBinariesFromDockerImage(config config.BinaryFromImage, destination stri
|
|||
|
||||
for i, destinationPath := range config.AllStorePathsForImage(image, destination) {
|
||||
path := config.PathsInImage[i]
|
||||
if err := copyBinaryFromContainer(containerName, path, destinationPath, config.Fingerprint()); err != nil {
|
||||
if err := copyBinaryFromContainer(containerName, path, destinationPath, config.Digest()); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
@ -170,7 +185,7 @@ func copyBinariesFromDockerImage(config config.BinaryFromImage, destination stri
|
|||
return nil
|
||||
}
|
||||
|
||||
func copyBinaryFromContainer(containerName, containerPath, destinationPath, fingerprint string) (err error) {
|
||||
func copyBinaryFromContainer(containerName, containerPath, destinationPath, digest string) (err error) {
|
||||
a := ui.Action{Msg: fmt.Sprintf("extract %s", containerPath)}
|
||||
a.Start()
|
||||
|
||||
|
@ -185,13 +200,24 @@ func copyBinaryFromContainer(containerName, containerPath, destinationPath, fing
|
|||
cmd := exec.Command("docker", "cp", fmt.Sprintf("%s:%s", containerName, containerPath), destinationPath) //nolint:gosec
|
||||
// reason for gosec exception: this is for processing test fixtures only, not used in production
|
||||
if err := cmd.Run(); err != nil {
|
||||
// attach stderr to output message
|
||||
var exitErr *exec.ExitError
|
||||
if errors.As(err, &exitErr) && len(exitErr.Stderr) > 0 {
|
||||
err = fmt.Errorf("%w:\n%s", err, exitErr.Stderr)
|
||||
}
|
||||
|
||||
return err
|
||||
}
|
||||
|
||||
// capture fingerprint file
|
||||
fingerprintPath := destinationPath + ".fingerprint"
|
||||
if err := os.WriteFile(fingerprintPath, []byte(fingerprint), 0600); err != nil {
|
||||
return fmt.Errorf("unable to write fingerprint file: %w", err)
|
||||
// ensure permissions are 600 for destination
|
||||
if err := os.Chmod(destinationPath, 0600); err != nil {
|
||||
return fmt.Errorf("unable to set permissions on file %q: %w", destinationPath, err)
|
||||
}
|
||||
|
||||
// capture digest file
|
||||
digestPath := destinationPath + digestFileSuffix
|
||||
if err := os.WriteFile(digestPath, []byte(digest), 0600); err != nil {
|
||||
return fmt.Errorf("unable to write digest file: %w", err)
|
||||
}
|
||||
|
||||
return nil
|
||||
|
|
|
@ -15,23 +15,23 @@ func TestIsDownloadStale(t *testing.T) {
|
|||
|
||||
cases := []struct {
|
||||
name string
|
||||
fingerprint string
|
||||
digest string
|
||||
expected bool
|
||||
}{
|
||||
{
|
||||
name: "no fingerprint",
|
||||
fingerprint: "",
|
||||
name: "no digest",
|
||||
digest: "",
|
||||
expected: true,
|
||||
},
|
||||
{
|
||||
name: "fingerprint matches",
|
||||
// this is the fingerprint for config in the loop body
|
||||
fingerprint: "5177d458eaca031ea16fa707841043df2e31b89be6bae7ea41290aa32f0251a6",
|
||||
name: "digest matches",
|
||||
// this is the digest for config in the loop body
|
||||
digest: "c9c8007f9c55c2f1",
|
||||
expected: false,
|
||||
},
|
||||
{
|
||||
name: "fingerprint does not match",
|
||||
fingerprint: "fingerprint",
|
||||
name: "digest does not match",
|
||||
digest: "bogus",
|
||||
expected: true,
|
||||
},
|
||||
}
|
||||
|
@ -39,10 +39,10 @@ func TestIsDownloadStale(t *testing.T) {
|
|||
for _, tt := range cases {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
binaryPath := filepath.Join(t.TempDir(), "binary")
|
||||
fh, err := os.Create(binaryPath + ".fingerprint")
|
||||
fh, err := os.Create(binaryPath + digestFileSuffix)
|
||||
require.NoError(t, err)
|
||||
|
||||
fh.Write([]byte(tt.fingerprint))
|
||||
fh.Write([]byte(tt.digest))
|
||||
require.NoError(t, fh.Close())
|
||||
|
||||
cfg := config.BinaryFromImage{
|
||||
|
|
|
@ -170,7 +170,7 @@ func getLogicalKey(managedBinaryPath string) (*LogicalEntryKey, error) {
|
|||
func allFilePaths(root string) ([]string, error) {
|
||||
var paths []string
|
||||
err := filepath.Walk(root, func(path string, info os.FileInfo, _ error) error {
|
||||
if info != nil && !info.IsDir() && !strings.HasSuffix(path, ".fingerprint") {
|
||||
if info != nil && !info.IsDir() && !strings.HasSuffix(path, digestFileSuffix) {
|
||||
paths = append(paths, path)
|
||||
}
|
||||
return nil
|
||||
|
|
|
@ -64,7 +64,7 @@ func TestPortageCataloger(t *testing.T) {
|
|||
var expectedRelationships []artifact.Relationship
|
||||
|
||||
pkgtest.NewCatalogTester().
|
||||
FromDirectory(t, "test-fixtures/image-portage").
|
||||
FromDirectory(t, "test-fixtures/layout").
|
||||
Expects(expectedPkgs, expectedRelationships).
|
||||
TestCataloger(t, NewPortageCataloger())
|
||||
|
||||
|
|
15
syft/pkg/cataloger/golang/test-fixtures/Makefile
Normal file
15
syft/pkg/cataloger/golang/test-fixtures/Makefile
Normal file
|
@ -0,0 +1,15 @@
|
|||
.DEFAULT_GOAL := default
|
||||
|
||||
default:
|
||||
@for dir in $(shell find . -mindepth 1 -maxdepth 1 -type d); do \
|
||||
if [ -f "$$dir/Makefile" ]; then \
|
||||
$(MAKE) -C $$dir; \
|
||||
fi; \
|
||||
done
|
||||
|
||||
%:
|
||||
@for dir in $(shell find . -mindepth 1 -maxdepth 1 -type d); do \
|
||||
if [ -f "$$dir/Makefile" ]; then \
|
||||
$(MAKE) -C $$dir $@; \
|
||||
fi; \
|
||||
done
|
|
@ -1,29 +1,39 @@
|
|||
DESTINATION=binaries
|
||||
FINGERPRINT_FILE=$(DESTINATION).fingerprint
|
||||
|
||||
all: $(DESTINATION)/hello-mach-o-arm64 $(DESTINATION)/hello-linux-arm $(DESTINATION)/hello-linux-ppc64le $(DESTINATION)/hello-win-amd64
|
||||
ifndef DESTINATION
|
||||
$(error DESTINATION is not set)
|
||||
endif
|
||||
|
||||
.DEFAULT_GOAL := fixtures
|
||||
|
||||
# requirement 1: 'fixtures' goal to generate any and all test fixtures
|
||||
fixtures: $(DESTINATION)
|
||||
|
||||
# requirement 2: 'fingerprint' goal to determine if the fixture input that indicates any existing cache should be busted
|
||||
fingerprint: $(DESTINATION).fingerprint
|
||||
|
||||
$(DESTINATION): $(DESTINATION)/hello-mach-o-arm64 $(DESTINATION)/hello-linux-arm $(DESTINATION)/hello-linux-ppc64le $(DESTINATION)/hello-win-amd64
|
||||
|
||||
$(DESTINATION)/hello-mach-o-arm64:
|
||||
mkdir -p $(DESTINATION)
|
||||
GOARCH=arm64 GOOS=darwin ./src/build.sh $(DESTINATION)/hello-mach-o-arm64
|
||||
|
||||
$(DESTINATION)/hello-linux-arm:
|
||||
mkdir -p $(DESTINATION)
|
||||
GOARCH=arm GOOS=linux ./src/build.sh $(DESTINATION)/hello-linux-arm
|
||||
|
||||
$(DESTINATION)/hello-linux-ppc64le:
|
||||
mkdir -p $(DESTINATION)
|
||||
GOARCH=ppc64le GOOS=linux ./src/build.sh $(DESTINATION)/hello-linux-ppc64le
|
||||
|
||||
$(DESTINATION)/hello-win-amd64:
|
||||
mkdir -p $(DESTINATION)
|
||||
GOARCH=amd64 GOOS=windows ./src/build.sh $(DESTINATION)/hello-win-amd64
|
||||
|
||||
# we need a way to determine if CI should bust the test cache based on the source material
|
||||
$(DESTINATION).fingerprint: clean
|
||||
mkdir -p $(DESTINATION)
|
||||
find src -type f -exec sha256sum {} \; | sort | tee /dev/stderr | tee $(DESTINATION).fingerprint
|
||||
sha256sum $(DESTINATION).fingerprint
|
||||
# requirement 3: we always need to recalculate the fingerprint based on source regardless of any existing fingerprint
|
||||
.PHONY: $(FINGERPRINT_FILE)
|
||||
$(FINGERPRINT_FILE):
|
||||
@find src -type f -exec sha256sum {} \; | sort -k2 > $(FINGERPRINT_FILE)
|
||||
@#cat $(FINGERPRINT_FILE) | sha256sum | awk '{print $$1}'
|
||||
|
||||
# requirement 4: 'clean' goal to remove all generated test fixtures
|
||||
.PHONY: clean
|
||||
clean:
|
||||
rm -f $(DESTINATION)/*
|
||||
rm -rf $(DESTINATION)
|
||||
|
|
|
@ -1,10 +1,13 @@
|
|||
#!/usr/bin/env bash
|
||||
set -uxe
|
||||
set -ue
|
||||
|
||||
# note: this can be easily done in a 1-liner, however circle CI does NOT allow volume mounts from the host in docker executors (since they are on remote hosts, where the host files are inaccessible)
|
||||
# note: gocache override is so we can run docker build not as root in a container without permission issues
|
||||
|
||||
BINARY=$1
|
||||
|
||||
mkdir -p "$(dirname "$BINARY")"
|
||||
|
||||
CTRID=$(docker create -e GOOS="${GOOS}" -e GOARCH="${GOARCH}" -u "$(id -u):$(id -g)" -e GOCACHE=/tmp -w /src golang:1.17 go build -o main main.go)
|
||||
|
||||
function cleanup() {
|
||||
|
|
15
syft/pkg/cataloger/java/test-fixtures/Makefile
Normal file
15
syft/pkg/cataloger/java/test-fixtures/Makefile
Normal file
|
@ -0,0 +1,15 @@
|
|||
.DEFAULT_GOAL := default
|
||||
|
||||
default:
|
||||
@for dir in $(shell find . -mindepth 1 -maxdepth 1 -type d); do \
|
||||
if [ -f "$$dir/Makefile" ]; then \
|
||||
$(MAKE) -C $$dir; \
|
||||
fi; \
|
||||
done
|
||||
|
||||
%:
|
||||
@for dir in $(shell find . -mindepth 1 -maxdepth 1 -type d); do \
|
||||
if [ -f "$$dir/Makefile" ]; then \
|
||||
$(MAKE) -C $$dir $@; \
|
||||
fi; \
|
||||
done
|
|
@ -1,5 +1,10 @@
|
|||
CACHE_DIR = cache
|
||||
CACHE_PATH = $(shell pwd)/cache
|
||||
FINGERPRINT_FILE=$(CACHE_DIR).fingerprint
|
||||
|
||||
ifndef CACHE_DIR
|
||||
$(error CACHE_DIR is not set)
|
||||
endif
|
||||
|
||||
JACKSON_CORE = jackson-core-2.15.2
|
||||
SBT_JACKSON_CORE = com.fasterxml.jackson.core.jackson-core-2.15.2
|
||||
|
@ -8,28 +13,53 @@ API_ALL_SOURCES = api-all-2.0.0-sources
|
|||
SPRING_INSTRUMENTATION = spring-instrumentation-4.3.0-1.0
|
||||
MULTIPLE_MATCHING = multiple-matching-2.11.5
|
||||
|
||||
$(CACHE_DIR):
|
||||
mkdir -p $(CACHE_DIR)
|
||||
|
||||
$(CACHE_DIR)/$(JACKSON_CORE).jar: $(CACHE_DIR)
|
||||
.DEFAULT_GOAL := fixtures
|
||||
|
||||
# requirement 1: 'fixtures' goal to generate any and all test fixtures
|
||||
fixtures: $(CACHE_DIR)
|
||||
|
||||
# requirement 2: 'fingerprint' goal to determine if the fixture input that indicates any existing cache should be busted
|
||||
fingerprint: $(FINGERPRINT_FILE)
|
||||
|
||||
$(CACHE_DIR): $(CACHE_DIR)/$(JACKSON_CORE).jar $(CACHE_DIR)/$(SBT_JACKSON_CORE).jar $(CACHE_DIR)/$(OPENSAML_CORE).jar $(CACHE_DIR)/$(API_ALL_SOURCES).jar $(CACHE_DIR)/$(SPRING_INSTRUMENTATION).jar $(CACHE_DIR)/$(MULTIPLE_MATCHING).jar
|
||||
|
||||
$(CACHE_DIR)/$(JACKSON_CORE).jar:
|
||||
mkdir -p $(CACHE_DIR)
|
||||
cd $(JACKSON_CORE) && zip -r $(CACHE_PATH)/$(JACKSON_CORE).jar .
|
||||
|
||||
$(CACHE_DIR)/$(SBT_JACKSON_CORE).jar: $(CACHE_DIR)
|
||||
$(CACHE_DIR)/$(SBT_JACKSON_CORE).jar:
|
||||
mkdir -p $(CACHE_DIR)
|
||||
cd $(SBT_JACKSON_CORE) && zip -r $(CACHE_PATH)/$(SBT_JACKSON_CORE).jar .
|
||||
|
||||
$(CACHE_DIR)/$(OPENSAML_CORE).jar: $(CACHE_DIR)
|
||||
$(CACHE_DIR)/$(OPENSAML_CORE).jar:
|
||||
mkdir -p $(CACHE_DIR)
|
||||
cd $(OPENSAML_CORE) && zip -r $(CACHE_PATH)/$(OPENSAML_CORE).jar .
|
||||
|
||||
$(CACHE_DIR)/$(API_ALL_SOURCES).jar: $(CACHE_DIR)
|
||||
$(CACHE_DIR)/$(API_ALL_SOURCES).jar:
|
||||
mkdir -p $(CACHE_DIR)
|
||||
cd $(API_ALL_SOURCES) && zip -r $(CACHE_PATH)/$(API_ALL_SOURCES).jar .
|
||||
|
||||
$(CACHE_DIR)/$(SPRING_INSTRUMENTATION).jar: $(CACHE_DIR)
|
||||
$(CACHE_DIR)/$(SPRING_INSTRUMENTATION).jar:
|
||||
mkdir -p $(CACHE_DIR)
|
||||
cd $(SPRING_INSTRUMENTATION) && zip -r $(CACHE_PATH)/$(SPRING_INSTRUMENTATION).jar .
|
||||
|
||||
$(CACHE_DIR)/$(MULTIPLE_MATCHING).jar: $(CACHE_DIR)
|
||||
$(CACHE_DIR)/$(MULTIPLE_MATCHING).jar:
|
||||
mkdir -p $(CACHE_DIR)
|
||||
cd $(MULTIPLE_MATCHING) && zip -r $(CACHE_PATH)/$(MULTIPLE_MATCHING).jar .
|
||||
|
||||
# Jenkins plugins typically do not have the version included in the archive name,
|
||||
# so it is important to not include it in the generated test fixture
|
||||
$(CACHE_DIR)/gradle.hpi: $(CACHE_DIR)
|
||||
$(CACHE_DIR)/gradle.hpi:
|
||||
mkdir -p $(CACHE_DIR)
|
||||
cd jenkins-plugins/gradle/2.11 && zip -r $(CACHE_PATH)/gradle.hpi .
|
||||
|
||||
# requirement 3: we always need to recalculate the fingerprint based on source regardless of any existing fingerprint
|
||||
.PHONY: $(FINGERPRINT_FILE)
|
||||
$(FINGERPRINT_FILE):
|
||||
@find . ! -path '*/cache*' -type f -exec sha256sum {} \; | sort -k2 > $(FINGERPRINT_FILE)
|
||||
@#cat $(FINGERPRINT_FILE) | sha256sum | awk '{print $$1}'
|
||||
|
||||
# requirement 4: 'clean' goal to remove all generated test fixtures
|
||||
clean:
|
||||
rm -rf $(CACHE_DIR)/* $(FINGERPRINT_FILE)
|
||||
|
|
|
@ -1,17 +1,18 @@
|
|||
PKGSDIR=packages
|
||||
FINGERPRINT_FILE=$(PKGSDIR).fingerprint
|
||||
|
||||
ifndef PKGSDIR
|
||||
$(error PKGSDIR is not set)
|
||||
endif
|
||||
|
||||
all: jars archives native-image
|
||||
|
||||
clean: clean-examples
|
||||
rm -f $(PKGSDIR)/*
|
||||
.DEFAULT_GOAL := fixtures
|
||||
|
||||
clean-examples: clean-gradle clean-maven clean-jenkins clean-nestedjar
|
||||
# requirement 1: 'fixtures' goal to generate any and all test fixtures
|
||||
fixtures: jars archives native-image
|
||||
|
||||
.PHONY: maven gradle clean clean-gradle clean-maven clean-jenkins clean-examples clean-nestedjar jars archives
|
||||
# requirement 2: 'fingerprint' goal to determine if the fixture input that indicates any existing cache should be busted
|
||||
fingerprint: $(FINGERPRINT_FILE)
|
||||
|
||||
jars: $(PKGSDIR)/example-java-app-maven-0.1.0.jar $(PKGSDIR)/example-java-app-gradle-0.1.0.jar $(PKGSDIR)/example-jenkins-plugin.hpi $(PKGSDIR)/spring-boot-0.0.1-SNAPSHOT.jar
|
||||
|
||||
|
@ -71,8 +72,16 @@ $(PKGSDIR)/example-java-app: $(PKGSDIR)/example-java-app-maven-0.1.0.jar
|
|||
$(PKGSDIR)/gcc-amd64-darwin-exec-debug:
|
||||
./build-example-macho-binary.sh $(PKGSDIR)
|
||||
|
||||
# we need a way to determine if CI should bust the test cache based on the source material
|
||||
.PHONY: cache.fingerprint
|
||||
cache.fingerprint:
|
||||
find example* build* gradle* Makefile -type f -exec sha256sum {} \; | sort | tee /dev/stderr | tee cache.fingerprint
|
||||
sha256sum cache.fingerprint
|
||||
# requirement 3: we always need to recalculate the fingerprint based on source regardless of any existing fingerprint
|
||||
.PHONY: $(FINGERPRINT_FILE)
|
||||
$(FINGERPRINT_FILE):
|
||||
@find example-* build-* Makefile -type f -exec sha256sum {} \; | sort -k2 > $(FINGERPRINT_FILE)
|
||||
@#cat $(FINGERPRINT_FILE) | sha256sum | awk '{print $$1}'
|
||||
|
||||
# requirement 4: 'clean' goal to remove all generated test fixtures
|
||||
clean: clean-examples
|
||||
rm -rf $(PKGSDIR) $(FINGERPRINT_FILE)
|
||||
|
||||
clean-examples: clean-gradle clean-maven clean-jenkins clean-nestedjar
|
||||
|
||||
.PHONY: maven gradle clean clean-gradle clean-maven clean-jenkins clean-examples clean-nestedjar jars archives
|
||||
|
|
|
@ -1,7 +1,21 @@
|
|||
all:
|
||||
FINGERPRINT_FILE=cache.fingerprint
|
||||
|
||||
# we need a way to determine if CI should bust the test cache based on the source material
|
||||
.PHONY: cache.fingerprint
|
||||
cache.fingerprint:
|
||||
find Makefile **/Dockerfile -type f -exec sha256sum {} \; | sort | tee /dev/stderr | tee cache.fingerprint
|
||||
sha256sum cache.fingerprint
|
||||
|
||||
.DEFAULT_GOAL := fixtures
|
||||
|
||||
# requirement 1: 'fixtures' goal to generate any and all test fixtures
|
||||
fixtures:
|
||||
@echo "nothing to do"
|
||||
|
||||
# requirement 2: 'fingerprint' goal to determine if the fixture input that indicates any existing cache should be busted
|
||||
fingerprint: $(FINGERPRINT_FILE)
|
||||
|
||||
# requirement 3: we always need to recalculate the fingerprint based on source regardless of any existing fingerprint
|
||||
.PHONY: $(FINGERPRINT_FILE)
|
||||
$(FINGERPRINT_FILE):
|
||||
@find Makefile **/Dockerfile -type f -exec sha256sum {} \; | sort -k2 > $(FINGERPRINT_FILE)
|
||||
@#cat $(FINGERPRINT_FILE) | sha256sum | awk '{print $$1}'
|
||||
|
||||
# requirement 4: 'clean' goal to remove all generated test fixtures
|
||||
clean:
|
||||
rm -f $(FINGERPRINT_FILE)
|
||||
|
|
|
@ -1,21 +1,38 @@
|
|||
RPMSDIR=rpms
|
||||
FINGERPRINT_FILE=$(RPMSDIR).fingerprint
|
||||
|
||||
ifndef RPMSDIR
|
||||
$(error RPMSDIR is not set)
|
||||
endif
|
||||
|
||||
all: rpms
|
||||
|
||||
clean:
|
||||
rm -rf $(RPMSDIR)
|
||||
.DEFAULT_GOAL := fixtures
|
||||
|
||||
# requirement 1: 'fixtures' goal to generate any and all test fixtures
|
||||
fixtures: rpms
|
||||
|
||||
# requirement 2: 'fingerprint' goal to determine if the fixture input that indicates any existing cache should be busted
|
||||
fingerprint: $(FINGERPRINT_FILE)
|
||||
|
||||
rpms:
|
||||
mkdir -p $(RPMSDIR)
|
||||
cd $(RPMSDIR) && curl https://dl.fedoraproject.org/pub/epel/7/x86_64/Packages/a/abc-1.01-9.hg20160905.el7.x86_64.rpm -O
|
||||
cd $(RPMSDIR) && curl https://dl.fedoraproject.org/pub/epel/7/x86_64/Packages/z/zork-1.0.3-1.el7.x86_64.rpm -O
|
||||
@# see note from https://dl.fedoraproject.org/pub/epel/7/README
|
||||
@# ATTENTION
|
||||
@# ======================================
|
||||
@# The contents of this directory have been moved to our archives available at:
|
||||
@#
|
||||
@# http://archives.fedoraproject.org/pub/archive/epel/
|
||||
|
||||
# we need a way to determine if CI should bust the test cache based on the source material
|
||||
.PHONY: $(RPMSDIR).fingerprint
|
||||
$(RPMSDIR).fingerprint:
|
||||
find Makefile -type f -exec sha256sum {} \; | sort | tee /dev/stderr | tee $(RPMSDIR).fingerprint
|
||||
sha256sum $(RPMSDIR).fingerprint
|
||||
cd $(RPMSDIR) && curl -LO https://archives.fedoraproject.org/pub/archive/epel/7/x86_64/Packages/a/abc-1.01-9.hg20160905.el7.x86_64.rpm
|
||||
cd $(RPMSDIR) && curl -LO https://archives.fedoraproject.org/pub/archive/epel/7/x86_64/Packages/z/zork-1.0.3-1.el7.x86_64.rpm
|
||||
|
||||
# requirement 3: we always need to recalculate the fingerprint based on source regardless of any existing fingerprint
|
||||
.PHONY: $(FINGERPRINT_FILE)
|
||||
$(FINGERPRINT_FILE):
|
||||
@find Makefile -type f -exec sha256sum {} \; | sort -k2 > $(FINGERPRINT_FILE)
|
||||
@#cat $(FINGERPRINT_FILE) | sha256sum | awk '{print $$1}'
|
||||
|
||||
# requirement 4: 'clean' goal to remove all generated test fixtures
|
||||
.PHONY: clean
|
||||
clean:
|
||||
rm -rf $(RPMSDIR) $(FINGERPRINT_FILE)
|
||||
|
|
|
@ -1,6 +1,22 @@
|
|||
# change these if you want CI to not use previous stored cache
|
||||
CLI_CACHE_BUSTER := "e5cdfd8"
|
||||
FINGERPRINT_FILE=cache.fingerprint
|
||||
|
||||
.DEFAULT_GOAL := fixtures
|
||||
|
||||
# requirement 1: 'fixtures' goal to generate any and all test fixtures
|
||||
fixtures:
|
||||
@echo "nothing to do"
|
||||
|
||||
# requirement 2: 'fingerprint' goal to determine if the fixture input that indicates any existing cache should be busted
|
||||
fingerprint: $(FINGERPRINT_FILE)
|
||||
|
||||
# requirement 3: we always need to recalculate the fingerprint based on source regardless of any existing fingerprint
|
||||
.PHONY: $(FINGERPRINT_FILE)
|
||||
$(FINGERPRINT_FILE):
|
||||
@find image-* -type f -exec sha256sum {} \; | sort -k2 > $(FINGERPRINT_FILE)
|
||||
@#cat $(FINGERPRINT_FILE) | sha256sum | awk '{print $$1}'
|
||||
|
||||
# requirement 4: 'clean' goal to remove all generated test fixtures
|
||||
.PHONY: clean
|
||||
clean:
|
||||
rm -f $(FINGERPRINT_FILE)
|
||||
|
||||
.PHONY: cache.fingerprint
|
||||
cache.fingerprint:
|
||||
find image-* -type f -exec md5sum {} + | awk '{print $1}' | sort | md5sum | tee cache.fingerprint && echo "$(CLI_CACHE_BUSTER)" >> cache.fingerprint
|
||||
|
|
|
@ -1,5 +1,7 @@
|
|||
NAME=syft
|
||||
|
||||
FINGERPRINT_FILE := cache.fingerprint
|
||||
|
||||
# for local testing (not testing within containers) use the binny-managed version of cosign.
|
||||
# this also means that the user does not need to install cosign on their system to run tests.
|
||||
COSIGN_BINARY=../../.tool/cosign
|
||||
|
@ -21,8 +23,6 @@ ACCEPTANCE_CMD=sh -c '../../install.sh -v -b /usr/local/bin && syft version && r
|
|||
PREVIOUS_RELEASE=v0.33.0
|
||||
ACCEPTANCE_PREVIOUS_RELEASE_CMD=sh -c "../../install.sh -b /usr/local/bin $(PREVIOUS_RELEASE) && syft version"
|
||||
|
||||
# CI cache busting values; change these if you want CI to not use previous stored cache
|
||||
INSTALL_TEST_CACHE_BUSTER=894d8ca
|
||||
|
||||
define title
|
||||
@printf '\n≡≡≡[ $(1) ]≡≡≡≡≡≡≡≡≡≡≡≡≡≡≡≡≡≡≡≡≡≡≡≡≡≡≡≡≡≡≡≡≡≡≡≡≡≡≡≡≡≡≡≡≡≡≡≡≡≡≡≡≡≡\n'
|
||||
|
@ -130,7 +130,8 @@ busybox-1.36:
|
|||
|
||||
## For CI ########################################################
|
||||
|
||||
.PHONY: cache.fingerprint
|
||||
cache.fingerprint:
|
||||
$(call title,Install test fixture fingerprint)
|
||||
@find ./environments/* -type f -exec md5sum {} + | awk '{print $1}' | sort | tee /dev/stderr | md5sum | tee cache.fingerprint && echo "$(INSTALL_TEST_CACHE_BUSTER)" >> cache.fingerprint
|
||||
# requirement 3: we always need to recalculate the fingerprint based on source regardless of any existing fingerprint
|
||||
.PHONY: $(FINGERPRINT_FILE)
|
||||
$(FINGERPRINT_FILE):
|
||||
@find ./environments/* -type f -exec sha256sum {} \; | sort -k2 > $(FINGERPRINT_FILE)
|
||||
@#cat $(FINGERPRINT_FILE) | sha256sum | awk '{print $$1}'
|
||||
|
|
Loading…
Reference in a new issue