From 82d6f56cfc3432f584632f761c44ec8efd2e7c67 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fran=C3=A7ois=20Mockers?= Date: Mon, 14 Oct 2024 07:24:38 +0200 Subject: [PATCH] Compare screenshots with main on PRs (#13248) # Objective - Compare screenshots for a few examples between PRs and main ## Solution - Send screenshots taken to a screenshot comparison service - Not completely sure every thing will work at once, but it shouldn't break anything at least - it needs a secret to work, I'll add it if enough people agree with this PR - this PR doesn't change anything on the screenshot selection (load_gltf and breakout currently), this will need rendering folks input and can happen later --------- Co-authored-by: Alice Cecile --- .github/workflows/ci-examples.yml | 77 +++++++++++++++ .github/workflows/ci.yml | 46 --------- .../send-screenshots-to-pixeleagle.yml | 94 +++++++++++++++++++ .github/workflows/validation-jobs.yml | 14 ++- 4 files changed, 184 insertions(+), 47 deletions(-) create mode 100644 .github/workflows/ci-examples.yml create mode 100644 .github/workflows/send-screenshots-to-pixeleagle.yml diff --git a/.github/workflows/ci-examples.yml b/.github/workflows/ci-examples.yml new file mode 100644 index 0000000000..d2dae725e6 --- /dev/null +++ b/.github/workflows/ci-examples.yml @@ -0,0 +1,77 @@ +name: CI - examples + +on: + merge_group: + pull_request_target: + push: + branches: + - main + - release-* + +# Unlike jobs in ci.yml, jobs in this workflow can access secrets. Their definitions are taken from the base branch of a PR, not from the PR itself. + +env: + CARGO_TERM_COLOR: always + # If nightly is breaking CI, modify this variable to target a specific nightly version. + NIGHTLY_TOOLCHAIN: nightly + +concurrency: + group: ${{github.workflow}}-${{github.ref}}-examples + cancel-in-progress: ${{github.event_name == 'pull_request_target'}} + +jobs: + run-examples-macos-metal: + # Explicity use macOS 14 to take advantage of M1 chip. + runs-on: macos-14 + timeout-minutes: 30 + steps: + - uses: actions/checkout@v4 + - uses: dtolnay/rust-toolchain@stable + - name: Disable audio + # Disable audio through a patch. on github m1 runners, audio timeouts after 15 minutes + run: git apply --ignore-whitespace tools/example-showcase/disable-audio.patch + - name: Build bevy + # this uses the same command as when running the example to ensure build is reused + run: | + TRACE_CHROME=trace-alien_cake_addict.json CI_TESTING_CONFIG=.github/example-run/alien_cake_addict.ron cargo build --example alien_cake_addict --features "bevy_ci_testing,trace,trace_chrome" + - name: Run examples + run: | + for example in .github/example-run/*.ron; do + example_name=`basename $example .ron` + echo -n $example_name > last_example_run + echo "running $example_name - "`date` + time TRACE_CHROME=trace-$example_name.json CI_TESTING_CONFIG=$example cargo run --example $example_name --features "bevy_ci_testing,trace,trace_chrome" + sleep 10 + if [ `find ./ -maxdepth 1 -name 'screenshot-*.png' -print -quit` ]; then + mkdir screenshots-$example_name + mv screenshot-*.png screenshots-$example_name/ + fi + done + mkdir traces && mv trace*.json traces/ + mkdir screenshots && mv screenshots-* screenshots/ + - name: save traces + uses: actions/upload-artifact@v4 + with: + name: example-traces-macos + path: traces + - name: save screenshots + uses: actions/upload-artifact@v4 + with: + name: screenshots-macos + path: screenshots + - uses: actions/upload-artifact@v4 + if: ${{ failure() && github.event_name == 'pull_request' }} + with: + name: example-run-macos + path: example-run/ + + compare-macos-screenshots: + name: Compare macOS screenshots + needs: [run-examples-macos-metal] + uses: ./.github/workflows/send-screenshots-to-pixeleagle.yml + with: + commit: ${{ github.sha }} + branch: ${{ github.ref_name }} + artifact: screenshots-macos + os: macos + secrets: inherit diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 0191064131..5b2f29e291 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -253,52 +253,6 @@ jobs: echo 'if you use VSCode, you can also install `Typos Spell Checker' echo 'You can find the extension here: https://marketplace.visualstudio.com/items?itemName=tekumara.typos-vscode' - - run-examples-macos-metal: - # Explicity use macOS 14 to take advantage of M1 chip. - runs-on: macos-14 - timeout-minutes: 30 - steps: - - uses: actions/checkout@v4 - - uses: dtolnay/rust-toolchain@stable - - name: Disable audio - # Disable audio through a patch. on github m1 runners, audio timeouts after 15 minutes - run: git apply --ignore-whitespace tools/example-showcase/disable-audio.patch - - name: Build bevy - # this uses the same command as when running the example to ensure build is reused - run: | - TRACE_CHROME=trace-alien_cake_addict.json CI_TESTING_CONFIG=.github/example-run/alien_cake_addict.ron cargo build --example alien_cake_addict --features "bevy_ci_testing,trace,trace_chrome" - - name: Run examples - run: | - for example in .github/example-run/*.ron; do - example_name=`basename $example .ron` - echo -n $example_name > last_example_run - echo "running $example_name - "`date` - time TRACE_CHROME=trace-$example_name.json CI_TESTING_CONFIG=$example cargo run --example $example_name --features "bevy_ci_testing,trace,trace_chrome" - sleep 10 - if [ `find ./ -maxdepth 1 -name 'screenshot-*.png' -print -quit` ]; then - mkdir screenshots-$example_name - mv screenshot-*.png screenshots-$example_name/ - fi - done - mkdir traces && mv trace*.json traces/ - mkdir screenshots && mv screenshots-* screenshots/ - - name: save traces - uses: actions/upload-artifact@v4 - with: - name: example-traces-macos - path: traces - - name: save screenshots - uses: actions/upload-artifact@v4 - with: - name: screenshots-macos - path: screenshots - - uses: actions/upload-artifact@v4 - if: ${{ failure() && github.event_name == 'pull_request' }} - with: - name: example-run-macos - path: example-run/ - check-doc: runs-on: ubuntu-latest timeout-minutes: 30 diff --git a/.github/workflows/send-screenshots-to-pixeleagle.yml b/.github/workflows/send-screenshots-to-pixeleagle.yml new file mode 100644 index 0000000000..5a6b7f4ea2 --- /dev/null +++ b/.github/workflows/send-screenshots-to-pixeleagle.yml @@ -0,0 +1,94 @@ +name: Send Screenshots to Pixel Eagle + +on: + workflow_call: + inputs: + artifact: + required: true + type: string + commit: + required: true + type: string + branch: + required: true + type: string + os: + required: true + type: string + +jobs: + send-to-pixel-eagle: + name: Send screenshots to Pixel Eagle + runs-on: ubuntu-latest + steps: + + - name: Download artifact + uses: actions/download-artifact@v4 + with: + pattern: ${{ inputs.artifact }} + + - name: Send to Pixel Eagle + env: + project: B04F67C0-C054-4A6F-92EC-F599FEC2FD1D + run: | + # Create a new run with its associated metadata + metadata='{"os":"${{ inputs.os }}", "commit": "${{ inputs.commit }}", "branch": "${{ inputs.branch }}"}' + run=`curl https://pixel-eagle.vleue.com/$project/runs --json "$metadata" --oauth2-bearer ${{ secrets.PIXELEAGLE_TOKEN }} | jq '.id'` + + SAVEIFS=$IFS + + cd ${{ inputs.artifact }} + + # Read the hashes of the screenshot for fast comparison when they are equal + IFS=$'\n' + # Build a json array of screenshots and their hashes + hashes='['; + for screenshot in $(find . -type f -name "*.png"); + do + name=${screenshot:14} + echo $name + hash=`shasum -a 256 $screenshot | awk '{print $1}'` + hashes="$hashes [\"$name\",\"$hash\"]," + done + hashes=`echo $hashes | rev | cut -c 2- | rev` + hashes="$hashes]" + + IFS=$SAVEIFS + + # Upload screenshots with unknown hashes + curl https://pixel-eagle.vleue.com/$project/runs/$run/hashes --json "$hashes" --oauth2-bearer ${{ secrets.PIXELEAGLE_TOKEN }} | jq '.[]|[.name] | @tsv' | + while IFS=$'\t' read -r name; do + name=`echo $name | tr -d '"'` + echo "Uploading $name" + curl https://pixel-eagle.vleue.com/$project/runs/$run/screenshots -F "data=@./$name" -F "screenshot=$name" --oauth2-bearer ${{ secrets.PIXELEAGLE_TOKEN }} + echo + done + + IFS=$SAVEIFS + + cd .. + + # Trigger comparison with the main branch on the same os + curl https://pixel-eagle.vleue.com/$project/runs/$run/compare/auto --json '{"os":"", "branch": "main"}' --oauth2-bearer ${{ secrets.PIXELEAGLE_TOKEN }} > pixeleagle.json + + # Log results + compared_with=`cat pixeleagle.json | jq '.to'` + + status=0 + missing=`cat pixeleagle.json | jq '.missing | length'` + if [ ! $missing -eq 0 ]; then + echo "There are $missing missing screenshots" + echo "::warning title=$missing missing screenshots on ${{ inputs.os }}::https://pixel-eagle.vleue.com/$project/runs/$run/compare/$to" + status=1 + fi + + diff=`cat pixeleagle.json | jq '.diff | length'` + if [ ! $diff -eq 0 ]; then + echo "There are $diff screenshots with a difference" + echo "::warning title=$diff different screenshots on ${{ inputs.os }}::https://pixel-eagle.vleue.com/$project/runs/$run/compare/$to" + status=1 + fi + + echo "created run $run: https://pixel-eagle.vleue.com/$project/runs/$run/compare/$to" + + exit $status diff --git a/.github/workflows/validation-jobs.yml b/.github/workflows/validation-jobs.yml index 0b3f4f0b11..00fc5de314 100644 --- a/.github/workflows/validation-jobs.yml +++ b/.github/workflows/validation-jobs.yml @@ -78,7 +78,8 @@ jobs: run: cd examples/mobile/android_example && chmod +x gradlew && ./gradlew build run-examples-linux-vulkan: - if: ${{ github.event_name == 'merge_group' }} + # also run when pushed to main to update reference screenshots + if: ${{ github.event_name != 'pull_request' }} runs-on: ubuntu-22.04 timeout-minutes: 30 steps: @@ -135,6 +136,17 @@ jobs: name: example-run-linux path: example-run/ + compare-linux-screenshots: + name: Compare Linux screenshots + needs: [run-examples-linux-vulkan] + uses: ./.github/workflows/send-screenshots-to-pixeleagle.yml + with: + commit: ${{ github.sha }} + branch: ${{ github.ref_name }} + artifact: screenshots-linux + os: linux + secrets: inherit + run-examples-on-windows-dx12: if: ${{ github.event_name == 'merge_group' }} runs-on: windows-latest