Compare screenshots with main on PRs (#13248)

# Objective

- Compare screenshots for a few examples between PRs and main

## Solution

- Send screenshots taken to a screenshot comparison service
- Not completely sure every thing will work at once, but it shouldn't
break anything at least
- it needs a secret to work, I'll add it if enough people agree with
this PR
- this PR doesn't change anything on the screenshot selection (load_gltf
and breakout currently), this will need rendering folks input and can
happen later

---------

Co-authored-by: Alice Cecile <alice.i.cecile@gmail.com>

.github/workflows/ci-examples.yml

@@ -0,0 +1,77 @@
+name: CI  - examples
+
+on:
+  merge_group:
+  pull_request_target:
+  push:
+    branches:
+      - main
+      - release-*
+
+# Unlike jobs in ci.yml, jobs in this workflow can access secrets. Their definitions are taken from the base branch of a PR, not from the PR itself.
+
+env:
+  CARGO_TERM_COLOR: always
+  # If nightly is breaking CI, modify this variable to target a specific nightly version.
+  NIGHTLY_TOOLCHAIN: nightly
+
+concurrency:
+  group: ${{github.workflow}}-${{github.ref}}-examples
+  cancel-in-progress: ${{github.event_name == 'pull_request_target'}}
+
+jobs:
+  run-examples-macos-metal:
+    # Explicity use macOS 14 to take advantage of M1 chip.
+    runs-on: macos-14
+    timeout-minutes: 30
+    steps:
+      - uses: actions/checkout@v4
+      - uses: dtolnay/rust-toolchain@stable
+      - name: Disable audio
+        # Disable audio through a patch. on github m1 runners, audio timeouts after 15 minutes
+        run: git apply --ignore-whitespace tools/example-showcase/disable-audio.patch
+      - name: Build bevy
+        # this uses the same command as when running the example to ensure build is reused
+        run: |
+          TRACE_CHROME=trace-alien_cake_addict.json CI_TESTING_CONFIG=.github/example-run/alien_cake_addict.ron cargo build --example alien_cake_addict --features "bevy_ci_testing,trace,trace_chrome"
+      - name: Run examples
+        run: |
+          for example in .github/example-run/*.ron; do
+            example_name=`basename $example .ron`
+            echo -n $example_name > last_example_run
+            echo "running $example_name - "`date`
+            time TRACE_CHROME=trace-$example_name.json CI_TESTING_CONFIG=$example cargo run --example $example_name --features "bevy_ci_testing,trace,trace_chrome"
+            sleep 10
+            if [ `find ./ -maxdepth 1 -name 'screenshot-*.png' -print -quit` ]; then
+              mkdir screenshots-$example_name
+              mv screenshot-*.png screenshots-$example_name/
+            fi
+          done
+          mkdir traces && mv trace*.json traces/
+          mkdir screenshots && mv screenshots-* screenshots/
+      - name: save traces
+        uses: actions/upload-artifact@v4
+        with:
+          name: example-traces-macos
+          path: traces
+      - name: save screenshots
+        uses: actions/upload-artifact@v4
+        with:
+          name: screenshots-macos
+          path: screenshots
+      - uses: actions/upload-artifact@v4
+        if: ${{ failure() && github.event_name == 'pull_request' }}
+        with:
+          name: example-run-macos
+          path: example-run/
+
+  compare-macos-screenshots:
+    name: Compare macOS screenshots
+    needs: [run-examples-macos-metal]
+    uses: ./.github/workflows/send-screenshots-to-pixeleagle.yml
+    with:
+      commit: ${{ github.sha }}
+      branch: ${{ github.ref_name }}
+      artifact: screenshots-macos
+      os: macos
+    secrets: inherit

.github/workflows/ci.yml

@@ -253,52 +253,6 @@ jobs:
           echo 'if you use VSCode, you can also install `Typos Spell Checker'
           echo 'You can find the extension here: https://marketplace.visualstudio.com/items?itemName=tekumara.typos-vscode'
 
-
-  run-examples-macos-metal:
-    # Explicity use macOS 14 to take advantage of M1 chip.
-    runs-on: macos-14
-    timeout-minutes: 30
-    steps:
-      - uses: actions/checkout@v4
-      - uses: dtolnay/rust-toolchain@stable
-      - name: Disable audio
-        # Disable audio through a patch. on github m1 runners, audio timeouts after 15 minutes
-        run: git apply --ignore-whitespace tools/example-showcase/disable-audio.patch
-      - name: Build bevy
-        # this uses the same command as when running the example to ensure build is reused
-        run: |
-          TRACE_CHROME=trace-alien_cake_addict.json CI_TESTING_CONFIG=.github/example-run/alien_cake_addict.ron cargo build --example alien_cake_addict --features "bevy_ci_testing,trace,trace_chrome"
-      - name: Run examples
-        run: |
-          for example in .github/example-run/*.ron; do
-            example_name=`basename $example .ron`
-            echo -n $example_name > last_example_run
-            echo "running $example_name - "`date`
-            time TRACE_CHROME=trace-$example_name.json CI_TESTING_CONFIG=$example cargo run --example $example_name --features "bevy_ci_testing,trace,trace_chrome"
-            sleep 10
-            if [ `find ./ -maxdepth 1 -name 'screenshot-*.png' -print -quit` ]; then
-              mkdir screenshots-$example_name
-              mv screenshot-*.png screenshots-$example_name/
-            fi
-          done
-          mkdir traces && mv trace*.json traces/
-          mkdir screenshots && mv screenshots-* screenshots/
-      - name: save traces
-        uses: actions/upload-artifact@v4
-        with:
-          name: example-traces-macos
-          path: traces
-      - name: save screenshots
-        uses: actions/upload-artifact@v4
-        with:
-          name: screenshots-macos
-          path: screenshots
-      - uses: actions/upload-artifact@v4
-        if: ${{ failure() && github.event_name == 'pull_request' }}
-        with:
-          name: example-run-macos
-          path: example-run/
-
   check-doc:
     runs-on: ubuntu-latest
     timeout-minutes: 30

.github/workflows/send-screenshots-to-pixeleagle.yml

@@ -0,0 +1,94 @@
+name: Send Screenshots to Pixel Eagle
+
+on:
+  workflow_call:
+    inputs:
+      artifact:
+        required: true
+        type: string
+      commit:
+        required: true
+        type: string
+      branch:
+        required: true
+        type: string
+      os:
+        required: true
+        type: string
+
+jobs:
+  send-to-pixel-eagle:
+    name: Send screenshots to Pixel Eagle
+    runs-on: ubuntu-latest
+    steps:
+      
+      - name: Download artifact
+        uses: actions/download-artifact@v4
+        with:
+          pattern: ${{ inputs.artifact }}
+      
+      - name: Send to Pixel Eagle
+        env:
+          project: B04F67C0-C054-4A6F-92EC-F599FEC2FD1D
+        run: |
+          # Create a new run with its associated metadata
+          metadata='{"os":"${{ inputs.os }}", "commit": "${{ inputs.commit }}", "branch": "${{ inputs.branch }}"}'
+          run=`curl https://pixel-eagle.vleue.com/$project/runs --json "$metadata" --oauth2-bearer ${{ secrets.PIXELEAGLE_TOKEN }} | jq '.id'`
+          
+          SAVEIFS=$IFS
+          
+          cd ${{ inputs.artifact }}
+          
+          # Read the hashes of the screenshot for fast comparison when they are equal
+          IFS=$'\n'
+          # Build a json array of screenshots and their hashes
+          hashes='[';
+          for screenshot in $(find . -type f -name "*.png");
+          do
+            name=${screenshot:14}
+            echo $name
+            hash=`shasum -a 256 $screenshot | awk '{print $1}'`
+            hashes="$hashes [\"$name\",\"$hash\"],"
+          done
+          hashes=`echo $hashes | rev | cut -c 2- | rev`
+          hashes="$hashes]"
+          
+          IFS=$SAVEIFS          
+
+          # Upload screenshots with unknown hashes
+          curl https://pixel-eagle.vleue.com/$project/runs/$run/hashes --json "$hashes" --oauth2-bearer ${{ secrets.PIXELEAGLE_TOKEN }} | jq '.[]|[.name] | @tsv' |
+            while IFS=$'\t' read -r name; do
+              name=`echo $name | tr -d '"'`
+              echo "Uploading $name"
+              curl https://pixel-eagle.vleue.com/$project/runs/$run/screenshots -F "data=@./$name" -F "screenshot=$name" --oauth2-bearer ${{ secrets.PIXELEAGLE_TOKEN }}
+              echo
+            done
+          
+          IFS=$SAVEIFS
+
+          cd ..
+
+          # Trigger comparison with the main branch on the same os
+          curl https://pixel-eagle.vleue.com/$project/runs/$run/compare/auto --json '{"os":"<equal>", "branch": "main"}' --oauth2-bearer ${{ secrets.PIXELEAGLE_TOKEN }} > pixeleagle.json
+
+          # Log results
+          compared_with=`cat pixeleagle.json | jq '.to'`
+
+          status=0
+          missing=`cat pixeleagle.json | jq '.missing | length'`
+          if [ ! $missing -eq 0 ]; then
+            echo "There are $missing missing screenshots"
+            echo "::warning title=$missing missing screenshots on ${{ inputs.os }}::https://pixel-eagle.vleue.com/$project/runs/$run/compare/$to"
+            status=1
+          fi
+
+          diff=`cat pixeleagle.json | jq '.diff | length'`
+          if [ ! $diff -eq 0 ]; then
+            echo "There are $diff screenshots with a difference"
+            echo "::warning title=$diff different screenshots on ${{ inputs.os }}::https://pixel-eagle.vleue.com/$project/runs/$run/compare/$to"
+            status=1
+          fi
+
+          echo "created run $run: https://pixel-eagle.vleue.com/$project/runs/$run/compare/$to"
+
+          exit $status

.github/workflows/validation-jobs.yml

@@ -78,7 +78,8 @@ jobs:
         run: cd examples/mobile/android_example && chmod +x gradlew && ./gradlew build
 
   run-examples-linux-vulkan:
-    if: ${{ github.event_name == 'merge_group' }}
+    # also run when pushed to main to update reference screenshots
+    if: ${{ github.event_name != 'pull_request' }}
     runs-on: ubuntu-22.04
     timeout-minutes: 30
     steps:
@@ -135,6 +136,17 @@ jobs:
           name: example-run-linux
           path: example-run/
 
+  compare-linux-screenshots:
+    name: Compare Linux screenshots
+    needs: [run-examples-linux-vulkan]
+    uses: ./.github/workflows/send-screenshots-to-pixeleagle.yml
+    with:
+      commit: ${{ github.sha }}
+      branch: ${{ github.ref_name }}
+      artifact: screenshots-linux
+      os: linux
+    secrets: inherit
+
   run-examples-on-windows-dx12:
     if: ${{ github.event_name == 'merge_group' }}
     runs-on: windows-latest