switch CI jobs between windows and linux for example execution (#9489)

# Objective

- Example execution on linux/vulkan on CI is segfaulting for unclear
reasons
- This makes a lot of noise on PRs

## Solution

- Switch example execution on Linux to validation jobs (on PR merged).
It will still crash but not block merging, and we'll know when it's
fixed
- Switch example execution on Windows to CI jobs (on PR push). It's a
bit longer than on Linux but provides a useful status
- Disable job commenting on PR with job execution to reduce noise

---------

Co-authored-by: Alice Cecile <alice.i.cecile@gmail.com>

.github/workflows/ci-comment-failures.yml

@@ -14,54 +14,6 @@ on:
       - completed
 
 jobs:
-  example-run:
-    runs-on: ubuntu-latest
-    if: >
-      github.event.workflow_run.event == 'pull_request' &&
-      github.event.workflow_run.conclusion == 'failure'
-    steps:
-      - name: 'Download artifact'
-        id: find-artifact
-        uses: actions/github-script@v6
-        with:
-          result-encoding: string
-          script: |
-            var artifacts = await github.rest.actions.listWorkflowRunArtifacts({
-               owner: context.repo.owner,
-               repo: context.repo.repo,
-               run_id: ${{github.event.workflow_run.id }},
-            });
-            var matchArtifacts = artifacts.data.artifacts.filter((artifact) => {
-              return artifact.name == "example-run"
-            });
-            if (matchArtifacts.length == 0) { return "false" }
-            var matchArtifact = matchArtifacts[0];
-            var download = await github.rest.actions.downloadArtifact({
-               owner: context.repo.owner,
-               repo: context.repo.repo,
-               artifact_id: matchArtifact.id,
-               archive_format: 'zip',
-            });
-            var fs = require('fs');
-            fs.writeFileSync('${{github.workspace}}/example-run.zip', Buffer.from(download.data));
-            return "true"
-      - run: unzip example-run.zip
-        if: ${{ steps.find-artifact.outputs.result == 'true' }}
-      - name: 'Comment on PR'
-        if: ${{ steps.find-artifact.outputs.result == 'true' }}
-        uses: actions/github-script@v6
-        with:
-          github-token: ${{ secrets.GITHUB_TOKEN }}
-          script: |
-            var fs = require('fs');
-            var issue_number = Number(fs.readFileSync('./NR'));
-            var last_example_run = fs.readFileSync('./last_example_run');
-            await github.rest.issues.createComment({
-              owner: context.repo.owner,
-              repo: context.repo.repo,
-              issue_number: issue_number,
-              body: 'Example `' + last_example_run + '` failed to run, please try running it locally and check the result.'
-            });
 
   missing-examples:
     runs-on: ubuntu-latest

.github/workflows/ci.yml

@@ -163,22 +163,14 @@ jobs:
           VALIDATE_MARKDOWN: true
           DEFAULT_BRANCH: main
 
-  run-examples:
-    runs-on: ubuntu-latest
-    timeout-minutes: 30
+  run-examples-on-windows-dx12:
+    runs-on: windows-latest
+    timeout-minutes: 60
     steps:
-      - name: Install Bevy dependencies
-        run: |
-          sudo apt-get update;
-          DEBIAN_FRONTEND=noninteractive sudo apt-get install --no-install-recommends -yq \
-            libasound2-dev libudev-dev;
-      - name: install xvfb, llvmpipe and lavapipe
-        run: |
-          sudo apt-get update -y -qq
-          sudo add-apt-repository ppa:oibaf/graphics-drivers -y
-          sudo apt-get update
-          sudo apt install -y xvfb libegl1-mesa libgl1-mesa-dri libxcb-xfixes0-dev mesa-vulkan-drivers
       - uses: actions/checkout@v3
+
+      - uses: dtolnay/rust-toolchain@stable
+
       - uses: actions/cache@v3
         with:
           path: |
@@ -187,48 +179,23 @@ jobs:
             ~/.cargo/registry/cache/
             ~/.cargo/git/db/
             target/
-          key: ${{ runner.os }}-cargo-run-examples-${{ hashFiles('**/Cargo.toml') }}
-      - uses: dtolnay/rust-toolchain@stable
+          key: ${{ runner.os }}-windows-run-examples-${{ hashFiles('**/Cargo.toml') }}
+
       - name: Build bevy
+        shell: bash
         # this uses the same command as when running the example to ensure build is reused
         run: |
-          TRACE_CHROME=trace-alien_cake_addict.json CI_TESTING_CONFIG=.github/example-run/alien_cake_addict.ron cargo build --example alien_cake_addict --features "bevy_ci_testing,trace,trace_chrome"
+          WGPU_BACKEND=dx12 CI_TESTING_CONFIG=.github/example-run/alien_cake_addict.ron cargo build --example alien_cake_addict --features "bevy_ci_testing"
+
       - name: Run examples
+        shell: bash
         run: |
           for example in .github/example-run/*.ron; do
             example_name=`basename $example .ron`
-            echo -n $example_name > last_example_run
             echo "running $example_name - "`date`
-            time TRACE_CHROME=trace-$example_name.json CI_TESTING_CONFIG=$example xvfb-run cargo run --example $example_name --features "bevy_ci_testing,trace,trace_chrome"
+            time WGPU_BACKEND=dx12 CI_TESTING_CONFIG=$example cargo run --example $example_name --features "bevy_ci_testing"
             sleep 10
-            if [ `find ./ -maxdepth 1 -name 'screenshot-*.png' -print -quit` ]; then
-              mkdir screenshots-$example_name
-              mv screenshot-*.png screenshots-$example_name/
-            fi
           done
-          zip traces.zip trace*.json
-          zip -r screenshots.zip screenshots-*
-      - name: save traces
-        uses: actions/upload-artifact@v3
-        with:
-          name: example-traces.zip
-          path: traces.zip
-      - name: save screenshots
-        uses: actions/upload-artifact@v3
-        with:
-          name: screenshots.zip
-          path: screenshots.zip
-      - name: Save PR number
-        if: ${{ failure() && github.event_name == 'pull_request' }}
-        run: |
-          mkdir -p ./example-run
-          echo ${{ github.event.number }} > ./example-run/NR
-          mv last_example_run ./example-run/
-      - uses: actions/upload-artifact@v2
-        if: ${{ failure() && github.event_name == 'pull_request' }}
-        with:
-          name: example-run
-          path: example-run/
 
   check-doc:
     runs-on: ubuntu-latest

.github/workflows/validation-jobs.yml

@@ -61,15 +61,23 @@ jobs:
       - name: Build APK
         run: ANDROID_NDK_ROOT=$ANDROID_NDK_LATEST_HOME cargo apk build --package bevy_mobile_example
 
-  run-examples-on-windows-dx12:
+  run-examples-linux-vulkan:
     if: ${{ github.event_name == 'merge_group' }}
-    runs-on: windows-latest
-    timeout-minutes: 60
+    runs-on: ubuntu-latest
+    timeout-minutes: 30
     steps:
+      - name: Install Bevy dependencies
+        run: |
+          sudo apt-get update;
+          DEBIAN_FRONTEND=noninteractive sudo apt-get install --no-install-recommends -yq \
+            libasound2-dev libudev-dev;
+      - name: install xvfb, llvmpipe and lavapipe
+        run: |
+          sudo apt-get update -y -qq
+          sudo add-apt-repository ppa:oibaf/graphics-drivers -y
+          sudo apt-get update
+          sudo apt install -y xvfb libegl1-mesa libgl1-mesa-dri libxcb-xfixes0-dev mesa-vulkan-drivers
       - uses: actions/checkout@v3
-
-      - uses: dtolnay/rust-toolchain@stable
-
       - uses: actions/cache@v3
         with:
           path: |
@@ -78,23 +86,42 @@ jobs:
             ~/.cargo/registry/cache/
             ~/.cargo/git/db/
             target/
-          key: ${{ runner.os }}-windows-run-examples-${{ hashFiles('**/Cargo.toml') }}
-
+          key: ${{ runner.os }}-cargo-run-examples-${{ hashFiles('**/Cargo.toml') }}
+      - uses: dtolnay/rust-toolchain@stable
       - name: Build bevy
-        shell: bash
         # this uses the same command as when running the example to ensure build is reused
         run: |
-          WGPU_BACKEND=dx12 CI_TESTING_CONFIG=.github/example-run/alien_cake_addict.ron cargo build --example alien_cake_addict --features "bevy_ci_testing"
-
+          TRACE_CHROME=trace-alien_cake_addict.json CI_TESTING_CONFIG=.github/example-run/alien_cake_addict.ron cargo build --example alien_cake_addict --features "bevy_ci_testing,trace,trace_chrome"
       - name: Run examples
-        shell: bash
         run: |
           for example in .github/example-run/*.ron; do
             example_name=`basename $example .ron`
+            echo -n $example_name > last_example_run
             echo "running $example_name - "`date`
-            time WGPU_BACKEND=dx12 CI_TESTING_CONFIG=$example cargo run --example $example_name --features "bevy_ci_testing"
+            time TRACE_CHROME=trace-$example_name.json CI_TESTING_CONFIG=$example xvfb-run cargo run --example $example_name --features "bevy_ci_testing,trace,trace_chrome"
             sleep 10
+            if [ `find ./ -maxdepth 1 -name 'screenshot-*.png' -print -quit` ]; then
+              mkdir screenshots-$example_name
+              mv screenshot-*.png screenshots-$example_name/
+            fi
           done
+          zip traces.zip trace*.json
+          zip -r screenshots.zip screenshots-*
+      - name: save traces
+        uses: actions/upload-artifact@v3
+        with:
+          name: example-traces.zip
+          path: traces.zip
+      - name: save screenshots
+        uses: actions/upload-artifact@v3
+        with:
+          name: screenshots.zip
+          path: screenshots.zip
+      - uses: actions/upload-artifact@v2
+        if: ${{ failure() && github.event_name == 'pull_request' }}
+        with:
+          name: example-run
+          path: example-run/
 
   run-examples-on-wasm:
     if: ${{ github.event_name == 'merge_group' }}