run examples in wasm in CI (#4818)

# Objective

- Run examples in WASM in CI
- Fix #4817 

## Solution

- on feature `bevy_ci_testing`
  - add an extra log message before exiting
  - when building for wasm, read CI config file at compile time
- add a simple [playwright](https://playwright.dev) test script that opens the browser then waits for the success log, and takes a screenshot
- add a CI job that runs the playwright test for Chromium and Firefox on one example (lighting) and save the screenshots
  - Firefox screenshot is good (with some clusters visible)
  - Chromium screenshot is gray, I don't know why but it's logging `GPU stall due to ReadPixels`
  - Webkit is not enabled for now, to revisit once https://bugs.webkit.org/show_bug.cgi?id=234926 is fixed or worked around
- the CI job only runs on bors validation

example run: https://github.com/mockersf/bevy/actions/runs/2361673465. The screenshots can be downloaded

.github/bors.toml

@@ -8,6 +8,7 @@ status = [
     "build-android",
     "markdownlint",
     "run-examples",
+    "run-examples-on-wasm",
     "check-doc",
     "check-missing-examples-in-docs",
     "check-unused-dependencies",

.github/start-wasm-example/.gitignore

@@ -0,0 +1,4 @@
+node_modules/
+/test-results/
+/playwright-report/
+/playwright/.cache/

.github/start-wasm-example/package-lock.json

@@ -0,0 +1,76 @@
+{
+  "name": "start-wasm-example",
+  "version": "1.0.0",
+  "lockfileVersion": 2,
+  "requires": true,
+  "packages": {
+    "": {
+      "name": "start-wasm-example",
+      "version": "1.0.0",
+      "license": "ISC",
+      "dependencies": {
+        "dotenv": "^16.0.1"
+      },
+      "devDependencies": {
+        "@playwright/test": "^1.22.1"
+      }
+    },
+    "node_modules/@playwright/test": {
+      "version": "1.22.1",
+      "resolved": "https://registry.npmjs.org/@playwright/test/-/test-1.22.1.tgz",
+      "integrity": "sha512-8ouMBUboYslHom41W8bnSEn0TwlAMHhCACwOZeuiAgzukj7KobpZ+UBwrGE0jJ0UblJbKAQNRHXL+z7sDSkb6g==",
+      "dev": true,
+      "dependencies": {
+        "playwright-core": "1.22.1"
+      },
+      "bin": {
+        "playwright": "cli.js"
+      },
+      "engines": {
+        "node": ">=14"
+      }
+    },
+    "node_modules/dotenv": {
+      "version": "16.0.1",
+      "resolved": "https://registry.npmjs.org/dotenv/-/dotenv-16.0.1.tgz",
+      "integrity": "sha512-1K6hR6wtk2FviQ4kEiSjFiH5rpzEVi8WW0x96aztHVMhEspNpc4DVOUTEHtEva5VThQ8IaBX1Pe4gSzpVVUsKQ==",
+      "engines": {
+        "node": ">=12"
+      }
+    },
+    "node_modules/playwright-core": {
+      "version": "1.22.1",
+      "resolved": "https://registry.npmjs.org/playwright-core/-/playwright-core-1.22.1.tgz",
+      "integrity": "sha512-H+ZUVYnceWNXrRf3oxTEKAr81QzFsCKu5Fp//fEjQvqgKkfA1iX3E9DBrPJpPNOrgVzcE+IqeI0fDmYJe6Ynnw==",
+      "dev": true,
+      "bin": {
+        "playwright": "cli.js"
+      },
+      "engines": {
+        "node": ">=14"
+      }
+    }
+  },
+  "dependencies": {
+    "@playwright/test": {
+      "version": "1.22.1",
+      "resolved": "https://registry.npmjs.org/@playwright/test/-/test-1.22.1.tgz",
+      "integrity": "sha512-8ouMBUboYslHom41W8bnSEn0TwlAMHhCACwOZeuiAgzukj7KobpZ+UBwrGE0jJ0UblJbKAQNRHXL+z7sDSkb6g==",
+      "dev": true,
+      "requires": {
+        "playwright-core": "1.22.1"
+      }
+    },
+    "dotenv": {
+      "version": "16.0.1",
+      "resolved": "https://registry.npmjs.org/dotenv/-/dotenv-16.0.1.tgz",
+      "integrity": "sha512-1K6hR6wtk2FviQ4kEiSjFiH5rpzEVi8WW0x96aztHVMhEspNpc4DVOUTEHtEva5VThQ8IaBX1Pe4gSzpVVUsKQ=="
+    },
+    "playwright-core": {
+      "version": "1.22.1",
+      "resolved": "https://registry.npmjs.org/playwright-core/-/playwright-core-1.22.1.tgz",
+      "integrity": "sha512-H+ZUVYnceWNXrRf3oxTEKAr81QzFsCKu5Fp//fEjQvqgKkfA1iX3E9DBrPJpPNOrgVzcE+IqeI0fDmYJe6Ynnw==",
+      "dev": true
+    }
+  }
+}

.github/start-wasm-example/package.json

@@ -0,0 +1,16 @@
+{
+  "name": "start-wasm-example",
+  "version": "1.0.0",
+  "description": "",
+  "main": "index.js",
+  "scripts": {},
+  "keywords": [],
+  "author": "",
+  "license": "ISC",
+  "devDependencies": {
+    "@playwright/test": "^1.22.1"
+  },
+  "dependencies": {
+    "dotenv": "^16.0.1"
+  }
+}

.github/start-wasm-example/playwright.config.ts

@@ -0,0 +1,107 @@
+import type { PlaywrightTestConfig } from '@playwright/test';
+import { devices } from '@playwright/test';
+
+/**
+ * Read environment variables from file.
+ * https://github.com/motdotla/dotenv
+ */
+require('dotenv').config();
+
+/**
+ * See https://playwright.dev/docs/test-configuration.
+ */
+const config: PlaywrightTestConfig = {
+  testDir: './tests',
+  /* Maximum time one test can run for. */
+  timeout: 300_000,
+  expect: {
+    /**
+     * Maximum time expect() should wait for the condition to be met.
+     * For example in `await expect(locator).toHaveText();`
+     */
+    timeout: 5000
+  },
+  /* Run tests in files in parallel */
+  fullyParallel: true,
+  /* Fail the build on CI if you accidentally left test.only in the source code. */
+  forbidOnly: !!process.env.CI,
+  /* Retry on CI only */
+  retries: 0,
+  /* Opt out of parallel tests on CI. */
+  workers: 1,
+  /* Reporter to use. See https://playwright.dev/docs/test-reporters */
+  reporter: 'list',
+  /* Shared settings for all the projects below. See https://playwright.dev/docs/api/class-testoptions. */
+  use: {
+    /* Maximum time each action such as `click()` can take. Defaults to 0 (no limit). */
+    actionTimeout: 0,
+    /* Base URL to use in actions like `await page.goto('/')`. */
+    // baseURL: 'http://localhost:3000',
+
+    /* Collect trace when retrying the failed test. See https://playwright.dev/docs/trace-viewer */
+    trace: 'on-first-retry',
+  },
+
+  /* Configure projects for major browsers */
+  projects: [
+    {
+      name: 'chromium',
+      use: {
+        ...devices['Desktop Chrome'],
+      },
+    },
+
+    {
+      name: 'firefox',
+      use: {
+        ...devices['Desktop Firefox'],
+      },
+    },
+
+    {
+      name: 'webkit',
+      use: {
+        ...devices['Desktop Safari'],
+      },
+    },
+
+    /* Test against mobile viewports. */
+    // {
+    //   name: 'Mobile Chrome',
+    //   use: {
+    //     ...devices['Pixel 5'],
+    //   },
+    // },
+    // {
+    //   name: 'Mobile Safari',
+    //   use: {
+    //     ...devices['iPhone 12'],
+    //   },
+    // },
+
+    /* Test against branded browsers. */
+    // {
+    //   name: 'Microsoft Edge',
+    //   use: {
+    //     channel: 'msedge',
+    //   },
+    // },
+    // {
+    //   name: 'Google Chrome',
+    //   use: {
+    //     channel: 'chrome',
+    //   },
+    // },
+  ],
+
+  /* Folder for test artifacts such as screenshots, videos, traces, etc. */
+  // outputDir: 'test-results/',
+
+  /* Run your local dev server before starting the tests */
+  // webServer: {
+  //   command: 'npm run start',
+  //   port: 3000,
+  // },
+};
+
+export default config;

.github/start-wasm-example/tests/wasm_example.spec.ts

@@ -0,0 +1,51 @@
+import { test, expect, Page } from '@playwright/test';
+
+test.beforeEach(async ({ page }) => {
+  await page.goto('http://localhost:8000/');
+});
+
+const MAX_TIMEOUT_FOR_TEST = 300_000;
+
+test.describe('WASM example', () => {
+  test('Wait for success', async ({ page }, test_info) => {
+    let start = new Date().getTime();
+
+    let found = false;
+    while (new Date().getTime() - start < MAX_TIMEOUT_FOR_TEST) {
+      let msg = await promise_with_timeout(100, on_console(page), "no log found");
+      if (msg.includes("no log found")) {
+        continue;
+      }
+      console.log(msg);
+      if (msg.includes("Test successful")) {
+        let prefix = process.env.SCREENSHOT_PREFIX === undefined ? "screenshot" : process.env.SCREENSHOT_PREFIX;
+        await page.screenshot({ path: `${prefix}-${test_info.project.name}.png`, fullPage: true });
+        found = true;
+        break;
+      }
+    }
+
+    expect(found).toBe(true);
+  });
+
+});
+
+function on_console(page) {
+  return new Promise(resolve => {
+    page.on('console', msg => resolve(msg.text()));
+  });
+}
+
+async function promise_with_timeout(time_limit, task, failure_value) {
+  let timeout;
+  const timeout_promise = new Promise((resolve, reject) => {
+    timeout = setTimeout(() => {
+      resolve(failure_value);
+    }, time_limit);
+  });
+  const response = await Promise.race([task, timeout_promise]);
+  if (timeout) {
+    clearTimeout(timeout);
+  }
+  return response;
+}

.github/workflows/validation-jobs.yml

@@ -97,3 +97,61 @@ jobs:
             time CI_TESTING_CONFIG=$example cargo run --example $example_name --features "bevy_ci_testing"
             sleep 10
           done
+
+  run-examples-on-wasm:
+    runs-on: ubuntu-latest
+    timeout-minutes: 60
+    steps:
+      - uses: actions/checkout@v3
+
+      - uses: actions-rs/toolchain@v1
+        with:
+          toolchain: stable
+          target: wasm32-unknown-unknown
+          override: true
+
+      - uses: actions/cache@v3
+        with:
+          path: |
+            ~/.cargo/bin/
+            ~/.cargo/registry/index/
+            ~/.cargo/registry/cache/
+            ~/.cargo/git/db/
+            ~/.github/start-wasm-example/node_modules
+            target/
+          key: ${{ runner.os }}-wasm-run-examples-${{ hashFiles('**/Cargo.toml') }}
+
+      - name: install xvfb, llvmpipe and lavapipe
+        run: |
+          sudo apt-get update -y -qq
+          sudo add-apt-repository ppa:oibaf/graphics-drivers -y
+          sudo apt-get update
+          sudo apt install -y xvfb libegl1-mesa libgl1-mesa-dri libxcb-xfixes0-dev mesa-vulkan-drivers
+
+      - name: Install wasm-bindgen
+        run: cargo install --force wasm-bindgen-cli
+
+      - name: Setup playwright
+        run: |
+          cd .github/start-wasm-example
+          npm install
+          npx playwright install --with-deps
+          cd ../..
+
+      - name: First WASM build
+        run: |
+          cargo build --release --example ui --target wasm32-unknown-unknown
+
+      - name: Run examples
+        shell: bash
+        run: |
+          # start a webserver
+          python3 -m http.server --directory examples/wasm &
+
+          xvfb-run cargo run -p build-wasm-example -- --browsers chromium --browsers firefox --frames 25 --test shapes lighting text_debug breakout
+
+      - name: Save screenshots
+        uses: actions/upload-artifact@v3
+        with:
+          name: screenshots
+          path: .github/start-wasm-example/screenshot-*.png

crates/bevy_app/src/ci_testing.rs

@@ -1,6 +1,8 @@
 use crate::{app::AppExit, App};
 use serde::Deserialize;
 
+use bevy_utils::tracing::info;
+
 /// A configuration struct for automated CI testing.
 ///
 /// It gets used when the `bevy_ci_testing` feature is enabled to automatically
@@ -20,18 +22,29 @@ fn ci_testing_exit_after(
     if let Some(exit_after) = ci_testing_config.exit_after {
         if *current_frame > exit_after {
             app_exit_events.send(AppExit);
+            info!("Exiting after {} frames. Test successful!", exit_after);
         }
     }
     *current_frame += 1;
 }
 
 pub(crate) fn setup_app(app: &mut App) -> &mut App {
-    let filename =
-        std::env::var("CI_TESTING_CONFIG").unwrap_or_else(|_| "ci_testing_config.ron".to_string());
-    let config: CiTestingConfig = ron::from_str(
-        &std::fs::read_to_string(filename).expect("error reading CI testing configuration file"),
+    #[cfg(not(target_arch = "wasm32"))]
+    let config: CiTestingConfig = {
+        let filename = std::env::var("CI_TESTING_CONFIG")
+            .unwrap_or_else(|_| "ci_testing_config.ron".to_string());
+        ron::from_str(
+            &std::fs::read_to_string(filename)
+                .expect("error reading CI testing configuration file"),
         )
-    .expect("error deserializing CI testing configuration file");
+        .expect("error deserializing CI testing configuration file")
+    };
+    #[cfg(target_arch = "wasm32")]
+    let config: CiTestingConfig = {
+        let config = include_str!("../../../ci_testing_config.ron");
+        ron::from_str(config).expect("error deserializing CI testing configuration file")
+    };
+
     app.insert_resource(config)
         .add_system(ci_testing_exit_after);
 

tools/build-wasm-example/Cargo.toml

@@ -9,3 +9,4 @@ license = "MIT OR Apache-2.0"
 
 [dependencies]
 xshell = "0.2"
+clap = { version = "3.1.12", features = ["derive"] }

tools/build-wasm-example/src/main.rs

@@ -1,11 +1,46 @@
+use std::{fs::File, io::Write};
+
+use clap::Parser;
 use xshell::{cmd, Shell};
 
+#[derive(Parser, Debug)]
+struct Args {
+    /// Examples to build
+    examples: Vec<String>,
+
+    #[clap(short, long)]
+    /// Run tests
+    test: bool,
+
+    #[clap(short, long)]
+    /// Run on the given browsers. By default, chromium, firefox, webkit
+    browsers: Vec<String>,
+
+    #[clap(short, long)]
+    /// Stop after this number of frames
+    frames: Option<usize>,
+}
+
 fn main() {
-    let example = std::env::args().nth(1).expect("abbb");
+    let cli = Args::parse();
+    eprintln!("{:?}", cli);
+
+    assert!(!cli.examples.is_empty(), "must have at least one example");
+
+    let mut bevy_ci_testing = vec![];
+    if let Some(frames) = cli.frames {
+        let mut file = File::create("ci_testing_config.ron").unwrap();
+        file.write_fmt(format_args!("(exit_after: Some({}))", frames))
+            .unwrap();
+        bevy_ci_testing = vec!["--features", "bevy_ci_testing"];
+    }
+
+    for example in cli.examples {
         let sh = Shell::new().unwrap();
+        let bevy_ci_testing = bevy_ci_testing.clone();
         cmd!(
             sh,
-        "cargo build --release --target wasm32-unknown-unknown --example {example}"
+            "cargo build {bevy_ci_testing...} --release --target wasm32-unknown-unknown --example {example}"
         )
         .run()
         .expect("Error building example");
@@ -15,4 +50,17 @@ fn main() {
         )
         .run()
         .expect("Error creating wasm binding");
+
+        if cli.test {
+            let _dir = sh.push_dir(".github/start-wasm-example");
+            let mut browsers = cli.browsers.clone();
+            if !browsers.is_empty() {
+                browsers.insert(0, "--project".to_string());
+            }
+            cmd!(sh, "npx playwright test --headed {browsers...}")
+                .env("SCREENSHOT_PREFIX", format!("screenshot-{example}"))
+                .run()
+                .expect("Error running playwright test");
+        }
+    }
 }