run examples in wasm in CI (#4818)

# Objective

- Run examples in WASM in CI
- Fix #4817 

## Solution

- on feature `bevy_ci_testing`
  - add an extra log message before exiting
  - when building for wasm, read CI config file at compile time
- add a simple [playwright](https://playwright.dev) test script that opens the browser then waits for the success log, and takes a screenshot
- add a CI job that runs the playwright test for Chromium and Firefox on one example (lighting) and save the screenshots
  - Firefox screenshot is good (with some clusters visible)
  - Chromium screenshot is gray, I don't know why but it's logging `GPU stall due to ReadPixels`
  - Webkit is not enabled for now, to revisit once https://bugs.webkit.org/show_bug.cgi?id=234926 is fixed or worked around
- the CI job only runs on bors validation

example run: https://github.com/mockersf/bevy/actions/runs/2361673465. The screenshots can be downloaded
This commit is contained in:
François 2022-06-06 20:22:51 +00:00
parent 193998b5d4
commit 39ea1bb9b7
10 changed files with 396 additions and 21 deletions

1
.github/bors.toml vendored
View file

@ -8,6 +8,7 @@ status = [
"build-android",
"markdownlint",
"run-examples",
"run-examples-on-wasm",
"check-doc",
"check-missing-examples-in-docs",
"check-unused-dependencies",

4
.github/start-wasm-example/.gitignore vendored Normal file
View file

@ -0,0 +1,4 @@
node_modules/
/test-results/
/playwright-report/
/playwright/.cache/

76
.github/start-wasm-example/package-lock.json generated vendored Normal file
View file

@ -0,0 +1,76 @@
{
"name": "start-wasm-example",
"version": "1.0.0",
"lockfileVersion": 2,
"requires": true,
"packages": {
"": {
"name": "start-wasm-example",
"version": "1.0.0",
"license": "ISC",
"dependencies": {
"dotenv": "^16.0.1"
},
"devDependencies": {
"@playwright/test": "^1.22.1"
}
},
"node_modules/@playwright/test": {
"version": "1.22.1",
"resolved": "https://registry.npmjs.org/@playwright/test/-/test-1.22.1.tgz",
"integrity": "sha512-8ouMBUboYslHom41W8bnSEn0TwlAMHhCACwOZeuiAgzukj7KobpZ+UBwrGE0jJ0UblJbKAQNRHXL+z7sDSkb6g==",
"dev": true,
"dependencies": {
"playwright-core": "1.22.1"
},
"bin": {
"playwright": "cli.js"
},
"engines": {
"node": ">=14"
}
},
"node_modules/dotenv": {
"version": "16.0.1",
"resolved": "https://registry.npmjs.org/dotenv/-/dotenv-16.0.1.tgz",
"integrity": "sha512-1K6hR6wtk2FviQ4kEiSjFiH5rpzEVi8WW0x96aztHVMhEspNpc4DVOUTEHtEva5VThQ8IaBX1Pe4gSzpVVUsKQ==",
"engines": {
"node": ">=12"
}
},
"node_modules/playwright-core": {
"version": "1.22.1",
"resolved": "https://registry.npmjs.org/playwright-core/-/playwright-core-1.22.1.tgz",
"integrity": "sha512-H+ZUVYnceWNXrRf3oxTEKAr81QzFsCKu5Fp//fEjQvqgKkfA1iX3E9DBrPJpPNOrgVzcE+IqeI0fDmYJe6Ynnw==",
"dev": true,
"bin": {
"playwright": "cli.js"
},
"engines": {
"node": ">=14"
}
}
},
"dependencies": {
"@playwright/test": {
"version": "1.22.1",
"resolved": "https://registry.npmjs.org/@playwright/test/-/test-1.22.1.tgz",
"integrity": "sha512-8ouMBUboYslHom41W8bnSEn0TwlAMHhCACwOZeuiAgzukj7KobpZ+UBwrGE0jJ0UblJbKAQNRHXL+z7sDSkb6g==",
"dev": true,
"requires": {
"playwright-core": "1.22.1"
}
},
"dotenv": {
"version": "16.0.1",
"resolved": "https://registry.npmjs.org/dotenv/-/dotenv-16.0.1.tgz",
"integrity": "sha512-1K6hR6wtk2FviQ4kEiSjFiH5rpzEVi8WW0x96aztHVMhEspNpc4DVOUTEHtEva5VThQ8IaBX1Pe4gSzpVVUsKQ=="
},
"playwright-core": {
"version": "1.22.1",
"resolved": "https://registry.npmjs.org/playwright-core/-/playwright-core-1.22.1.tgz",
"integrity": "sha512-H+ZUVYnceWNXrRf3oxTEKAr81QzFsCKu5Fp//fEjQvqgKkfA1iX3E9DBrPJpPNOrgVzcE+IqeI0fDmYJe6Ynnw==",
"dev": true
}
}
}

16
.github/start-wasm-example/package.json vendored Normal file
View file

@ -0,0 +1,16 @@
{
"name": "start-wasm-example",
"version": "1.0.0",
"description": "",
"main": "index.js",
"scripts": {},
"keywords": [],
"author": "",
"license": "ISC",
"devDependencies": {
"@playwright/test": "^1.22.1"
},
"dependencies": {
"dotenv": "^16.0.1"
}
}

View file

@ -0,0 +1,107 @@
import type { PlaywrightTestConfig } from '@playwright/test';
import { devices } from '@playwright/test';
/**
* Read environment variables from file.
* https://github.com/motdotla/dotenv
*/
require('dotenv').config();
/**
* See https://playwright.dev/docs/test-configuration.
*/
const config: PlaywrightTestConfig = {
testDir: './tests',
/* Maximum time one test can run for. */
timeout: 300_000,
expect: {
/**
* Maximum time expect() should wait for the condition to be met.
* For example in `await expect(locator).toHaveText();`
*/
timeout: 5000
},
/* Run tests in files in parallel */
fullyParallel: true,
/* Fail the build on CI if you accidentally left test.only in the source code. */
forbidOnly: !!process.env.CI,
/* Retry on CI only */
retries: 0,
/* Opt out of parallel tests on CI. */
workers: 1,
/* Reporter to use. See https://playwright.dev/docs/test-reporters */
reporter: 'list',
/* Shared settings for all the projects below. See https://playwright.dev/docs/api/class-testoptions. */
use: {
/* Maximum time each action such as `click()` can take. Defaults to 0 (no limit). */
actionTimeout: 0,
/* Base URL to use in actions like `await page.goto('/')`. */
// baseURL: 'http://localhost:3000',
/* Collect trace when retrying the failed test. See https://playwright.dev/docs/trace-viewer */
trace: 'on-first-retry',
},
/* Configure projects for major browsers */
projects: [
{
name: 'chromium',
use: {
...devices['Desktop Chrome'],
},
},
{
name: 'firefox',
use: {
...devices['Desktop Firefox'],
},
},
{
name: 'webkit',
use: {
...devices['Desktop Safari'],
},
},
/* Test against mobile viewports. */
// {
// name: 'Mobile Chrome',
// use: {
// ...devices['Pixel 5'],
// },
// },
// {
// name: 'Mobile Safari',
// use: {
// ...devices['iPhone 12'],
// },
// },
/* Test against branded browsers. */
// {
// name: 'Microsoft Edge',
// use: {
// channel: 'msedge',
// },
// },
// {
// name: 'Google Chrome',
// use: {
// channel: 'chrome',
// },
// },
],
/* Folder for test artifacts such as screenshots, videos, traces, etc. */
// outputDir: 'test-results/',
/* Run your local dev server before starting the tests */
// webServer: {
// command: 'npm run start',
// port: 3000,
// },
};
export default config;

View file

@ -0,0 +1,51 @@
import { test, expect, Page } from '@playwright/test';
test.beforeEach(async ({ page }) => {
await page.goto('http://localhost:8000/');
});
const MAX_TIMEOUT_FOR_TEST = 300_000;
test.describe('WASM example', () => {
test('Wait for success', async ({ page }, test_info) => {
let start = new Date().getTime();
let found = false;
while (new Date().getTime() - start < MAX_TIMEOUT_FOR_TEST) {
let msg = await promise_with_timeout(100, on_console(page), "no log found");
if (msg.includes("no log found")) {
continue;
}
console.log(msg);
if (msg.includes("Test successful")) {
let prefix = process.env.SCREENSHOT_PREFIX === undefined ? "screenshot" : process.env.SCREENSHOT_PREFIX;
await page.screenshot({ path: `${prefix}-${test_info.project.name}.png`, fullPage: true });
found = true;
break;
}
}
expect(found).toBe(true);
});
});
function on_console(page) {
return new Promise(resolve => {
page.on('console', msg => resolve(msg.text()));
});
}
async function promise_with_timeout(time_limit, task, failure_value) {
let timeout;
const timeout_promise = new Promise((resolve, reject) => {
timeout = setTimeout(() => {
resolve(failure_value);
}, time_limit);
});
const response = await Promise.race([task, timeout_promise]);
if (timeout) {
clearTimeout(timeout);
}
return response;
}

View file

@ -97,3 +97,61 @@ jobs:
time CI_TESTING_CONFIG=$example cargo run --example $example_name --features "bevy_ci_testing"
sleep 10
done
run-examples-on-wasm:
runs-on: ubuntu-latest
timeout-minutes: 60
steps:
- uses: actions/checkout@v3
- uses: actions-rs/toolchain@v1
with:
toolchain: stable
target: wasm32-unknown-unknown
override: true
- uses: actions/cache@v3
with:
path: |
~/.cargo/bin/
~/.cargo/registry/index/
~/.cargo/registry/cache/
~/.cargo/git/db/
~/.github/start-wasm-example/node_modules
target/
key: ${{ runner.os }}-wasm-run-examples-${{ hashFiles('**/Cargo.toml') }}
- name: install xvfb, llvmpipe and lavapipe
run: |
sudo apt-get update -y -qq
sudo add-apt-repository ppa:oibaf/graphics-drivers -y
sudo apt-get update
sudo apt install -y xvfb libegl1-mesa libgl1-mesa-dri libxcb-xfixes0-dev mesa-vulkan-drivers
- name: Install wasm-bindgen
run: cargo install --force wasm-bindgen-cli
- name: Setup playwright
run: |
cd .github/start-wasm-example
npm install
npx playwright install --with-deps
cd ../..
- name: First WASM build
run: |
cargo build --release --example ui --target wasm32-unknown-unknown
- name: Run examples
shell: bash
run: |
# start a webserver
python3 -m http.server --directory examples/wasm &
xvfb-run cargo run -p build-wasm-example -- --browsers chromium --browsers firefox --frames 25 --test shapes lighting text_debug breakout
- name: Save screenshots
uses: actions/upload-artifact@v3
with:
name: screenshots
path: .github/start-wasm-example/screenshot-*.png

View file

@ -1,6 +1,8 @@
use crate::{app::AppExit, App};
use serde::Deserialize;
use bevy_utils::tracing::info;
/// A configuration struct for automated CI testing.
///
/// It gets used when the `bevy_ci_testing` feature is enabled to automatically
@ -20,18 +22,29 @@ fn ci_testing_exit_after(
if let Some(exit_after) = ci_testing_config.exit_after {
if *current_frame > exit_after {
app_exit_events.send(AppExit);
info!("Exiting after {} frames. Test successful!", exit_after);
}
}
*current_frame += 1;
}
pub(crate) fn setup_app(app: &mut App) -> &mut App {
let filename =
std::env::var("CI_TESTING_CONFIG").unwrap_or_else(|_| "ci_testing_config.ron".to_string());
let config: CiTestingConfig = ron::from_str(
&std::fs::read_to_string(filename).expect("error reading CI testing configuration file"),
)
.expect("error deserializing CI testing configuration file");
#[cfg(not(target_arch = "wasm32"))]
let config: CiTestingConfig = {
let filename = std::env::var("CI_TESTING_CONFIG")
.unwrap_or_else(|_| "ci_testing_config.ron".to_string());
ron::from_str(
&std::fs::read_to_string(filename)
.expect("error reading CI testing configuration file"),
)
.expect("error deserializing CI testing configuration file")
};
#[cfg(target_arch = "wasm32")]
let config: CiTestingConfig = {
let config = include_str!("../../../ci_testing_config.ron");
ron::from_str(config).expect("error deserializing CI testing configuration file")
};
app.insert_resource(config)
.add_system(ci_testing_exit_after);

View file

@ -9,3 +9,4 @@ license = "MIT OR Apache-2.0"
[dependencies]
xshell = "0.2"
clap = { version = "3.1.12", features = ["derive"] }

View file

@ -1,18 +1,66 @@
use std::{fs::File, io::Write};
use clap::Parser;
use xshell::{cmd, Shell};
fn main() {
let example = std::env::args().nth(1).expect("abbb");
let sh = Shell::new().unwrap();
cmd!(
sh,
"cargo build --release --target wasm32-unknown-unknown --example {example}"
)
.run()
.expect("Error building example");
cmd!(
sh,
"wasm-bindgen --out-dir examples/wasm/target --out-name wasm_example --target web target/wasm32-unknown-unknown/release/examples/{example}.wasm"
)
.run()
.expect("Error creating wasm binding");
#[derive(Parser, Debug)]
struct Args {
/// Examples to build
examples: Vec<String>,
#[clap(short, long)]
/// Run tests
test: bool,
#[clap(short, long)]
/// Run on the given browsers. By default, chromium, firefox, webkit
browsers: Vec<String>,
#[clap(short, long)]
/// Stop after this number of frames
frames: Option<usize>,
}
fn main() {
let cli = Args::parse();
eprintln!("{:?}", cli);
assert!(!cli.examples.is_empty(), "must have at least one example");
let mut bevy_ci_testing = vec![];
if let Some(frames) = cli.frames {
let mut file = File::create("ci_testing_config.ron").unwrap();
file.write_fmt(format_args!("(exit_after: Some({}))", frames))
.unwrap();
bevy_ci_testing = vec!["--features", "bevy_ci_testing"];
}
for example in cli.examples {
let sh = Shell::new().unwrap();
let bevy_ci_testing = bevy_ci_testing.clone();
cmd!(
sh,
"cargo build {bevy_ci_testing...} --release --target wasm32-unknown-unknown --example {example}"
)
.run()
.expect("Error building example");
cmd!(
sh,
"wasm-bindgen --out-dir examples/wasm/target --out-name wasm_example --target web target/wasm32-unknown-unknown/release/examples/{example}.wasm"
)
.run()
.expect("Error creating wasm binding");
if cli.test {
let _dir = sh.push_dir(".github/start-wasm-example");
let mut browsers = cli.browsers.clone();
if !browsers.is_empty() {
browsers.insert(0, "--project".to_string());
}
cmd!(sh, "npx playwright test --headed {browsers...}")
.env("SCREENSHOT_PREFIX", format!("screenshot-{example}"))
.run()
.expect("Error running playwright test");
}
}
}