mirror of
https://github.com/bevyengine/bevy
synced 2024-11-24 21:53:07 +00:00
run examples in wasm in CI (#4818)
# Objective - Run examples in WASM in CI - Fix #4817 ## Solution - on feature `bevy_ci_testing` - add an extra log message before exiting - when building for wasm, read CI config file at compile time - add a simple [playwright](https://playwright.dev) test script that opens the browser then waits for the success log, and takes a screenshot - add a CI job that runs the playwright test for Chromium and Firefox on one example (lighting) and save the screenshots - Firefox screenshot is good (with some clusters visible) - Chromium screenshot is gray, I don't know why but it's logging `GPU stall due to ReadPixels` - Webkit is not enabled for now, to revisit once https://bugs.webkit.org/show_bug.cgi?id=234926 is fixed or worked around - the CI job only runs on bors validation example run: https://github.com/mockersf/bevy/actions/runs/2361673465. The screenshots can be downloaded
This commit is contained in:
parent
193998b5d4
commit
39ea1bb9b7
10 changed files with 396 additions and 21 deletions
1
.github/bors.toml
vendored
1
.github/bors.toml
vendored
|
@ -8,6 +8,7 @@ status = [
|
|||
"build-android",
|
||||
"markdownlint",
|
||||
"run-examples",
|
||||
"run-examples-on-wasm",
|
||||
"check-doc",
|
||||
"check-missing-examples-in-docs",
|
||||
"check-unused-dependencies",
|
||||
|
|
4
.github/start-wasm-example/.gitignore
vendored
Normal file
4
.github/start-wasm-example/.gitignore
vendored
Normal file
|
@ -0,0 +1,4 @@
|
|||
node_modules/
|
||||
/test-results/
|
||||
/playwright-report/
|
||||
/playwright/.cache/
|
76
.github/start-wasm-example/package-lock.json
generated
vendored
Normal file
76
.github/start-wasm-example/package-lock.json
generated
vendored
Normal file
|
@ -0,0 +1,76 @@
|
|||
{
|
||||
"name": "start-wasm-example",
|
||||
"version": "1.0.0",
|
||||
"lockfileVersion": 2,
|
||||
"requires": true,
|
||||
"packages": {
|
||||
"": {
|
||||
"name": "start-wasm-example",
|
||||
"version": "1.0.0",
|
||||
"license": "ISC",
|
||||
"dependencies": {
|
||||
"dotenv": "^16.0.1"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@playwright/test": "^1.22.1"
|
||||
}
|
||||
},
|
||||
"node_modules/@playwright/test": {
|
||||
"version": "1.22.1",
|
||||
"resolved": "https://registry.npmjs.org/@playwright/test/-/test-1.22.1.tgz",
|
||||
"integrity": "sha512-8ouMBUboYslHom41W8bnSEn0TwlAMHhCACwOZeuiAgzukj7KobpZ+UBwrGE0jJ0UblJbKAQNRHXL+z7sDSkb6g==",
|
||||
"dev": true,
|
||||
"dependencies": {
|
||||
"playwright-core": "1.22.1"
|
||||
},
|
||||
"bin": {
|
||||
"playwright": "cli.js"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">=14"
|
||||
}
|
||||
},
|
||||
"node_modules/dotenv": {
|
||||
"version": "16.0.1",
|
||||
"resolved": "https://registry.npmjs.org/dotenv/-/dotenv-16.0.1.tgz",
|
||||
"integrity": "sha512-1K6hR6wtk2FviQ4kEiSjFiH5rpzEVi8WW0x96aztHVMhEspNpc4DVOUTEHtEva5VThQ8IaBX1Pe4gSzpVVUsKQ==",
|
||||
"engines": {
|
||||
"node": ">=12"
|
||||
}
|
||||
},
|
||||
"node_modules/playwright-core": {
|
||||
"version": "1.22.1",
|
||||
"resolved": "https://registry.npmjs.org/playwright-core/-/playwright-core-1.22.1.tgz",
|
||||
"integrity": "sha512-H+ZUVYnceWNXrRf3oxTEKAr81QzFsCKu5Fp//fEjQvqgKkfA1iX3E9DBrPJpPNOrgVzcE+IqeI0fDmYJe6Ynnw==",
|
||||
"dev": true,
|
||||
"bin": {
|
||||
"playwright": "cli.js"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">=14"
|
||||
}
|
||||
}
|
||||
},
|
||||
"dependencies": {
|
||||
"@playwright/test": {
|
||||
"version": "1.22.1",
|
||||
"resolved": "https://registry.npmjs.org/@playwright/test/-/test-1.22.1.tgz",
|
||||
"integrity": "sha512-8ouMBUboYslHom41W8bnSEn0TwlAMHhCACwOZeuiAgzukj7KobpZ+UBwrGE0jJ0UblJbKAQNRHXL+z7sDSkb6g==",
|
||||
"dev": true,
|
||||
"requires": {
|
||||
"playwright-core": "1.22.1"
|
||||
}
|
||||
},
|
||||
"dotenv": {
|
||||
"version": "16.0.1",
|
||||
"resolved": "https://registry.npmjs.org/dotenv/-/dotenv-16.0.1.tgz",
|
||||
"integrity": "sha512-1K6hR6wtk2FviQ4kEiSjFiH5rpzEVi8WW0x96aztHVMhEspNpc4DVOUTEHtEva5VThQ8IaBX1Pe4gSzpVVUsKQ=="
|
||||
},
|
||||
"playwright-core": {
|
||||
"version": "1.22.1",
|
||||
"resolved": "https://registry.npmjs.org/playwright-core/-/playwright-core-1.22.1.tgz",
|
||||
"integrity": "sha512-H+ZUVYnceWNXrRf3oxTEKAr81QzFsCKu5Fp//fEjQvqgKkfA1iX3E9DBrPJpPNOrgVzcE+IqeI0fDmYJe6Ynnw==",
|
||||
"dev": true
|
||||
}
|
||||
}
|
||||
}
|
16
.github/start-wasm-example/package.json
vendored
Normal file
16
.github/start-wasm-example/package.json
vendored
Normal file
|
@ -0,0 +1,16 @@
|
|||
{
|
||||
"name": "start-wasm-example",
|
||||
"version": "1.0.0",
|
||||
"description": "",
|
||||
"main": "index.js",
|
||||
"scripts": {},
|
||||
"keywords": [],
|
||||
"author": "",
|
||||
"license": "ISC",
|
||||
"devDependencies": {
|
||||
"@playwright/test": "^1.22.1"
|
||||
},
|
||||
"dependencies": {
|
||||
"dotenv": "^16.0.1"
|
||||
}
|
||||
}
|
107
.github/start-wasm-example/playwright.config.ts
vendored
Normal file
107
.github/start-wasm-example/playwright.config.ts
vendored
Normal file
|
@ -0,0 +1,107 @@
|
|||
import type { PlaywrightTestConfig } from '@playwright/test';
|
||||
import { devices } from '@playwright/test';
|
||||
|
||||
/**
|
||||
* Read environment variables from file.
|
||||
* https://github.com/motdotla/dotenv
|
||||
*/
|
||||
require('dotenv').config();
|
||||
|
||||
/**
|
||||
* See https://playwright.dev/docs/test-configuration.
|
||||
*/
|
||||
const config: PlaywrightTestConfig = {
|
||||
testDir: './tests',
|
||||
/* Maximum time one test can run for. */
|
||||
timeout: 300_000,
|
||||
expect: {
|
||||
/**
|
||||
* Maximum time expect() should wait for the condition to be met.
|
||||
* For example in `await expect(locator).toHaveText();`
|
||||
*/
|
||||
timeout: 5000
|
||||
},
|
||||
/* Run tests in files in parallel */
|
||||
fullyParallel: true,
|
||||
/* Fail the build on CI if you accidentally left test.only in the source code. */
|
||||
forbidOnly: !!process.env.CI,
|
||||
/* Retry on CI only */
|
||||
retries: 0,
|
||||
/* Opt out of parallel tests on CI. */
|
||||
workers: 1,
|
||||
/* Reporter to use. See https://playwright.dev/docs/test-reporters */
|
||||
reporter: 'list',
|
||||
/* Shared settings for all the projects below. See https://playwright.dev/docs/api/class-testoptions. */
|
||||
use: {
|
||||
/* Maximum time each action such as `click()` can take. Defaults to 0 (no limit). */
|
||||
actionTimeout: 0,
|
||||
/* Base URL to use in actions like `await page.goto('/')`. */
|
||||
// baseURL: 'http://localhost:3000',
|
||||
|
||||
/* Collect trace when retrying the failed test. See https://playwright.dev/docs/trace-viewer */
|
||||
trace: 'on-first-retry',
|
||||
},
|
||||
|
||||
/* Configure projects for major browsers */
|
||||
projects: [
|
||||
{
|
||||
name: 'chromium',
|
||||
use: {
|
||||
...devices['Desktop Chrome'],
|
||||
},
|
||||
},
|
||||
|
||||
{
|
||||
name: 'firefox',
|
||||
use: {
|
||||
...devices['Desktop Firefox'],
|
||||
},
|
||||
},
|
||||
|
||||
{
|
||||
name: 'webkit',
|
||||
use: {
|
||||
...devices['Desktop Safari'],
|
||||
},
|
||||
},
|
||||
|
||||
/* Test against mobile viewports. */
|
||||
// {
|
||||
// name: 'Mobile Chrome',
|
||||
// use: {
|
||||
// ...devices['Pixel 5'],
|
||||
// },
|
||||
// },
|
||||
// {
|
||||
// name: 'Mobile Safari',
|
||||
// use: {
|
||||
// ...devices['iPhone 12'],
|
||||
// },
|
||||
// },
|
||||
|
||||
/* Test against branded browsers. */
|
||||
// {
|
||||
// name: 'Microsoft Edge',
|
||||
// use: {
|
||||
// channel: 'msedge',
|
||||
// },
|
||||
// },
|
||||
// {
|
||||
// name: 'Google Chrome',
|
||||
// use: {
|
||||
// channel: 'chrome',
|
||||
// },
|
||||
// },
|
||||
],
|
||||
|
||||
/* Folder for test artifacts such as screenshots, videos, traces, etc. */
|
||||
// outputDir: 'test-results/',
|
||||
|
||||
/* Run your local dev server before starting the tests */
|
||||
// webServer: {
|
||||
// command: 'npm run start',
|
||||
// port: 3000,
|
||||
// },
|
||||
};
|
||||
|
||||
export default config;
|
51
.github/start-wasm-example/tests/wasm_example.spec.ts
vendored
Normal file
51
.github/start-wasm-example/tests/wasm_example.spec.ts
vendored
Normal file
|
@ -0,0 +1,51 @@
|
|||
import { test, expect, Page } from '@playwright/test';
|
||||
|
||||
test.beforeEach(async ({ page }) => {
|
||||
await page.goto('http://localhost:8000/');
|
||||
});
|
||||
|
||||
const MAX_TIMEOUT_FOR_TEST = 300_000;
|
||||
|
||||
test.describe('WASM example', () => {
|
||||
test('Wait for success', async ({ page }, test_info) => {
|
||||
let start = new Date().getTime();
|
||||
|
||||
let found = false;
|
||||
while (new Date().getTime() - start < MAX_TIMEOUT_FOR_TEST) {
|
||||
let msg = await promise_with_timeout(100, on_console(page), "no log found");
|
||||
if (msg.includes("no log found")) {
|
||||
continue;
|
||||
}
|
||||
console.log(msg);
|
||||
if (msg.includes("Test successful")) {
|
||||
let prefix = process.env.SCREENSHOT_PREFIX === undefined ? "screenshot" : process.env.SCREENSHOT_PREFIX;
|
||||
await page.screenshot({ path: `${prefix}-${test_info.project.name}.png`, fullPage: true });
|
||||
found = true;
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
expect(found).toBe(true);
|
||||
});
|
||||
|
||||
});
|
||||
|
||||
function on_console(page) {
|
||||
return new Promise(resolve => {
|
||||
page.on('console', msg => resolve(msg.text()));
|
||||
});
|
||||
}
|
||||
|
||||
async function promise_with_timeout(time_limit, task, failure_value) {
|
||||
let timeout;
|
||||
const timeout_promise = new Promise((resolve, reject) => {
|
||||
timeout = setTimeout(() => {
|
||||
resolve(failure_value);
|
||||
}, time_limit);
|
||||
});
|
||||
const response = await Promise.race([task, timeout_promise]);
|
||||
if (timeout) {
|
||||
clearTimeout(timeout);
|
||||
}
|
||||
return response;
|
||||
}
|
58
.github/workflows/validation-jobs.yml
vendored
58
.github/workflows/validation-jobs.yml
vendored
|
@ -97,3 +97,61 @@ jobs:
|
|||
time CI_TESTING_CONFIG=$example cargo run --example $example_name --features "bevy_ci_testing"
|
||||
sleep 10
|
||||
done
|
||||
|
||||
run-examples-on-wasm:
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 60
|
||||
steps:
|
||||
- uses: actions/checkout@v3
|
||||
|
||||
- uses: actions-rs/toolchain@v1
|
||||
with:
|
||||
toolchain: stable
|
||||
target: wasm32-unknown-unknown
|
||||
override: true
|
||||
|
||||
- uses: actions/cache@v3
|
||||
with:
|
||||
path: |
|
||||
~/.cargo/bin/
|
||||
~/.cargo/registry/index/
|
||||
~/.cargo/registry/cache/
|
||||
~/.cargo/git/db/
|
||||
~/.github/start-wasm-example/node_modules
|
||||
target/
|
||||
key: ${{ runner.os }}-wasm-run-examples-${{ hashFiles('**/Cargo.toml') }}
|
||||
|
||||
- name: install xvfb, llvmpipe and lavapipe
|
||||
run: |
|
||||
sudo apt-get update -y -qq
|
||||
sudo add-apt-repository ppa:oibaf/graphics-drivers -y
|
||||
sudo apt-get update
|
||||
sudo apt install -y xvfb libegl1-mesa libgl1-mesa-dri libxcb-xfixes0-dev mesa-vulkan-drivers
|
||||
|
||||
- name: Install wasm-bindgen
|
||||
run: cargo install --force wasm-bindgen-cli
|
||||
|
||||
- name: Setup playwright
|
||||
run: |
|
||||
cd .github/start-wasm-example
|
||||
npm install
|
||||
npx playwright install --with-deps
|
||||
cd ../..
|
||||
|
||||
- name: First WASM build
|
||||
run: |
|
||||
cargo build --release --example ui --target wasm32-unknown-unknown
|
||||
|
||||
- name: Run examples
|
||||
shell: bash
|
||||
run: |
|
||||
# start a webserver
|
||||
python3 -m http.server --directory examples/wasm &
|
||||
|
||||
xvfb-run cargo run -p build-wasm-example -- --browsers chromium --browsers firefox --frames 25 --test shapes lighting text_debug breakout
|
||||
|
||||
- name: Save screenshots
|
||||
uses: actions/upload-artifact@v3
|
||||
with:
|
||||
name: screenshots
|
||||
path: .github/start-wasm-example/screenshot-*.png
|
||||
|
|
|
@ -1,6 +1,8 @@
|
|||
use crate::{app::AppExit, App};
|
||||
use serde::Deserialize;
|
||||
|
||||
use bevy_utils::tracing::info;
|
||||
|
||||
/// A configuration struct for automated CI testing.
|
||||
///
|
||||
/// It gets used when the `bevy_ci_testing` feature is enabled to automatically
|
||||
|
@ -20,18 +22,29 @@ fn ci_testing_exit_after(
|
|||
if let Some(exit_after) = ci_testing_config.exit_after {
|
||||
if *current_frame > exit_after {
|
||||
app_exit_events.send(AppExit);
|
||||
info!("Exiting after {} frames. Test successful!", exit_after);
|
||||
}
|
||||
}
|
||||
*current_frame += 1;
|
||||
}
|
||||
|
||||
pub(crate) fn setup_app(app: &mut App) -> &mut App {
|
||||
let filename =
|
||||
std::env::var("CI_TESTING_CONFIG").unwrap_or_else(|_| "ci_testing_config.ron".to_string());
|
||||
let config: CiTestingConfig = ron::from_str(
|
||||
&std::fs::read_to_string(filename).expect("error reading CI testing configuration file"),
|
||||
)
|
||||
.expect("error deserializing CI testing configuration file");
|
||||
#[cfg(not(target_arch = "wasm32"))]
|
||||
let config: CiTestingConfig = {
|
||||
let filename = std::env::var("CI_TESTING_CONFIG")
|
||||
.unwrap_or_else(|_| "ci_testing_config.ron".to_string());
|
||||
ron::from_str(
|
||||
&std::fs::read_to_string(filename)
|
||||
.expect("error reading CI testing configuration file"),
|
||||
)
|
||||
.expect("error deserializing CI testing configuration file")
|
||||
};
|
||||
#[cfg(target_arch = "wasm32")]
|
||||
let config: CiTestingConfig = {
|
||||
let config = include_str!("../../../ci_testing_config.ron");
|
||||
ron::from_str(config).expect("error deserializing CI testing configuration file")
|
||||
};
|
||||
|
||||
app.insert_resource(config)
|
||||
.add_system(ci_testing_exit_after);
|
||||
|
||||
|
|
|
@ -9,3 +9,4 @@ license = "MIT OR Apache-2.0"
|
|||
|
||||
[dependencies]
|
||||
xshell = "0.2"
|
||||
clap = { version = "3.1.12", features = ["derive"] }
|
||||
|
|
|
@ -1,18 +1,66 @@
|
|||
use std::{fs::File, io::Write};
|
||||
|
||||
use clap::Parser;
|
||||
use xshell::{cmd, Shell};
|
||||
|
||||
fn main() {
|
||||
let example = std::env::args().nth(1).expect("abbb");
|
||||
let sh = Shell::new().unwrap();
|
||||
cmd!(
|
||||
sh,
|
||||
"cargo build --release --target wasm32-unknown-unknown --example {example}"
|
||||
)
|
||||
.run()
|
||||
.expect("Error building example");
|
||||
cmd!(
|
||||
sh,
|
||||
"wasm-bindgen --out-dir examples/wasm/target --out-name wasm_example --target web target/wasm32-unknown-unknown/release/examples/{example}.wasm"
|
||||
)
|
||||
.run()
|
||||
.expect("Error creating wasm binding");
|
||||
#[derive(Parser, Debug)]
|
||||
struct Args {
|
||||
/// Examples to build
|
||||
examples: Vec<String>,
|
||||
|
||||
#[clap(short, long)]
|
||||
/// Run tests
|
||||
test: bool,
|
||||
|
||||
#[clap(short, long)]
|
||||
/// Run on the given browsers. By default, chromium, firefox, webkit
|
||||
browsers: Vec<String>,
|
||||
|
||||
#[clap(short, long)]
|
||||
/// Stop after this number of frames
|
||||
frames: Option<usize>,
|
||||
}
|
||||
|
||||
fn main() {
|
||||
let cli = Args::parse();
|
||||
eprintln!("{:?}", cli);
|
||||
|
||||
assert!(!cli.examples.is_empty(), "must have at least one example");
|
||||
|
||||
let mut bevy_ci_testing = vec![];
|
||||
if let Some(frames) = cli.frames {
|
||||
let mut file = File::create("ci_testing_config.ron").unwrap();
|
||||
file.write_fmt(format_args!("(exit_after: Some({}))", frames))
|
||||
.unwrap();
|
||||
bevy_ci_testing = vec!["--features", "bevy_ci_testing"];
|
||||
}
|
||||
|
||||
for example in cli.examples {
|
||||
let sh = Shell::new().unwrap();
|
||||
let bevy_ci_testing = bevy_ci_testing.clone();
|
||||
cmd!(
|
||||
sh,
|
||||
"cargo build {bevy_ci_testing...} --release --target wasm32-unknown-unknown --example {example}"
|
||||
)
|
||||
.run()
|
||||
.expect("Error building example");
|
||||
cmd!(
|
||||
sh,
|
||||
"wasm-bindgen --out-dir examples/wasm/target --out-name wasm_example --target web target/wasm32-unknown-unknown/release/examples/{example}.wasm"
|
||||
)
|
||||
.run()
|
||||
.expect("Error creating wasm binding");
|
||||
|
||||
if cli.test {
|
||||
let _dir = sh.push_dir(".github/start-wasm-example");
|
||||
let mut browsers = cli.browsers.clone();
|
||||
if !browsers.is_empty() {
|
||||
browsers.insert(0, "--project".to_string());
|
||||
}
|
||||
cmd!(sh, "npx playwright test --headed {browsers...}")
|
||||
.env("SCREENSHOT_PREFIX", format!("screenshot-{example}"))
|
||||
.run()
|
||||
.expect("Error running playwright test");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
Loading…
Reference in a new issue