daily job running mobile example on real devices (#8216)

# Objective - Test mobile example on real devices ## Solution - Use [BrowserStack](https://www.browserstack.com) to have access to [real devices](https://www.browserstack.com/list-of-browsers-and-platforms/app_automate) - [App Automate](https://www.browserstack.com/app-automate) to run the example - [App Percy](https://www.browserstack.com/app-percy) to compare the screenshot - Added a daily/manual CI job that will build for iOS and Android, send the apps to BrowserStack, run the app on one iOS device and one Android device, capture a screenshot, send it for visual validation, and archive it in the GitHub action Example run: https://github.com/mockersf/bevy/actions/runs/4521883534 They currently have a bug with the settings to view snapshots, they should be public. I'll raise it to them, and if they don't fix it in time it's possible to work around for everyone to view the results through their API. @cart to get this to work, you'll need - to set up an account on BrowserStack - add the secrets `BROWSERSTACK_USERNAME` and `BROWSERSTACK_ACCESS_KEY` to the Bevy repo - create a project in Percy - add the secret `PERCY_TOKEN` to the Bevy repo and modify the project name line 122 in the `Daily.yml` file
2024-11-22 04:33:37 +00:00 · 2023-03-29 01:16:07 +02:00 · 2023-03-29 01:16:07 +02:00 · 21dc3abe1b
commit 21dc3abe1b
parent 0893852c40
10 changed files with 4958 additions and 16 deletions
--- a/.github/start-mobile-example/.gitignore
+++ b/.github/start-mobile-example/.gitignore
@ -0,0 +1,5 @@
+node_modules/
+/test-results/
+/playwright-report/
+/playwright/.cache/
+screenshot.png
--- a/.github/start-mobile-example/mobile.conf.js
+++ b/.github/start-mobile-example/mobile.conf.js
@ -0,0 +1,35 @@
+exports.config = {
+    user: process.env.BROWSERSTACK_USERNAME,
+    key: process.env.BROWSERSTACK_ACCESS_KEY,
+
+    updateJob: false,
+    specs: [
+        './specs/screenshot.js'
+    ],
+    exclude: [],
+
+    capabilities: [{
+        project: "Bevy Example",
+        build: 'Bevy Example Runner',
+        name: 'run_example',
+        device: process.env.DEVICE || 'Samsung Galaxy S23',
+        os_version: process.env.OS_VERSION || "13.0",
+        app: process.env.BROWSERSTACK_APP_ID,
+        'browserstack.debug': true,
+        orientation: 'LANDSCAPE'
+    }],
+
+    logLevel: 'info',
+    coloredLogs: true,
+    screenshotPath: './screenshots/',
+    baseUrl: '',
+    waitforTimeout: 10000,
+    connectionRetryTimeout: 90000,
+    connectionRetryCount: 3,
+
+    framework: 'mocha',
+    mochaOpts: {
+        ui: 'bdd',
+        timeout: 20000
+    }
+};
--- a/.github/start-mobile-example/package-lock.json
+++ b/.github/start-mobile-example/package-lock.json
--- a/.github/start-mobile-example/package.json
+++ b/.github/start-mobile-example/package.json
@ -0,0 +1,25 @@
+{
+  "name": "start-mobile-example",
+  "version": "1.0.0",
+  "description": "",
+  "main": "index.js",
+  "scripts": {
+    "mobile": "./node_modules/.bin/wdio mobile.conf.js",
+    "android": "OS_VERSION='13.0' DEVICE='Samsung Galaxy S23' ./node_modules/.bin/wdio mobile.conf.js",
+    "ios": "OS_VERSION='15' DEVICE='iPhone 13' ./node_modules/.bin/wdio mobile.conf.js",
+    "clean": "rm -rf node_modules && rm -f package-lock.json && npm install"
+  },
+  "keywords": [],
+  "author": "",
+  "license": "MIT/Apache2",
+  "devDependencies": {
+    "@wdio/cli": "^5.20.1",
+    "@wdio/local-runner": "^5.20.1",
+    "@wdio/mocha-framework": "^5.18.7",
+    "browserstack-local": "^1.4.5",
+    "@percy/appium-app": "^0.0.7"
+  },
+  "dependencies": {
+    "dotenv": "^16.0.1"
+  }
+}
--- a/.github/start-mobile-example/specs/screenshot.js
+++ b/.github/start-mobile-example/specs/screenshot.js
@ -0,0 +1,18 @@
+var assert = require('assert');
+const percyScreenshot = require('@percy/appium-app');
+
+
+describe('Running Bevy Example', () => {
+  it('can take a screenshot', async () => {
+
+    // Sleep to wait for app startup, device rotation, ...
+    await new Promise(r => setTimeout(r, 2000));
+
+    // Take local screenshot
+    await browser.saveScreenshot('./screenshot.png');
+
+    // Take screenshot for visual testing
+    await percyScreenshot(`Bevy Mobile Example`);
+
+  });
+});
--- a/.github/workflows/daily.yml
+++ b/.github/workflows/daily.yml
@ -0,0 +1,124 @@
+name: Daily Jobs
+
+on:
+  schedule:
+    - cron:  '0 12 * * *'
+  workflow_dispatch:
+
+env:
+  CARGO_TERM_COLOR: always
+  NIGHTLY_TOOLCHAIN: nightly
+
+jobs:
+  build-for-iOS:
+    runs-on: macos-latest
+    timeout-minutes: 30
+    steps:
+      - uses: actions/checkout@v3
+
+      - uses: dtolnay/rust-toolchain@stable
+
+      - name: Add iOS targets
+        run: rustup target add aarch64-apple-ios x86_64-apple-ios
+
+      - name: Build app for iOS
+        run: |
+          cd examples/mobile
+          make xcodebuild-iphone
+          mkdir Payload
+          mv build/Build/Products/Debug-iphoneos/bevy_mobile_example.app Payload
+          zip -r bevy_mobile_example.zip Payload
+          mv bevy_mobile_example.zip bevy_mobile_example.ipa
+
+      - name: Upload to Browser Stack
+        run: |
+          curl -u "${{ secrets.BROWSERSTACK_USERNAME }}:${{ secrets.BROWSERSTACK_ACCESS_KEY }}" \
+            -X POST "https://api-cloud.browserstack.com/app-automate/upload" \
+            -F "file=@examples/mobile/bevy_mobile_example.ipa" \
+            -F "custom_id=$GITHUB_RUN_ID"
+
+  build-for-Android:
+    runs-on: ubuntu-latest
+    timeout-minutes: 30
+    steps:
+      - uses: actions/checkout@v3
+
+      - uses: dtolnay/rust-toolchain@stable
+
+      - name: Add Android targets
+        run: rustup target add aarch64-linux-android armv7-linux-androideabi
+
+      - name: Install Cargo APK
+        run: cargo install --force cargo-apk
+
+      - name: Build app for Android
+        run: ANDROID_NDK_ROOT=$ANDROID_NDK_LATEST_HOME cargo apk build --package bevy_mobile_example
+        env: 
+          # This will reduce the APK size from 1GB to ~200MB
+          CARGO_PROFILE_DEV_DEBUG: false
+
+      - name: Upload to Browser Stack
+        run: |
+          curl -u "${{ secrets.BROWSERSTACK_USERNAME }}:${{ secrets.BROWSERSTACK_ACCESS_KEY }}" \
+            -X POST "https://api-cloud.browserstack.com/app-automate/upload" \
+            -F "file=@target/debug/apk/bevyexample.apk" \
+            -F "custom_id=$GITHUB_RUN_ID"
+
+  nonce:
+    runs-on: ubuntu-latest
+    timeout-minutes: 30
+    outputs:
+      result: ${{ steps.nonce.outputs.result }}
+    steps:
+    - id: nonce
+      run: echo "result=${{ github.run_id }}-$(date +%s)" >> $GITHUB_OUTPUT
+
+  run:
+    runs-on: ubuntu-latest
+    timeout-minutes: 30
+    needs: [nonce, build-for-iOS, build-for-Android]
+    env: 
+      PERCY_PARALLEL_NONCE: ${{ needs.nonce.outputs.result }}
+      PERCY_PARALLEL_TOTAL: ${{ strategy.job-total }}
+    strategy:
+      matrix:
+        include:
+          - device: "iPhone 13"
+            os_version: "15"
+          - device: "Samsung Galaxy S23"
+            os_version: "13.0"
+    steps:
+      - uses: actions/checkout@v3
+
+      - name: Run Example
+        run: |
+          cd .github/start-mobile-example
+          npm install
+          npm install -g @percy/cli@latest
+          npx percy app:exec --parallel -- npm run mobile
+        env:
+          BROWSERSTACK_APP_ID: ${{ github.run_id }}
+          BROWSERSTACK_USERNAME: ${{ secrets.BROWSERSTACK_USERNAME }}
+          BROWSERSTACK_ACCESS_KEY: ${{ secrets.BROWSERSTACK_ACCESS_KEY }}
+          PERCY_TOKEN: ${{ secrets.PERCY_TOKEN }}
+          DEVICE: ${{ matrix.device }}
+          OS_VERSION: ${{ matrix.os_version }}
+
+      - name: Save screenshots
+        if: ${{ always() }}
+        uses: actions/upload-artifact@v3
+        with:
+          name: screenshots-${{ matrix.device }}-${{ matrix.os_version }}
+          path: .github/start-mobile-example/*.png
+
+  check-result:
+    runs-on: ubuntu-latest
+    timeout-minutes: 30
+    needs: [run]
+    steps:
+      - name: Wait for screenshots comparison
+        run: |
+          npm install -g @percy/cli@latest
+          npx percy build:wait --project dede4209/Bevy-Mobile-Example --commit ${{ github.sha }} --fail-on-changes --pass-if-approved
+        env:
+          PERCY_TOKEN: ${{ secrets.PERCY_TOKEN }}
--- a/README.md
+++ b/README.md
@ -107,6 +107,9 @@ Plugins are very welcome to extend Bevy's features. [Guidelines][plugin_guidelin

 Additionally, we would like to thank the [Amethyst](https://github.com/amethyst/amethyst), [macroquad](https://github.com/not-fl3/macroquad), [coffee](https://github.com/hecrj/coffee), [ggez](https://github.com/ggez/ggez), [Fyrox](https://github.com/FyroxEngine/Fyrox), and [Piston](https://github.com/PistonDevelopers/piston) projects for providing solid examples of game engine development in Rust. If you are looking for a Rust game engine, it is worth considering all of your options. Each engine has different design goals, and some will likely resonate with you more than others.

+<!-- This next line need to stay exactly as is. It is required for BrowserStack sponsorship. -->
+This project is tested with BrowserStack.
+
 ## License

 Bevy is free, open source and permissively licensed!
--- a/examples/mobile/Makefile
+++ b/examples/mobile/Makefile
@ -6,7 +6,7 @@ ifndef DEVICE_ID
 endif

 run: install
-	xcrun simctl launch --console $(DEVICE) com.rust.bevy_mobile_example
+	xcrun simctl launch --console $(DEVICE) org.bevyengine.example

 boot-sim:
 	xcrun simctl boot $(DEVICE) || true
--- a/examples/mobile/bevy_mobile_example.xcodeproj/project.pbxproj
+++ b/examples/mobile/bevy_mobile_example.xcodeproj/project.pbxproj
@ -229,6 +229,7 @@
 				CLANG_WARN_UNGUARDED_AVAILABILITY = YES_AGGRESSIVE;
 				CLANG_WARN_UNREACHABLE_CODE = YES;
 				CLANG_WARN__DUPLICATE_METHOD_MATCH = YES;
+				CODE_SIGN_IDENTITY = "";
 				COPY_PHASE_STRIP = NO;
 				DEBUG_INFORMATION_FORMAT = dwarf;
 				ENABLE_STRICT_OBJC_MSGSEND = YES;
@ -263,7 +264,7 @@
 				ASSETCATALOG_COMPILER_APPICON_NAME = AppIcon;
 				CLANG_CXX_LANGUAGE_STANDARD = "c++11";
 				CLANG_CXX_LIBRARY = "libc++";
-				CODE_SIGN_IDENTITY = "iPhone Developer";
+				CODE_SIGN_IDENTITY = "";
 				DEVELOPMENT_TEAM = "";
 				ENABLE_BITCODE = NO;
 				HEADER_SEARCH_PATHS = (
@ -293,7 +294,7 @@
 					"-lbevy_mobile_example",
 					"-lc++abi",
 				);
-				PRODUCT_BUNDLE_IDENTIFIER = "com.rust.bevy_mobile_example";
+				PRODUCT_BUNDLE_IDENTIFIER = "org.bevyengine.example";
 				SDKROOT = iphoneos;
 				TARGETED_DEVICE_FAMILY = "1,2";
 			};
@ -331,6 +332,7 @@
 				CLANG_WARN_UNGUARDED_AVAILABILITY = YES_AGGRESSIVE;
 				CLANG_WARN_UNREACHABLE_CODE = YES;
 				CLANG_WARN__DUPLICATE_METHOD_MATCH = YES;
+				CODE_SIGN_IDENTITY = "";
 				COPY_PHASE_STRIP = NO;
 				DEBUG_INFORMATION_FORMAT = "dwarf-with-dsym";
 				ENABLE_NS_ASSERTIONS = NO;
@ -370,7 +372,7 @@
 				ASSETCATALOG_COMPILER_APPICON_NAME = AppIcon;
 				CLANG_CXX_LANGUAGE_STANDARD = "c++11";
 				CLANG_CXX_LIBRARY = "libc++";
-				CODE_SIGN_IDENTITY = "iPhone Developer";
+				CODE_SIGN_IDENTITY = "";
 				DEVELOPMENT_TEAM = "";
 				ENABLE_BITCODE = NO;
 				HEADER_SEARCH_PATHS = (
@ -400,7 +402,7 @@
 					"-lbevy_mobile_example",
 					"-lc++abi",
 				);
-				PRODUCT_BUNDLE_IDENTIFIER = "com.rust.bevy_mobile_example";
+				PRODUCT_BUNDLE_IDENTIFIER = "org.bevyengine.example";
 				SDKROOT = iphoneos;
 				TARGETED_DEVICE_FAMILY = "1,2";
 			};
--- a/examples/mobile/src/lib.rs
+++ b/examples/mobile/src/lib.rs
@ -3,18 +3,24 @@ use bevy::{input::touch::TouchPhase, prelude::*, window::WindowMode};
 // the `bevy_main` proc_macro generates the required boilerplate for iOS and Android
 #[bevy_main]
 fn main() {
-    App::new()
-        .add_plugins(DefaultPlugins.set(WindowPlugin {
-            primary_window: Some(Window {
-                resizable: false,
-                mode: WindowMode::BorderlessFullscreen,
-                ..default()
-            }),
+    let mut app = App::new();
+    app.add_plugins(DefaultPlugins.set(WindowPlugin {
+        primary_window: Some(Window {
+            resizable: false,
+            mode: WindowMode::BorderlessFullscreen,
            ..default()
-        }))
-        .add_systems(Startup, (setup_scene, setup_music))
-        .add_systems(Update, (touch_camera, button_handler))
-        .run();
+        }),
+        ..default()
+    }))
+    .add_systems(Startup, (setup_scene, setup_music))
+    .add_systems(Update, (touch_camera, button_handler));
+
+    // MSAA makes some Android devices panic, this is under investigation
+    // https://github.com/bevyengine/bevy/issues/8229
+    #[cfg(target_os = "android")]
+    app.insert_resource(Msaa::Off);
+
+    app.run();
 }

 fn touch_camera(
@ -82,6 +88,9 @@ fn setup_scene(
        transform: Transform::from_xyz(4.0, 8.0, 4.0),
        point_light: PointLight {
            intensity: 5000.0,
+            // Shadows makes some Android devices segfault, this is under investigation
+            // https://github.com/bevyengine/bevy/issues/8214
+            #[cfg(not(target_os = "android"))]
            shadows_enabled: true,
            ..default()
        },