From 78345a2f7a5a6b633621db47ccc1e6d8b329b973 Mon Sep 17 00:00:00 2001 From: Gino Valente <49806985+MrGVSV@users.noreply.github.com> Date: Fri, 12 Apr 2024 18:48:37 -0700 Subject: [PATCH] tools: Refactor CI to use `argh` (#12923) # Objective The CI tool currently parses input manually. This has worked fine, but makes it just a bit more difficult to maintain and extend. Additionally, it provides no usage help for devs wanting to run the tool locally. It would be better if parsing was handled by a dedicated CLI library like [`clap`](https://github.com/clap-rs/clap) or [`argh`](https://github.com/google/argh). ## Solution Use `argh` to parse command line input for CI. `argh` was chosen over `clap` and other tools due to being more lightweight and already existing in our dependency tree. Using `argh`, the usage notes are generated automatically: ``` $ cargo run -p ci --quiet -- --help Usage: ci [--keep-going] [] [] The CI command line tool for Bevy. Options: --keep-going continue running commands even if one fails --help display usage information Commands: lints Alias for running the `format` and `clippy` subcommands. doc Alias for running the `doc-test` and `doc-check` subcommands. compile Alias for running the `compile-fail`, `bench-check`, `example-check`, `compile-check`, and `test-check` subcommands. format Check code formatting. clippy Check for clippy warnings and errors. test Runs all tests (except for doc tests). test-check Checks that all tests compile. doc-check Checks that all docs compile. doc-test Runs all doc tests. compile-check Checks that the project compiles. cfg-check Checks that the project compiles using the nightly compiler with cfg checks enabled. compile-fail Runs the compile-fail tests. bench-check Checks that the benches compile. example-check Checks that the examples compile. ``` This PR makes each subcommand more modular, allowing them to be called from other subcommands. This also makes it much easier to extract them out of `main.rs` and into their own dedicated modules. Additionally, this PR improves failure output: ``` $ cargo run -p ci -- lints ... One or more CI commands failed: format: Please run 'cargo fmt --all' to format your code. ``` Including when run with the `--keep-going` flag: ``` $ cargo run -p ci -- --keep-going lints ... One or more CI commands failed: - format: Please run 'cargo fmt --all' to format your code. - clippy: Please fix clippy errors in output above. ``` ### Future Work There are a lot of other things we could possibly clean up. I chose to try and keep the API surface as unchanged as I could (for this PR at least). For example, now that each subcommand is an actual command, we can specify custom arguments for each. The `format` subcommand could include a `--check` (making the default fun `cargo fmt` as normal). Or the `compile-fail` subcommand could include `--ecs`, `--reflect`, and `--macros` flags for specifying which set of compile fail tests to run. The `--keep-going` flag could be split so that it doesn't do double-duty where it also enables `--no-fail-fast` for certain commands. Or at least make it more explicit via renaming or using alternative flags. --- ## Changelog - Improved the CI CLI tool - Now includes usage info with the `--help` option! - [Internal] Cleaned up and refactored the `tools/ci` crate using the `argh` crate ## Migration Guide The CI tool no longer supports running multiple subcommands in a single call. Users who are currently doing so will need to split them across multiple commands: ```bash # BEFORE cargo run -p ci -- lints doc compile # AFTER cargo run -p ci -- lints && cargo run -p ci -- doc && cargo run -p ci -- compile # or cargo run -p ci -- lints; cargo run -p ci -- doc; cargo run -p ci -- compile # or cargo run -p ci -- lints cargo run -p ci -- doc cargo run -p ci -- compile ``` --- tools/ci/Cargo.toml | 1 + tools/ci/src/commands/ci.rs | 124 ++++++ tools/ci/src/commands/mod.rs | 6 + tools/ci/src/commands/prepare.rs | 57 +++ .../subcommands/bench_check_command.rs | 17 + .../commands/subcommands/cfg_check_command.rs | 18 + .../commands/subcommands/clippy_command.rs | 20 + .../subcommands/compile_check_command.rs | 17 + .../commands/subcommands/compile_command.rs | 23 ++ .../subcommands/compile_fail_command.rs | 54 +++ .../commands/subcommands/doc_check_command.rs | 20 + .../src/commands/subcommands/doc_command.rs | 17 + .../commands/subcommands/doc_test_command.rs | 22 + .../subcommands/example_check_command.rs | 17 + .../commands/subcommands/format_command.rs | 17 + .../src/commands/subcommands/lints_command.rs | 17 + tools/ci/src/commands/subcommands/mod.rs | 29 ++ .../subcommands/test_check_command.rs | 17 + .../src/commands/subcommands/test_command.rs | 25 ++ tools/ci/src/main.rs | 391 +----------------- 20 files changed, 520 insertions(+), 389 deletions(-) create mode 100644 tools/ci/src/commands/ci.rs create mode 100644 tools/ci/src/commands/mod.rs create mode 100644 tools/ci/src/commands/prepare.rs create mode 100644 tools/ci/src/commands/subcommands/bench_check_command.rs create mode 100644 tools/ci/src/commands/subcommands/cfg_check_command.rs create mode 100644 tools/ci/src/commands/subcommands/clippy_command.rs create mode 100644 tools/ci/src/commands/subcommands/compile_check_command.rs create mode 100644 tools/ci/src/commands/subcommands/compile_command.rs create mode 100644 tools/ci/src/commands/subcommands/compile_fail_command.rs create mode 100644 tools/ci/src/commands/subcommands/doc_check_command.rs create mode 100644 tools/ci/src/commands/subcommands/doc_command.rs create mode 100644 tools/ci/src/commands/subcommands/doc_test_command.rs create mode 100644 tools/ci/src/commands/subcommands/example_check_command.rs create mode 100644 tools/ci/src/commands/subcommands/format_command.rs create mode 100644 tools/ci/src/commands/subcommands/lints_command.rs create mode 100644 tools/ci/src/commands/subcommands/mod.rs create mode 100644 tools/ci/src/commands/subcommands/test_check_command.rs create mode 100644 tools/ci/src/commands/subcommands/test_command.rs diff --git a/tools/ci/Cargo.toml b/tools/ci/Cargo.toml index dd3c0d265d..611a6f318d 100644 --- a/tools/ci/Cargo.toml +++ b/tools/ci/Cargo.toml @@ -7,6 +7,7 @@ publish = false license = "MIT OR Apache-2.0" [dependencies] +argh = "0.1" xshell = "0.2" bitflags = "2.3" diff --git a/tools/ci/src/commands/ci.rs b/tools/ci/src/commands/ci.rs new file mode 100644 index 0000000000..4be530d048 --- /dev/null +++ b/tools/ci/src/commands/ci.rs @@ -0,0 +1,124 @@ +use crate::commands::prepare::{Flag, Prepare, PreparedCommand}; +use crate::commands::subcommands; +use argh::FromArgs; + +/// The CI command line tool for Bevy. +#[derive(FromArgs)] +pub(crate) struct CI { + #[argh(subcommand)] + command: Option, + /// continue running commands even if one fails + #[argh(switch)] + keep_going: bool, +} + +impl CI { + /// Runs the specified commands or all commands if none are specified. + /// + /// When run locally, results may differ from actual CI runs triggered by `.github/workflows/ci.yml`. + /// This is because the official CI runs latest stable, while local runs use whatever the default Rust is locally. + pub fn run(self) { + let sh = xshell::Shell::new().unwrap(); + + let prepared_commands = self.prepare(&sh); + + let mut failures = vec![]; + + for command in prepared_commands { + // If the CI test is to be executed in a subdirectory, we move there before running the command. + // This will automatically move back to the original directory once dropped. + let _subdir_hook = command.subdir.map(|path| sh.push_dir(path)); + + if command.command.envs(command.env_vars).run().is_err() { + let name = command.name; + let message = command.failure_message; + if self.keep_going { + failures.push(format!("- {name}: {message}")); + } else { + failures.push(format!("{name}: {message}")); + break; + } + } + } + + if !failures.is_empty() { + let failures = failures.join("\n"); + panic!( + "One or more CI commands failed:\n\ + {failures}" + ); + } + } + + fn prepare<'a>(&self, sh: &'a xshell::Shell) -> Vec> { + let mut flags = Flag::empty(); + + if self.keep_going { + flags |= Flag::KEEP_GOING; + } + + match &self.command { + Some(command) => command.prepare(sh, flags), + None => { + // Note that we are running the subcommands directly rather than using any aliases + let mut cmds = vec![]; + cmds.append(&mut subcommands::FormatCommand::default().prepare(sh, flags)); + cmds.append(&mut subcommands::ClippyCommand::default().prepare(sh, flags)); + cmds.append(&mut subcommands::TestCommand::default().prepare(sh, flags)); + cmds.append(&mut subcommands::TestCheckCommand::default().prepare(sh, flags)); + cmds.append(&mut subcommands::DocCheckCommand::default().prepare(sh, flags)); + cmds.append(&mut subcommands::DocTestCommand::default().prepare(sh, flags)); + cmds.append(&mut subcommands::CompileCheckCommand::default().prepare(sh, flags)); + cmds.append(&mut subcommands::CfgCheckCommand::default().prepare(sh, flags)); + cmds.append(&mut subcommands::CompileFailCommand::default().prepare(sh, flags)); + cmds.append(&mut subcommands::BenchCheckCommand::default().prepare(sh, flags)); + cmds.append(&mut subcommands::ExampleCheckCommand::default().prepare(sh, flags)); + cmds + } + } + } +} + +/// The subcommands that can be run by the CI script. +#[derive(FromArgs)] +#[argh(subcommand)] +enum Commands { + // Aliases (subcommands that run other subcommands) + Lints(subcommands::LintsCommand), + Doc(subcommands::DocCommand), + Compile(subcommands::CompileCommand), + // Actual subcommands + Format(subcommands::FormatCommand), + Clippy(subcommands::ClippyCommand), + Test(subcommands::TestCommand), + TestCheck(subcommands::TestCheckCommand), + DocCheck(subcommands::DocCheckCommand), + DocTest(subcommands::DocTestCommand), + CompileCheck(subcommands::CompileCheckCommand), + CfgCheck(subcommands::CfgCheckCommand), + CompileFail(subcommands::CompileFailCommand), + BenchCheck(subcommands::BenchCheckCommand), + ExampleCheck(subcommands::ExampleCheckCommand), +} + +impl Prepare for Commands { + fn prepare<'a>(&self, sh: &'a xshell::Shell, flags: Flag) -> Vec> { + match self { + Commands::Lints(subcommand) => subcommand.prepare(sh, flags), + Commands::Doc(subcommand) => subcommand.prepare(sh, flags), + Commands::Compile(subcommand) => subcommand.prepare(sh, flags), + + Commands::Format(subcommand) => subcommand.prepare(sh, flags), + Commands::Clippy(subcommand) => subcommand.prepare(sh, flags), + Commands::Test(subcommand) => subcommand.prepare(sh, flags), + Commands::TestCheck(subcommand) => subcommand.prepare(sh, flags), + Commands::DocCheck(subcommand) => subcommand.prepare(sh, flags), + Commands::DocTest(subcommand) => subcommand.prepare(sh, flags), + Commands::CompileCheck(subcommand) => subcommand.prepare(sh, flags), + Commands::CfgCheck(subcommand) => subcommand.prepare(sh, flags), + Commands::CompileFail(subcommand) => subcommand.prepare(sh, flags), + Commands::BenchCheck(subcommand) => subcommand.prepare(sh, flags), + Commands::ExampleCheck(subcommand) => subcommand.prepare(sh, flags), + } + } +} diff --git a/tools/ci/src/commands/mod.rs b/tools/ci/src/commands/mod.rs new file mode 100644 index 0000000000..6940923c94 --- /dev/null +++ b/tools/ci/src/commands/mod.rs @@ -0,0 +1,6 @@ +pub(crate) use ci::*; +use prepare::*; + +mod ci; +mod prepare; +mod subcommands; diff --git a/tools/ci/src/commands/prepare.rs b/tools/ci/src/commands/prepare.rs new file mode 100644 index 0000000000..46d09c3531 --- /dev/null +++ b/tools/ci/src/commands/prepare.rs @@ -0,0 +1,57 @@ +use bitflags::bitflags; + +/// Trait for preparing a subcommand to be run. +pub(crate) trait Prepare { + fn prepare<'a>(&self, sh: &'a xshell::Shell, flags: Flag) -> Vec>; +} + +bitflags! { + #[derive(Clone, Copy, Debug, PartialEq, Eq)] + pub(crate) struct Flag: u32 { + /// Forces certain checks to continue running even if they hit an error. + const KEEP_GOING = 1 << 0; + } +} + +#[derive(Debug)] +pub(crate) struct PreparedCommand<'a> { + /// The name of the command. + pub name: &'static str, + + /// The command to execute + pub command: xshell::Cmd<'a>, + + /// The message to display if the test command fails + pub failure_message: &'static str, + + /// The subdirectory path to run the test command within + pub subdir: Option<&'static str>, + + /// Environment variables that need to be set before the test runs + pub env_vars: Vec<(&'static str, &'static str)>, +} + +impl<'a> PreparedCommand<'a> { + pub fn new( + command: xshell::Cmd<'a>, + failure_message: &'static str, + ) -> Self { + Self { + command, + name: T::COMMAND.name, + failure_message, + subdir: None, + env_vars: vec![], + } + } + + pub fn with_subdir(mut self, subdir: &'static str) -> Self { + self.subdir = Some(subdir); + self + } + + pub fn with_env_var(mut self, key: &'static str, value: &'static str) -> Self { + self.env_vars.push((key, value)); + self + } +} diff --git a/tools/ci/src/commands/subcommands/bench_check_command.rs b/tools/ci/src/commands/subcommands/bench_check_command.rs new file mode 100644 index 0000000000..0a3a0f5507 --- /dev/null +++ b/tools/ci/src/commands/subcommands/bench_check_command.rs @@ -0,0 +1,17 @@ +use crate::commands::{Flag, Prepare, PreparedCommand}; +use argh::FromArgs; +use xshell::cmd; + +/// Checks that the benches compile. +#[derive(FromArgs, Default)] +#[argh(subcommand, name = "bench-check")] +pub(crate) struct BenchCheckCommand {} + +impl Prepare for BenchCheckCommand { + fn prepare<'a>(&self, sh: &'a xshell::Shell, _flags: Flag) -> Vec> { + vec![PreparedCommand::new::( + cmd!(sh, "cargo check --benches --target-dir ../target"), + "Failed to check the benches.", + )] + } +} diff --git a/tools/ci/src/commands/subcommands/cfg_check_command.rs b/tools/ci/src/commands/subcommands/cfg_check_command.rs new file mode 100644 index 0000000000..2678834e77 --- /dev/null +++ b/tools/ci/src/commands/subcommands/cfg_check_command.rs @@ -0,0 +1,18 @@ +use crate::commands::{Flag, Prepare, PreparedCommand}; +use argh::FromArgs; +use xshell::cmd; + +/// Checks that the project compiles using the nightly compiler with cfg checks enabled. +#[derive(FromArgs, Default)] +#[argh(subcommand, name = "cfg-check")] +pub(crate) struct CfgCheckCommand {} + +impl Prepare for CfgCheckCommand { + fn prepare<'a>(&self, sh: &'a xshell::Shell, _flags: Flag) -> Vec> { + vec![PreparedCommand::new::( + cmd!(sh, "cargo +nightly check -Zcheck-cfg --workspace"), + "Please fix failing cfg checks in output above.", + ) + .with_env_var("RUSTFLAGS", "-D warnings")] + } +} diff --git a/tools/ci/src/commands/subcommands/clippy_command.rs b/tools/ci/src/commands/subcommands/clippy_command.rs new file mode 100644 index 0000000000..7ef055b155 --- /dev/null +++ b/tools/ci/src/commands/subcommands/clippy_command.rs @@ -0,0 +1,20 @@ +use crate::commands::{Flag, Prepare, PreparedCommand}; +use argh::FromArgs; +use xshell::cmd; + +/// Check for clippy warnings and errors. +#[derive(FromArgs, Default)] +#[argh(subcommand, name = "clippy")] +pub(crate) struct ClippyCommand {} + +impl Prepare for ClippyCommand { + fn prepare<'a>(&self, sh: &'a xshell::Shell, _flags: Flag) -> Vec> { + vec![PreparedCommand::new::( + cmd!( + sh, + "cargo clippy --workspace --all-targets --all-features -- -Dwarnings" + ), + "Please fix clippy errors in output above.", + )] + } +} diff --git a/tools/ci/src/commands/subcommands/compile_check_command.rs b/tools/ci/src/commands/subcommands/compile_check_command.rs new file mode 100644 index 0000000000..81d277074a --- /dev/null +++ b/tools/ci/src/commands/subcommands/compile_check_command.rs @@ -0,0 +1,17 @@ +use crate::commands::{Flag, Prepare, PreparedCommand}; +use argh::FromArgs; +use xshell::cmd; + +/// Checks that the project compiles. +#[derive(FromArgs, Default)] +#[argh(subcommand, name = "compile-check")] +pub(crate) struct CompileCheckCommand {} + +impl Prepare for CompileCheckCommand { + fn prepare<'a>(&self, sh: &'a xshell::Shell, _flags: Flag) -> Vec> { + vec![PreparedCommand::new::( + cmd!(sh, "cargo check --workspace"), + "Please fix compiler errors in output above.", + )] + } +} diff --git a/tools/ci/src/commands/subcommands/compile_command.rs b/tools/ci/src/commands/subcommands/compile_command.rs new file mode 100644 index 0000000000..3f954e4fdb --- /dev/null +++ b/tools/ci/src/commands/subcommands/compile_command.rs @@ -0,0 +1,23 @@ +use crate::commands::subcommands::{ + BenchCheckCommand, CompileCheckCommand, CompileFailCommand, ExampleCheckCommand, + TestCheckCommand, +}; +use crate::commands::{Flag, Prepare, PreparedCommand}; +use argh::FromArgs; + +/// Alias for running the `compile-fail`, `bench-check`, `example-check`, `compile-check`, and `test-check` subcommands. +#[derive(FromArgs, Default)] +#[argh(subcommand, name = "compile")] +pub(crate) struct CompileCommand {} + +impl Prepare for CompileCommand { + fn prepare<'a>(&self, sh: &'a xshell::Shell, flags: Flag) -> Vec> { + let mut commands = vec![]; + commands.append(&mut CompileFailCommand::default().prepare(sh, flags)); + commands.append(&mut BenchCheckCommand::default().prepare(sh, flags)); + commands.append(&mut ExampleCheckCommand::default().prepare(sh, flags)); + commands.append(&mut CompileCheckCommand::default().prepare(sh, flags)); + commands.append(&mut TestCheckCommand::default().prepare(sh, flags)); + commands + } +} diff --git a/tools/ci/src/commands/subcommands/compile_fail_command.rs b/tools/ci/src/commands/subcommands/compile_fail_command.rs new file mode 100644 index 0000000000..9bc89e9bb8 --- /dev/null +++ b/tools/ci/src/commands/subcommands/compile_fail_command.rs @@ -0,0 +1,54 @@ +use crate::commands::{Flag, Prepare, PreparedCommand}; +use argh::FromArgs; +use xshell::cmd; + +/// Runs the compile-fail tests. +#[derive(FromArgs, Default)] +#[argh(subcommand, name = "compile-fail")] +pub(crate) struct CompileFailCommand {} + +impl Prepare for CompileFailCommand { + fn prepare<'a>(&self, sh: &'a xshell::Shell, flags: Flag) -> Vec> { + let no_fail_fast = flags + .contains(Flag::KEEP_GOING) + .then_some("--no-fail-fast") + .unwrap_or_default(); + + let mut commands = vec![]; + + // ECS Compile Fail Tests + // Run UI tests (they do not get executed with the workspace tests) + // - See crates/bevy_ecs_compile_fail_tests/README.md + commands.push( + PreparedCommand::new::( + cmd!(sh, "cargo test --target-dir ../../target {no_fail_fast}"), + "Compiler errors of the ECS compile fail tests seem to be different than expected! Check locally and compare rust versions.", + ) + .with_subdir("crates/bevy_ecs_compile_fail_tests"), + ); + + // Reflect Compile Fail Tests + // Run tests (they do not get executed with the workspace tests) + // - See crates/bevy_reflect_compile_fail_tests/README.md + commands.push( + PreparedCommand::new::( + cmd!(sh, "cargo test --target-dir ../../target {no_fail_fast}"), + "Compiler errors of the Reflect compile fail tests seem to be different than expected! Check locally and compare rust versions.", + ) + .with_subdir("crates/bevy_reflect_compile_fail_tests"), + ); + + // Macro Compile Fail Tests + // Run tests (they do not get executed with the workspace tests) + // - See crates/bevy_macros_compile_fail_tests/README.md + commands.push( + PreparedCommand::new::( + cmd!(sh, "cargo test --target-dir ../../target {no_fail_fast}"), + "Compiler errors of the macros compile fail tests seem to be different than expected! Check locally and compare rust versions.", + ) + .with_subdir("crates/bevy_macros_compile_fail_tests"), + ); + + commands + } +} diff --git a/tools/ci/src/commands/subcommands/doc_check_command.rs b/tools/ci/src/commands/subcommands/doc_check_command.rs new file mode 100644 index 0000000000..96a48c737f --- /dev/null +++ b/tools/ci/src/commands/subcommands/doc_check_command.rs @@ -0,0 +1,20 @@ +use crate::commands::{Flag, Prepare, PreparedCommand}; +use argh::FromArgs; +use xshell::cmd; + +/// Checks that all docs compile. +#[derive(FromArgs, Default)] +#[argh(subcommand, name = "doc-check")] +pub(crate) struct DocCheckCommand {} + +impl Prepare for DocCheckCommand { + fn prepare<'a>(&self, sh: &'a xshell::Shell, _flags: Flag) -> Vec> { + vec![PreparedCommand::new::( + cmd!( + sh, + "cargo doc --workspace --all-features --no-deps --document-private-items" + ), + "Please fix doc warnings in output above.", + )] + } +} diff --git a/tools/ci/src/commands/subcommands/doc_command.rs b/tools/ci/src/commands/subcommands/doc_command.rs new file mode 100644 index 0000000000..8dddd06d52 --- /dev/null +++ b/tools/ci/src/commands/subcommands/doc_command.rs @@ -0,0 +1,17 @@ +use crate::commands::subcommands::{DocCheckCommand, DocTestCommand}; +use crate::commands::{Flag, Prepare, PreparedCommand}; +use argh::FromArgs; + +/// Alias for running the `doc-test` and `doc-check` subcommands. +#[derive(FromArgs, Default)] +#[argh(subcommand, name = "doc")] +pub(crate) struct DocCommand {} + +impl Prepare for DocCommand { + fn prepare<'a>(&self, sh: &'a xshell::Shell, flags: Flag) -> Vec> { + let mut commands = vec![]; + commands.append(&mut DocTestCommand::default().prepare(sh, flags)); + commands.append(&mut DocCheckCommand::default().prepare(sh, flags)); + commands + } +} diff --git a/tools/ci/src/commands/subcommands/doc_test_command.rs b/tools/ci/src/commands/subcommands/doc_test_command.rs new file mode 100644 index 0000000000..04a8ec97df --- /dev/null +++ b/tools/ci/src/commands/subcommands/doc_test_command.rs @@ -0,0 +1,22 @@ +use crate::commands::{Flag, Prepare, PreparedCommand}; +use argh::FromArgs; +use xshell::cmd; + +/// Runs all doc tests. +#[derive(FromArgs, Default)] +#[argh(subcommand, name = "doc-test")] +pub(crate) struct DocTestCommand {} + +impl Prepare for DocTestCommand { + fn prepare<'a>(&self, sh: &'a xshell::Shell, flags: Flag) -> Vec> { + let no_fail_fast = flags + .contains(Flag::KEEP_GOING) + .then_some("--no-fail-fast") + .unwrap_or_default(); + + vec![PreparedCommand::new::( + cmd!(sh, "cargo test --workspace --doc {no_fail_fast}"), + "Please fix failing doc tests in output above.", + )] + } +} diff --git a/tools/ci/src/commands/subcommands/example_check_command.rs b/tools/ci/src/commands/subcommands/example_check_command.rs new file mode 100644 index 0000000000..1fc4467301 --- /dev/null +++ b/tools/ci/src/commands/subcommands/example_check_command.rs @@ -0,0 +1,17 @@ +use crate::commands::{Flag, Prepare, PreparedCommand}; +use argh::FromArgs; +use xshell::cmd; + +/// Checks that the examples compile. +#[derive(FromArgs, Default)] +#[argh(subcommand, name = "example-check")] +pub(crate) struct ExampleCheckCommand {} + +impl Prepare for ExampleCheckCommand { + fn prepare<'a>(&self, sh: &'a xshell::Shell, _flags: Flag) -> Vec> { + vec![PreparedCommand::new::( + cmd!(sh, "cargo check --workspace --examples"), + "Please fix compiler errors for examples in output above.", + )] + } +} diff --git a/tools/ci/src/commands/subcommands/format_command.rs b/tools/ci/src/commands/subcommands/format_command.rs new file mode 100644 index 0000000000..2f3c6e5fda --- /dev/null +++ b/tools/ci/src/commands/subcommands/format_command.rs @@ -0,0 +1,17 @@ +use crate::commands::{Flag, Prepare, PreparedCommand}; +use argh::FromArgs; +use xshell::cmd; + +/// Check code formatting. +#[derive(FromArgs, Default)] +#[argh(subcommand, name = "format")] +pub(crate) struct FormatCommand {} + +impl Prepare for FormatCommand { + fn prepare<'a>(&self, sh: &'a xshell::Shell, _flags: Flag) -> Vec> { + vec![PreparedCommand::new::( + cmd!(sh, "cargo fmt --all -- --check"), + "Please run 'cargo fmt --all' to format your code.", + )] + } +} diff --git a/tools/ci/src/commands/subcommands/lints_command.rs b/tools/ci/src/commands/subcommands/lints_command.rs new file mode 100644 index 0000000000..04275a7327 --- /dev/null +++ b/tools/ci/src/commands/subcommands/lints_command.rs @@ -0,0 +1,17 @@ +use crate::commands::subcommands::{ClippyCommand, FormatCommand}; +use crate::commands::{Flag, Prepare, PreparedCommand}; +use argh::FromArgs; + +/// Alias for running the `format` and `clippy` subcommands. +#[derive(FromArgs, Default)] +#[argh(subcommand, name = "lints")] +pub(crate) struct LintsCommand {} + +impl Prepare for LintsCommand { + fn prepare<'a>(&self, sh: &'a xshell::Shell, flags: Flag) -> Vec> { + let mut commands = vec![]; + commands.append(&mut FormatCommand::default().prepare(sh, flags)); + commands.append(&mut ClippyCommand::default().prepare(sh, flags)); + commands + } +} diff --git a/tools/ci/src/commands/subcommands/mod.rs b/tools/ci/src/commands/subcommands/mod.rs new file mode 100644 index 0000000000..7f05e9b3a2 --- /dev/null +++ b/tools/ci/src/commands/subcommands/mod.rs @@ -0,0 +1,29 @@ +pub(crate) use bench_check_command::*; +pub(crate) use cfg_check_command::*; +pub(crate) use clippy_command::*; +pub(crate) use compile_check_command::*; +pub(crate) use compile_command::*; +pub(crate) use compile_fail_command::*; +pub(crate) use doc_check_command::*; +pub(crate) use doc_command::*; +pub(crate) use doc_test_command::*; +pub(crate) use example_check_command::*; +pub(crate) use format_command::*; +pub(crate) use lints_command::*; +pub(crate) use test_check_command::*; +pub(crate) use test_command::*; + +mod bench_check_command; +mod cfg_check_command; +mod clippy_command; +mod compile_check_command; +mod compile_command; +mod compile_fail_command; +mod doc_check_command; +mod doc_command; +mod doc_test_command; +mod example_check_command; +mod format_command; +mod lints_command; +mod test_check_command; +mod test_command; diff --git a/tools/ci/src/commands/subcommands/test_check_command.rs b/tools/ci/src/commands/subcommands/test_check_command.rs new file mode 100644 index 0000000000..5398480601 --- /dev/null +++ b/tools/ci/src/commands/subcommands/test_check_command.rs @@ -0,0 +1,17 @@ +use crate::commands::{Flag, Prepare, PreparedCommand}; +use argh::FromArgs; +use xshell::cmd; + +/// Checks that all tests compile. +#[derive(FromArgs, Default)] +#[argh(subcommand, name = "test-check")] +pub(crate) struct TestCheckCommand {} + +impl Prepare for TestCheckCommand { + fn prepare<'a>(&self, sh: &'a xshell::Shell, _flags: Flag) -> Vec> { + vec![PreparedCommand::new::( + cmd!(sh, "cargo check --workspace --tests"), + "Please fix compiler examples for tests in output above.", + )] + } +} diff --git a/tools/ci/src/commands/subcommands/test_command.rs b/tools/ci/src/commands/subcommands/test_command.rs new file mode 100644 index 0000000000..d903922197 --- /dev/null +++ b/tools/ci/src/commands/subcommands/test_command.rs @@ -0,0 +1,25 @@ +use crate::commands::{Flag, Prepare, PreparedCommand}; +use argh::FromArgs; +use xshell::cmd; + +/// Runs all tests (except for doc tests). +#[derive(FromArgs, Default)] +#[argh(subcommand, name = "test")] +pub(crate) struct TestCommand {} + +impl Prepare for TestCommand { + fn prepare<'a>(&self, sh: &'a xshell::Shell, flags: Flag) -> Vec> { + let no_fail_fast = flags + .contains(Flag::KEEP_GOING) + .then_some("--no-fail-fast") + .unwrap_or_default(); + + vec![PreparedCommand::new::( + cmd!( + sh, + "cargo test --workspace --lib --bins --tests --benches {no_fail_fast}" + ), + "Please fix failing tests in output above.", + )] + } +} diff --git a/tools/ci/src/main.rs b/tools/ci/src/main.rs index e78215f3c1..2c3798d659 100644 --- a/tools/ci/src/main.rs +++ b/tools/ci/src/main.rs @@ -1,394 +1,7 @@ //! CI script used for Bevy. -use bitflags::bitflags; -use std::collections::BTreeMap; -use xshell::{cmd, Cmd, Shell}; - -bitflags! { - /// A collection of individual checks that can be run in CI. - #[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord)] - struct Check: u32 { - const FORMAT = 1 << 0; - const CLIPPY = 1 << 1; - const COMPILE_FAIL = 1 << 2; - const TEST = 1 << 3; - const DOC_TEST = 1 << 4; - const DOC_CHECK = 1 << 5; - const BENCH_CHECK = 1 << 6; - const EXAMPLE_CHECK = 1 << 7; - const COMPILE_CHECK = 1 << 8; - const CFG_CHECK = 1 << 9; - const TEST_CHECK = 1 << 10; - } - - /// A collection of flags that can modify how checks are run. - #[derive(Clone, Copy, Debug, PartialEq, Eq)] - struct Flag: u32 { - /// Forces certain checks to continue running even if they hit an error. - const KEEP_GOING = 1 << 0; - } -} - -// None of the CI tests require any information at runtime other than the options that have been set, -// which is why all of these are 'static; we could easily update this to use more flexible types. -struct CITest<'a> { - /// The command to execute - command: Cmd<'a>, - - /// The message to display if the test command fails - failure_message: &'static str, - - /// The subdirectory path to run the test command within - subdir: Option<&'static str>, - - /// Environment variables that need to be set before the test runs - env_vars: Vec<(&'static str, &'static str)>, -} +mod commands; fn main() { - // When run locally, results may differ from actual CI runs triggered by - // .github/workflows/ci.yml - // - Official CI runs latest stable - // - Local runs use whatever the default Rust is locally - - let arguments = [ - ("lints", Check::FORMAT | Check::CLIPPY), - ("test", Check::TEST), - ("doc", Check::DOC_TEST | Check::DOC_CHECK), - ( - "compile", - Check::COMPILE_FAIL - | Check::BENCH_CHECK - | Check::EXAMPLE_CHECK - | Check::COMPILE_CHECK - | Check::TEST_CHECK, - ), - ("format", Check::FORMAT), - ("clippy", Check::CLIPPY), - ("compile-fail", Check::COMPILE_FAIL), - ("bench-check", Check::BENCH_CHECK), - ("example-check", Check::EXAMPLE_CHECK), - ("test-check", Check::TEST_CHECK), - ("cfg-check", Check::CFG_CHECK), - ("doc-check", Check::DOC_CHECK), - ("doc-test", Check::DOC_TEST), - ]; - - let flag_arguments = [("--keep-going", Flag::KEEP_GOING)]; - - // Parameters are parsed disregarding their order. Note that the first arg is generally the name of - // the executable, so it is ignored. Any parameter may either be a flag or the name of a battery of tests - // to include. - let (mut checks, mut flags) = (Check::empty(), Flag::empty()); - - // Skip first argument, which is usually the path of the executable. - for arg in std::env::args().skip(1) { - if let Some((_, check)) = arguments.iter().find(|(check_arg, _)| *check_arg == arg) { - // Note that this actually adds all of the constituent checks to the test suite. - checks.insert(*check); - } else if let Some((_, flag)) = flag_arguments.iter().find(|(flag_arg, _)| *flag_arg == arg) - { - flags.insert(*flag); - } else { - // We encountered an invalid parameter: - eprintln!( - "Invalid argument: {arg:?}.\n\ - Valid parameters: {}.", - // Skip first argument so it can be added in fold(), when displaying as a comma separated list. - arguments[1..] - .iter() - .map(|(s, _)| s) - .chain(flag_arguments.iter().map(|(s, _)| s)) - .fold(arguments[0].0.to_owned(), |c, v| c + ", " + v) - ); - - return; - } - } - - // If no checks are specified, we run every check - if checks.is_empty() { - checks = Check::all(); - } - - let sh = Shell::new().unwrap(); - - // Each check contains a 'battery' (vector) that can include more than one command, but almost all of them - // just contain a single command. - let mut test_suite: BTreeMap> = BTreeMap::new(); - - if checks.contains(Check::FORMAT) { - // See if any code needs to be formatted - test_suite.insert( - Check::FORMAT, - vec![CITest { - command: cmd!(sh, "cargo fmt --all -- --check"), - failure_message: "Please run 'cargo fmt --all' to format your code.", - subdir: None, - env_vars: vec![], - }], - ); - } - - if checks.contains(Check::CLIPPY) { - // See if clippy has any complaints. - // - Type complexity must be ignored because we use huge templates for queries - test_suite.insert( - Check::CLIPPY, - vec![CITest { - command: cmd!( - sh, - "cargo clippy --workspace --all-targets --all-features -- -Dwarnings" - ), - failure_message: "Please fix clippy errors in output above.", - subdir: None, - env_vars: vec![], - }], - ); - } - - if checks.contains(Check::COMPILE_FAIL) { - let mut args = vec!["--target-dir", "../../target"]; - if flags.contains(Flag::KEEP_GOING) { - args.push("--no-fail-fast"); - } - - // ECS Compile Fail Tests - // Run UI tests (they do not get executed with the workspace tests) - // - See crates/bevy_ecs_compile_fail_tests/README.md - - // (These must be cloned because of move semantics in `cmd!`) - let args_clone = args.clone(); - - test_suite.insert(Check::COMPILE_FAIL, vec![CITest { - command: cmd!(sh, "cargo test {args_clone...}"), - failure_message: "Compiler errors of the ECS compile fail tests seem to be different than expected! Check locally and compare rust versions.", - subdir: Some("crates/bevy_ecs_compile_fail_tests"), - env_vars: vec![], - }]); - - // Reflect Compile Fail Tests - // Run tests (they do not get executed with the workspace tests) - // - See crates/bevy_reflect_compile_fail_tests/README.md - let args_clone = args.clone(); - - test_suite.entry(Check::COMPILE_FAIL).and_modify(|tests| tests.push( CITest { - command: cmd!(sh, "cargo test {args_clone...}"), - failure_message: "Compiler errors of the Reflect compile fail tests seem to be different than expected! Check locally and compare rust versions.", - subdir: Some("crates/bevy_reflect_compile_fail_tests"), - env_vars: vec![], - })); - - // Macro Compile Fail Tests - // Run tests (they do not get executed with the workspace tests) - // - See crates/bevy_macros_compile_fail_tests/README.md - let args_clone = args.clone(); - - test_suite.entry(Check::COMPILE_FAIL).and_modify(|tests| tests.push( CITest { - command: cmd!(sh, "cargo test {args_clone...}"), - failure_message: "Compiler errors of the macros compile fail tests seem to be different than expected! Check locally and compare rust versions.", - subdir: Some("crates/bevy_macros_compile_fail_tests"), - env_vars: vec![], - })); - } - - if checks.contains(Check::TEST) { - // Run tests (except doc tests and without building examples) - let mut args = vec!["--workspace", "--lib", "--bins", "--tests", "--benches"]; - if flags.contains(Flag::KEEP_GOING) { - args.push("--no-fail-fast"); - } - - test_suite.insert( - Check::TEST, - vec![CITest { - command: cmd!(sh, "cargo test {args...}"), - failure_message: "Please fix failing tests in output above.", - subdir: None, - env_vars: vec![], - }], - ); - } - - if checks.contains(Check::DOC_TEST) { - // Run doc tests - let mut args = vec!["--workspace", "--doc"]; - if flags.contains(Flag::KEEP_GOING) { - args.push("--no-fail-fast"); - } - - test_suite.insert( - Check::DOC_TEST, - vec![CITest { - command: cmd!(sh, "cargo test {args...}"), - failure_message: "Please fix failing doc-tests in output above.", - subdir: None, - env_vars: vec![], - }], - ); - } - - if checks.contains(Check::DOC_CHECK) { - // Check that building docs work and does not emit warnings - let mut args = vec![ - "--workspace", - "--all-features", - "--no-deps", - "--document-private-items", - ]; - if flags.contains(Flag::KEEP_GOING) { - args.push("--keep-going"); - } - - test_suite.insert( - Check::DOC_CHECK, - vec![CITest { - command: cmd!(sh, "cargo doc {args...}"), - failure_message: "Please fix doc warnings in output above.", - subdir: None, - env_vars: vec![("RUSTDOCFLAGS", "-D warnings")], - }], - ); - } - - if checks.contains(Check::BENCH_CHECK) { - // Check that benches are building - let mut args = vec!["--benches", "--target-dir", "../target"]; - if flags.contains(Flag::KEEP_GOING) { - args.push("--keep-going"); - } - - test_suite.insert( - Check::BENCH_CHECK, - vec![CITest { - command: cmd!(sh, "cargo check {args...}"), - failure_message: "Failed to check the benches.", - subdir: Some("benches"), - env_vars: vec![], - }], - ); - } - - if checks.contains(Check::EXAMPLE_CHECK) { - // Build examples and check they compile - let mut args = vec!["--workspace", "--examples"]; - if flags.contains(Flag::KEEP_GOING) { - args.push("--keep-going"); - } - - test_suite.insert( - Check::EXAMPLE_CHECK, - vec![CITest { - command: cmd!(sh, "cargo check {args...}"), - failure_message: "Please fix compiler errors for examples in output above.", - subdir: None, - env_vars: vec![], - }], - ); - } - - if checks.contains(Check::COMPILE_CHECK) { - // Build bevy and check that it compiles - let mut args = vec!["--workspace"]; - if flags.contains(Flag::KEEP_GOING) { - args.push("--keep-going"); - } - - test_suite.insert( - Check::COMPILE_CHECK, - vec![CITest { - command: cmd!(sh, "cargo check {args...}"), - failure_message: "Please fix compiler errors in output above.", - subdir: None, - env_vars: vec![], - }], - ); - } - - if checks.contains(Check::CFG_CHECK) { - // Check cfg and imports - let mut args = vec!["-Zcheck-cfg", "--workspace"]; - if flags.contains(Flag::KEEP_GOING) { - args.push("--keep-going"); - } - - test_suite.insert( - Check::CFG_CHECK, - vec![CITest { - command: cmd!(sh, "cargo +nightly check {args...}"), - failure_message: "Please fix failing cfg checks in output above.", - subdir: None, - env_vars: vec![("RUSTFLAGS", "-D warnings")], - }], - ); - } - - if checks.contains(Check::TEST_CHECK) { - let mut args = vec!["--workspace", "--tests"]; - - if flags.contains(Flag::KEEP_GOING) { - args.push("--keep-going"); - } - - test_suite.insert( - Check::TEST_CHECK, - vec![CITest { - command: cmd!(sh, "cargo check {args...}"), - failure_message: "Please fix compiler examples for tests in output above.", - subdir: None, - env_vars: Vec::new(), - }], - ); - } - - // Actually run the tests: - - let mut failed_checks: Check = Check::empty(); - let mut failure_message: String = String::new(); - - // In KEEP_GOING-mode, we save all errors until the end; otherwise, we just - // panic with the given message for test failure. - fn fail( - current_check: Check, - failure_message: &'static str, - failed_checks: &mut Check, - existing_fail_message: &mut String, - flags: &Flag, - ) { - if flags.contains(Flag::KEEP_GOING) { - failed_checks.insert(current_check); - if !existing_fail_message.is_empty() { - existing_fail_message.push('\n'); - } - existing_fail_message.push_str(failure_message); - } else { - panic!("{failure_message}"); - } - } - - for (check, battery) in test_suite { - for ci_test in battery { - // If the CI test is to be executed in a subdirectory, we move there before running the command - let _subdir_hook = ci_test.subdir.map(|path| sh.push_dir(path)); - - // Actually run the test, setting environment variables temporarily - if ci_test.command.envs(ci_test.env_vars).run().is_err() { - fail( - check, - ci_test.failure_message, - &mut failed_checks, - &mut failure_message, - &flags, - ); - } - // ^ This must run while `_subdir_hook` is in scope; it is dropped on loop iteration. - } - } - - if !failed_checks.is_empty() { - panic!( - "One or more CI checks failed.\n\ - {failure_message}" - ); - } + argh::from_env::().run(); }