From e5e44888c64022411d6593096669eb65889d7411 Mon Sep 17 00:00:00 2001 From: MiniaczQ Date: Tue, 15 Oct 2024 04:38:22 +0200 Subject: [PATCH] Validate param benchmarks (#15885) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit # Objective Benchmark overhead of validation for: - `DynSystemParam`, - `ParamSet`, - combinator systems. Needed for #15606 ## Solution As noted in objective, I've added 3 benchmarks, where each uses an excessive amount of the specific functionality. I benchmark on the level of schedules, rather than individual `validate_param` calls, so we get a better idea how changes to the code impact memory-lookup, etc. related side effects. ## Testing ``` param/combinator_system/8_piped_systems time: [1.7560 µs 1.7865 µs 1.8180 µs] change: [+4.5244% +6.7955% +9.1413%] (p = 0.00 < 0.05) Performance has regressed. Found 2 outliers among 100 measurements (2.00%) 1 (1.00%) high mild 1 (1.00%) high severe param/combinator_system/8_dyn_params_system time: [89.354 ns 89.790 ns 90.300 ns] change: [+0.6751% +1.6825% +2.6842%] (p = 0.00 < 0.05) Change within noise threshold. Found 9 outliers among 100 measurements (9.00%) 6 (6.00%) high mild 3 (3.00%) high severe param/combinator_system/8_variant_param_set_system time: [88.295 ns 89.202 ns 90.208 ns] change: [+0.1320% +1.0060% +1.8482%] (p = 0.02 < 0.05) Change within noise threshold. Found 4 outliers among 100 measurements (4.00%) 4 (4.00%) high mild ``` 2 back-to-back runs of the benchmarks, there is quire a lot of noise, can use feedback on fixing that --- benches/benches/bevy_ecs/benches.rs | 2 + .../bevy_ecs/param/combinator_system.rs | 31 ++++++++++++ benches/benches/bevy_ecs/param/dyn_param.rs | 49 +++++++++++++++++++ benches/benches/bevy_ecs/param/mod.rs | 11 +++++ benches/benches/bevy_ecs/param/param_set.rs | 36 ++++++++++++++ benches/benches/bevy_ecs/world/commands.rs | 1 - 6 files changed, 129 insertions(+), 1 deletion(-) create mode 100644 benches/benches/bevy_ecs/param/combinator_system.rs create mode 100644 benches/benches/bevy_ecs/param/dyn_param.rs create mode 100644 benches/benches/bevy_ecs/param/mod.rs create mode 100644 benches/benches/bevy_ecs/param/param_set.rs diff --git a/benches/benches/bevy_ecs/benches.rs b/benches/benches/bevy_ecs/benches.rs index 1392536a7d..af5a38c963 100644 --- a/benches/benches/bevy_ecs/benches.rs +++ b/benches/benches/bevy_ecs/benches.rs @@ -5,6 +5,7 @@ mod events; mod fragmentation; mod iteration; mod observers; +mod param; mod scheduling; mod world; @@ -16,4 +17,5 @@ criterion_main!( observers::observer_benches, scheduling::scheduling_benches, world::world_benches, + param::param_benches, ); diff --git a/benches/benches/bevy_ecs/param/combinator_system.rs b/benches/benches/bevy_ecs/param/combinator_system.rs new file mode 100644 index 0000000000..5d15c30a72 --- /dev/null +++ b/benches/benches/bevy_ecs/param/combinator_system.rs @@ -0,0 +1,31 @@ +use bevy_ecs::prelude::*; +use criterion::Criterion; + +pub fn combinator_system(criterion: &mut Criterion) { + let mut world = World::new(); + let mut group = criterion.benchmark_group("param/combinator_system"); + + group.warm_up_time(core::time::Duration::from_millis(500)); + group.measurement_time(core::time::Duration::from_secs(3)); + + let mut schedule = Schedule::default(); + schedule.add_systems( + (|| {}) + .pipe(|| {}) + .pipe(|| {}) + .pipe(|| {}) + .pipe(|| {}) + .pipe(|| {}) + .pipe(|| {}) + .pipe(|| {}), + ); + // run once to initialize systems + schedule.run(&mut world); + group.bench_function("8_piped_systems", |bencher| { + bencher.iter(|| { + schedule.run(&mut world); + }); + }); + + group.finish(); +} diff --git a/benches/benches/bevy_ecs/param/dyn_param.rs b/benches/benches/bevy_ecs/param/dyn_param.rs new file mode 100644 index 0000000000..33de52bf13 --- /dev/null +++ b/benches/benches/bevy_ecs/param/dyn_param.rs @@ -0,0 +1,49 @@ +use bevy_ecs::{ + prelude::*, + system::{DynParamBuilder, DynSystemParam, ParamBuilder}, +}; +use criterion::Criterion; + +pub fn dyn_param(criterion: &mut Criterion) { + let mut world = World::new(); + let mut group = criterion.benchmark_group("param/combinator_system"); + + group.warm_up_time(core::time::Duration::from_millis(500)); + group.measurement_time(core::time::Duration::from_secs(3)); + + #[derive(Resource)] + struct R; + + let mut schedule = Schedule::default(); + let system = ( + DynParamBuilder::new::>(ParamBuilder), + DynParamBuilder::new::>(ParamBuilder), + DynParamBuilder::new::>(ParamBuilder), + DynParamBuilder::new::>(ParamBuilder), + DynParamBuilder::new::>(ParamBuilder), + DynParamBuilder::new::>(ParamBuilder), + DynParamBuilder::new::>(ParamBuilder), + DynParamBuilder::new::>(ParamBuilder), + ) + .build_state(&mut world) + .build_system( + |_: DynSystemParam, + _: DynSystemParam, + _: DynSystemParam, + _: DynSystemParam, + _: DynSystemParam, + _: DynSystemParam, + _: DynSystemParam, + _: DynSystemParam| {}, + ); + schedule.add_systems(system); + // run once to initialize systems + schedule.run(&mut world); + group.bench_function("8_dyn_params_system", |bencher| { + bencher.iter(|| { + schedule.run(&mut world); + }); + }); + + group.finish(); +} diff --git a/benches/benches/bevy_ecs/param/mod.rs b/benches/benches/bevy_ecs/param/mod.rs new file mode 100644 index 0000000000..20955fc29e --- /dev/null +++ b/benches/benches/bevy_ecs/param/mod.rs @@ -0,0 +1,11 @@ +use criterion::criterion_group; + +mod combinator_system; +mod dyn_param; +mod param_set; + +use combinator_system::*; +use dyn_param::*; +use param_set::*; + +criterion_group!(param_benches, combinator_system, dyn_param, param_set); diff --git a/benches/benches/bevy_ecs/param/param_set.rs b/benches/benches/bevy_ecs/param/param_set.rs new file mode 100644 index 0000000000..0521561b6b --- /dev/null +++ b/benches/benches/bevy_ecs/param/param_set.rs @@ -0,0 +1,36 @@ +use bevy_ecs::prelude::*; +use criterion::Criterion; + +pub fn param_set(criterion: &mut Criterion) { + let mut world = World::new(); + let mut group = criterion.benchmark_group("param/combinator_system"); + + group.warm_up_time(core::time::Duration::from_millis(500)); + group.measurement_time(core::time::Duration::from_secs(3)); + + #[derive(Resource)] + struct R; + + let mut schedule = Schedule::default(); + schedule.add_systems( + |_: ParamSet<( + ResMut, + ResMut, + ResMut, + ResMut, + ResMut, + ResMut, + ResMut, + ResMut, + )>| {}, + ); + // run once to initialize systems + schedule.run(&mut world); + group.bench_function("8_variant_param_set_system", |bencher| { + bencher.iter(|| { + schedule.run(&mut world); + }); + }); + + group.finish(); +} diff --git a/benches/benches/bevy_ecs/world/commands.rs b/benches/benches/bevy_ecs/world/commands.rs index ea220cf1ce..a1d7cdb09e 100644 --- a/benches/benches/bevy_ecs/world/commands.rs +++ b/benches/benches/bevy_ecs/world/commands.rs @@ -1,6 +1,5 @@ use bevy_ecs::{ component::Component, - entity::Entity, system::Commands, world::{Command, CommandQueue, World}, };