mirror of
https://github.com/nushell/nushell
synced 2024-12-26 13:03:07 +00:00
Tango migration (#12469)
# Description This PR migrates the benchmark suit to Tango. Its different compared to other framework because it require 2 binaries, to run to do A/B benchmarking, this is currently limited to Linux, Max, (Windows require rustc nightly flag), by switching between two suits it can reduce noise and run the code "almost" concurrently. I have have been in contact with the maintainer, and bases this on the dev branch, as it had a newer API simular to criterion. This framework compared to Divan also have a simple file dump system if we want to generate graphs, do other analysis on later. I think overall this crate is very nice, a lot faster to compile and run then criterion, that's for sure.
This commit is contained in:
Filip Andersson
GitHub
parent
ce3bc470ba
commit
3143ded374
5 changed files with 920 additions and 675 deletions
769
Cargo.lock
generated
769
Cargo.lock
generated
File diff suppressed because it is too large
Load diff
|
|
@ -227,7 +227,7 @@ nu-plugin-protocol = { path = "./crates/nu-plugin-protocol", version = "0.93.1"
|
||||||
nu-plugin-core = { path = "./crates/nu-plugin-core", version = "0.93.1" }
|
nu-plugin-core = { path = "./crates/nu-plugin-core", version = "0.93.1" }
|
||||||
assert_cmd = "2.0"
|
assert_cmd = "2.0"
|
||||||
dirs-next = { workspace = true }
|
dirs-next = { workspace = true }
|
||||||
divan = "0.1.14"
|
tango-bench = "0.5"
|
||||||
pretty_assertions = { workspace = true }
|
pretty_assertions = { workspace = true }
|
||||||
rstest = { workspace = true, default-features = false }
|
rstest = { workspace = true, default-features = false }
|
||||||
serial_test = "3.1"
|
serial_test = "3.1"
|
||||||
|
|
|
||||||
|
|
@ -1,7 +1,7 @@
|
||||||
use nu_cli::{eval_source, evaluate_commands};
|
use nu_cli::{eval_source, evaluate_commands};
|
||||||
use nu_parser::parse;
|
|
||||||
use nu_plugin_core::{Encoder, EncodingType};
|
use nu_plugin_core::{Encoder, EncodingType};
|
||||||
use nu_plugin_protocol::{PluginCallResponse, PluginOutput};
|
use nu_plugin_protocol::{PluginCallResponse, PluginOutput};
|
||||||
|
|
||||||
use nu_protocol::{
|
use nu_protocol::{
|
||||||
engine::{EngineState, Stack},
|
engine::{EngineState, Stack},
|
||||||
eval_const::create_nu_constant,
|
eval_const::create_nu_constant,
|
||||||
|
|
@ -9,12 +9,14 @@ use nu_protocol::{
|
||||||
};
|
};
|
||||||
use nu_std::load_standard_library;
|
use nu_std::load_standard_library;
|
||||||
use nu_utils::{get_default_config, get_default_env};
|
use nu_utils::{get_default_config, get_default_env};
|
||||||
use std::path::{Path, PathBuf};
|
use std::{
|
||||||
|
path::{Path, PathBuf},
|
||||||
|
rc::Rc,
|
||||||
|
};
|
||||||
|
|
||||||
fn main() {
|
use std::hint::black_box;
|
||||||
// Run registered benchmarks.
|
|
||||||
divan::main();
|
use tango_bench::{benchmark_fn, tango_benchmarks, tango_main, IntoBenchmarks};
|
||||||
}
|
|
||||||
|
|
||||||
fn load_bench_commands() -> EngineState {
|
fn load_bench_commands() -> EngineState {
|
||||||
nu_command::add_shell_command_context(nu_cmd_lang::create_default_context())
|
nu_command::add_shell_command_context(nu_cmd_lang::create_default_context())
|
||||||
|
|
@ -57,42 +59,6 @@ fn setup_engine() -> EngineState {
|
||||||
engine_state
|
engine_state
|
||||||
}
|
}
|
||||||
|
|
||||||
fn bench_command(bencher: divan::Bencher, scaled_command: String) {
|
|
||||||
bench_command_with_custom_stack_and_engine(
|
|
||||||
bencher,
|
|
||||||
scaled_command,
|
|
||||||
Stack::new(),
|
|
||||||
setup_engine(),
|
|
||||||
)
|
|
||||||
}
|
|
||||||
|
|
||||||
fn bench_command_with_custom_stack_and_engine(
|
|
||||||
bencher: divan::Bencher,
|
|
||||||
scaled_command: String,
|
|
||||||
stack: nu_protocol::engine::Stack,
|
|
||||||
mut engine: EngineState,
|
|
||||||
) {
|
|
||||||
load_standard_library(&mut engine).unwrap();
|
|
||||||
let commands = Spanned {
|
|
||||||
span: Span::unknown(),
|
|
||||||
item: scaled_command,
|
|
||||||
};
|
|
||||||
|
|
||||||
bencher
|
|
||||||
.with_inputs(|| engine.clone())
|
|
||||||
.bench_values(|mut engine| {
|
|
||||||
evaluate_commands(
|
|
||||||
&commands,
|
|
||||||
&mut engine,
|
|
||||||
&mut stack.clone(),
|
|
||||||
PipelineData::empty(),
|
|
||||||
None,
|
|
||||||
false,
|
|
||||||
)
|
|
||||||
.unwrap();
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
fn setup_stack_and_engine_from_command(command: &str) -> (Stack, EngineState) {
|
fn setup_stack_and_engine_from_command(command: &str) -> (Stack, EngineState) {
|
||||||
let mut engine = setup_engine();
|
let mut engine = setup_engine();
|
||||||
let commands = Spanned {
|
let commands = Spanned {
|
||||||
|
|
@ -114,261 +80,6 @@ fn setup_stack_and_engine_from_command(command: &str) -> (Stack, EngineState) {
|
||||||
(stack, engine)
|
(stack, engine)
|
||||||
}
|
}
|
||||||
|
|
||||||
// FIXME: All benchmarks live in this 1 file to speed up build times when benchmarking.
|
|
||||||
// When the *_benchmarks functions were in different files, `cargo bench` would build
|
|
||||||
// an executable for every single one - incredibly slowly. Would be nice to figure out
|
|
||||||
// a way to split things up again.
|
|
||||||
|
|
||||||
#[divan::bench]
|
|
||||||
fn load_standard_lib(bencher: divan::Bencher) {
|
|
||||||
let engine = setup_engine();
|
|
||||||
bencher
|
|
||||||
.with_inputs(|| engine.clone())
|
|
||||||
.bench_values(|mut engine| {
|
|
||||||
load_standard_library(&mut engine).unwrap();
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
#[divan::bench_group]
|
|
||||||
mod record {
|
|
||||||
|
|
||||||
use super::*;
|
|
||||||
|
|
||||||
fn create_flat_record_string(n: i32) -> String {
|
|
||||||
let mut s = String::from("let record = {");
|
|
||||||
for i in 0..n {
|
|
||||||
s.push_str(&format!("col_{}: {}", i, i));
|
|
||||||
if i < n - 1 {
|
|
||||||
s.push_str(", ");
|
|
||||||
}
|
|
||||||
}
|
|
||||||
s.push('}');
|
|
||||||
s
|
|
||||||
}
|
|
||||||
|
|
||||||
fn create_nested_record_string(depth: i32) -> String {
|
|
||||||
let mut s = String::from("let record = {");
|
|
||||||
for _ in 0..depth {
|
|
||||||
s.push_str("col: {");
|
|
||||||
}
|
|
||||||
s.push_str("col_final: 0");
|
|
||||||
for _ in 0..depth {
|
|
||||||
s.push('}');
|
|
||||||
}
|
|
||||||
s.push('}');
|
|
||||||
s
|
|
||||||
}
|
|
||||||
|
|
||||||
#[divan::bench(args = [1, 10, 100, 1000])]
|
|
||||||
fn create(bencher: divan::Bencher, n: i32) {
|
|
||||||
bench_command(bencher, create_flat_record_string(n));
|
|
||||||
}
|
|
||||||
|
|
||||||
#[divan::bench(args = [1, 10, 100, 1000])]
|
|
||||||
fn flat_access(bencher: divan::Bencher, n: i32) {
|
|
||||||
let (stack, engine) = setup_stack_and_engine_from_command(&create_flat_record_string(n));
|
|
||||||
bench_command_with_custom_stack_and_engine(
|
|
||||||
bencher,
|
|
||||||
"$record.col_0 | ignore".to_string(),
|
|
||||||
stack,
|
|
||||||
engine,
|
|
||||||
);
|
|
||||||
}
|
|
||||||
|
|
||||||
#[divan::bench(args = [1, 2, 4, 8, 16, 32, 64, 128])]
|
|
||||||
fn nest_access(bencher: divan::Bencher, depth: i32) {
|
|
||||||
let (stack, engine) =
|
|
||||||
setup_stack_and_engine_from_command(&create_nested_record_string(depth));
|
|
||||||
let nested_access = ".col".repeat(depth as usize);
|
|
||||||
bench_command_with_custom_stack_and_engine(
|
|
||||||
bencher,
|
|
||||||
format!("$record{} | ignore", nested_access),
|
|
||||||
stack,
|
|
||||||
engine,
|
|
||||||
);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
#[divan::bench_group]
|
|
||||||
mod table {
|
|
||||||
|
|
||||||
use super::*;
|
|
||||||
|
|
||||||
fn create_example_table_nrows(n: i32) -> String {
|
|
||||||
let mut s = String::from("let table = [[foo bar baz]; ");
|
|
||||||
for i in 0..n {
|
|
||||||
s.push_str(&format!("[0, 1, {i}]"));
|
|
||||||
if i < n - 1 {
|
|
||||||
s.push_str(", ");
|
|
||||||
}
|
|
||||||
}
|
|
||||||
s.push(']');
|
|
||||||
s
|
|
||||||
}
|
|
||||||
|
|
||||||
#[divan::bench(args = [1, 10, 100, 1000])]
|
|
||||||
fn create(bencher: divan::Bencher, n: i32) {
|
|
||||||
bench_command(bencher, create_example_table_nrows(n));
|
|
||||||
}
|
|
||||||
|
|
||||||
#[divan::bench(args = [1, 10, 100, 1000])]
|
|
||||||
fn get(bencher: divan::Bencher, n: i32) {
|
|
||||||
let (stack, engine) = setup_stack_and_engine_from_command(&create_example_table_nrows(n));
|
|
||||||
bench_command_with_custom_stack_and_engine(
|
|
||||||
bencher,
|
|
||||||
"$table | get bar | math sum | ignore".to_string(),
|
|
||||||
stack,
|
|
||||||
engine,
|
|
||||||
);
|
|
||||||
}
|
|
||||||
|
|
||||||
#[divan::bench(args = [1, 10, 100, 1000])]
|
|
||||||
fn select(bencher: divan::Bencher, n: i32) {
|
|
||||||
let (stack, engine) = setup_stack_and_engine_from_command(&create_example_table_nrows(n));
|
|
||||||
bench_command_with_custom_stack_and_engine(
|
|
||||||
bencher,
|
|
||||||
"$table | select foo baz | ignore".to_string(),
|
|
||||||
stack,
|
|
||||||
engine,
|
|
||||||
);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
#[divan::bench_group]
|
|
||||||
mod eval_commands {
|
|
||||||
use super::*;
|
|
||||||
|
|
||||||
#[divan::bench(args = [100, 1_000, 10_000])]
|
|
||||||
fn interleave(bencher: divan::Bencher, n: i32) {
|
|
||||||
bench_command(
|
|
||||||
bencher,
|
|
||||||
format!("seq 1 {n} | wrap a | interleave {{ seq 1 {n} | wrap b }} | ignore"),
|
|
||||||
)
|
|
||||||
}
|
|
||||||
|
|
||||||
#[divan::bench(args = [100, 1_000, 10_000])]
|
|
||||||
fn interleave_with_ctrlc(bencher: divan::Bencher, n: i32) {
|
|
||||||
let mut engine = setup_engine();
|
|
||||||
engine.ctrlc = Some(std::sync::Arc::new(std::sync::atomic::AtomicBool::new(
|
|
||||||
false,
|
|
||||||
)));
|
|
||||||
load_standard_library(&mut engine).unwrap();
|
|
||||||
let commands = Spanned {
|
|
||||||
span: Span::unknown(),
|
|
||||||
item: format!("seq 1 {n} | wrap a | interleave {{ seq 1 {n} | wrap b }} | ignore"),
|
|
||||||
};
|
|
||||||
|
|
||||||
bencher
|
|
||||||
.with_inputs(|| engine.clone())
|
|
||||||
.bench_values(|mut engine| {
|
|
||||||
evaluate_commands(
|
|
||||||
&commands,
|
|
||||||
&mut engine,
|
|
||||||
&mut nu_protocol::engine::Stack::new(),
|
|
||||||
PipelineData::empty(),
|
|
||||||
None,
|
|
||||||
false,
|
|
||||||
)
|
|
||||||
.unwrap();
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
#[divan::bench(args = [1, 5, 10, 100, 1_000])]
|
|
||||||
fn for_range(bencher: divan::Bencher, n: i32) {
|
|
||||||
bench_command(bencher, format!("(for $x in (1..{}) {{ sleep 50ns }})", n))
|
|
||||||
}
|
|
||||||
|
|
||||||
#[divan::bench(args = [1, 5, 10, 100, 1_000])]
|
|
||||||
fn each(bencher: divan::Bencher, n: i32) {
|
|
||||||
bench_command(
|
|
||||||
bencher,
|
|
||||||
format!("(1..{}) | each {{|_| sleep 50ns }} | ignore", n),
|
|
||||||
)
|
|
||||||
}
|
|
||||||
|
|
||||||
#[divan::bench(args = [1, 5, 10, 100, 1_000])]
|
|
||||||
fn par_each_1t(bencher: divan::Bencher, n: i32) {
|
|
||||||
bench_command(
|
|
||||||
bencher,
|
|
||||||
format!("(1..{}) | par-each -t 1 {{|_| sleep 50ns }} | ignore", n),
|
|
||||||
)
|
|
||||||
}
|
|
||||||
|
|
||||||
#[divan::bench(args = [1, 5, 10, 100, 1_000])]
|
|
||||||
fn par_each_2t(bencher: divan::Bencher, n: i32) {
|
|
||||||
bench_command(
|
|
||||||
bencher,
|
|
||||||
format!("(1..{}) | par-each -t 2 {{|_| sleep 50ns }} | ignore", n),
|
|
||||||
)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
#[divan::bench_group()]
|
|
||||||
mod parser_benchmarks {
|
|
||||||
use super::*;
|
|
||||||
|
|
||||||
#[divan::bench()]
|
|
||||||
fn parse_default_config_file(bencher: divan::Bencher) {
|
|
||||||
let engine_state = setup_engine();
|
|
||||||
let default_env = get_default_config().as_bytes();
|
|
||||||
|
|
||||||
bencher
|
|
||||||
.with_inputs(|| nu_protocol::engine::StateWorkingSet::new(&engine_state))
|
|
||||||
.bench_refs(|working_set| parse(working_set, None, default_env, false))
|
|
||||||
}
|
|
||||||
|
|
||||||
#[divan::bench()]
|
|
||||||
fn parse_default_env_file(bencher: divan::Bencher) {
|
|
||||||
let engine_state = setup_engine();
|
|
||||||
let default_env = get_default_env().as_bytes();
|
|
||||||
|
|
||||||
bencher
|
|
||||||
.with_inputs(|| nu_protocol::engine::StateWorkingSet::new(&engine_state))
|
|
||||||
.bench_refs(|working_set| parse(working_set, None, default_env, false))
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
#[divan::bench_group()]
|
|
||||||
mod eval_benchmarks {
|
|
||||||
use super::*;
|
|
||||||
|
|
||||||
#[divan::bench()]
|
|
||||||
fn eval_default_env(bencher: divan::Bencher) {
|
|
||||||
let default_env = get_default_env().as_bytes();
|
|
||||||
let fname = "default_env.nu";
|
|
||||||
bencher
|
|
||||||
.with_inputs(|| (setup_engine(), nu_protocol::engine::Stack::new()))
|
|
||||||
.bench_values(|(mut engine_state, mut stack)| {
|
|
||||||
eval_source(
|
|
||||||
&mut engine_state,
|
|
||||||
&mut stack,
|
|
||||||
default_env,
|
|
||||||
fname,
|
|
||||||
PipelineData::empty(),
|
|
||||||
false,
|
|
||||||
)
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
#[divan::bench()]
|
|
||||||
fn eval_default_config(bencher: divan::Bencher) {
|
|
||||||
let default_env = get_default_config().as_bytes();
|
|
||||||
let fname = "default_config.nu";
|
|
||||||
bencher
|
|
||||||
.with_inputs(|| (setup_engine(), nu_protocol::engine::Stack::new()))
|
|
||||||
.bench_values(|(mut engine_state, mut stack)| {
|
|
||||||
eval_source(
|
|
||||||
&mut engine_state,
|
|
||||||
&mut stack,
|
|
||||||
default_env,
|
|
||||||
fname,
|
|
||||||
PipelineData::empty(),
|
|
||||||
false,
|
|
||||||
)
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// generate a new table data with `row_cnt` rows, `col_cnt` columns.
|
// generate a new table data with `row_cnt` rows, `col_cnt` columns.
|
||||||
fn encoding_test_data(row_cnt: usize, col_cnt: usize) -> Value {
|
fn encoding_test_data(row_cnt: usize, col_cnt: usize) -> Value {
|
||||||
let record = Value::test_record(
|
let record = Value::test_record(
|
||||||
|
|
@ -380,76 +91,423 @@ fn encoding_test_data(row_cnt: usize, col_cnt: usize) -> Value {
|
||||||
Value::list(vec![record; row_cnt], Span::test_data())
|
Value::list(vec![record; row_cnt], Span::test_data())
|
||||||
}
|
}
|
||||||
|
|
||||||
#[divan::bench_group()]
|
fn bench_command(
|
||||||
mod encoding_benchmarks {
|
name: &str,
|
||||||
use super::*;
|
command: &str,
|
||||||
|
stack: Stack,
|
||||||
#[divan::bench(args = [(100, 5), (10000, 15)])]
|
engine: EngineState,
|
||||||
fn json_encode(bencher: divan::Bencher, (row_cnt, col_cnt): (usize, usize)) {
|
) -> impl IntoBenchmarks {
|
||||||
let test_data = PluginOutput::CallResponse(
|
let commands = Spanned {
|
||||||
0,
|
span: Span::unknown(),
|
||||||
PluginCallResponse::value(encoding_test_data(row_cnt, col_cnt)),
|
item: command.to_string(),
|
||||||
);
|
};
|
||||||
let encoder = EncodingType::try_from_bytes(b"json").unwrap();
|
[benchmark_fn(name, move |b| {
|
||||||
bencher
|
let commands = commands.clone();
|
||||||
.with_inputs(Vec::new)
|
let stack = stack.clone();
|
||||||
.bench_values(|mut res| encoder.encode(&test_data, &mut res))
|
let engine = engine.clone();
|
||||||
}
|
b.iter(move || {
|
||||||
|
let mut stack = stack.clone();
|
||||||
#[divan::bench(args = [(100, 5), (10000, 15)])]
|
let mut engine = engine.clone();
|
||||||
fn msgpack_encode(bencher: divan::Bencher, (row_cnt, col_cnt): (usize, usize)) {
|
black_box(
|
||||||
let test_data = PluginOutput::CallResponse(
|
evaluate_commands(
|
||||||
0,
|
&commands,
|
||||||
PluginCallResponse::value(encoding_test_data(row_cnt, col_cnt)),
|
&mut engine,
|
||||||
);
|
&mut stack,
|
||||||
let encoder = EncodingType::try_from_bytes(b"msgpack").unwrap();
|
PipelineData::empty(),
|
||||||
bencher
|
None,
|
||||||
.with_inputs(Vec::new)
|
false,
|
||||||
.bench_values(|mut res| encoder.encode(&test_data, &mut res))
|
)
|
||||||
}
|
.unwrap(),
|
||||||
|
);
|
||||||
|
})
|
||||||
|
})]
|
||||||
}
|
}
|
||||||
|
|
||||||
#[divan::bench_group()]
|
fn bench_eval_source(
|
||||||
mod decoding_benchmarks {
|
name: &str,
|
||||||
use super::*;
|
fname: String,
|
||||||
|
source: Vec<u8>,
|
||||||
#[divan::bench(args = [(100, 5), (10000, 15)])]
|
stack: Stack,
|
||||||
fn json_decode(bencher: divan::Bencher, (row_cnt, col_cnt): (usize, usize)) {
|
engine: EngineState,
|
||||||
let test_data = PluginOutput::CallResponse(
|
) -> impl IntoBenchmarks {
|
||||||
0,
|
[benchmark_fn(name, move |b| {
|
||||||
PluginCallResponse::value(encoding_test_data(row_cnt, col_cnt)),
|
let stack = stack.clone();
|
||||||
);
|
let engine = engine.clone();
|
||||||
let encoder = EncodingType::try_from_bytes(b"json").unwrap();
|
let fname = fname.clone();
|
||||||
let mut res = vec![];
|
let source = source.clone();
|
||||||
encoder.encode(&test_data, &mut res).unwrap();
|
b.iter(move || {
|
||||||
bencher
|
let mut stack = stack.clone();
|
||||||
.with_inputs(|| {
|
let mut engine = engine.clone();
|
||||||
let mut binary_data = std::io::Cursor::new(res.clone());
|
let fname: &str = &fname.clone();
|
||||||
binary_data.set_position(0);
|
let source: &[u8] = &source.clone();
|
||||||
binary_data
|
black_box(eval_source(
|
||||||
})
|
&mut engine,
|
||||||
.bench_values(|mut binary_data| -> Result<Option<PluginOutput>, _> {
|
&mut stack,
|
||||||
encoder.decode(&mut binary_data)
|
source,
|
||||||
})
|
fname,
|
||||||
}
|
PipelineData::empty(),
|
||||||
|
false,
|
||||||
#[divan::bench(args = [(100, 5), (10000, 15)])]
|
));
|
||||||
fn msgpack_decode(bencher: divan::Bencher, (row_cnt, col_cnt): (usize, usize)) {
|
})
|
||||||
let test_data = PluginOutput::CallResponse(
|
})]
|
||||||
0,
|
|
||||||
PluginCallResponse::value(encoding_test_data(row_cnt, col_cnt)),
|
|
||||||
);
|
|
||||||
let encoder = EncodingType::try_from_bytes(b"msgpack").unwrap();
|
|
||||||
let mut res = vec![];
|
|
||||||
encoder.encode(&test_data, &mut res).unwrap();
|
|
||||||
bencher
|
|
||||||
.with_inputs(|| {
|
|
||||||
let mut binary_data = std::io::Cursor::new(res.clone());
|
|
||||||
binary_data.set_position(0);
|
|
||||||
binary_data
|
|
||||||
})
|
|
||||||
.bench_values(|mut binary_data| -> Result<Option<PluginOutput>, _> {
|
|
||||||
encoder.decode(&mut binary_data)
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Load the standard library into the engine.
|
||||||
|
fn bench_load_standard_lib() -> impl IntoBenchmarks {
|
||||||
|
[benchmark_fn("load_standard_lib", move |b| {
|
||||||
|
let engine = setup_engine();
|
||||||
|
b.iter(move || {
|
||||||
|
let mut engine = engine.clone();
|
||||||
|
load_standard_library(&mut engine)
|
||||||
|
})
|
||||||
|
})]
|
||||||
|
}
|
||||||
|
|
||||||
|
fn create_flat_record_string(n: i32) -> String {
|
||||||
|
let mut s = String::from("let record = {");
|
||||||
|
for i in 0..n {
|
||||||
|
s.push_str(&format!("col_{}: {}", i, i));
|
||||||
|
if i < n - 1 {
|
||||||
|
s.push_str(", ");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
s.push('}');
|
||||||
|
s
|
||||||
|
}
|
||||||
|
|
||||||
|
fn create_nested_record_string(depth: i32) -> String {
|
||||||
|
let mut s = String::from("let record = {");
|
||||||
|
for _ in 0..depth {
|
||||||
|
s.push_str("col: {");
|
||||||
|
}
|
||||||
|
s.push_str("col_final: 0");
|
||||||
|
for _ in 0..depth {
|
||||||
|
s.push('}');
|
||||||
|
}
|
||||||
|
s.push('}');
|
||||||
|
s
|
||||||
|
}
|
||||||
|
|
||||||
|
fn create_example_table_nrows(n: i32) -> String {
|
||||||
|
let mut s = String::from("let table = [[foo bar baz]; ");
|
||||||
|
for i in 0..n {
|
||||||
|
s.push_str(&format!("[0, 1, {i}]"));
|
||||||
|
if i < n - 1 {
|
||||||
|
s.push_str(", ");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
s.push(']');
|
||||||
|
s
|
||||||
|
}
|
||||||
|
|
||||||
|
fn bench_record_create(n: i32) -> impl IntoBenchmarks {
|
||||||
|
bench_command(
|
||||||
|
&format!("record_create_{n}"),
|
||||||
|
&create_flat_record_string(n),
|
||||||
|
Stack::new(),
|
||||||
|
setup_engine(),
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
fn bench_record_flat_access(n: i32) -> impl IntoBenchmarks {
|
||||||
|
let setup_command = create_flat_record_string(n);
|
||||||
|
let (stack, engine) = setup_stack_and_engine_from_command(&setup_command);
|
||||||
|
bench_command(
|
||||||
|
&format!("record_flat_access_{n}"),
|
||||||
|
"$record.col_0 | ignore",
|
||||||
|
stack,
|
||||||
|
engine,
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
fn bench_record_nested_access(n: i32) -> impl IntoBenchmarks {
|
||||||
|
let setup_command = create_nested_record_string(n);
|
||||||
|
let (stack, engine) = setup_stack_and_engine_from_command(&setup_command);
|
||||||
|
let nested_access = ".col".repeat(n as usize);
|
||||||
|
bench_command(
|
||||||
|
&format!("record_nested_access_{n}"),
|
||||||
|
&format!("$record{} | ignore", nested_access),
|
||||||
|
stack,
|
||||||
|
engine,
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
fn bench_table_create(n: i32) -> impl IntoBenchmarks {
|
||||||
|
bench_command(
|
||||||
|
&format!("table_create_{n}"),
|
||||||
|
&create_example_table_nrows(n),
|
||||||
|
Stack::new(),
|
||||||
|
setup_engine(),
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
fn bench_table_get(n: i32) -> impl IntoBenchmarks {
|
||||||
|
let setup_command = create_example_table_nrows(n);
|
||||||
|
let (stack, engine) = setup_stack_and_engine_from_command(&setup_command);
|
||||||
|
bench_command(
|
||||||
|
&format!("table_get_{n}"),
|
||||||
|
"$table | get bar | math sum | ignore",
|
||||||
|
stack,
|
||||||
|
engine,
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
fn bench_table_select(n: i32) -> impl IntoBenchmarks {
|
||||||
|
let setup_command = create_example_table_nrows(n);
|
||||||
|
let (stack, engine) = setup_stack_and_engine_from_command(&setup_command);
|
||||||
|
bench_command(
|
||||||
|
&format!("table_select_{n}"),
|
||||||
|
"$table | select foo baz | ignore",
|
||||||
|
stack,
|
||||||
|
engine,
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
fn bench_eval_interleave(n: i32) -> impl IntoBenchmarks {
|
||||||
|
let engine = setup_engine();
|
||||||
|
let stack = Stack::new();
|
||||||
|
bench_command(
|
||||||
|
&format!("eval_interleave_{n}"),
|
||||||
|
&format!("seq 1 {n} | wrap a | interleave {{ seq 1 {n} | wrap b }} | ignore"),
|
||||||
|
stack,
|
||||||
|
engine,
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
fn bench_eval_interleave_with_ctrlc(n: i32) -> impl IntoBenchmarks {
|
||||||
|
let mut engine = setup_engine();
|
||||||
|
engine.ctrlc = Some(std::sync::Arc::new(std::sync::atomic::AtomicBool::new(
|
||||||
|
false,
|
||||||
|
)));
|
||||||
|
let stack = Stack::new();
|
||||||
|
bench_command(
|
||||||
|
&format!("eval_interleave_with_ctrlc_{n}"),
|
||||||
|
&format!("seq 1 {n} | wrap a | interleave {{ seq 1 {n} | wrap b }} | ignore"),
|
||||||
|
stack,
|
||||||
|
engine,
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
fn bench_eval_for(n: i32) -> impl IntoBenchmarks {
|
||||||
|
let engine = setup_engine();
|
||||||
|
let stack = Stack::new();
|
||||||
|
bench_command(
|
||||||
|
&format!("eval_for_{n}"),
|
||||||
|
&format!("(for $x in (1..{n}) {{ 1 }}) | ignore"),
|
||||||
|
stack,
|
||||||
|
engine,
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
fn bench_eval_each(n: i32) -> impl IntoBenchmarks {
|
||||||
|
let engine = setup_engine();
|
||||||
|
let stack = Stack::new();
|
||||||
|
bench_command(
|
||||||
|
&format!("eval_each_{n}"),
|
||||||
|
&format!("(1..{n}) | each {{|_| 1 }} | ignore"),
|
||||||
|
stack,
|
||||||
|
engine,
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
fn bench_eval_par_each(n: i32) -> impl IntoBenchmarks {
|
||||||
|
let engine = setup_engine();
|
||||||
|
let stack = Stack::new();
|
||||||
|
bench_command(
|
||||||
|
&format!("eval_par_each_{n}"),
|
||||||
|
&format!("(1..{}) | par-each -t 2 {{|_| 1 }} | ignore", n),
|
||||||
|
stack,
|
||||||
|
engine,
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
fn bench_eval_default_config() -> impl IntoBenchmarks {
|
||||||
|
let default_env = get_default_config().as_bytes().to_vec();
|
||||||
|
let fname = "default_config.nu".to_string();
|
||||||
|
bench_eval_source(
|
||||||
|
"eval_default_config",
|
||||||
|
fname,
|
||||||
|
default_env,
|
||||||
|
Stack::new(),
|
||||||
|
setup_engine(),
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
fn bench_eval_default_env() -> impl IntoBenchmarks {
|
||||||
|
let default_env = get_default_env().as_bytes().to_vec();
|
||||||
|
let fname = "default_env.nu".to_string();
|
||||||
|
bench_eval_source(
|
||||||
|
"eval_default_env",
|
||||||
|
fname,
|
||||||
|
default_env,
|
||||||
|
Stack::new(),
|
||||||
|
setup_engine(),
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
fn encode_json(row_cnt: usize, col_cnt: usize) -> impl IntoBenchmarks {
|
||||||
|
let test_data = Rc::new(PluginOutput::CallResponse(
|
||||||
|
0,
|
||||||
|
PluginCallResponse::value(encoding_test_data(row_cnt, col_cnt)),
|
||||||
|
));
|
||||||
|
let encoder = Rc::new(EncodingType::try_from_bytes(b"json").unwrap());
|
||||||
|
|
||||||
|
[benchmark_fn(
|
||||||
|
format!("encode_json_{}_{}", row_cnt, col_cnt),
|
||||||
|
move |b| {
|
||||||
|
let encoder = encoder.clone();
|
||||||
|
let test_data = test_data.clone();
|
||||||
|
b.iter(move || {
|
||||||
|
let mut res = Vec::new();
|
||||||
|
encoder.encode(&*test_data, &mut res).unwrap();
|
||||||
|
})
|
||||||
|
},
|
||||||
|
)]
|
||||||
|
}
|
||||||
|
|
||||||
|
fn encode_msgpack(row_cnt: usize, col_cnt: usize) -> impl IntoBenchmarks {
|
||||||
|
let test_data = Rc::new(PluginOutput::CallResponse(
|
||||||
|
0,
|
||||||
|
PluginCallResponse::value(encoding_test_data(row_cnt, col_cnt)),
|
||||||
|
));
|
||||||
|
let encoder = Rc::new(EncodingType::try_from_bytes(b"msgpack").unwrap());
|
||||||
|
|
||||||
|
[benchmark_fn(
|
||||||
|
format!("encode_msgpack_{}_{}", row_cnt, col_cnt),
|
||||||
|
move |b| {
|
||||||
|
let encoder = encoder.clone();
|
||||||
|
let test_data = test_data.clone();
|
||||||
|
b.iter(move || {
|
||||||
|
let mut res = Vec::new();
|
||||||
|
encoder.encode(&*test_data, &mut res).unwrap();
|
||||||
|
})
|
||||||
|
},
|
||||||
|
)]
|
||||||
|
}
|
||||||
|
|
||||||
|
fn decode_json(row_cnt: usize, col_cnt: usize) -> impl IntoBenchmarks {
|
||||||
|
let test_data = PluginOutput::CallResponse(
|
||||||
|
0,
|
||||||
|
PluginCallResponse::value(encoding_test_data(row_cnt, col_cnt)),
|
||||||
|
);
|
||||||
|
let encoder = EncodingType::try_from_bytes(b"json").unwrap();
|
||||||
|
let mut res = vec![];
|
||||||
|
encoder.encode(&test_data, &mut res).unwrap();
|
||||||
|
|
||||||
|
[benchmark_fn(
|
||||||
|
format!("decode_json_{}_{}", row_cnt, col_cnt),
|
||||||
|
move |b| {
|
||||||
|
let res = res.clone();
|
||||||
|
b.iter(move || {
|
||||||
|
let mut binary_data = std::io::Cursor::new(res.clone());
|
||||||
|
binary_data.set_position(0);
|
||||||
|
let _: Result<Option<PluginOutput>, _> =
|
||||||
|
black_box(encoder.decode(&mut binary_data));
|
||||||
|
})
|
||||||
|
},
|
||||||
|
)]
|
||||||
|
}
|
||||||
|
|
||||||
|
fn decode_msgpack(row_cnt: usize, col_cnt: usize) -> impl IntoBenchmarks {
|
||||||
|
let test_data = PluginOutput::CallResponse(
|
||||||
|
0,
|
||||||
|
PluginCallResponse::value(encoding_test_data(row_cnt, col_cnt)),
|
||||||
|
);
|
||||||
|
let encoder = EncodingType::try_from_bytes(b"msgpack").unwrap();
|
||||||
|
let mut res = vec![];
|
||||||
|
encoder.encode(&test_data, &mut res).unwrap();
|
||||||
|
|
||||||
|
[benchmark_fn(
|
||||||
|
format!("decode_msgpack_{}_{}", row_cnt, col_cnt),
|
||||||
|
move |b| {
|
||||||
|
let res = res.clone();
|
||||||
|
b.iter(move || {
|
||||||
|
let mut binary_data = std::io::Cursor::new(res.clone());
|
||||||
|
binary_data.set_position(0);
|
||||||
|
let _: Result<Option<PluginOutput>, _> =
|
||||||
|
black_box(encoder.decode(&mut binary_data));
|
||||||
|
})
|
||||||
|
},
|
||||||
|
)]
|
||||||
|
}
|
||||||
|
|
||||||
|
tango_benchmarks!(
|
||||||
|
bench_load_standard_lib(),
|
||||||
|
// Data types
|
||||||
|
// Record
|
||||||
|
bench_record_create(1),
|
||||||
|
bench_record_create(10),
|
||||||
|
bench_record_create(100),
|
||||||
|
bench_record_create(1_000),
|
||||||
|
bench_record_flat_access(1),
|
||||||
|
bench_record_flat_access(10),
|
||||||
|
bench_record_flat_access(100),
|
||||||
|
bench_record_flat_access(1_000),
|
||||||
|
bench_record_nested_access(1),
|
||||||
|
bench_record_nested_access(2),
|
||||||
|
bench_record_nested_access(4),
|
||||||
|
bench_record_nested_access(8),
|
||||||
|
bench_record_nested_access(16),
|
||||||
|
bench_record_nested_access(32),
|
||||||
|
bench_record_nested_access(64),
|
||||||
|
bench_record_nested_access(128),
|
||||||
|
// Table
|
||||||
|
bench_table_create(1),
|
||||||
|
bench_table_create(10),
|
||||||
|
bench_table_create(100),
|
||||||
|
bench_table_create(1_000),
|
||||||
|
bench_table_get(1),
|
||||||
|
bench_table_get(10),
|
||||||
|
bench_table_get(100),
|
||||||
|
bench_table_get(1_000),
|
||||||
|
bench_table_select(1),
|
||||||
|
bench_table_select(10),
|
||||||
|
bench_table_select(100),
|
||||||
|
bench_table_select(1_000),
|
||||||
|
// Eval
|
||||||
|
// Interleave
|
||||||
|
bench_eval_interleave(100),
|
||||||
|
bench_eval_interleave(1_000),
|
||||||
|
bench_eval_interleave(10_000),
|
||||||
|
bench_eval_interleave_with_ctrlc(100),
|
||||||
|
bench_eval_interleave_with_ctrlc(1_000),
|
||||||
|
bench_eval_interleave_with_ctrlc(10_000),
|
||||||
|
// For
|
||||||
|
bench_eval_for(1),
|
||||||
|
bench_eval_for(10),
|
||||||
|
bench_eval_for(100),
|
||||||
|
bench_eval_for(1_000),
|
||||||
|
bench_eval_for(10_000),
|
||||||
|
// Each
|
||||||
|
bench_eval_each(1),
|
||||||
|
bench_eval_each(10),
|
||||||
|
bench_eval_each(100),
|
||||||
|
bench_eval_each(1_000),
|
||||||
|
bench_eval_each(10_000),
|
||||||
|
// Par-Each
|
||||||
|
bench_eval_par_each(1),
|
||||||
|
bench_eval_par_each(10),
|
||||||
|
bench_eval_par_each(100),
|
||||||
|
bench_eval_par_each(1_000),
|
||||||
|
bench_eval_par_each(10_000),
|
||||||
|
// Config
|
||||||
|
bench_eval_default_config(),
|
||||||
|
// Env
|
||||||
|
bench_eval_default_env(),
|
||||||
|
// Encode
|
||||||
|
// Json
|
||||||
|
encode_json(100, 5),
|
||||||
|
encode_json(10000, 15),
|
||||||
|
// MsgPack
|
||||||
|
encode_msgpack(100, 5),
|
||||||
|
encode_msgpack(10000, 15),
|
||||||
|
// Decode
|
||||||
|
// Json
|
||||||
|
decode_json(100, 5),
|
||||||
|
decode_json(10000, 15),
|
||||||
|
// MsgPack
|
||||||
|
decode_msgpack(100, 5),
|
||||||
|
decode_msgpack(10000, 15)
|
||||||
|
);
|
||||||
|
|
||||||
|
tango_main!();
|
||||||
|
|
|
||||||
|
|
@ -10,4 +10,9 @@ fn main() {
|
||||||
}
|
}
|
||||||
|
|
||||||
#[cfg(not(windows))]
|
#[cfg(not(windows))]
|
||||||
fn main() {}
|
fn main() {
|
||||||
|
// Tango uses dynamic linking, to allow us to dynamically change between two bench suit at runtime.
|
||||||
|
// This is currently not supported on non nightly rust, on windows.
|
||||||
|
println!("cargo:rustc-link-arg-benches=-rdynamic");
|
||||||
|
println!("cargo:rerun-if-changed=build.rs");
|
||||||
|
}
|
||||||
|
|
|
||||||
25
toolkit.nu
25
toolkit.nu
|
|
@ -491,6 +491,31 @@ export def cov [] {
|
||||||
print $"Coverage generation took ($end - $start)."
|
print $"Coverage generation took ($end - $start)."
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
|
# Benchmark the current branch against the main branch
|
||||||
|
# Ensure you have `cargo-export` installed to generate separate artifacts for each branch.
|
||||||
|
export def "benchmark-current-branch-with-main" [] {
|
||||||
|
let main = "main"
|
||||||
|
let current_branch = (git branch --show-current)
|
||||||
|
|
||||||
|
cargo export $"target/($current_branch)" -- bench
|
||||||
|
git checkout $main
|
||||||
|
cargo export $"target/($main)" -- bench
|
||||||
|
git checkout $current_branch
|
||||||
|
^$"./target/($current_branch)/benchmarks" compare $"./target/($main)/benchmarks" -o -s 50
|
||||||
|
}
|
||||||
|
|
||||||
|
# Benchmark the current branch and logs the result in `./target/samples`
|
||||||
|
# Ensure you have `cargo-export` installed to generate separate artifacts for each branch.
|
||||||
|
export def "benchmark-and-log-result" [] {
|
||||||
|
let current_branch = (git branch --show-current)
|
||||||
|
let current_dir = "./" | path expand
|
||||||
|
let res_path = $"($current_dir)/target/samples"
|
||||||
|
|
||||||
|
cargo export $"target/($current_branch)" -- bench
|
||||||
|
^$"./target/($current_branch)/benchmarks" compare -o -s 50 --dump $res_path
|
||||||
|
}
|
||||||
|
|
||||||
# Build all Windows archives and MSIs for release manually
|
# Build all Windows archives and MSIs for release manually
|
||||||
#
|
#
|
||||||
# This builds std and full distributions for both aarch64 and x86_64.
|
# This builds std and full distributions for both aarch64 and x86_64.
|
||||||
|
|
|
||||||
Loading…
Reference in a new issue