diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 62ef1aa..06b925c 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -61,6 +61,11 @@ jobs: with: toolchain: ${{ matrix.rust-version }} + - name: Generate MSRV-compatible lockfile + if: matrix.rust-version == needs.resolve.outputs.MSRV + run: | + CARGO_RESOLVER_INCOMPATIBLE_RUST_VERSIONS=fallback cargo +stable update + - uses: Swatinem/rust-cache@v2 - uses: taiki-e/install-action@cargo-llvm-cov diff --git a/crates/jiter/Cargo.toml b/crates/jiter/Cargo.toml index c616968..1e953c8 100644 --- a/crates/jiter/Cargo.toml +++ b/crates/jiter/Cargo.toml @@ -27,7 +27,6 @@ python = ["dep:pyo3", "dep:pyo3-build-config"] num-bigint = ["dep:num-bigint", "pyo3?/num-bigint"] [dev-dependencies] -bencher = "0.1.5" paste = "1.0.7" serde_json = { version = "1.0.87", features = [ "preserve_order", @@ -36,7 +35,7 @@ serde_json = { version = "1.0.87", features = [ ] } serde = "1.0.147" pyo3 = { workspace = true, features = ["auto-initialize"] } -codspeed-bencher-compat = "2.7.1" +codspeed-criterion-compat = "2.7.2" [build-dependencies] pyo3-build-config = { workspace = true, optional = true } diff --git a/crates/jiter/benches/README.md b/crates/jiter/benches/README.md new file mode 100644 index 0000000..bfd6465 --- /dev/null +++ b/crates/jiter/benches/README.md @@ -0,0 +1,13 @@ +# Benchmarks + +Before running benchmarks, make sure to generate `big.json`: + +```shell +python3 ./crates/jiter/benches/generate_big.py +``` + +To run benchmarks, run: + +```shell +cargo bench -p jiter +``` diff --git a/crates/jiter/benches/main.rs b/crates/jiter/benches/main.rs index 6a5368a..85225d3 100644 --- a/crates/jiter/benches/main.rs +++ b/crates/jiter/benches/main.rs @@ -1,12 +1,20 @@ -use codspeed_bencher_compat::{benchmark_group, benchmark_main, Bencher}; -use std::hint::black_box; +use codspeed_criterion_compat::{criterion_group, criterion_main, Criterion}; use std::fs::File; +use std::hint::black_box; use std::io::Read; +use std::path::Path; use jiter::{Jiter, JsonValue, PartialMode, Peek}; use serde_json::Value; +fn read_title(path: &str) -> String { + let path = Path::new(path); + let file_stem = path.file_stem().unwrap(); + + file_stem.to_str().unwrap().to_owned() +} + fn read_file(path: &str) -> String { let mut file = File::open(path).unwrap(); let mut contents = String::new(); @@ -14,44 +22,56 @@ fn read_file(path: &str) -> String { contents } -fn jiter_value(path: &str, bench: &mut Bencher) { +fn jiter_value(path: &str, c: &mut Criterion) { + let title = read_title(path) + "_jiter_value"; let json = read_file(path); let json_data = json.as_bytes(); - bench.iter(|| { - let v = JsonValue::parse(black_box(json_data), false).unwrap(); - black_box(v) + + c.bench_function(&title, |bench| { + bench.iter(|| { + let v = JsonValue::parse(black_box(json_data), false).unwrap(); + black_box(v) + }); }); } -fn jiter_skip(path: &str, bench: &mut Bencher) { +fn jiter_skip(path: &str, c: &mut Criterion) { + let title = read_title(path) + "_jiter_skip"; let json = read_file(path); let json_data = black_box(json.as_bytes()); - bench.iter(|| { - let mut jiter = Jiter::new(json_data); - jiter.next_skip().unwrap(); + + c.bench_function(&title, |bench| { + bench.iter(|| { + let mut jiter = Jiter::new(json_data); + jiter.next_skip().unwrap(); + }); }); } -fn jiter_iter_big(path: &str, bench: &mut Bencher) { +fn jiter_iter_big(path: &str, c: &mut Criterion) { + let title = read_title(path) + "_jiter_iter"; let json = read_file(path); let json_data = black_box(json.as_bytes()); - bench.iter(|| { - let mut jiter = Jiter::new(json_data); - jiter.next_array().unwrap(); - loop { - if let Some(peek) = jiter.next_array().unwrap() { - let i = jiter.known_float(peek).unwrap(); - black_box(i); - while let Some(peek) = jiter.array_step().unwrap() { + c.bench_function(&title, |bench| { + bench.iter(|| { + let mut jiter = Jiter::new(json_data); + jiter.next_array().unwrap(); + + loop { + if let Some(peek) = jiter.next_array().unwrap() { let i = jiter.known_float(peek).unwrap(); black_box(i); + while let Some(peek) = jiter.array_step().unwrap() { + let i = jiter.known_float(peek).unwrap(); + black_box(i); + } + } + if jiter.array_step().unwrap().is_none() { + break; } } - if jiter.array_step().unwrap().is_none() { - break; - } - } + }); }); } @@ -69,165 +89,201 @@ fn find_string(jiter: &mut Jiter) -> String { } } -fn jiter_iter_pass2(path: &str, bench: &mut Bencher) { +fn jiter_iter_pass2(path: &str, c: &mut Criterion) { + let title = read_title(path) + "_jiter_iter"; let json = read_file(path); let json_data = black_box(json.as_bytes()); - bench.iter(|| { - let mut jiter = Jiter::new(json_data); - let string = find_string(&mut jiter); - jiter.finish().unwrap(); - black_box(string) + + c.bench_function(&title, |bench| { + bench.iter(|| { + let mut jiter = Jiter::new(json_data); + let string = find_string(&mut jiter); + jiter.finish().unwrap(); + black_box(string) + }); }); } -fn jiter_iter_string_array(path: &str, bench: &mut Bencher) { +fn jiter_iter_string_array(path: &str, c: &mut Criterion) { + let title = read_title(path) + "_jiter_iter"; let json = read_file(path); let json_data = black_box(json.as_bytes()); - bench.iter(|| { - let mut jiter = Jiter::new(json_data); - jiter.next_array().unwrap(); - let i = jiter.known_str().unwrap(); - // record len instead of allocating the string to simulate something like constructing a PyString - black_box(i.len()); - while jiter.array_step().unwrap().is_some() { + + c.bench_function(&title, |bench| { + bench.iter(|| { + let mut jiter = Jiter::new(json_data); + jiter.next_array().unwrap(); let i = jiter.known_str().unwrap(); + // record len instead of allocating the string to simulate something like constructing a PyString black_box(i.len()); - } - jiter.finish().unwrap(); + while jiter.array_step().unwrap().is_some() { + let i = jiter.known_str().unwrap(); + black_box(i.len()); + } + jiter.finish().unwrap(); + }); }); } -fn jiter_iter_true_array(path: &str, bench: &mut Bencher) { +fn jiter_iter_true_array(path: &str, c: &mut Criterion) { + let title = read_title(path) + "_jiter_iter"; let json = read_file(path); let json_data = black_box(json.as_bytes()); - bench.iter(|| { - let mut jiter = Jiter::new(json_data); - let first_peek = jiter.next_array().unwrap().unwrap(); - let i = jiter.known_bool(first_peek).unwrap(); - black_box(i); - while let Some(peek) = jiter.array_step().unwrap() { - let i = jiter.known_bool(peek).unwrap(); + + c.bench_function(&title, |bench| { + bench.iter(|| { + let mut jiter = Jiter::new(json_data); + let first_peek = jiter.next_array().unwrap().unwrap(); + let i = jiter.known_bool(first_peek).unwrap(); black_box(i); - } + while let Some(peek) = jiter.array_step().unwrap() { + let i = jiter.known_bool(peek).unwrap(); + black_box(i); + } + }); }); } -fn jiter_iter_true_object(path: &str, bench: &mut Bencher) { +fn jiter_iter_true_object(path: &str, c: &mut Criterion) { + let title = read_title(path) + "_jiter_iter"; let json = read_file(path); let json_data = black_box(json.as_bytes()); - bench.iter(|| { - let mut jiter = Jiter::new(json_data); - if let Some(first_key) = jiter.next_object().unwrap() { - let first_key = first_key.to_string(); - let first_value = jiter.next_bool().unwrap(); - black_box((first_key, first_value)); - while let Some(key) = jiter.next_key().unwrap() { - let key = key.to_string(); - let value = jiter.next_bool().unwrap(); - black_box((key, value)); + + c.bench_function(&title, |bench| { + bench.iter(|| { + let mut jiter = Jiter::new(json_data); + if let Some(first_key) = jiter.next_object().unwrap() { + let first_key = first_key.to_string(); + let first_value = jiter.next_bool().unwrap(); + black_box((first_key, first_value)); + while let Some(key) = jiter.next_key().unwrap() { + let key = key.to_string(); + let value = jiter.next_bool().unwrap(); + black_box((key, value)); + } } - } + }); }); } -fn jiter_iter_ints_array(path: &str, bench: &mut Bencher) { +fn jiter_iter_ints_array(path: &str, c: &mut Criterion) { + let title = read_title(path) + "_jiter_iter"; let json = read_file(path); let json_data = black_box(json.as_bytes()); - bench.iter(|| { - let mut jiter = Jiter::new(json_data); - let first_peek = jiter.next_array().unwrap().unwrap(); - let i = jiter.known_int(first_peek).unwrap(); - black_box(i); - while let Some(peek) = jiter.array_step().unwrap() { - let i = jiter.known_int(peek).unwrap(); + + c.bench_function(&title, |bench| { + bench.iter(|| { + let mut jiter = Jiter::new(json_data); + let first_peek = jiter.next_array().unwrap().unwrap(); + let i = jiter.known_int(first_peek).unwrap(); black_box(i); - } + while let Some(peek) = jiter.array_step().unwrap() { + let i = jiter.known_int(peek).unwrap(); + black_box(i); + } + }); }); } -fn jiter_iter_floats_array(path: &str, bench: &mut Bencher) { +fn jiter_iter_floats_array(path: &str, c: &mut Criterion) { + let title = read_title(path) + "_jiter_iter"; let json = read_file(path); let json_data = black_box(json.as_bytes()); - bench.iter(|| { - let mut jiter = Jiter::new(json_data); - let first_peek = jiter.next_array().unwrap().unwrap(); - let i = jiter.known_float(first_peek).unwrap(); - black_box(i); - while let Some(peek) = jiter.array_step().unwrap() { - let i = jiter.known_float(peek).unwrap(); + + c.bench_function(&title, |bench| { + bench.iter(|| { + let mut jiter = Jiter::new(json_data); + let first_peek = jiter.next_array().unwrap().unwrap(); + let i = jiter.known_float(first_peek).unwrap(); black_box(i); - } + while let Some(peek) = jiter.array_step().unwrap() { + let i = jiter.known_float(peek).unwrap(); + black_box(i); + } + }); }); } -fn jiter_string(path: &str, bench: &mut Bencher) { +fn jiter_string(path: &str, c: &mut Criterion) { + let title = read_title(path) + "_jiter_iter"; let json = read_file(path); let json_data = black_box(json.as_bytes()); - bench.iter(|| { - let mut jiter = Jiter::new(json_data); - let string = jiter.next_str().unwrap(); - black_box(string); - jiter.finish().unwrap(); + + c.bench_function(&title, |bench| { + bench.iter(|| { + let mut jiter = Jiter::new(json_data); + let string = jiter.next_str().unwrap(); + black_box(string); + jiter.finish().unwrap(); + }); }); } -fn serde_value(path: &str, bench: &mut Bencher) { +fn serde_value(path: &str, c: &mut Criterion) { + let title = read_title(path) + "_serde_value"; let json = read_file(path); let json_data = black_box(json.as_bytes()); - bench.iter(|| { - let value: Value = serde_json::from_slice(json_data).unwrap(); - black_box(value); + + c.bench_function(&title, |bench| { + bench.iter(|| { + let value: Value = serde_json::from_slice(json_data).unwrap(); + black_box(value); + }); }); } -fn serde_str(path: &str, bench: &mut Bencher) { +fn serde_str(path: &str, c: &mut Criterion) { + let title = read_title(path) + "_serde_iter"; let json = read_file(path); let json_data = black_box(json.as_bytes()); - bench.iter(|| { - let value: String = serde_json::from_slice(json_data).unwrap(); - black_box(value); + + c.bench_function(&title, |bench| { + bench.iter(|| { + let value: String = serde_json::from_slice(json_data).unwrap(); + black_box(value); + }); }); } macro_rules! test_cases { ($file_name:ident) => { paste::item! { - fn [< $file_name _jiter_value >](bench: &mut Bencher) { + fn [< $file_name _jiter_value >](c: &mut Criterion) { let file_path = format!("./benches/{}.json", stringify!($file_name)); - jiter_value(&file_path, bench); + jiter_value(&file_path, c); } - fn [< $file_name _jiter_iter >](bench: &mut Bencher) { + fn [< $file_name _jiter_iter >](c: &mut Criterion) { let file_name = stringify!($file_name); let file_path = format!("./benches/{}.json", file_name); if file_name == "big" { - jiter_iter_big(&file_path, bench); + jiter_iter_big(&file_path, c); } else if file_name == "pass2" { - jiter_iter_pass2(&file_path, bench); + jiter_iter_pass2(&file_path, c); } else if file_name == "string_array" { - jiter_iter_string_array(&file_path, bench); + jiter_iter_string_array(&file_path, c); } else if file_name == "true_array" { - jiter_iter_true_array(&file_path, bench); + jiter_iter_true_array(&file_path, c); } else if file_name == "true_object" { - jiter_iter_true_object(&file_path, bench); + jiter_iter_true_object(&file_path, c); } else if file_name == "bigints_array" { - jiter_iter_ints_array(&file_path, bench); + jiter_iter_ints_array(&file_path, c); } else if file_name == "massive_ints_array" { - jiter_iter_ints_array(&file_path, bench); + jiter_iter_ints_array(&file_path, c); } else if file_name == "floats_array" { - jiter_iter_floats_array(&file_path, bench); + jiter_iter_floats_array(&file_path, c); } else if file_name == "x100" || file_name == "sentence" || file_name == "unicode" { - jiter_string(&file_path, bench); + jiter_string(&file_path, c); } } - fn [< $file_name _jiter_skip >](bench: &mut Bencher) { + fn [< $file_name _jiter_skip >](c: &mut Criterion) { let file_path = format!("./benches/{}.json", stringify!($file_name)); - jiter_skip(&file_path, bench); + jiter_skip(&file_path, c); } - fn [< $file_name _serde_value >](bench: &mut Bencher) { + fn [< $file_name _serde_value >](c: &mut Criterion) { let file_path = format!("./benches/{}.json", stringify!($file_name)); - serde_value(&file_path, bench); + serde_value(&file_path, c); } } }; @@ -254,29 +310,35 @@ test_cases!(sentence); test_cases!(unicode); test_cases!(short_numbers); -fn string_array_jiter_value_owned(bench: &mut Bencher) { +fn string_array_jiter_value_owned(c: &mut Criterion) { let json = read_file("./benches/string_array.json"); let json_data = json.as_bytes(); - bench.iter(|| { - let v = JsonValue::parse_owned(black_box(json_data), false, PartialMode::Off).unwrap(); - black_box(v) + + c.bench_function("string_array_jiter_value_owned", |bench| { + bench.iter(|| { + let v = JsonValue::parse_owned(black_box(json_data), false, PartialMode::Off).unwrap(); + black_box(v) + }); }); } -fn medium_response_jiter_value_owned(bench: &mut Bencher) { +fn medium_response_jiter_value_owned(c: &mut Criterion) { let json = read_file("./benches/medium_response.json"); let json_data = json.as_bytes(); - bench.iter(|| { - let v = JsonValue::parse_owned(black_box(json_data), false, PartialMode::Off).unwrap(); - black_box(v) + + c.bench_function("medium_response_jiter_value_owned", |bench| { + bench.iter(|| { + let v = JsonValue::parse_owned(black_box(json_data), false, PartialMode::Off).unwrap(); + black_box(v) + }); }); } -fn x100_serde_iter(bench: &mut Bencher) { - serde_str("./benches/x100.json", bench); +fn x100_serde_iter(c: &mut Criterion) { + serde_str("./benches/x100.json", c); } -benchmark_group!( +criterion_group!( benches, big_jiter_iter, big_jiter_skip, @@ -338,4 +400,4 @@ benchmark_group!( short_numbers_jiter_value, short_numbers_serde_value, ); -benchmark_main!(benches); +criterion_main!(benches); diff --git a/crates/jiter/benches/python.rs b/crates/jiter/benches/python.rs index 77796db..038c7bc 100644 --- a/crates/jiter/benches/python.rs +++ b/crates/jiter/benches/python.rs @@ -1,106 +1,126 @@ -use codspeed_bencher_compat::{benchmark_group, benchmark_main, Bencher}; +use codspeed_criterion_compat::{criterion_group, criterion_main, Criterion}; use std::fs::File; use std::io::Read; +use std::path::Path; use pyo3::Python; use jiter::{cache_clear, PythonParse, StringCacheMode}; -fn python_parse_numeric(bench: &mut Bencher) { +fn python_parse_numeric(c: &mut Criterion) { Python::with_gil(|py| { cache_clear(); - bench.iter(|| { - PythonParse::default() - .python_parse( - py, - br#" { "int": 1, "bigint": 123456789012345678901234567890, "float": 1.2} "#, - ) - .unwrap() + c.bench_function("python_parse_numeric", |bench| { + bench.iter(|| { + PythonParse::default() + .python_parse( + py, + br#" { "int": 1, "bigint": 123456789012345678901234567890, "float": 1.2} "#, + ) + .unwrap() + }); }); }); } -fn python_parse_other(bench: &mut Bencher) { +fn python_parse_other(c: &mut Criterion) { Python::with_gil(|py| { cache_clear(); - bench.iter(|| { - PythonParse::default() - .python_parse(py, br#"["string", true, false, null]"#) - .unwrap() + c.bench_function("python_parse_other", |bench| { + bench.iter(|| { + PythonParse::default() + .python_parse(py, br#"["string", true, false, null]"#) + .unwrap() + }); }); }); } -fn python_parse_file(path: &str, bench: &mut Bencher, cache_mode: StringCacheMode) { +fn python_parse_file(path: &str, c: &mut Criterion, cache_mode: StringCacheMode) { + let path = Path::new(path); let mut file = File::open(path).unwrap(); let mut contents = String::new(); file.read_to_string(&mut contents).unwrap(); let json_data = contents.as_bytes(); + let title = { + let file_stem = path.file_stem().unwrap().to_str().unwrap(); + + let cache_mode = match cache_mode { + StringCacheMode::None => "_not_cached", + _ => "", + }; + + "python_parse_".to_owned() + file_stem + cache_mode + }; + Python::with_gil(|py| { cache_clear(); - bench.iter(|| { - PythonParse { - cache_mode, - ..Default::default() - } - .python_parse(py, json_data) - .unwrap() + + c.bench_function(&title, |bench| { + bench.iter(|| { + PythonParse { + cache_mode, + ..Default::default() + } + .python_parse(py, json_data) + .unwrap() + }); }); }); } -fn python_parse_massive_ints_array(bench: &mut Bencher) { - python_parse_file("./benches/massive_ints_array.json", bench, StringCacheMode::All); +fn python_parse_massive_ints_array(c: &mut Criterion) { + python_parse_file("./benches/massive_ints_array.json", c, StringCacheMode::All); } -fn python_parse_medium_response_not_cached(bench: &mut Bencher) { - python_parse_file("./benches/medium_response.json", bench, StringCacheMode::None); +fn python_parse_medium_response_not_cached(c: &mut Criterion) { + python_parse_file("./benches/medium_response.json", c, StringCacheMode::None); } -fn python_parse_medium_response(bench: &mut Bencher) { - python_parse_file("./benches/medium_response.json", bench, StringCacheMode::All); +fn python_parse_medium_response(c: &mut Criterion) { + python_parse_file("./benches/medium_response.json", c, StringCacheMode::All); } -fn python_parse_true_object_not_cached(bench: &mut Bencher) { - python_parse_file("./benches/true_object.json", bench, StringCacheMode::None); +fn python_parse_true_object_not_cached(c: &mut Criterion) { + python_parse_file("./benches/true_object.json", c, StringCacheMode::None); } -fn python_parse_string_array_not_cached(bench: &mut Bencher) { - python_parse_file("./benches/string_array.json", bench, StringCacheMode::None); +fn python_parse_string_array_not_cached(c: &mut Criterion) { + python_parse_file("./benches/string_array.json", c, StringCacheMode::None); } -fn python_parse_string_array(bench: &mut Bencher) { - python_parse_file("./benches/string_array.json", bench, StringCacheMode::All); +fn python_parse_string_array(c: &mut Criterion) { + python_parse_file("./benches/string_array.json", c, StringCacheMode::All); } -fn python_parse_x100_not_cached(bench: &mut Bencher) { - python_parse_file("./benches/x100.json", bench, StringCacheMode::None); +fn python_parse_x100_not_cached(c: &mut Criterion) { + python_parse_file("./benches/x100.json", c, StringCacheMode::None); } -fn python_parse_x100(bench: &mut Bencher) { - python_parse_file("./benches/x100.json", bench, StringCacheMode::All); +fn python_parse_x100(c: &mut Criterion) { + python_parse_file("./benches/x100.json", c, StringCacheMode::All); } -fn python_parse_string_array_unique_not_cached(bench: &mut Bencher) { - python_parse_file("./benches/string_array_unique.json", bench, StringCacheMode::None); +fn python_parse_string_array_unique_not_cached(c: &mut Criterion) { + python_parse_file("./benches/string_array_unique.json", c, StringCacheMode::None); } -fn python_parse_string_array_unique(bench: &mut Bencher) { - python_parse_file("./benches/string_array_unique.json", bench, StringCacheMode::All); +fn python_parse_string_array_unique(c: &mut Criterion) { + python_parse_file("./benches/string_array_unique.json", c, StringCacheMode::All); } -fn python_parse_true_object(bench: &mut Bencher) { - python_parse_file("./benches/true_object.json", bench, StringCacheMode::All); +fn python_parse_true_object(c: &mut Criterion) { + python_parse_file("./benches/true_object.json", c, StringCacheMode::All); } /// Note - caching strings should make no difference here -fn python_parse_true_array(bench: &mut Bencher) { - python_parse_file("./benches/true_array.json", bench, StringCacheMode::All); +fn python_parse_true_array(c: &mut Criterion) { + python_parse_file("./benches/true_array.json", c, StringCacheMode::All); } -benchmark_group!( +criterion_group!( benches, python_parse_numeric, python_parse_other, @@ -117,4 +137,4 @@ benchmark_group!( python_parse_true_array, python_parse_massive_ints_array, ); -benchmark_main!(benches); +criterion_main!(benches);