From 687784e74373f99c7d8d9601662753dc3cc9e275 Mon Sep 17 00:00:00 2001 From: not-matthias Date: Thu, 20 Nov 2025 18:47:37 +0100 Subject: [PATCH 01/12] chore: add example with allocations --- crates/divan_compat/examples/Cargo.toml | 4 ++ crates/divan_compat/examples/benches/alloc.rs | 40 +++++++++++++++++++ 2 files changed, 44 insertions(+) create mode 100644 crates/divan_compat/examples/benches/alloc.rs diff --git a/crates/divan_compat/examples/Cargo.toml b/crates/divan_compat/examples/Cargo.toml index 1e9ea793..40c57f3c 100644 --- a/crates/divan_compat/examples/Cargo.toml +++ b/crates/divan_compat/examples/Cargo.toml @@ -44,3 +44,7 @@ harness = false [[bench]] name = "counters" harness = false + +[[bench]] +name = "alloc" +harness = false diff --git a/crates/divan_compat/examples/benches/alloc.rs b/crates/divan_compat/examples/benches/alloc.rs new file mode 100644 index 00000000..e32d380f --- /dev/null +++ b/crates/divan_compat/examples/benches/alloc.rs @@ -0,0 +1,40 @@ +use std::{ + alloc::Layout, + collections::{HashMap, HashSet}, +}; + +#[divan::bench] +fn allocate() { + println!("Hello, world!"); + + let vec = vec![1, 2, 3]; + println!("{vec:?}"); + + let mut map = HashMap::new(); + map.insert("key", "value"); + println!("{map:?}"); + + let mut set = HashSet::new(); + set.insert("apple"); + set.insert("banana"); + println!("{set:?}"); + + std::thread::sleep(std::time::Duration::from_secs(1)); + + let mut bytes_vec = vec![0u8; 0x100]; + println!("{:?}", bytes_vec.len()); + + bytes_vec.extend(&vec![0u8; 0x1000]); + + // Alloc 42 bytes of memory per iteration (4200 bytes total) + for _ in 0..100 { + let layout = Layout::new::<[u8; 42]>(); + let memory = unsafe { std::alloc::alloc(layout) }; + core::hint::black_box(memory); + unsafe { std::alloc::dealloc(memory, layout) }; + } +} + +fn main() { + divan::main(); +} From c11500dffa5ab1dcee1750c6021e903dc1df81c3 Mon Sep 17 00:00:00 2001 From: not-matthias Date: Fri, 21 Nov 2025 16:24:43 +0100 Subject: [PATCH 02/12] chore: bump instrument-hooks --- crates/codspeed/instrument-hooks | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/codspeed/instrument-hooks b/crates/codspeed/instrument-hooks index 1752e9e4..504dc86b 160000 --- a/crates/codspeed/instrument-hooks +++ b/crates/codspeed/instrument-hooks @@ -1 +1 @@ -Subproject commit 1752e9e4eae585e26703932d0055a1473dd77048 +Subproject commit 504dc86bacb8e0ae44e08b247c2eee09d8d4f6df From 56ca6f03536935d9b53fc6850f98cfd0b80e206f Mon Sep 17 00:00:00 2001 From: not-matthias Date: Wed, 19 Nov 2025 19:31:47 +0100 Subject: [PATCH 03/12] feat: add support for analysis mode --- crates/codspeed/src/codspeed.rs | 9 +++++++-- crates/codspeed/src/instrument_hooks/mod.rs | 14 ++++++++++++++ crates/codspeed/src/measurement.rs | 11 ----------- crates/codspeed/src/request/mod.rs | 1 - 4 files changed, 21 insertions(+), 14 deletions(-) diff --git a/crates/codspeed/src/codspeed.rs b/crates/codspeed/src/codspeed.rs index 937036be..82597fc5 100644 --- a/crates/codspeed/src/codspeed.rs +++ b/crates/codspeed/src/codspeed.rs @@ -1,4 +1,4 @@ -use crate::measurement; +use crate::{instrument_hooks::InstrumentHooks, measurement}; use colored::Colorize; use std::ffi::CString; @@ -19,7 +19,8 @@ pub struct CodSpeed { impl CodSpeed { pub fn new() -> Self { - let is_instrumented = measurement::is_instrumented(); + let hooks_instance = InstrumentHooks::instance(); + let is_instrumented = hooks_instance.is_instrumented(); if !is_instrumented { println!( "{} codspeed is enabled, but no performance measurement will be made since it's running in an unknown environment.", @@ -46,12 +47,16 @@ impl CodSpeed { #[inline(always)] pub fn start_benchmark(&mut self, name: &str) { self.current_benchmark = CString::new(name).expect("CString::new failed"); + let _ = InstrumentHooks::instance().start_benchmark(); measurement::start(); } #[inline(always)] pub fn end_benchmark(&mut self) { measurement::stop(&self.current_benchmark); + let _ = InstrumentHooks::instance().stop_benchmark(); + let _ = InstrumentHooks::instance() + .set_executed_benchmark(&self.current_benchmark.to_string_lossy()); self.benchmarked .push(self.current_benchmark.to_str().unwrap().to_string()); let action_str = if self.is_instrumented { diff --git a/crates/codspeed/src/instrument_hooks/mod.rs b/crates/codspeed/src/instrument_hooks/mod.rs index 2c47ebb4..2ec408f0 100644 --- a/crates/codspeed/src/instrument_hooks/mod.rs +++ b/crates/codspeed/src/instrument_hooks/mod.rs @@ -34,6 +34,9 @@ mod linux_impl { instance .set_integration("codspeed-rust", env!("CARGO_PKG_VERSION")) .expect("Failed to set integration"); + // We're using inline assembly to control callgrind, so we shouldn't do it in + // instrument-hooks to avoid sending the same events twice. + InstrumentHooks::disable_callgrind_markers(); instance }) } @@ -129,6 +132,15 @@ mod linux_impl { ffi::instrument_hooks_current_timestamp() } } + + pub fn disable_callgrind_markers() { + unsafe { + ffi::instrument_hooks_set_feature( + ffi::instrument_hooks_feature_t_FEATURE_DISABLE_CALLGRIND_MARKERS, + true, + ) + }; + } } impl Drop for InstrumentHooks { @@ -175,6 +187,8 @@ mod other_impl { pub fn current_timestamp() -> u64 { 0 } + + pub fn disable_callgrind_markers() {} } } diff --git a/crates/codspeed/src/measurement.rs b/crates/codspeed/src/measurement.rs index 9780e667..1169789f 100644 --- a/crates/codspeed/src/measurement.rs +++ b/crates/codspeed/src/measurement.rs @@ -2,17 +2,6 @@ use std::ffi::CString; use crate::request::{send_client_request, ClientRequest, Value}; -#[inline(always)] -pub fn is_instrumented() -> bool { - let valgrind_depth = unsafe { - send_client_request( - 0, - &[ClientRequest::RunningOnValgrind as Value, 0, 0, 0, 0, 0], - ) - }; - valgrind_depth > 0 -} - #[inline(always)] pub fn set_metadata() { let full_metadata = CString::new(format!( diff --git a/crates/codspeed/src/request/mod.rs b/crates/codspeed/src/request/mod.rs index a59fd322..6377cb94 100644 --- a/crates/codspeed/src/request/mod.rs +++ b/crates/codspeed/src/request/mod.rs @@ -3,7 +3,6 @@ const CG_BASE: u32 = ((b'C' as u32) << 24) + ((b'T' as u32) << 16); #[allow(non_camel_case_types)] #[repr(u32)] pub enum ClientRequest { - RunningOnValgrind = 0x1001, ZeroStatistics = CG_BASE + 1, DumpStatisticsAt = CG_BASE + 3, StartInstrumentation = CG_BASE + 4, From 238398ee98490d8439bce5de6e47e4e5dfa13171 Mon Sep 17 00:00:00 2001 From: not-matthias Date: Fri, 21 Nov 2025 16:22:12 +0100 Subject: [PATCH 04/12] feat(cargo-codspeed): add analysis build mode --- crates/cargo-codspeed/src/build.rs | 6 ++++-- crates/cargo-codspeed/src/measurement_mode.rs | 3 +++ 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/crates/cargo-codspeed/src/build.rs b/crates/cargo-codspeed/src/build.rs index 4cc28b83..f0e6f26e 100644 --- a/crates/cargo-codspeed/src/build.rs +++ b/crates/cargo-codspeed/src/build.rs @@ -225,8 +225,10 @@ See `cargo codspeed build --help` for more information."); "-Cstrip=none".to_owned(), ]; - // Add the codspeed cfg flag if simulation mode is enabled - if measurement_mode == MeasurementMode::Simulation { + // Add the codspeed cfg flag if the benchmark should only run once + if measurement_mode == MeasurementMode::Simulation + || measurement_mode == MeasurementMode::Analysis + { flags.push("--cfg=codspeed".to_owned()); } diff --git a/crates/cargo-codspeed/src/measurement_mode.rs b/crates/cargo-codspeed/src/measurement_mode.rs index 171ac611..f9f64dae 100644 --- a/crates/cargo-codspeed/src/measurement_mode.rs +++ b/crates/cargo-codspeed/src/measurement_mode.rs @@ -9,6 +9,8 @@ pub enum MeasurementMode { #[value(alias = "instrumentation")] Simulation, Walltime, + #[value(alias = "memory")] + Analysis, } impl fmt::Display for MeasurementMode { @@ -19,6 +21,7 @@ impl fmt::Display for MeasurementMode { match self { MeasurementMode::Simulation => "simulation", MeasurementMode::Walltime => "walltime", + MeasurementMode::Analysis => "analysis", } ) } From 99785ae69292b7585c891f7e4c4b190f689fdffa Mon Sep 17 00:00:00 2001 From: not-matthias Date: Thu, 20 Nov 2025 18:50:30 +0100 Subject: [PATCH 05/12] chore(ci): add memory profiling --- .github/workflows/ci.yml | 60 +++++++++++++--------------------------- 1 file changed, 19 insertions(+), 41 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 3ec0b066..587f04ca 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -127,47 +127,20 @@ jobs: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - run: cargo nextest run -p cargo-codspeed --partition hash:${{ matrix.partition }}/5 - compat-integration-test-instrumentation: - runs-on: ubuntu-latest - strategy: - matrix: - build-args: - - "-p codspeed" - - "-p codspeed-bencher-compat" - - "--features async_futures -p codspeed-criterion-compat" - - "-p codspeed-divan-compat" - - "-p codspeed-divan-compat-examples" - steps: - - uses: actions/checkout@v4 - with: - submodules: true - - uses: moonrepo/setup-rust@v1 - with: - cache-target: release - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - - - run: cargo install --path crates/cargo-codspeed --locked - - - run: cargo codspeed build ${{ matrix.build-args }} - - - name: Run the benchmarks - uses: CodSpeedHQ/action@main - env: - MY_ENV_VAR: "YES" - with: - run: cargo codspeed run - mode: instrumentation - token: ${{ secrets.CODSPEED_TOKEN }} - - compat-integration-test-walltime: - runs-on: codspeed-macro + compat-integration-test: strategy: matrix: package: + - codspeed + - codspeed-bencher-compat - codspeed-divan-compat - codspeed-divan-compat-examples - codspeed-criterion-compat + mode: + - walltime + - instrumentation + - memory + runs-on: ${{ matrix.mode == 'walltime' && 'codspeed-macro' || 'ubuntu-latest' }} steps: - uses: actions/checkout@v4 with: @@ -180,10 +153,16 @@ jobs: - run: cargo install --path crates/cargo-codspeed --locked + - name: Remove .cargo/config.toml to not force instrumentation mode + if: ${{ matrix.mode != 'instrumentation' }} + run: rm -f .cargo/config.toml + - run: | - # Remove the cargo config else it forces instrumentation mode - rm -f .cargo/config.toml - cargo codspeed build -p ${{ matrix.package }} + if [ "${{ matrix.package }}" = "codspeed-criterion-compat" ]; then + cargo codspeed build -p ${{ matrix.package }} --features async_futures + else + cargo codspeed build -p ${{ matrix.package }} + fi - name: Run the benchmarks uses: CodSpeedHQ/action@main @@ -191,7 +170,7 @@ jobs: MY_ENV_VAR: "YES" with: run: cargo codspeed run - mode: walltime + mode: ${{ matrix.mode }} token: ${{ secrets.CODSPEED_TOKEN }} musl-build-check: @@ -228,8 +207,7 @@ jobs: - tests-without-cargo-codspeed - test-cargo-codspeed - msrv-check - - compat-integration-test-instrumentation - - compat-integration-test-walltime + - compat-integration-test - musl-build-check steps: - uses: re-actors/alls-green@release/v1 From 984f949570cbd2e117ecd22c2a14f89ff365e0cf Mon Sep 17 00:00:00 2001 From: not-matthias Date: Thu, 27 Nov 2025 16:04:51 +0100 Subject: [PATCH 06/12] chore(ci): switch to OIDC auth --- .github/workflows/ci.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 587f04ca..ec10ec8b 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -5,6 +5,10 @@ on: pull_request: workflow_dispatch: +permissions: + contents: read + id-token: write + jobs: lint: runs-on: ubuntu-latest @@ -171,7 +175,6 @@ jobs: with: run: cargo codspeed run mode: ${{ matrix.mode }} - token: ${{ secrets.CODSPEED_TOKEN }} musl-build-check: strategy: From 3d968aa8e7ee0e72904102a20a716ae5f6708027 Mon Sep 17 00:00:00 2001 From: not-matthias Date: Thu, 18 Dec 2025 18:07:00 +0100 Subject: [PATCH 07/12] wip: use latest runner; run on staging --- .github/workflows/ci.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index ec10ec8b..b698a77d 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -168,13 +168,18 @@ jobs: cargo codspeed build -p ${{ matrix.package }} fi + - name: Install memtrack (temp until runner is released) + run: curl --proto '=https' --tlsv1.2 -LsSf https://github.com/CodSpeedHQ/runner/releases/download/memtrack-v1.0.1-alpha.1/memtrack-installer.sh | sh + - name: Run the benchmarks uses: CodSpeedHQ/action@main env: MY_ENV_VAR: "YES" with: + runner-version: branch:main run: cargo codspeed run mode: ${{ matrix.mode }} + upload-url: https://api.staging.preview.codspeed.io/upload musl-build-check: strategy: From 14c139f2193116b649bba5dd6f36ae922e142a0a Mon Sep 17 00:00:00 2001 From: not-matthias Date: Thu, 18 Dec 2025 18:20:02 +0100 Subject: [PATCH 08/12] fix: set runner mode --- .github/workflows/ci.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index b698a77d..e75e42dc 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -145,6 +145,8 @@ jobs: - instrumentation - memory runs-on: ${{ matrix.mode == 'walltime' && 'codspeed-macro' || 'ubuntu-latest' }} + env: + CODSPEED_RUNNER_MODE: ${{ matrix.mode == 'memory' && 'analysis' || matrix.mode }} steps: - uses: actions/checkout@v4 with: From d0a7e829aef77c753bd6b0bae41229f5313730f3 Mon Sep 17 00:00:00 2001 From: not-matthias Date: Thu, 18 Dec 2025 19:12:06 +0100 Subject: [PATCH 09/12] wip: use alpha runner release --- .github/workflows/ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index e75e42dc..98e856e2 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -171,14 +171,14 @@ jobs: fi - name: Install memtrack (temp until runner is released) - run: curl --proto '=https' --tlsv1.2 -LsSf https://github.com/CodSpeedHQ/runner/releases/download/memtrack-v1.0.1-alpha.1/memtrack-installer.sh | sh + run: curl --proto '=https' --tlsv1.2 -LsSf https://github.com/CodSpeedHQ/runner/releases/download/memtrack-v1.0.1-alpha.3/memtrack-installer.sh | sh - name: Run the benchmarks uses: CodSpeedHQ/action@main env: MY_ENV_VAR: "YES" with: - runner-version: branch:main + runner-version: 4.4.2-alpha.3 run: cargo codspeed run mode: ${{ matrix.mode }} upload-url: https://api.staging.preview.codspeed.io/upload From c9b8f38c92686de2a6c6d9d433b4da16db18b20d Mon Sep 17 00:00:00 2001 From: Guillaume Lagrange Date: Fri, 19 Dec 2025 12:17:24 +0100 Subject: [PATCH 10/12] chore: test without memory to trigger error? --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 98e856e2..e0151cb8 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -143,7 +143,7 @@ jobs: mode: - walltime - instrumentation - - memory + # - memory runs-on: ${{ matrix.mode == 'walltime' && 'codspeed-macro' || 'ubuntu-latest' }} env: CODSPEED_RUNNER_MODE: ${{ matrix.mode == 'memory' && 'analysis' || matrix.mode }} From 564bccd257c3639abc091f70bd6d126e0309d5a6 Mon Sep 17 00:00:00 2001 From: Guillaume Lagrange Date: Fri, 19 Dec 2025 12:36:48 +0100 Subject: [PATCH 11/12] --wip-- [skip ci] --- .github/workflows/ci.yml | 44 +++++++++++++++++++++++++++++++++++----- 1 file changed, 39 insertions(+), 5 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index e0151cb8..d1f8c534 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -131,7 +131,7 @@ jobs: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - run: cargo nextest run -p cargo-codspeed --partition hash:${{ matrix.partition }}/5 - compat-integration-test: + analysis-integration-test: strategy: matrix: package: @@ -143,10 +143,9 @@ jobs: mode: - walltime - instrumentation - # - memory - runs-on: ${{ matrix.mode == 'walltime' && 'codspeed-macro' || 'ubuntu-latest' }} + runs-on: ubuntu-latest env: - CODSPEED_RUNNER_MODE: ${{ matrix.mode == 'memory' && 'analysis' || matrix.mode }} + CODSPEED_RUNNER_MODE: ${{ matrix.mode }} steps: - uses: actions/checkout@v4 with: @@ -160,7 +159,6 @@ jobs: - run: cargo install --path crates/cargo-codspeed --locked - name: Remove .cargo/config.toml to not force instrumentation mode - if: ${{ matrix.mode != 'instrumentation' }} run: rm -f .cargo/config.toml - run: | @@ -181,6 +179,42 @@ jobs: runner-version: 4.4.2-alpha.3 run: cargo codspeed run mode: ${{ matrix.mode }} + # TODO: Dont merge this + upload-url: https://api.staging.preview.codspeed.io/upload + + walltime-integration-test: + runs-on: codspeed-macro + strategy: + matrix: + package: + - codspeed-divan-compat + - codspeed-divan-compat-examples + - codspeed-criterion-compat + steps: + - uses: actions/checkout@v4 + with: + submodules: true + - uses: moonrepo/setup-rust@v1 + with: + cache-target: release + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + + - run: cargo install --path crates/cargo-codspeed --locked + + - run: | + # Remove the cargo config else it forces instrumentation mode + rm -f .cargo/config.toml + cargo codspeed build -p ${{ matrix.package }} + + - name: Run the benchmarks + uses: CodSpeedHQ/action@main + env: + MY_ENV_VAR: "YES" + with: + run: cargo codspeed run + mode: walltime + # TODO: Dont merge this upload-url: https://api.staging.preview.codspeed.io/upload musl-build-check: From 046c8b38a1bc0ebeec92cd46be5e5d621f2c388d Mon Sep 17 00:00:00 2001 From: Guillaume Lagrange Date: Fri, 19 Dec 2025 12:42:50 +0100 Subject: [PATCH 12/12] fix: dont run bencher in walltime --- .github/workflows/ci.yml | 28 ++++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index d1f8c534..1dcc3e0a 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -141,11 +141,12 @@ jobs: - codspeed-divan-compat-examples - codspeed-criterion-compat mode: - - walltime + - memory - instrumentation + include: + - package: codspeed-criterion-compat + features: async_futures runs-on: ubuntu-latest - env: - CODSPEED_RUNNER_MODE: ${{ matrix.mode }} steps: - uses: actions/checkout@v4 with: @@ -158,16 +159,13 @@ jobs: - run: cargo install --path crates/cargo-codspeed --locked - - name: Remove .cargo/config.toml to not force instrumentation mode - run: rm -f .cargo/config.toml - - - run: | - if [ "${{ matrix.package }}" = "codspeed-criterion-compat" ]; then - cargo codspeed build -p ${{ matrix.package }} --features async_futures - else - cargo codspeed build -p ${{ matrix.package }} - fi + - name: Build the benchmarks + run: | + # Remove the cargo config else it forces instrumentation mode + rm -f .cargo/config.toml + cargo codspeed build -m ${{matrix.mode}} -p ${{ matrix.package }} ${{ matrix.features && format('--features {0}', matrix.features) || '' }} + # TODO: Dont merge this - name: Install memtrack (temp until runner is released) run: curl --proto '=https' --tlsv1.2 -LsSf https://github.com/CodSpeedHQ/runner/releases/download/memtrack-v1.0.1-alpha.3/memtrack-installer.sh | sh @@ -176,10 +174,10 @@ jobs: env: MY_ENV_VAR: "YES" with: - runner-version: 4.4.2-alpha.3 run: cargo codspeed run mode: ${{ matrix.mode }} # TODO: Dont merge this + runner-version: 4.4.2-alpha.3 upload-url: https://api.staging.preview.codspeed.io/upload walltime-integration-test: @@ -215,6 +213,7 @@ jobs: run: cargo codspeed run mode: walltime # TODO: Dont merge this + runner-version: 4.4.2-alpha.3 upload-url: https://api.staging.preview.codspeed.io/upload musl-build-check: @@ -251,7 +250,8 @@ jobs: - tests-without-cargo-codspeed - test-cargo-codspeed - msrv-check - - compat-integration-test + - analysis-integration-test + - walltime-integration-test - musl-build-check steps: - uses: re-actors/alls-green@release/v1