Skip to content

Commit 88c2612

Browse files
not-matthiasnot-matthias
committed
feat: add codspeed benchmarks for valgrind
1 parent c9593ee commit 88c2612

File tree

8 files changed

+405
-0
lines changed

8 files changed

+405
-0
lines changed

‎.gitattributes‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
bench/testdata/* filter=lfs diff=lfs merge=lfs -text

‎.github/workflows/ci.yml‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,17 @@ jobs:
2222
steps:
2323
- uses: actions/checkout@v4
2424

25+
# Skip installing package docs to avoid wasting time when installing valgrind
26+
# See: https://github.com/actions/runner-images/issues/10977#issuecomment-2810713336
27+
- name: Skip installing package docs
28+
if: runner.os == 'Linux'
29+
run: |
30+
sudo tee /etc/dpkg/dpkg.cfg.d/01_nodoc > /dev/null << 'EOF'
31+
path-exclude /usr/share/doc/*
32+
path-exclude /usr/share/man/*
33+
path-exclude /usr/share/info/*
34+
EOF
35+
2536
- name: Update apt-get cache
2637
run: sudo apt-get update
2738

‎.github/workflows/codspeed.yml‎

Lines changed: 99 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,99 @@
1+
name: CodSpeed Benchmarks
2+
3+
on:
4+
push:
5+
branches:
6+
- master
7+
pull_request:
8+
workflow_dispatch:
9+
10+
jobs:
11+
benchmarks:
12+
runs-on: codspeed-macro
13+
strategy:
14+
matrix:
15+
# IMPORTANT: The binary has to match the architecture of the runner!
16+
cmd:
17+
- bench/testdata/take_strings-aarch64
18+
- ls -alh NEWS NEWS.old NEWS.older
19+
- echo Hello, World!
20+
- tar czf /tmp/news.tar.gz NEWS NEWS.old NEWS.older
21+
- python3 bench/fib.py
22+
valgrind:
23+
- "3.26.0"
24+
- "3.25.1"
25+
- "local"
26+
exclude:
27+
# Skip take_strings with 3.25.0 as it takes too long to complete
28+
- cmd: bench/testdata/take_strings-aarch64
29+
valgrind: "3.25.1"
30+
steps:
31+
- uses: actions/checkout@v4
32+
with:
33+
lfs: true
34+
- uses: extractions/setup-just@v3
35+
36+
# Skip installing package docs to avoid wasting time when installing build dependencies
37+
# See: https://github.com/actions/runner-images/issues/10977#issuecomment-2810713336
38+
- name: Skip installing package docs
39+
if: runner.os == 'Linux'
40+
run: |
41+
sudo tee /etc/dpkg/dpkg.cfg.d/01_nodoc > /dev/null << 'EOF'
42+
path-exclude /usr/share/doc/*
43+
path-exclude /usr/share/man/*
44+
path-exclude /usr/share/info/*
45+
EOF
46+
47+
- name: Cache Valgrind build
48+
uses: actions/cache@v4
49+
id: valgrind-cache
50+
with:
51+
path: /tmp/valgrind-upstream
52+
key: valgrind-${{ matrix.valgrind }}-${{ runner.os }}-build
53+
54+
# Build and install Valgrind
55+
- name: Update apt-get cache
56+
if: steps.valgrind-cache.outputs.cache-hit != 'true'
57+
run: |
58+
sudo apt-get update
59+
60+
# Remove existing Valgrind installation
61+
sudo apt-get remove -y valgrind || true
62+
63+
- name: Install build dependencies
64+
if: steps.valgrind-cache.outputs.cache-hit != 'true'
65+
run: |
66+
sudo apt-get install -y \
67+
build-essential \
68+
automake \
69+
autoconf \
70+
gdb \
71+
docbook \
72+
docbook-xsl \
73+
docbook-xml \
74+
xsltproc
75+
76+
- name: Build Valgrind (${{ matrix.valgrind }})
77+
if: steps.valgrind-cache.outputs.cache-hit != 'true'
78+
run: just build ${{ matrix.valgrind }}
79+
80+
- name: Install Valgrind (${{ matrix.valgrind }})
81+
run: |
82+
just install ${{ matrix.valgrind }}
83+
84+
# Ensure libc6-dev is installed for Valgrind to work properly
85+
sudo apt-get update
86+
sudo apt-get install -y libc6-dev
87+
88+
- name: Verify Valgrind build
89+
run: /usr/local/bin/valgrind --version
90+
91+
# Setup benchmarks and run them
92+
- name: Install uv
93+
uses: astral-sh/setup-uv@v5
94+
95+
- name: Run the benchmarks
96+
uses: CodSpeedHQ/action@main
97+
with:
98+
mode: walltime
99+
run: ./bench/bench.py --cmd "${{ matrix.cmd }}" --valgrind-path /usr/local/bin/valgrind

‎Justfile‎

Lines changed: 61 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,61 @@
1+
# Builds a specific valgrind version
2+
# Usage:
3+
# - just build 3.24.0: Downloads the specified version from sourceware.org, builds and installs it
4+
# - just build local: Builds the local Valgrind source in the current directory
5+
build version:
6+
#!/usr/bin/env bash
7+
set -euo pipefail
8+
9+
if [ "{{ version }}" = "local" ]; then
10+
just build-in "."
11+
else
12+
just build-upstream "{{ version }}"
13+
fi
14+
15+
build-in dir:
16+
#!/usr/bin/env bash
17+
set -euo pipefail
18+
cd "{{ dir }}"
19+
20+
# Check if we need to run autogen.sh (for git checkouts)
21+
if [ -f "autogen.sh" ] && [ ! -f "configure" ]; then
22+
./autogen.sh
23+
fi
24+
25+
./configure
26+
make include/vgversion.h
27+
make -j$(nproc) -C VEX
28+
make -j$(nproc) -C coregrind
29+
make -j$(nproc) -C callgrind
30+
31+
# Download, build and install upstream Valgrind from sourceware.org
32+
build-upstream version:
33+
#!/usr/bin/env bash
34+
set -euo pipefail
35+
36+
# Download and extract upstream Valgrind
37+
mkdir -p /tmp/valgrind-upstream
38+
rm -rf /tmp/valgrind-upstream/valgrind-{{ version }}*
39+
wget -q -O /tmp/valgrind-upstream/valgrind-{{ version }}.tar.bz2 \
40+
https://sourceware.org/pub/valgrind/valgrind-{{ version }}.tar.bz2
41+
tar -xjf /tmp/valgrind-upstream/valgrind-{{ version }}.tar.bz2 \
42+
-C /tmp/valgrind-upstream
43+
44+
# Build and install using build-in
45+
just build-in "/tmp/valgrind-upstream/valgrind-{{ version }}"
46+
47+
install version:
48+
#!/usr/bin/env bash
49+
set -euo pipefail
50+
51+
if [ "{{ version }}" = "local" ]; then
52+
just install-in "."
53+
else
54+
just install-in "/tmp/valgrind-upstream/valgrind-{{ version }}"
55+
fi
56+
57+
install-in dir:
58+
#!/usr/bin/env bash
59+
set -euo pipefail
60+
cd "{{ dir }}"
61+
sudo make install

‎bench/bench.py‎

Lines changed: 213 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,213 @@
1+
#!/usr/bin/env -S uv run --script
2+
# /// script
3+
# requires-python = ">=3.9"
4+
# dependencies = [
5+
# "pytest>=8.4.2",
6+
# "pytest-codspeed>=4.2.0",
7+
# ]
8+
# ///
9+
10+
import argparse
11+
import os
12+
import shlex
13+
import subprocess
14+
from pathlib import Path
15+
import time
16+
17+
import pytest
18+
19+
20+
class ValgrindRunner:
21+
"""Run Valgrind with different configurations."""
22+
23+
def __init__(
24+
self,
25+
cmd: str,
26+
valgrind_path: str = "valgrind",
27+
output_dir: str = "/tmp",
28+
):
29+
"""Initialize valgrind runner.
30+
31+
Args:
32+
cmd: Command to profile (can be a path or arbitrary shell command)
33+
valgrind_path: Path to valgrind executable
34+
output_dir: Directory for callgrind output files
35+
"""
36+
self.cmd = cmd
37+
self.valgrind_path = valgrind_path
38+
self.output_dir = Path(output_dir)
39+
self.output_dir.mkdir(parents=True, exist_ok=True)
40+
41+
# Verify valgrind is available
42+
result = subprocess.run(
43+
[self.valgrind_path, "--version"],
44+
capture_output=True,
45+
text=True,
46+
)
47+
if result.returncode != 0:
48+
raise RuntimeError(f"Valgrind not found at: {self.valgrind_path}")
49+
self.valgrind_version = result.stdout.strip()
50+
51+
def run_valgrind(self, *args: str) -> None:
52+
"""Execute valgrind with given arguments.
53+
54+
Args:
55+
*args: Valgrind arguments
56+
"""
57+
callgrind_output = self.output_dir / f"callgrind.{os.getpid()}.{time.time_ns()}"
58+
59+
cmd = [
60+
self.valgrind_path,
61+
"--tool=callgrind",
62+
f"--callgrind-out-file={callgrind_output}",
63+
*args,
64+
*shlex.split(self.cmd),
65+
]
66+
67+
# Add timeout and capture output to prevent hanging and buffering issues
68+
result = subprocess.run(
69+
cmd,
70+
capture_output=True,
71+
text=True,
72+
)
73+
if result.returncode != 0:
74+
raise RuntimeError(
75+
f"Valgrind execution failed with code {result.returncode}\n"
76+
f"stderr: {result.stderr}"
77+
)
78+
79+
# Clean up
80+
if callgrind_output.exists():
81+
callgrind_output.unlink()
82+
83+
84+
@pytest.fixture
85+
def runner(request):
86+
"""Fixture to provide runner instance to tests."""
87+
return request.config._valgrind_runner
88+
89+
90+
def pytest_generate_tests(metafunc):
91+
"""Parametrize tests with valgrind configurations."""
92+
if "valgrind_args" in metafunc.fixturenames:
93+
runner = getattr(metafunc.config, "_valgrind_runner", None)
94+
if not runner:
95+
return
96+
97+
# Define valgrind configurations
98+
configs = [
99+
(["--read-inline-info=no"], "no-inline"),
100+
(["--read-inline-info=yes"], "inline"),
101+
(
102+
[
103+
"--trace-children=yes",
104+
"--cache-sim=yes",
105+
"--I1=32768,8,64",
106+
"--D1=32768,8,64",
107+
"--LL=8388608,16,64",
108+
"--collect-systime=nsec",
109+
"--compress-strings=no",
110+
"--combine-dumps=yes",
111+
"--dump-line=no",
112+
"--read-inline-info=yes",
113+
],
114+
"full-with-inline",
115+
),
116+
(
117+
[
118+
"--trace-children=yes",
119+
"--cache-sim=yes",
120+
"--I1=32768,8,64",
121+
"--D1=32768,8,64",
122+
"--LL=8388608,16,64",
123+
"--collect-systime=nsec",
124+
"--compress-strings=no",
125+
"--combine-dumps=yes",
126+
"--dump-line=no",
127+
],
128+
"full-no-inline",
129+
),
130+
]
131+
132+
# Create test IDs with format: valgrind-version, command, config-name
133+
test_ids = [
134+
f"{runner.valgrind_version}, {runner.cmd}, {config_name}"
135+
for _, config_name in configs
136+
]
137+
138+
# Parametrize with just the args
139+
metafunc.parametrize(
140+
"valgrind_args",
141+
[args for args, _ in configs],
142+
ids=test_ids,
143+
)
144+
145+
146+
@pytest.mark.benchmark
147+
def test_valgrind(runner, valgrind_args):
148+
if runner:
149+
runner.run_valgrind(*valgrind_args)
150+
151+
152+
def main():
153+
parser = argparse.ArgumentParser(
154+
description="Benchmark Valgrind with pytest-codspeed",
155+
formatter_class=argparse.RawDescriptionHelpFormatter,
156+
epilog="""
157+
Examples:
158+
# Run with a binary path
159+
uv run bench.py --cmd /path/to/binary
160+
161+
# Run with an arbitrary command
162+
uv run bench.py --cmd 'echo "hello world"'
163+
164+
# Run with custom valgrind installation
165+
uv run bench.py --cmd /usr/bin/ls --valgrind-path /usr/local/bin/valgrind
166+
""",
167+
)
168+
169+
parser.add_argument(
170+
"--cmd",
171+
type=str,
172+
required=True,
173+
help="Command to profile (can be a path to a binary or any arbitrary command)",
174+
)
175+
parser.add_argument(
176+
"--valgrind-path",
177+
type=str,
178+
default="valgrind",
179+
help="Path to valgrind executable (default: valgrind)",
180+
)
181+
parser.add_argument(
182+
"--output-dir",
183+
type=str,
184+
default="/tmp",
185+
help="Directory for callgrind files (default: /tmp)",
186+
)
187+
188+
args = parser.parse_args()
189+
190+
# Create runner instance
191+
runner = ValgrindRunner(
192+
cmd=args.cmd,
193+
valgrind_path=args.valgrind_path,
194+
output_dir=args.output_dir,
195+
)
196+
print(f"Valgrind version: {runner.valgrind_version}")
197+
print(f"Command: {args.cmd}")
198+
199+
# Plugin to pass runner to tests
200+
class RunnerPlugin:
201+
def pytest_configure(self, config):
202+
config._valgrind_runner = runner
203+
204+
exit_code = pytest.main(
205+
[__file__, "-v", "--codspeed"],
206+
plugins=[RunnerPlugin()],
207+
)
208+
if exit_code != 0 and exit_code != 5:
209+
print(f"Benchmark execution returned exit code: {exit_code}")
210+
211+
212+
if __name__ == "__main__":
213+
main()

0 commit comments

Comments
 (0)