Skip to content

Commit 37a69f9

Browse files
not-matthiasnot-matthias
committed
feat: add codspeed benchmarks for valgrind
1 parent c9593ee commit 37a69f9

File tree

8 files changed

+398
-0
lines changed

8 files changed

+398
-0
lines changed

‎.gitattributes‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
bench/testdata/* filter=lfs diff=lfs merge=lfs -text

‎.github/workflows/ci.yml‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,17 @@ jobs:
2222
steps:
2323
- uses: actions/checkout@v4
2424

25+
# Skip installing package docs to avoid wasting time when installing valgrind
26+
# See: https://github.com/actions/runner-images/issues/10977#issuecomment-2810713336
27+
- name: Skip installing package docs
28+
if: runner.os == 'Linux'
29+
run: |
30+
sudo tee /etc/dpkg/dpkg.cfg.d/01_nodoc > /dev/null << 'EOF'
31+
path-exclude /usr/share/doc/*
32+
path-exclude /usr/share/man/*
33+
path-exclude /usr/share/info/*
34+
EOF
35+
2536
- name: Update apt-get cache
2637
run: sudo apt-get update
2738

‎.github/workflows/codspeed.yml‎

Lines changed: 99 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,99 @@
1+
name: CodSpeed Benchmarks
2+
3+
on:
4+
push:
5+
branches:
6+
- master
7+
pull_request:
8+
workflow_dispatch:
9+
10+
jobs:
11+
benchmarks:
12+
runs-on: codspeed-macro
13+
strategy:
14+
matrix:
15+
# IMPORTANT: The binary has to match the architecture of the runner!
16+
cmd:
17+
- bench/testdata/take_strings-aarch64
18+
- ls -alh NEWS NEWS.old NEWS.older
19+
- echo Hello, World!
20+
- tar czf /tmp/news.tar.gz NEWS NEWS.old NEWS.older
21+
- python3 bench/fib.py
22+
valgrind:
23+
- "3.26.0"
24+
- "3.25.1"
25+
- "local"
26+
exclude:
27+
# Skip take_strings with 3.25.0 as it takes too long to complete
28+
- cmd: bench/testdata/take_strings-aarch64
29+
valgrind: "3.25.1"
30+
steps:
31+
- uses: actions/checkout@v4
32+
with:
33+
lfs: true
34+
- uses: extractions/setup-just@v3
35+
36+
# Skip installing package docs to avoid wasting time when installing build dependencies
37+
# See: https://github.com/actions/runner-images/issues/10977#issuecomment-2810713336
38+
- name: Skip installing package docs
39+
if: runner.os == 'Linux'
40+
run: |
41+
sudo tee /etc/dpkg/dpkg.cfg.d/01_nodoc > /dev/null << 'EOF'
42+
path-exclude /usr/share/doc/*
43+
path-exclude /usr/share/man/*
44+
path-exclude /usr/share/info/*
45+
EOF
46+
47+
- name: Cache Valgrind build
48+
uses: actions/cache@v4
49+
id: valgrind-cache
50+
with:
51+
path: /tmp/valgrind-upstream
52+
key: valgrind-${{ matrix.valgrind }}-${{ runner.os }}-build
53+
54+
# Build and install Valgrind
55+
- name: Update apt-get cache
56+
if: steps.valgrind-cache.outputs.cache-hit != 'true'
57+
run: |
58+
sudo apt-get update
59+
60+
# Remove existing Valgrind installation
61+
sudo apt-get remove -y valgrind || true
62+
63+
- name: Install build dependencies
64+
if: steps.valgrind-cache.outputs.cache-hit != 'true'
65+
run: |
66+
sudo apt-get install -y \
67+
build-essential \
68+
automake \
69+
autoconf \
70+
gdb \
71+
docbook \
72+
docbook-xsl \
73+
docbook-xml \
74+
xsltproc
75+
76+
- name: Build Valgrind (${{ matrix.valgrind }})
77+
if: steps.valgrind-cache.outputs.cache-hit != 'true'
78+
run: just build ${{ matrix.valgrind }}
79+
80+
- name: Install Valgrind (${{ matrix.valgrind }})
81+
run: |
82+
just install ${{ matrix.valgrind }}
83+
84+
# Ensure libc6-dev is installed for Valgrind to work properly
85+
sudo apt-get update
86+
sudo apt-get install -y libc6-dev
87+
88+
- name: Verify Valgrind build
89+
run: /usr/local/bin/valgrind --version
90+
91+
# Setup benchmarks and run them
92+
- name: Install uv
93+
uses: astral-sh/setup-uv@v5
94+
95+
- name: Run the benchmarks
96+
uses: CodSpeedHQ/action@main
97+
with:
98+
mode: walltime
99+
run: ./bench/bench.py --cmd "${{ matrix.cmd }}" --valgrind-path /usr/local/bin/valgrind

‎Justfile‎

Lines changed: 61 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,61 @@
1+
# Builds a specific valgrind version
2+
# Usage:
3+
# - just build 3.24.0: Downloads the specified version from sourceware.org, builds and installs it
4+
# - just build local: Builds the local Valgrind source in the current directory
5+
build version:
6+
#!/usr/bin/env bash
7+
set -euo pipefail
8+
9+
if [ "{{ version }}" = "local" ]; then
10+
just build-in "."
11+
else
12+
just build-upstream "{{ version }}"
13+
fi
14+
15+
build-in dir:
16+
#!/usr/bin/env bash
17+
set -euo pipefail
18+
cd "{{ dir }}"
19+
20+
# Check if we need to run autogen.sh (for git checkouts)
21+
if [ -f "autogen.sh" ] && [ ! -f "configure" ]; then
22+
./autogen.sh
23+
fi
24+
25+
./configure
26+
make include/vgversion.h
27+
make -j$(nproc) -C VEX
28+
make -j$(nproc) -C coregrind
29+
make -j$(nproc) -C callgrind
30+
31+
# Download, build and install upstream Valgrind from sourceware.org
32+
build-upstream version:
33+
#!/usr/bin/env bash
34+
set -euo pipefail
35+
36+
# Download and extract upstream Valgrind
37+
mkdir -p /tmp/valgrind-upstream
38+
rm -rf /tmp/valgrind-upstream/valgrind-{{ version }}*
39+
wget -q -O /tmp/valgrind-upstream/valgrind-{{ version }}.tar.bz2 \
40+
https://sourceware.org/pub/valgrind/valgrind-{{ version }}.tar.bz2
41+
tar -xjf /tmp/valgrind-upstream/valgrind-{{ version }}.tar.bz2 \
42+
-C /tmp/valgrind-upstream
43+
44+
# Build and install using build-in
45+
just build-in "/tmp/valgrind-upstream/valgrind-{{ version }}"
46+
47+
install version:
48+
#!/usr/bin/env bash
49+
set -euo pipefail
50+
51+
if [ "{{ version }}" = "local" ]; then
52+
just install-in "."
53+
else
54+
just install-in "/tmp/valgrind-upstream/valgrind-{{ version }}"
55+
fi
56+
57+
install-in dir:
58+
#!/usr/bin/env bash
59+
set -euo pipefail
60+
cd "{{ dir }}"
61+
sudo make install

‎bench/bench.py‎

Lines changed: 206 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,206 @@
1+
#!/usr/bin/env -S uv run --script
2+
# /// script
3+
# requires-python = ">=3.9"
4+
# dependencies = [
5+
# "pytest>=8.0",
6+
# "pytest-codspeed>=4.2.0",
7+
# ]
8+
# ///
9+
10+
import argparse
11+
import os
12+
import shlex
13+
import subprocess
14+
from pathlib import Path
15+
16+
import pytest
17+
18+
19+
class ValgrindRunner:
20+
"""Run Valgrind with different configurations."""
21+
22+
def __init__(
23+
self,
24+
cmd: str,
25+
valgrind_path: str = "valgrind",
26+
output_dir: str = "/tmp",
27+
):
28+
"""Initialize valgrind runner.
29+
30+
Args:
31+
cmd: Command to profile (can be a path or arbitrary shell command)
32+
valgrind_path: Path to valgrind executable
33+
output_dir: Directory for callgrind output files
34+
"""
35+
self.cmd = cmd
36+
self.valgrind_path = valgrind_path
37+
self.output_dir = Path(output_dir)
38+
self.output_dir.mkdir(parents=True, exist_ok=True)
39+
40+
# Verify valgrind is available
41+
result = subprocess.run(
42+
[self.valgrind_path, "--version"],
43+
capture_output=True,
44+
text=True,
45+
)
46+
if result.returncode != 0:
47+
raise RuntimeError(f"Valgrind not found at: {self.valgrind_path}")
48+
self.valgrind_version = result.stdout.strip()
49+
50+
def run_valgrind(self, *args: str) -> None:
51+
"""Execute valgrind with given arguments.
52+
53+
Args:
54+
*args: Valgrind arguments
55+
"""
56+
callgrind_output = self.output_dir / f"callgrind.{os.getpid()}"
57+
58+
cmd = [
59+
self.valgrind_path,
60+
"--tool=callgrind",
61+
f"--callgrind-out-file={callgrind_output}",
62+
*args,
63+
*shlex.split(self.cmd),
64+
]
65+
66+
result = subprocess.run(cmd)
67+
if result.returncode != 0:
68+
raise RuntimeError(
69+
f"Valgrind execution failed with code {result.returncode}"
70+
)
71+
72+
# Clean up
73+
if callgrind_output.exists():
74+
callgrind_output.unlink()
75+
76+
77+
@pytest.fixture
78+
def runner(request):
79+
"""Fixture to provide runner instance to tests."""
80+
return request.config._valgrind_runner
81+
82+
83+
def pytest_generate_tests(metafunc):
84+
"""Parametrize tests with valgrind configurations."""
85+
if "valgrind_args" in metafunc.fixturenames:
86+
runner = getattr(metafunc.config, "_valgrind_runner", None)
87+
if not runner:
88+
return
89+
90+
# Define valgrind configurations
91+
configs = [
92+
(["--read-inline-info=no"], "no-inline"),
93+
(["--read-inline-info=yes"], "inline"),
94+
(
95+
[
96+
"--trace-children=yes",
97+
"--cache-sim=yes",
98+
"--I1=32768,8,64",
99+
"--D1=32768,8,64",
100+
"--LL=8388608,16,64",
101+
"--collect-systime=nsec",
102+
"--compress-strings=no",
103+
"--combine-dumps=yes",
104+
"--dump-line=no",
105+
"--read-inline-info=yes",
106+
],
107+
"full-with-inline",
108+
),
109+
(
110+
[
111+
"--trace-children=yes",
112+
"--cache-sim=yes",
113+
"--I1=32768,8,64",
114+
"--D1=32768,8,64",
115+
"--LL=8388608,16,64",
116+
"--collect-systime=nsec",
117+
"--compress-strings=no",
118+
"--combine-dumps=yes",
119+
"--dump-line=no",
120+
],
121+
"full-no-inline",
122+
),
123+
]
124+
125+
# Create test IDs with format: valgrind-version, command, config-name
126+
test_ids = [
127+
f"{runner.valgrind_version}, {runner.cmd}, {config_name}"
128+
for _, config_name in configs
129+
]
130+
131+
# Parametrize with just the args
132+
metafunc.parametrize(
133+
"valgrind_args",
134+
[args for args, _ in configs],
135+
ids=test_ids,
136+
)
137+
138+
139+
@pytest.mark.benchmark
140+
def test_valgrind(runner, valgrind_args):
141+
if runner:
142+
runner.run_valgrind(*valgrind_args)
143+
144+
145+
def main():
146+
parser = argparse.ArgumentParser(
147+
description="Benchmark Valgrind with pytest-codspeed",
148+
formatter_class=argparse.RawDescriptionHelpFormatter,
149+
epilog="""
150+
Examples:
151+
# Run with a binary path
152+
uv run bench.py --cmd /path/to/binary
153+
154+
# Run with an arbitrary command
155+
uv run bench.py --cmd 'echo "hello world"'
156+
157+
# Run with custom valgrind installation
158+
uv run bench.py --cmd /usr/bin/ls --valgrind-path /usr/local/bin/valgrind
159+
""",
160+
)
161+
162+
parser.add_argument(
163+
"--cmd",
164+
type=str,
165+
required=True,
166+
help="Command to profile (can be a path to a binary or any arbitrary command)",
167+
)
168+
parser.add_argument(
169+
"--valgrind-path",
170+
type=str,
171+
default="valgrind",
172+
help="Path to valgrind executable (default: valgrind)",
173+
)
174+
parser.add_argument(
175+
"--output-dir",
176+
type=str,
177+
default="/tmp",
178+
help="Directory for callgrind files (default: /tmp)",
179+
)
180+
181+
args = parser.parse_args()
182+
183+
# Create runner instance
184+
runner = ValgrindRunner(
185+
cmd=args.cmd,
186+
valgrind_path=args.valgrind_path,
187+
output_dir=args.output_dir,
188+
)
189+
print(f"Valgrind version: {runner.valgrind_version}")
190+
print(f"Command: {args.cmd}")
191+
192+
# Plugin to pass runner to tests
193+
class RunnerPlugin:
194+
def pytest_configure(self, config):
195+
config._valgrind_runner = runner
196+
197+
exit_code = pytest.main(
198+
[__file__, "-v", "--codspeed"],
199+
plugins=[RunnerPlugin()],
200+
)
201+
if exit_code != 0 and exit_code != 5:
202+
print(f"Benchmark execution returned exit code: {exit_code}")
203+
204+
205+
if __name__ == "__main__":
206+
main()

0 commit comments

Comments
 (0)