Skip to content

Commit c84c8bc

Browse files
pranaygppranaygp
committed
add benchmarking
1 parent 3aee6d7 commit c84c8bc

File tree

6 files changed

+381
-0
lines changed

6 files changed

+381
-0
lines changed

‎.github/workflows/benchmarks.yml‎

Lines changed: 213 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,213 @@
1+
name: Performance Benchmarks
2+
3+
on:
4+
pull_request:
5+
branches: [main]
6+
push:
7+
branches: [main]
8+
workflow_dispatch: # Allow manual triggers
9+
10+
jobs:
11+
benchmark-local-prod:
12+
name: Benchmark Local Prod (${{ matrix.app.name }} - ${{ matrix.backend }})
13+
runs-on: ubuntu-latest
14+
timeout-minutes: 60
15+
strategy:
16+
fail-fast: false
17+
matrix:
18+
# Start with subset, expand later
19+
app:
20+
- name: "nextjs-turbopack"
21+
- name: "nitro"
22+
- name: "express"
23+
backend: [local, postgres]
24+
25+
services:
26+
postgres:
27+
image: postgres:18-alpine
28+
env:
29+
POSTGRES_USER: world
30+
POSTGRES_PASSWORD: world
31+
POSTGRES_DB: world
32+
ports:
33+
- 5432:5432
34+
options: >-
35+
--health-cmd pg_isready
36+
--health-interval 10s
37+
--health-timeout 5s
38+
--health-retries 5
39+
40+
env:
41+
TURBO_TOKEN: ${{ secrets.TURBO_TOKEN }}
42+
TURBO_TEAM: ${{ vars.TURBO_TEAM }}
43+
WORKFLOW_TARGET_WORLD: ${{ matrix.backend == 'postgres' && '@workflow/world-postgres' || '@workflow/world-local' }}
44+
WORKFLOW_POSTGRES_URL: ${{ matrix.backend == 'postgres' && 'postgres://world:world@localhost:5432/world' || '' }}
45+
46+
steps:
47+
- uses: actions/checkout@v4
48+
49+
- uses: pnpm/action-setup@v3
50+
with:
51+
version: 10.14.0
52+
53+
- uses: actions/setup-node@v4
54+
with:
55+
node-version: 22.x
56+
cache: 'pnpm'
57+
58+
- name: Install dependencies
59+
run: pnpm install --frozen-lockfile
60+
61+
- name: Build all packages
62+
run: pnpm turbo run build --filter='!./workbench/*'
63+
64+
- name: Setup PostgreSQL database
65+
if: matrix.backend == 'postgres'
66+
run: ./packages/world-postgres/bin/setup.js
67+
68+
- name: Build workbench
69+
run: cd workbench/${{ matrix.app.name }} && pnpm build
70+
71+
- name: Start production server in background
72+
run: |
73+
cd workbench/${{ matrix.app.name }}
74+
pnpm start > /tmp/prod.log 2>&1 &
75+
echo $! > /tmp/prod.pid
76+
sleep 10
77+
78+
- name: Wait for server to be ready
79+
run: |
80+
timeout 60 bash -c 'until curl -f http://localhost:3000 > /dev/null 2>&1; do sleep 2; done'
81+
82+
- name: Run benchmarks
83+
env:
84+
DEPLOYMENT_URL: "http://localhost:3000"
85+
APP_NAME: ${{ matrix.app.name }}
86+
run: |
87+
pnpm vitest bench packages/core/e2e/bench.bench.ts \
88+
--reporter=default --reporter=json \
89+
--outputFile=bench-results-${{ matrix.app.name }}-${{ matrix.backend }}.json
90+
91+
- name: Stop production server
92+
if: always()
93+
run: |
94+
if [ -f /tmp/prod.pid ]; then
95+
kill $(cat /tmp/prod.pid) || true
96+
fi
97+
98+
- name: Upload benchmark results
99+
uses: actions/upload-artifact@v4
100+
with:
101+
name: bench-results-${{ matrix.app.name }}-${{ matrix.backend }}
102+
path: bench-results-${{ matrix.app.name }}-${{ matrix.backend }}.json
103+
104+
- name: Store benchmark result
105+
uses: benchmark-action/github-action-benchmark@v1
106+
if: github.event_name == 'push' && github.ref == 'refs/heads/main'
107+
with:
108+
tool: 'vitest'
109+
output-file-path: bench-results-${{ matrix.app.name }}-${{ matrix.backend }}.json
110+
github-token: ${{ secrets.GITHUB_TOKEN }}
111+
auto-push: true
112+
benchmark-data-dir-path: 'benchmarks/${{ matrix.app.name }}-${{ matrix.backend }}'
113+
114+
- name: Comment benchmark on PR
115+
uses: benchmark-action/github-action-benchmark@v1
116+
if: github.event_name == 'pull_request'
117+
with:
118+
tool: 'vitest'
119+
output-file-path: bench-results-${{ matrix.app.name }}-${{ matrix.backend }}.json
120+
github-token: ${{ secrets.GITHUB_TOKEN }}
121+
comment-on-alert: true
122+
alert-threshold: '150%'
123+
fail-on-alert: false
124+
benchmark-data-dir-path: 'benchmarks/${{ matrix.app.name }}-${{ matrix.backend }}'
125+
126+
benchmark-vercel-prod:
127+
name: Benchmark Vercel Prod (${{ matrix.app.name }})
128+
runs-on: ubuntu-latest
129+
timeout-minutes: 60
130+
strategy:
131+
fail-fast: false
132+
matrix:
133+
app:
134+
- name: "nextjs-turbopack"
135+
project-id: "prj_yjkM7UdHliv8bfxZ1sMJQf1pMpdi"
136+
- name: "nitro"
137+
project-id: "prj_e7DZirYdLrQKXNrlxg7KmA6ABx8r"
138+
- name: "express"
139+
project-id: "prj_cCZjpBy92VRbKHHbarDMhOHtkuIr"
140+
141+
env:
142+
TURBO_TOKEN: ${{ secrets.TURBO_TOKEN }}
143+
TURBO_TEAM: ${{ vars.TURBO_TEAM }}
144+
145+
steps:
146+
- uses: actions/checkout@v4
147+
148+
- uses: pnpm/action-setup@v3
149+
with:
150+
version: 10.14.0
151+
152+
- uses: actions/setup-node@v4
153+
with:
154+
node-version: 22.x
155+
cache: 'pnpm'
156+
157+
- name: Install dependencies
158+
run: pnpm install --frozen-lockfile
159+
160+
- name: Build CLI
161+
run: pnpm turbo run build --filter='@workflow/cli'
162+
163+
- name: Wait for Vercel deployment
164+
id: waitForDeployment
165+
uses: ./.github/actions/wait-for-vercel-project
166+
with:
167+
team-id: "team_nO2mCG4W8IxPIeKoSsqwAxxB"
168+
project-id: ${{ matrix.app.project-id }}
169+
vercel-token: ${{ secrets.VERCEL_LABS_TOKEN }}
170+
timeout: 1000
171+
check-interval: 15
172+
environment: ${{ github.ref == 'refs/heads/main' && 'production' || 'preview' }}
173+
174+
- name: Run benchmarks
175+
env:
176+
DEPLOYMENT_URL: ${{ steps.waitForDeployment.outputs.deployment-url }}
177+
APP_NAME: ${{ matrix.app.name }}
178+
WORKFLOW_VERCEL_ENV: ${{ github.ref == 'refs/heads/main' && 'production' || 'preview' }}
179+
WORKFLOW_VERCEL_AUTH_TOKEN: ${{ secrets.VERCEL_LABS_TOKEN }}
180+
WORKFLOW_VERCEL_TEAM: "team_nO2mCG4W8IxPIeKoSsqwAxxB"
181+
WORKFLOW_VERCEL_PROJECT: ${{ matrix.app.project-id }}
182+
run: |
183+
pnpm vitest bench packages/core/e2e/bench.bench.ts \
184+
--reporter=default --reporter=json \
185+
--outputFile=bench-results-${{ matrix.app.name }}-vercel.json
186+
187+
- name: Upload benchmark results
188+
uses: actions/upload-artifact@v4
189+
with:
190+
name: bench-results-${{ matrix.app.name }}-vercel
191+
path: bench-results-${{ matrix.app.name }}-vercel.json
192+
193+
- name: Store benchmark result
194+
uses: benchmark-action/github-action-benchmark@v1
195+
if: github.event_name == 'push' && github.ref == 'refs/heads/main'
196+
with:
197+
tool: 'vitest'
198+
output-file-path: bench-results-${{ matrix.app.name }}-vercel.json
199+
github-token: ${{ secrets.GITHUB_TOKEN }}
200+
auto-push: true
201+
benchmark-data-dir-path: 'benchmarks/${{ matrix.app.name }}-vercel'
202+
203+
- name: Comment benchmark on PR
204+
uses: benchmark-action/github-action-benchmark@v1
205+
if: github.event_name == 'pull_request'
206+
with:
207+
tool: 'vitest'
208+
output-file-path: bench-results-${{ matrix.app.name }}-vercel.json
209+
github-token: ${{ secrets.GITHUB_TOKEN }}
210+
comment-on-alert: true
211+
alert-threshold: '150%'
212+
fail-on-alert: false
213+
benchmark-data-dir-path: 'benchmarks/${{ matrix.app.name }}-vercel'

‎package.json‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -32,6 +32,8 @@
3232
"clean": "turbo clean",
3333
"typecheck": "turbo typecheck",
3434
"test:e2e": "vitest run packages/core/e2e/e2e.test.ts",
35+
"bench": "vitest bench packages/core/e2e/bench.bench.ts",
36+
"bench:local": "DEPLOYMENT_URL=http://localhost:3000 APP_NAME=nextjs-turbopack vitest bench packages/core/e2e/bench.bench.ts",
3537
"lint": "biome check",
3638
"format": "biome format --write",
3739
"changeset": "changeset",

‎packages/core/e2e/bench.bench.ts‎

Lines changed: 113 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,113 @@
1+
import { withResolvers } from '@workflow/utils';
2+
import { bench, describe } from 'vitest';
3+
import { dehydrateWorkflowArguments } from '../src/serialization';
4+
5+
const deploymentUrl = process.env.DEPLOYMENT_URL;
6+
if (!deploymentUrl) {
7+
throw new Error('`DEPLOYMENT_URL` environment variable is not set');
8+
}
9+
10+
async function triggerWorkflow(
11+
workflow: string | { workflowFile: string; workflowFn: string },
12+
args: any[]
13+
): Promise<{ runId: string }> {
14+
const url = new URL('/api/trigger', deploymentUrl);
15+
const workflowFn =
16+
typeof workflow === 'string' ? workflow : workflow.workflowFn;
17+
const workflowFile =
18+
typeof workflow === 'string'
19+
? 'workflows/97_bench.ts'
20+
: workflow.workflowFile;
21+
22+
url.searchParams.set('workflowFile', workflowFile);
23+
url.searchParams.set('workflowFn', workflowFn);
24+
25+
const ops: Promise<void>[] = [];
26+
const { promise: runIdPromise, resolve: resolveRunId } =
27+
withResolvers<string>();
28+
const dehydratedArgs = dehydrateWorkflowArguments(args, ops, runIdPromise);
29+
30+
const res = await fetch(url, {
31+
method: 'POST',
32+
body: JSON.stringify(dehydratedArgs),
33+
});
34+
if (!res.ok) {
35+
throw new Error(
36+
`Failed to trigger workflow: ${res.url} ${
37+
res.status
38+
}: ${await res.text()}`
39+
);
40+
}
41+
const run = await res.json();
42+
resolveRunId(run.runId);
43+
44+
// Resolve and wait for any stream operations
45+
await Promise.all(ops);
46+
47+
return run;
48+
}
49+
50+
async function getWorkflowReturnValue(runId: string) {
51+
// We need to poll the GET endpoint until the workflow run is completed.
52+
while (true) {
53+
const url = new URL('/api/trigger', deploymentUrl);
54+
url.searchParams.set('runId', runId);
55+
56+
const res = await fetch(url);
57+
58+
if (res.status === 202) {
59+
// Workflow run is still running, so we need to wait and poll again
60+
await new Promise((resolve) => setTimeout(resolve, 100));
61+
continue;
62+
}
63+
const contentType = res.headers.get('Content-Type');
64+
65+
if (contentType?.includes('application/json')) {
66+
return await res.json();
67+
}
68+
69+
if (contentType?.includes('application/octet-stream')) {
70+
return res.body;
71+
}
72+
73+
throw new Error(`Unexpected content type: ${contentType}`);
74+
}
75+
}
76+
77+
describe('Workflow Performance Benchmarks', () => {
78+
bench(
79+
'workflow with no steps',
80+
async () => {
81+
const { runId } = await triggerWorkflow('noStepsWorkflow', [42]);
82+
await getWorkflowReturnValue(runId);
83+
},
84+
{ time: 5000 }
85+
);
86+
87+
bench(
88+
'workflow with 1 step',
89+
async () => {
90+
const { runId } = await triggerWorkflow('oneStepWorkflow', [100]);
91+
await getWorkflowReturnValue(runId);
92+
},
93+
{ time: 5000 }
94+
);
95+
96+
bench(
97+
'workflow with 10 sequential steps',
98+
async () => {
99+
const { runId } = await triggerWorkflow('tenSequentialStepsWorkflow', []);
100+
await getWorkflowReturnValue(runId);
101+
},
102+
{ time: 5000 }
103+
);
104+
105+
bench(
106+
'workflow with 10 parallel steps',
107+
async () => {
108+
const { runId } = await triggerWorkflow('tenParallelStepsWorkflow', []);
109+
await getWorkflowReturnValue(runId);
110+
},
111+
{ time: 5000 }
112+
);
113+
});

‎vitest.config.ts‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,12 @@
1+
import { defineConfig } from 'vitest/config';
2+
3+
export default defineConfig({
4+
test: {
5+
testTimeout: 60_000,
6+
},
7+
benchmark: {
8+
include: ['**/*.bench.test.ts'],
9+
reporters: ['default', 'json'],
10+
outputFile: './bench-results.json',
11+
},
12+
});

‎workbench/example/workflows/97_bench.ts‎

Lines changed: 40 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,40 @@
1+
// Benchmark workflows for performance testing
2+
3+
async function doWork() {
4+
'use step';
5+
return 42;
6+
}
7+
8+
// Workflow with no steps - pure orchestration
9+
export async function noStepsWorkflow(input: number) {
10+
'use workflow';
11+
return input * 2;
12+
}
13+
14+
// Workflow with 1 step
15+
export async function oneStepWorkflow(input: number) {
16+
'use workflow';
17+
const result = await doWork();
18+
return result + input;
19+
}
20+
21+
// Workflow with 10 sequential steps
22+
export async function tenSequentialStepsWorkflow() {
23+
'use workflow';
24+
let result = 0;
25+
for (let i = 0; i < 10; i++) {
26+
result = await doWork();
27+
}
28+
return result;
29+
}
30+
31+
// Workflow with 10 parallel steps
32+
export async function tenParallelStepsWorkflow() {
33+
'use workflow';
34+
const promises = [];
35+
for (let i = 0; i < 10; i++) {
36+
promises.push(doWork());
37+
}
38+
const results = await Promise.all(promises);
39+
return results.reduce((sum, val) => sum + val, 0);
40+
}

‎workbench/nextjs-turbopack/workflows/97_bench.ts‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
../../example/workflows/97_bench.ts

0 commit comments

Comments
 (0)