Skip to content

Commit 13b0aa9

Browse files
pranaygppranaygp
committed
add benchmarking
1 parent 3aee6d7 commit 13b0aa9

File tree

6 files changed

+331
-0
lines changed

6 files changed

+331
-0
lines changed

‎.github/workflows/benchmarks.yml‎

Lines changed: 165 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,165 @@
1+
name: Performance Benchmarks
2+
3+
on:
4+
pull_request:
5+
branches: [main]
6+
push:
7+
branches: [main]
8+
workflow_dispatch: # Allow manual triggers
9+
10+
jobs:
11+
benchmark-local-prod:
12+
name: Benchmark Local Prod (${{ matrix.app.name }} - ${{ matrix.backend }})
13+
runs-on: ubuntu-latest
14+
timeout-minutes: 60
15+
strategy:
16+
fail-fast: false
17+
matrix:
18+
# Start with subset, expand later
19+
app:
20+
- name: "nextjs-turbopack"
21+
- name: "nitro"
22+
- name: "express"
23+
backend: [local, postgres]
24+
25+
services:
26+
postgres:
27+
image: postgres:18-alpine
28+
env:
29+
POSTGRES_USER: world
30+
POSTGRES_PASSWORD: world
31+
POSTGRES_DB: world
32+
ports:
33+
- 5432:5432
34+
options: >-
35+
--health-cmd pg_isready
36+
--health-interval 10s
37+
--health-timeout 5s
38+
--health-retries 5
39+
40+
env:
41+
TURBO_TOKEN: ${{ secrets.TURBO_TOKEN }}
42+
TURBO_TEAM: ${{ vars.TURBO_TEAM }}
43+
WORKFLOW_TARGET_WORLD: ${{ matrix.backend == 'postgres' && '@workflow/world-postgres' || '@workflow/world-local' }}
44+
WORKFLOW_POSTGRES_URL: ${{ matrix.backend == 'postgres' && 'postgres://world:world@localhost:5432/world' || '' }}
45+
46+
steps:
47+
- uses: actions/checkout@v4
48+
49+
- uses: pnpm/action-setup@v3
50+
with:
51+
version: 10.14.0
52+
53+
- uses: actions/setup-node@v4
54+
with:
55+
node-version: 22.x
56+
cache: 'pnpm'
57+
58+
- name: Install dependencies
59+
run: pnpm install --frozen-lockfile
60+
61+
- name: Build all packages
62+
run: pnpm turbo run build --filter='!./workbench/*'
63+
64+
- name: Setup PostgreSQL database
65+
if: matrix.backend == 'postgres'
66+
run: ./packages/world-postgres/bin/setup.js
67+
68+
- name: Build workbench
69+
run: cd workbench/${{ matrix.app.name }} && pnpm build
70+
71+
- name: Start production server in background
72+
run: |
73+
cd workbench/${{ matrix.app.name }}
74+
pnpm start > /tmp/prod.log 2>&1 &
75+
echo $! > /tmp/prod.pid
76+
sleep 10
77+
78+
- name: Wait for server to be ready
79+
run: |
80+
timeout 120 bash -c 'until curl -f http://localhost:3000 > /dev/null 2>&1; do sleep 2; done'
81+
82+
- name: Run benchmarks
83+
env:
84+
DEPLOYMENT_URL: "http://localhost:3000"
85+
APP_NAME: ${{ matrix.app.name }}
86+
run: |
87+
pnpm vitest bench packages/core/e2e/bench.bench.ts --run > bench-results-${{ matrix.app.name }}-${{ matrix.backend }}.txt
88+
89+
- name: Stop production server
90+
if: always()
91+
run: |
92+
if [ -f /tmp/prod.pid ]; then
93+
kill $(cat /tmp/prod.pid) || true
94+
fi
95+
96+
- name: Upload benchmark results
97+
uses: actions/upload-artifact@v4
98+
with:
99+
name: bench-results-${{ matrix.app.name }}-${{ matrix.backend }}
100+
path: bench-results-${{ matrix.app.name }}-${{ matrix.backend }}.txt
101+
102+
benchmark-vercel-prod:
103+
name: Benchmark Vercel Prod (${{ matrix.app.name }})
104+
runs-on: ubuntu-latest
105+
timeout-minutes: 60
106+
strategy:
107+
fail-fast: false
108+
matrix:
109+
app:
110+
- name: "nextjs-turbopack"
111+
project-id: "prj_yjkM7UdHliv8bfxZ1sMJQf1pMpdi"
112+
- name: "nitro"
113+
project-id: "prj_e7DZirYdLrQKXNrlxg7KmA6ABx8r"
114+
- name: "express"
115+
project-id: "prj_cCZjpBy92VRbKHHbarDMhOHtkuIr"
116+
117+
env:
118+
TURBO_TOKEN: ${{ secrets.TURBO_TOKEN }}
119+
TURBO_TEAM: ${{ vars.TURBO_TEAM }}
120+
121+
steps:
122+
- uses: actions/checkout@v4
123+
124+
- uses: pnpm/action-setup@v3
125+
with:
126+
version: 10.14.0
127+
128+
- uses: actions/setup-node@v4
129+
with:
130+
node-version: 22.x
131+
cache: 'pnpm'
132+
133+
- name: Install dependencies
134+
run: pnpm install --frozen-lockfile
135+
136+
- name: Build CLI
137+
run: pnpm turbo run build --filter='@workflow/cli'
138+
139+
- name: Wait for Vercel deployment
140+
id: waitForDeployment
141+
uses: ./.github/actions/wait-for-vercel-project
142+
with:
143+
team-id: "team_nO2mCG4W8IxPIeKoSsqwAxxB"
144+
project-id: ${{ matrix.app.project-id }}
145+
vercel-token: ${{ secrets.VERCEL_LABS_TOKEN }}
146+
timeout: 1000
147+
check-interval: 15
148+
environment: ${{ github.ref == 'refs/heads/main' && 'production' || 'preview' }}
149+
150+
- name: Run benchmarks
151+
env:
152+
DEPLOYMENT_URL: ${{ steps.waitForDeployment.outputs.deployment-url }}
153+
APP_NAME: ${{ matrix.app.name }}
154+
WORKFLOW_VERCEL_ENV: ${{ github.ref == 'refs/heads/main' && 'production' || 'preview' }}
155+
WORKFLOW_VERCEL_AUTH_TOKEN: ${{ secrets.VERCEL_LABS_TOKEN }}
156+
WORKFLOW_VERCEL_TEAM: "team_nO2mCG4W8IxPIeKoSsqwAxxB"
157+
WORKFLOW_VERCEL_PROJECT: ${{ matrix.app.project-id }}
158+
run: |
159+
pnpm vitest bench packages/core/e2e/bench.bench.ts --run > bench-results-${{ matrix.app.name }}-vercel.txt
160+
161+
- name: Upload benchmark results
162+
uses: actions/upload-artifact@v4
163+
with:
164+
name: bench-results-${{ matrix.app.name }}-vercel
165+
path: bench-results-${{ matrix.app.name }}-vercel.txt

‎package.json‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -32,6 +32,8 @@
3232
"clean": "turbo clean",
3333
"typecheck": "turbo typecheck",
3434
"test:e2e": "vitest run packages/core/e2e/e2e.test.ts",
35+
"bench": "vitest bench packages/core/e2e/bench.bench.ts",
36+
"bench:local": "DEPLOYMENT_URL=http://localhost:3000 APP_NAME=nextjs-turbopack vitest bench packages/core/e2e/bench.bench.ts",
3537
"lint": "biome check",
3638
"format": "biome format --write",
3739
"changeset": "changeset",

‎packages/core/e2e/bench.bench.ts‎

Lines changed: 113 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,113 @@
1+
import { withResolvers } from '@workflow/utils';
2+
import { bench, describe } from 'vitest';
3+
import { dehydrateWorkflowArguments } from '../src/serialization';
4+
5+
const deploymentUrl = process.env.DEPLOYMENT_URL;
6+
if (!deploymentUrl) {
7+
throw new Error('`DEPLOYMENT_URL` environment variable is not set');
8+
}
9+
10+
async function triggerWorkflow(
11+
workflow: string | { workflowFile: string; workflowFn: string },
12+
args: any[]
13+
): Promise<{ runId: string }> {
14+
const url = new URL('/api/trigger', deploymentUrl);
15+
const workflowFn =
16+
typeof workflow === 'string' ? workflow : workflow.workflowFn;
17+
const workflowFile =
18+
typeof workflow === 'string'
19+
? 'workflows/97_bench.ts'
20+
: workflow.workflowFile;
21+
22+
url.searchParams.set('workflowFile', workflowFile);
23+
url.searchParams.set('workflowFn', workflowFn);
24+
25+
const ops: Promise<void>[] = [];
26+
const { promise: runIdPromise, resolve: resolveRunId } =
27+
withResolvers<string>();
28+
const dehydratedArgs = dehydrateWorkflowArguments(args, ops, runIdPromise);
29+
30+
const res = await fetch(url, {
31+
method: 'POST',
32+
body: JSON.stringify(dehydratedArgs),
33+
});
34+
if (!res.ok) {
35+
throw new Error(
36+
`Failed to trigger workflow: ${res.url} ${
37+
res.status
38+
}: ${await res.text()}`
39+
);
40+
}
41+
const run = await res.json();
42+
resolveRunId(run.runId);
43+
44+
// Resolve and wait for any stream operations
45+
await Promise.all(ops);
46+
47+
return run;
48+
}
49+
50+
async function getWorkflowReturnValue(runId: string) {
51+
// We need to poll the GET endpoint until the workflow run is completed.
52+
while (true) {
53+
const url = new URL('/api/trigger', deploymentUrl);
54+
url.searchParams.set('runId', runId);
55+
56+
const res = await fetch(url);
57+
58+
if (res.status === 202) {
59+
// Workflow run is still running, so we need to wait and poll again
60+
await new Promise((resolve) => setTimeout(resolve, 100));
61+
continue;
62+
}
63+
const contentType = res.headers.get('Content-Type');
64+
65+
if (contentType?.includes('application/json')) {
66+
return await res.json();
67+
}
68+
69+
if (contentType?.includes('application/octet-stream')) {
70+
return res.body;
71+
}
72+
73+
throw new Error(`Unexpected content type: ${contentType}`);
74+
}
75+
}
76+
77+
describe('Workflow Performance Benchmarks', () => {
78+
bench(
79+
'workflow with no steps',
80+
async () => {
81+
const { runId } = await triggerWorkflow('noStepsWorkflow', [42]);
82+
await getWorkflowReturnValue(runId);
83+
},
84+
{ time: 5000 }
85+
);
86+
87+
bench(
88+
'workflow with 1 step',
89+
async () => {
90+
const { runId } = await triggerWorkflow('oneStepWorkflow', [100]);
91+
await getWorkflowReturnValue(runId);
92+
},
93+
{ time: 5000 }
94+
);
95+
96+
bench(
97+
'workflow with 10 sequential steps',
98+
async () => {
99+
const { runId } = await triggerWorkflow('tenSequentialStepsWorkflow', []);
100+
await getWorkflowReturnValue(runId);
101+
},
102+
{ time: 5000 }
103+
);
104+
105+
bench(
106+
'workflow with 10 parallel steps',
107+
async () => {
108+
const { runId } = await triggerWorkflow('tenParallelStepsWorkflow', []);
109+
await getWorkflowReturnValue(runId);
110+
},
111+
{ time: 5000 }
112+
);
113+
});

‎vitest.config.ts‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,10 @@
1+
import { defineConfig } from 'vitest/config';
2+
3+
export default defineConfig({
4+
test: {
5+
testTimeout: 60_000,
6+
},
7+
benchmark: {
8+
include: ['**/*.bench.test.ts'],
9+
},
10+
});

‎workbench/example/workflows/97_bench.ts‎

Lines changed: 40 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,40 @@
1+
// Benchmark workflows for performance testing
2+
3+
async function doWork() {
4+
'use step';
5+
return 42;
6+
}
7+
8+
// Workflow with no steps - pure orchestration
9+
export async function noStepsWorkflow(input: number) {
10+
'use workflow';
11+
return input * 2;
12+
}
13+
14+
// Workflow with 1 step
15+
export async function oneStepWorkflow(input: number) {
16+
'use workflow';
17+
const result = await doWork();
18+
return result + input;
19+
}
20+
21+
// Workflow with 10 sequential steps
22+
export async function tenSequentialStepsWorkflow() {
23+
'use workflow';
24+
let result = 0;
25+
for (let i = 0; i < 10; i++) {
26+
result = await doWork();
27+
}
28+
return result;
29+
}
30+
31+
// Workflow with 10 parallel steps
32+
export async function tenParallelStepsWorkflow() {
33+
'use workflow';
34+
const promises = [];
35+
for (let i = 0; i < 10; i++) {
36+
promises.push(doWork());
37+
}
38+
const results = await Promise.all(promises);
39+
return results.reduce((sum, val) => sum + val, 0);
40+
}

‎workbench/nextjs-turbopack/workflows/97_bench.ts‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
../../example/workflows/97_bench.ts

0 commit comments

Comments
 (0)