Skip to content

Commit dd56a00

Browse files
hillarysandershillarysanders
authored
Merge pull request #19 from heroku/integration_tests
Integration tests
2 parents 2025eff + f0f2b84 commit dd56a00

File tree

9 files changed

+443
-1
lines changed

9 files changed

+443
-1
lines changed

‎.github/workflows/test.yml

Lines changed: 160 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,160 @@
1+
name: MCP Tests
2+
3+
on:
4+
pull_request:
5+
jobs:
6+
###########################################################################
7+
# 1 - Local integration tests (always run)
8+
###########################################################################
9+
local:
10+
runs-on: ubuntu-latest
11+
# Dummy key lets the clients authenticate against the local servers.
12+
env:
13+
API_KEY: ci-test-key
14+
15+
steps:
16+
- uses: actions/checkout@v4
17+
18+
- name: Read Python version from .python-version
19+
id: python-version
20+
run: |
21+
PY_VERSION=$(cat .python-version)
22+
echo "version=$PY_VERSION" >> $GITHUB_OUTPUT
23+
24+
- uses: actions/setup-python@v5
25+
with:
26+
python-version: ${{ steps.python-version.outputs.version }}
27+
28+
- name: Read Node.js version from package.json
29+
id: node-version
30+
run: |
31+
NODE_VERSION=$(jq -r '.engines.node' package.json)
32+
echo "version=$NODE_VERSION" >> $GITHUB_OUTPUT
33+
34+
- uses: actions/setup-node@v4
35+
with:
36+
node-version: ${{ steps.node-version.outputs.version }}
37+
38+
- uses: actions/cache@v4
39+
with:
40+
path: ~/.cache/pip
41+
key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements*.txt') }}
42+
43+
- name: Install dependencies
44+
run: |
45+
python -m pip install --upgrade pip
46+
pip install -r requirements.txt
47+
48+
- name: Run pytest (local transports)
49+
run: pytest -q
50+
51+
###########################################################################
52+
# 2 - Deploy this PR to a temp Heroku app and run tests against deployed app (in addition to 'local')
53+
###########################################################################
54+
remote:
55+
runs-on: ubuntu-latest
56+
env:
57+
HEROKU_API_KEY: ${{ secrets.HEROKU_API_KEY }}
58+
API_KEY: ci-test-key
59+
# also note that github CI doesn't have access to your app's config vars, so here we're setting the remote
60+
# server type to streamable HTTP. Folks using SSE would need to change this line for their e2e remote integration
61+
# tests to test SSE instead of streamable HTTP.
62+
REMOTE_SERVER_TRANSPORT_MODULE: streamable_http_server
63+
# $APP_NAME is set below because we need to shorten the repo owner's name, as a precaution
64+
65+
steps:
66+
- uses: actions/checkout@v4
67+
with:
68+
fetch-depth: 0 # <-- disables shallow clone, which heroku is upset by when running git push heroku later on
69+
70+
# Setting a short $APP_NAME that will be unique even if folks choose to fork this repo --> avoids clashes.
71+
# Needs to be shortened if the github repo owner has a long name (max 30 char app name heroku limit).
72+
- name: Generate short APP_NAME
73+
id: appname
74+
run: |
75+
OWNER_SHORT=${GITHUB_REPOSITORY_OWNER:0:5}
76+
REPO_NAME=$(basename "$GITHUB_REPOSITORY")
77+
PR_NUMBER=$(jq .number "$GITHUB_EVENT_PATH")
78+
APP_NAME="${OWNER_SHORT}-${REPO_NAME}-${PR_NUMBER}"
79+
echo "APP_NAME=$APP_NAME"
80+
echo "APP_NAME=$APP_NAME" >> $GITHUB_ENV
81+
82+
- name: Read Python version from .python-version
83+
id: python-version
84+
run: |
85+
PY_VERSION=$(cat .python-version)
86+
echo "version=$PY_VERSION" >> $GITHUB_OUTPUT
87+
88+
- uses: actions/setup-python@v5
89+
with:
90+
python-version: ${{ steps.python-version.outputs.version }}
91+
92+
- name: Read Node.js version from package.json
93+
id: node-version
94+
run: |
95+
NODE_VERSION=$(jq -r '.engines.node' package.json)
96+
echo "version=$NODE_VERSION" >> $GITHUB_OUTPUT
97+
98+
- uses: actions/setup-node@v4
99+
with:
100+
node-version: ${{ steps.node-version.outputs.version }}
101+
102+
- name: Install Heroku CLI
103+
run: |
104+
curl https://cli-assets.heroku.com/install.sh | sh
105+
106+
- name: Log in to Heroku
107+
run: |
108+
echo "$HEROKU_API_KEY" | heroku auth:token
109+
110+
- name: Pre-cleanup (destroy app if it exists)
111+
continue-on-error: true
112+
run: |
113+
heroku apps:destroy --app $APP_NAME --confirm $APP_NAME
114+
115+
# github CI can't use our app.json, so the config etc bits must be set manually.
116+
# note WEB_CONCURRENCY is important! You get non-deterministic errors w/out it.
117+
- name: Create temp Heroku app for this PR
118+
run: |
119+
heroku create $APP_NAME
120+
heroku buildpacks:add --index 1 heroku/nodejs -a $APP_NAME
121+
heroku buildpacks:add --index 2 heroku/python -a $APP_NAME
122+
heroku config:set API_KEY=$API_KEY --app $APP_NAME
123+
heroku config:set STDIO_MODE_ONLY=false
124+
heroku config:set REMOTE_SERVER_TRANSPORT_MODULE=$REMOTE_SERVER_TRANSPORT_MODULE --app $APP_NAME
125+
heroku config:set WEB_CONCURRENCY=1 --app $APP_NAME
126+
127+
- name: Deploy this branch to Heroku
128+
run: |
129+
git push https://heroku:[email protected]/$APP_NAME.git HEAD:refs/heads/main --force
130+
131+
- uses: actions/cache@v4
132+
with:
133+
path: ~/.cache/pip
134+
key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements*.txt') }}
135+
136+
- name: Install test dependencies
137+
run: |
138+
python -m pip install --upgrade pip
139+
pip install -r requirements.txt
140+
141+
- name: Get Heroku env vars
142+
id: heroku_env
143+
run: |
144+
url=$(heroku info -s -a $APP_NAME | grep web_url | cut -d= -f2 | tr -d '\n')
145+
echo "url=$url" >> "$GITHUB_OUTPUT"
146+
147+
- name: Run pytest against deployed app
148+
env:
149+
MCP_SERVER_URL: ${{ steps.heroku_env.outputs.url }}
150+
run: |
151+
echo "APP_NAME = $APP_NAME"
152+
echo "MCP_SERVER_URL = $MCP_SERVER_URL"
153+
echo "REMOTE_SERVER_TRANSPORT_MODULE = $REMOTE_SERVER_TRANSPORT_MODULE"
154+
echo "API_KEY is ${API_KEY:+set}" # won't print the key, just confirms it's non-empty
155+
pytest -q
156+
157+
- name: Destroy Heroku app after test
158+
if: always()
159+
run: |
160+
heroku apps:destroy --app $APP_NAME --confirm $APP_NAME

‎package.json

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,9 @@
33
"version": "1.0.0",
44
"description": "Minimal Node.js setup for Heroku runtime",
55
"main": "index.js",
6+
"engines": {
7+
"node": "22.x"
8+
},
69
"dependencies": {
710
}
811
}

‎pytest.ini

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,4 @@
1+
[pytest]
2+
addopts = -ra
3+
asyncio_mode = auto
4+
asyncio_default_fixture_loop_scope = function

‎requirements.txt

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,8 @@
1-
mcp==1.9.3
1+
mcp[client,server]==1.9.2
22
fastapi==0.115.12
33
uvicorn==0.34.3
44
python-dotenv==1.1.0
55
mando==0.8.2
6+
# --- dev / CI only -------------------------------------------------
7+
pytest>=8.0
8+
pytest-asyncio>=0.23

‎tests/README.md

Lines changed: 53 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,53 @@
1+
# End-to-End Test Suite (`tests/`)
2+
3+
You know what's better than unit tests? End to end integration tests.
4+
5+
These pytest tests run **every MCP transport** this repo supports, both locally and (optionally) against a deployed Heroku app.
6+
7+
| Context ID | Transport exercised | Needs running **web** dyno? |
8+
|-------------------|-----------------------------------------------------------|-----------------------------|
9+
| `http_local` | Streamable HTTP on `localhost:8000/mcp/` | — |
10+
| `sse_local` | SSE on `localhost:8000/mcp/sse` | — |
11+
| `stdio_local` | STDIO (example client boots its own server) | — |
12+
| `remote` | Transport named in **`$REMOTE_SERVER_TRANSPORT_MODULE`**&nbsp;(`streamable_http_server` or `sse_server`) served by your web dyno | **Yes** |
13+
| `remote_stdio` | STDIO via a **one-off Heroku dyno** | **No** – works even at `web=0` |
14+
15+
*If the web dyno is asleep or scaled to `0`, the `remote` tests auto-skip.
16+
`remote_stdio` still runs, because it spins up its own one-off dyno.*
17+
18+
To scale up the number of web dynos running in your app to 1, run:
19+
```bash
20+
heroku ps:scale web=1 -a "$APP_NAME"
21+
```
22+
23+
---
24+
All of these tests will run automatically via github CI when you push up a pull request.
25+
However, if you wish to run these end to end integration tests locally, you can do that too.
26+
27+
## 1 · Install dependencies
28+
29+
```bash
30+
# inside an activated venv
31+
pip install -r requirements.txt
32+
```
33+
34+
## Run E2E Integration Tests
35+
Next, deploy your app. This is required for all E2E tests to run (some tests will be skipped if an app is not deployed & `MCP_SERVER_URL` is not set).
36+
```bash
37+
git push heroku <your-branch>:main
38+
```
39+
40+
## 1 · Run local & one-off-dyno (STDIO) deployed transports
41+
```bash
42+
REMOTE_SERVER_TRANSPORT_MODULE=$(heroku config:get REMOTE_SERVER_TRANSPORT_MODULE) pytest tests -q
43+
```
44+
45+
## 2 - Run local & all deployed transports
46+
```bash
47+
REMOTE_SERVER_TRANSPORT_MODULE=$(heroku config:get REMOTE_SERVER_TRANSPORT_MODULE) \
48+
MCP_SERVER_URL=$(heroku info -s -a "$APP_NAME" | grep web_url | cut -d= -f2 | tr -d '\n') \
49+
API_KEY=$(heroku config:get API_KEY -a "$APP_NAME") \
50+
pytest tests -q
51+
```
52+
53+
*NOTE: if your `REMOTE_SERVER_TRANSPORT_MODULE` is set to `sse_server` and not the default `streamable_http_server`, you'll need to change the `REMOTE_SERVER_TRANSPORT_MODULE` declaration line in `.github/workflows/test.yml` to make sure that the end to end integration tests against the temporary deployed remote server are using the appropriate client code.*

‎tests/__init__.py

Whitespace-only changes.

‎tests/client_runner.py

Lines changed: 54 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,54 @@
1+
"""
2+
Launch one of the example_clients in a fresh subprocess and return STDOUT.
3+
"""
4+
from __future__ import annotations
5+
import asyncio, os, sys, subprocess, textwrap
6+
from pathlib import Path
7+
from typing import Sequence, Mapping
8+
9+
ROOT = Path(__file__).resolve().parents[1]
10+
EXAMPLES = ROOT / "example_clients"
11+
PYTHON_EXE = sys.executable
12+
13+
14+
async def call_client(
15+
module_name: str,
16+
cli_args: Sequence[str],
17+
extra_env: Mapping[str, str] | None = None,
18+
) -> str:
19+
env = os.environ.copy() | (extra_env or {})
20+
21+
# ----- NEW: run STDIO client inside a Heroku one-off dyno ----------
22+
if module_name == "remote_stdio":
23+
app = env.get("APP_NAME")
24+
if not app:
25+
raise RuntimeError("APP_NAME env-var required for remote_stdio context")
26+
cmd = [
27+
"heroku", "run", "--exit-code", "--app", app, "--",
28+
"python", "-m", "example_clients.stdio_client", "mcp", *cli_args,
29+
]
30+
# -------------------------------------------------------------------
31+
else:
32+
cmd = [
33+
PYTHON_EXE,
34+
"-m", f"example_clients.{module_name}",
35+
"mcp", *cli_args,
36+
]
37+
38+
proc = await asyncio.create_subprocess_exec(
39+
*cmd, cwd=ROOT, env=env,
40+
stdout=subprocess.PIPE, stderr=subprocess.PIPE
41+
)
42+
out_b, err_b = await proc.communicate()
43+
out, err = out_b.decode(), err_b.decode()
44+
if proc.returncode:
45+
raise RuntimeError(
46+
textwrap.dedent(
47+
f"""
48+
Client {module_name} exited with {proc.returncode}
49+
CMD : {' '.join(cmd)}
50+
STDERR:
51+
{err}"""
52+
)
53+
)
54+
return out

0 commit comments

Comments
 (0)