Skip to content

Commit ca3cbc3

Browse files
sleepyStickNoahStapp
sleepyStick
and
NoahStapp
authored
PYTHON-5253 Automated Spec Test Sync (#2409)
Co-authored-by: Noah Stapp <[email protected]>
1 parent 84db915 commit ca3cbc3

11 files changed

+493
-6
lines changed

‎.evergreen/config.yml

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -42,3 +42,23 @@ post:
4242
- func: "upload mo artifacts"
4343
- func: "upload test results"
4444
- func: "cleanup"
45+
46+
tasks:
47+
- name: resync_specs
48+
commands:
49+
- command: subprocess.exec
50+
params:
51+
binary: bash
52+
include_expansions_in_env: [AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY, AWS_SESSION_TOKEN]
53+
args:
54+
- .evergreen/scripts/resync-all-specs.sh
55+
working_dir: src
56+
57+
buildvariants:
58+
- name: resync_specs
59+
display_name: "Resync Specs"
60+
run_on: rhel80-small
61+
cron: '0 16 * * MON'
62+
patchable: true
63+
tasks:
64+
- name: resync_specs

‎.evergreen/remove-unimplemented-tests.sh

Lines changed: 45 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,45 @@
1+
#!/bin/bash
2+
PYMONGO=$(dirname "$(cd "$(dirname "$0")" || exit; pwd)")
3+
4+
rm $PYMONGO/test/transactions/legacy/errors-client.json # PYTHON-1894
5+
rm $PYMONGO/test/connection_monitoring/wait-queue-fairness.json # PYTHON-1873
6+
rm $PYMONGO/test/client-side-encryption/spec/unified/fle2v2-BypassQueryAnalysis.json # PYTHON-5143
7+
rm $PYMONGO/test/client-side-encryption/spec/unified/fle2v2-EncryptedFields-vs-EncryptedFieldsMap.json # PYTHON-5143
8+
rm $PYMONGO/test/client-side-encryption/spec/unified/localSchema.json # PYTHON-5143
9+
rm $PYMONGO/test/client-side-encryption/spec/unified/maxWireVersion.json # PYTHON-5143
10+
rm $PYMONGO/test/unified-test-format/valid-pass/poc-queryable-encryption.json # PYTHON-5143
11+
rm $PYMONGO/test/gridfs/rename.json # PYTHON-4931
12+
rm $PYMONGO/test/discovery_and_monitoring/unified/pool-clear-application-error.json # PYTHON-4918
13+
rm $PYMONGO/test/discovery_and_monitoring/unified/pool-clear-checkout-error.json # PYTHON-4918
14+
rm $PYMONGO/test/discovery_and_monitoring/unified/pool-clear-min-pool-size-error.json # PYTHON-4918
15+
16+
# Python doesn't implement DRIVERS-3064
17+
rm $PYMONGO/test/collection_management/listCollections-rawdata.json
18+
rm $PYMONGO/test/crud/unified/aggregate-rawdata.json
19+
rm $PYMONGO/test/crud/unified/bulkWrite-deleteMany-rawdata.json
20+
rm $PYMONGO/test/crud/unified/bulkWrite-deleteOne-rawdata.json
21+
rm $PYMONGO/test/crud/unified/bulkWrite-replaceOne-rawdata.json
22+
rm $PYMONGO/test/crud/unified/bulkWrite-updateMany-rawdata.json
23+
rm $PYMONGO/test/crud/unified/bulkWrite-updateOne-rawdata.json
24+
rm $PYMONGO/test/crud/unified/client-bulkWrite-delete-rawdata.json
25+
rm $PYMONGO/test/crud/unified/client-bulkWrite-replaceOne-rawdata.json
26+
rm $PYMONGO/test/crud/unified/client-bulkWrite-update-rawdata.json
27+
rm $PYMONGO/test/crud/unified/count-rawdata.json
28+
rm $PYMONGO/test/crud/unified/countDocuments-rawdata.json
29+
rm $PYMONGO/test/crud/unified/db-aggregate-rawdata.json
30+
rm $PYMONGO/test/crud/unified/deleteMany-rawdata.json
31+
rm $PYMONGO/test/crud/unified/deleteOne-rawdata.json
32+
rm $PYMONGO/test/crud/unified/distinct-rawdata.json
33+
rm $PYMONGO/test/crud/unified/estimatedDocumentCount-rawdata.json
34+
rm $PYMONGO/test/crud/unified/find-rawdata.json
35+
rm $PYMONGO/test/crud/unified/findOneAndDelete-rawdata.json
36+
rm $PYMONGO/test/crud/unified/findOneAndReplace-rawdata.json
37+
rm $PYMONGO/test/crud/unified/findOneAndUpdate-rawdata.json
38+
rm $PYMONGO/test/crud/unified/insertMany-rawdata.json
39+
rm $PYMONGO/test/crud/unified/insertOne-rawdata.json
40+
rm $PYMONGO/test/crud/unified/replaceOne-rawdata.json
41+
rm $PYMONGO/test/crud/unified/updateMany-rawdata.json
42+
rm $PYMONGO/test/crud/unified/updateOne-rawdata.json
43+
rm $PYMONGO/test/index_management/index-rawdata.json
44+
45+
echo "Done removing unimplemented tests\n"

‎.evergreen/resync-specs.sh

Lines changed: 7 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -45,9 +45,12 @@ then
4545
fi
4646

4747
# Ensure the JSON files are up to date.
48-
cd $SPECS/source
49-
make
50-
cd -
48+
if ! [ -n "${CI:-}" ]
49+
then
50+
cd $SPECS/source
51+
make
52+
cd -
53+
fi
5154
# cpjson unified-test-format/tests/invalid unified-test-format/invalid
5255
# * param1: Path to spec tests dir in specifications repo
5356
# * param2: Path to where the corresponding tests live in Python.
@@ -110,7 +113,6 @@ do
110113
cmap|CMAP|connection-monitoring-and-pooling)
111114
cpjson connection-monitoring-and-pooling/tests/logging connection_logging
112115
cpjson connection-monitoring-and-pooling/tests/cmap-format connection_monitoring
113-
rm $PYMONGO/test/connection_monitoring/wait-queue-fairness.json # PYTHON-1873
114116
;;
115117
apm|APM|command-monitoring|command_monitoring)
116118
cpjson command-logging-and-monitoring/tests/monitoring command_monitoring
@@ -174,7 +176,7 @@ do
174176
;;
175177
server-selection|server_selection)
176178
cpjson server-selection/tests/ server_selection
177-
rm -rf $PYMONGO/test/server_selection/logging
179+
rm -rf $PYMONGO/test/server_selection/logging # these tests live in server_selection_logging
178180
cpjson server-selection/tests/logging server_selection_logging
179181
;;
180182
server-selection-logging|server_selection_logging)
@@ -186,7 +188,6 @@ do
186188
transactions|transactions-convenient-api)
187189
cpjson transactions/tests/ transactions
188190
cpjson transactions-convenient-api/tests/ transactions-convenient-api
189-
rm $PYMONGO/test/transactions/legacy/errors-client.json # PYTHON-1894
190191
;;
191192
unified|unified-test-format)
192193
cpjson unified-test-format/tests/ unified-test-format/

‎.evergreen/scripts/create-spec-pr.sh

Lines changed: 50 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,50 @@
1+
#!/usr/bin/env bash
2+
3+
tools="$(realpath -s "../drivers-tools")"
4+
pushd $tools/.evergreen/github_app || exit
5+
6+
owner="mongodb"
7+
repo="mongo-python-driver"
8+
9+
# Bootstrap the app.
10+
echo "bootstrapping"
11+
source utils.sh
12+
bootstrap drivers/comment-bot
13+
14+
# Run the app.
15+
source ./secrets-export.sh
16+
17+
# Get a github access token for the git checkout.
18+
echo "Getting github token..."
19+
20+
token=$(bash ./get-access-token.sh $repo $owner)
21+
if [ -z "${token}" ]; then
22+
echo "Failed to get github access token!"
23+
popd || exit
24+
exit 1
25+
fi
26+
echo "Getting github token... done."
27+
popd || exit
28+
29+
# Make the git checkout and create a new branch.
30+
echo "Creating the git checkout..."
31+
branch="spec-resync-"$(date '+%m-%d-%Y')
32+
33+
git remote set-url origin https://x-access-token:${token}@github.com/$owner/$repo.git
34+
git checkout -b $branch "origin/master"
35+
git add ./test
36+
git commit -am "resyncing specs $(date '+%m-%d-%Y')"
37+
echo "Creating the git checkout... done."
38+
39+
git push origin $branch
40+
resp=$(curl -L \
41+
-X POST \
42+
-H "Accept: application/vnd.github+json" \
43+
-H "Authorization: Bearer $token" \
44+
-H "X-GitHub-Api-Version: 2022-11-28" \
45+
-d "{\"title\":\"[Spec Resync] $(date '+%m-%d-%Y')\",\"body\":\"$(cat "$1")\",\"head\":\"${branch}\",\"base\":\"master\"}" \
46+
--url https://api.github.com/repos/$owner/$repo/pulls)
47+
echo $resp | jq '.html_url'
48+
echo "Creating the PR... done."
49+
50+
rm -rf $tools

‎.evergreen/scripts/resync-all-specs.py

Lines changed: 119 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,119 @@
1+
from __future__ import annotations
2+
3+
import argparse
4+
import os
5+
import pathlib
6+
import subprocess
7+
from argparse import Namespace
8+
from subprocess import CalledProcessError
9+
from typing import Optional
10+
11+
12+
def resync_specs(directory: pathlib.Path, errored: dict[str, str]) -> None:
13+
"""Actually sync the specs"""
14+
print("Beginning to sync specs") # noqa: T201
15+
for spec in os.scandir(directory):
16+
if not spec.is_dir():
17+
continue
18+
19+
if spec.name in ["asynchronous"]:
20+
continue
21+
try:
22+
subprocess.run(
23+
["bash", "./.evergreen/resync-specs.sh", spec.name], # noqa: S603, S607
24+
capture_output=True,
25+
text=True,
26+
check=True,
27+
)
28+
except CalledProcessError as exc:
29+
errored[spec.name] = exc.stderr
30+
print("Done syncing specs") # noqa: T201
31+
32+
33+
def apply_patches():
34+
print("Beginning to apply patches") # noqa: T201
35+
subprocess.run(["bash", "./.evergreen/remove-unimplemented-tests.sh"], check=True) # noqa: S603, S607
36+
subprocess.run(["git apply -R --allow-empty ./.evergreen/spec-patch/*"], shell=True, check=True) # noqa: S602, S607
37+
38+
39+
def check_new_spec_directories(directory: pathlib.Path) -> list[str]:
40+
"""Check to see if there are any directories in the spec repo that don't exist in pymongo/test"""
41+
spec_dir = pathlib.Path(os.environ["MDB_SPECS"]) / "source"
42+
spec_set = {
43+
entry.name.replace("-", "_")
44+
for entry in os.scandir(spec_dir)
45+
if entry.is_dir()
46+
and (pathlib.Path(entry.path) / "tests").is_dir()
47+
and len(list(os.scandir(pathlib.Path(entry.path) / "tests"))) > 1
48+
}
49+
test_set = {entry.name.replace("-", "_") for entry in os.scandir(directory) if entry.is_dir()}
50+
known_mappings = {
51+
"ocsp_support": "ocsp",
52+
"client_side_operations_timeout": "csot",
53+
"mongodb_handshake": "handshake",
54+
"load_balancers": "load_balancer",
55+
"atlas_data_lake_testing": "atlas",
56+
"connection_monitoring_and_pooling": "connection_monitoring",
57+
"command_logging_and_monitoring": "command_logging",
58+
"initial_dns_seedlist_discovery": "srv_seedlist",
59+
"server_discovery_and_monitoring": "sdam_monitoring",
60+
}
61+
62+
for k, v in known_mappings.items():
63+
if k in spec_set:
64+
spec_set.remove(k)
65+
spec_set.add(v)
66+
return list(spec_set - test_set)
67+
68+
69+
def write_summary(errored: dict[str, str], new: list[str], filename: Optional[str]) -> None:
70+
"""Generate the PR description"""
71+
pr_body = ""
72+
process = subprocess.run(
73+
["git diff --name-only | awk -F'/' '{print $2}' | sort | uniq"], # noqa: S607
74+
shell=True, # noqa: S602
75+
capture_output=True,
76+
text=True,
77+
check=True,
78+
)
79+
succeeded = process.stdout.strip().split()
80+
if len(succeeded) > 0:
81+
pr_body += "The following specs were changed:\n -"
82+
pr_body += "\n -".join(succeeded)
83+
pr_body += "\n"
84+
if len(errored) > 0:
85+
pr_body += "\n\nThe following spec syncs encountered errors:\n -"
86+
for k, v in errored.items():
87+
pr_body += f"\n -{k}\n```{v}\n```"
88+
pr_body += "\n"
89+
if len(new) > 0:
90+
pr_body += "\n\nThe following directories are in the specification repository and not in our test directory:\n -"
91+
pr_body += "\n -".join(new)
92+
pr_body += "\n"
93+
if pr_body != "":
94+
if filename is None:
95+
print(f"\n{pr_body}") # noqa: T201
96+
else:
97+
with open(filename, "w") as f:
98+
# replacements made for proper json
99+
f.write(pr_body.replace("\n", "\\n").replace("\t", "\\t"))
100+
101+
102+
def main(args: Namespace):
103+
directory = pathlib.Path("./test")
104+
errored: dict[str, str] = {}
105+
resync_specs(directory, errored)
106+
apply_patches()
107+
new = check_new_spec_directories(directory)
108+
write_summary(errored, new, args.filename)
109+
110+
111+
if __name__ == "__main__":
112+
parser = argparse.ArgumentParser(
113+
description="Python Script to resync all specs and generate summary for PR."
114+
)
115+
parser.add_argument(
116+
"--filename", help="Name of file for the summary to be written into.", default=None
117+
)
118+
args = parser.parse_args()
119+
main(args)

‎.evergreen/scripts/resync-all-specs.sh

Lines changed: 42 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,42 @@
1+
#!/usr/bin/env bash
2+
# Run spec syncing script and create PR
3+
4+
# SETUP
5+
SRC_URL="https://github.com/mongodb/specifications.git"
6+
# needs to be set for resync-specs.sh
7+
SPEC_SRC="$(realpath "../specifications")"
8+
SCRIPT="$(realpath "./.evergreen/resync-specs.sh")"
9+
10+
# Clone the spec repo if the directory does not exist
11+
if [[ ! -d $SPEC_SRC ]]; then
12+
git clone $SRC_URL $SPEC_SRC
13+
if [[ $? -ne 0 ]]; then
14+
echo "Error: Failed to clone repository."
15+
exit 1
16+
fi
17+
fi
18+
19+
# Set environment variable to the cloned spec repo for resync-specs.sh
20+
export MDB_SPECS="$SPEC_SRC"
21+
22+
# Check that resync-specs.sh exists and is executable
23+
if [[ ! -x $SCRIPT ]]; then
24+
echo "Error: $SCRIPT not found or is not executable."
25+
exit 1
26+
fi
27+
28+
PR_DESC="spec_sync.txt"
29+
30+
# run python script that actually does all the resyncing
31+
if ! [ -n "${CI:-}" ]
32+
then
33+
# we're running locally
34+
python3 ./.evergreen/scripts/resync-all-specs.py
35+
else
36+
/opt/devtools/bin/python3.11 ./.evergreen/scripts/resync-all-specs.py "$PR_DESC"
37+
if [[ -f $PR_DESC ]]; then
38+
# changes were made -> call scrypt to create PR for us
39+
.evergreen/scripts/create-spec-pr.sh "$PR_DESC"
40+
rm "$PR_DESC"
41+
fi
42+
fi

‎.evergreen/spec-patch/PYTHON-4884.patch

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,12 @@
1+
diff --git a/test/bson_corpus/datetime.json b/test/bson_corpus/datetime.json
2+
index f857afdc..1554341d 100644
3+
--- a/test/bson_corpus/datetime.json
4+
+++ b/test/bson_corpus/datetime.json
5+
@@ -24,6 +24,7 @@
6+
{
7+
"description" : "Y10K",
8+
"canonical_bson" : "1000000009610000DC1FD277E6000000",
9+
+ "relaxed_extjson" : "{\"a\":{\"$date\":{\"$numberLong\":\"253402300800000\"}}}",
10+
"canonical_extjson" : "{\"a\":{\"$date\":{\"$numberLong\":\"253402300800000\"}}}"
11+
},
12+
{

‎.evergreen/spec-patch/PYTHON-4918.patch

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,24 @@
1+
diff --git a/test/connection_monitoring/pool-create-min-size-error.json b/test/connection_monitoring/pool-create-min-size-error.json
2+
index 1c744b85..509b2a23 100644
3+
--- a/test/connection_monitoring/pool-create-min-size-error.json
4+
+++ b/test/connection_monitoring/pool-create-min-size-error.json
5+
@@ -49,15 +49,15 @@
6+
"type": "ConnectionCreated",
7+
"address": 42
8+
},
9+
+ {
10+
+ "type": "ConnectionPoolCleared",
11+
+ "address": 42
12+
+ },
13+
{
14+
"type": "ConnectionClosed",
15+
"address": 42,
16+
"connectionId": 42,
17+
"reason": "error"
18+
- },
19+
- {
20+
- "type": "ConnectionPoolCleared",
21+
- "address": 42
22+
}
23+
],
24+
"ignore": [

0 commit comments

Comments
 (0)