PYTHON-5253 Automated Spec Test Sync #2409
There are no files selected for viewing
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,46 @@ | ||
| #!/bin/bash | ||
| PYMONGO=$(dirname "$(cd "$(dirname "$0")" || exit; pwd)") | ||
|
|
||
| rm -rf $PYMONGO/test/server_selection/logging | ||
| rm $PYMONGO/test/transactions/legacy/errors-client.json # PYTHON-1894 | ||
| rm $PYMONGO/test/connection_monitoring/wait-queue-fairness.json # PYTHON-1873 | ||
| rm $PYMONGO/test/client-side-encryption/spec/unified/fle2v2-BypassQueryAnalysis.json # PYTHON-5143 | ||
| rm $PYMONGO/test/client-side-encryption/spec/unified/fle2v2-EncryptedFields-vs-EncryptedFieldsMap.json # PYTHON-5143 | ||
| rm $PYMONGO/test/client-side-encryption/spec/unified/localSchema.json # PYTHON-5143 | ||
| rm $PYMONGO/test/client-side-encryption/spec/unified/maxWireVersion.json # PYTHON-5143 | ||
| rm $PYMONGO/test/unified-test-format/valid-pass/poc-queryable-encryption.json # PYTHON-5143 | ||
| rm $PYMONGO/test/gridfs/rename.json # PYTHON-4931 | ||
| rm $PYMONGO/test/discovery_and_monitoring/unified/pool-clear-application-error.json # PYTHON-4918 | ||
| rm $PYMONGO/test/discovery_and_monitoring/unified/pool-clear-checkout-error.json # PYTHON-4918 | ||
| rm $PYMONGO/test/discovery_and_monitoring/unified/pool-clear-min-pool-size-error.json # PYTHON-4918 | ||
|
|
||
| # Python doesn't implement DRIVERS-3064 | ||
| rm $PYMONGO/test/collection_management/listCollections-rawdata.json | ||
| rm $PYMONGO/test/crud/unified/aggregate-rawdata.json | ||
| rm $PYMONGO/test/crud/unified/bulkWrite-deleteMany-rawdata.json | ||
| rm $PYMONGO/test/crud/unified/bulkWrite-deleteOne-rawdata.json | ||
| rm $PYMONGO/test/crud/unified/bulkWrite-replaceOne-rawdata.json | ||
| rm $PYMONGO/test/crud/unified/bulkWrite-updateMany-rawdata.json | ||
| rm $PYMONGO/test/crud/unified/bulkWrite-updateOne-rawdata.json | ||
| rm $PYMONGO/test/crud/unified/client-bulkWrite-delete-rawdata.json | ||
| rm $PYMONGO/test/crud/unified/client-bulkWrite-replaceOne-rawdata.json | ||
| rm $PYMONGO/test/crud/unified/client-bulkWrite-update-rawdata.json | ||
| rm $PYMONGO/test/crud/unified/count-rawdata.json | ||
| rm $PYMONGO/test/crud/unified/countDocuments-rawdata.json | ||
| rm $PYMONGO/test/crud/unified/db-aggregate-rawdata.json | ||
| rm $PYMONGO/test/crud/unified/deleteMany-rawdata.json | ||
| rm $PYMONGO/test/crud/unified/deleteOne-rawdata.json | ||
| rm $PYMONGO/test/crud/unified/distinct-rawdata.json | ||
| rm $PYMONGO/test/crud/unified/estimatedDocumentCount-rawdata.json | ||
| rm $PYMONGO/test/crud/unified/find-rawdata.json | ||
| rm $PYMONGO/test/crud/unified/findOneAndDelete-rawdata.json | ||
| rm $PYMONGO/test/crud/unified/findOneAndReplace-rawdata.json | ||
| rm $PYMONGO/test/crud/unified/findOneAndUpdate-rawdata.json | ||
| rm $PYMONGO/test/crud/unified/insertMany-rawdata.json | ||
| rm $PYMONGO/test/crud/unified/insertOne-rawdata.json | ||
| rm $PYMONGO/test/crud/unified/replaceOne-rawdata.json | ||
| rm $PYMONGO/test/crud/unified/updateMany-rawdata.json | ||
| rm $PYMONGO/test/crud/unified/updateOne-rawdata.json | ||
| rm $PYMONGO/test/index_management/index-rawdata.json | ||
|
|
||
| echo "Done removing unimplemented tests\n" |
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,50 @@ | ||
| #!/usr/bin/env bash | ||
|
|
||
| tools="$(realpath -s "../drivers-tools")" | ||
| pushd $tools/.evergreen/github_app || exit | ||
|
|
||
| owner="mongodb" | ||
| repo="mongo-python-driver" | ||
|
|
||
| # Bootstrap the app. | ||
| echo "bootstrapping" | ||
| source utils.sh | ||
| bootstrap drivers/comment-bot | ||
|
|
||
| # Run the app. | ||
| source ./secrets-export.sh | ||
|
|
||
| # Get a github access token for the git checkout. | ||
| echo "Getting github token..." | ||
|
|
||
| token=$(bash ./get-access-token.sh $repo $owner) | ||
| if [ -z "${token}" ]; then | ||
| echo "Failed to get github access token!" | ||
| popd || exit | ||
| exit 1 | ||
| fi | ||
| echo "Getting github token... done." | ||
| popd || exit | ||
|
|
||
| # Make the git checkout and create a new branch. | ||
| echo "Creating the git checkout..." | ||
| branch="spec-resync-"$(date '+%m-%d-%Y') | ||
|
|
||
| git remote set-url origin https://x-access-token:${token}@github.com/$owner/$repo.git | ||
| git checkout -b $branch "origin/master" | ||
| git add ./test | ||
| git commit -am "resyncing specs $(date '+%m-%d-%Y')" | ||
| echo "Creating the git checkout... done." | ||
|
|
||
| git push origin $branch | ||
| resp=$(curl -L \ | ||
| -X POST \ | ||
| -H "Accept: application/vnd.github+json" \ | ||
| -H "Authorization: Bearer $token" \ | ||
| -H "X-GitHub-Api-Version: 2022-11-28" \ | ||
| -d "{\"title\":\"[Spec Resync] $(date '+%m-%d-%Y')\",\"body\":\"$(cat "$1")\",\"head\":\"${branch}\",\"base\":\"master\"}" \ | ||
| --url https://api.github.com/repos/$owner/$repo/pulls) | ||
| echo $resp | jq '.html_url' | ||
| echo "Creating the PR... done." | ||
|
|
||
| rm -rf $tools |
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,119 @@ | ||
| from __future__ import annotations | ||
|
|
||
| import argparse | ||
| import os | ||
| import pathlib | ||
| import subprocess | ||
| from argparse import Namespace | ||
| from subprocess import CalledProcessError | ||
| from typing import Optional | ||
|
|
||
|
|
||
| def resync_specs(directory: pathlib.Path, errored: dict[str, str]) -> None: | ||
| """Actually sync the specs""" | ||
| print("Beginning to sync specs") # noqa: T201 | ||
| for spec in os.scandir(directory): | ||
| if not spec.is_dir(): | ||
| continue | ||
|
|
||
| if spec.name in ["asynchronous"]: | ||
| continue | ||
| try: | ||
| subprocess.run( | ||
| ["bash", "./.evergreen/resync-specs.sh", spec.name], # noqa: S603, S607 | ||
| capture_output=True, | ||
| text=True, | ||
| check=True, | ||
| ) | ||
| except CalledProcessError as exc: | ||
| errored[spec.name] = exc.stderr | ||
| print("Done syncing specs") # noqa: T201 | ||
|
|
||
|
|
||
| def apply_patches(): | ||
| print("Beginning to apply patches") # noqa: T201 | ||
| subprocess.run(["bash", "./.evergreen/remove-unimplemented-tests.sh"], check=True) # noqa: S603, S607 | ||
| subprocess.run(["git apply -R --allow-empty ./.evergreen/spec-patch/*"], shell=True, check=True) # noqa: S602, S607 | ||
|
|
||
|
|
||
| def check_new_spec_directories(directory: pathlib.Path) -> list[str]: | ||
| """Check to see if there are any directories in the spec repo that don't exist in pymongo/test""" | ||
| spec_dir = pathlib.Path(os.environ["MDB_SPECS"]) / "source" | ||
| spec_set = { | ||
| entry.name.replace("-", "_") | ||
| for entry in os.scandir(spec_dir) | ||
| if entry.is_dir() | ||
| and (pathlib.Path(entry.path) / "tests").is_dir() | ||
| and len(list(os.scandir(pathlib.Path(entry.path) / "tests"))) > 1 | ||
| } | ||
| test_set = {entry.name.replace("-", "_") for entry in os.scandir(directory) if entry.is_dir()} | ||
| known_mappings = { | ||
|
There was a problem hiding this comment. I think there's a slight difference in how we name the directories vs how the spec names them sometimes? (Not 100% sure if these are correct though, so please let me know if these need to be changed!) There was a problem hiding this comment. Yes we often have different names than the spec repo, that's expected. |
||
| "ocsp_support": "ocsp", | ||
| "client_side_operations_timeout": "csot", | ||
| "mongodb_handshake": "handshake", | ||
| "load_balancers": "load_balancer", | ||
| "atlas_data_lake_testing": "atlas", | ||
| "connection_monitoring_and_pooling": "connection_monitoring", | ||
| "command_logging_and_monitoring": "command_logging", | ||
| "initial_dns_seedlist_discovery": "srv_seedlist", | ||
| "server_discovery_and_monitoring": "sdam_monitoring", | ||
| } | ||
|
|
||
| for k, v in known_mappings.items(): | ||
| if k in spec_set: | ||
| spec_set.remove(k) | ||
| spec_set.add(v) | ||
| return list(spec_set - test_set) | ||
|
|
||
|
|
||
| def write_summary(errored: dict[str, str], new: list[str], filename: Optional[str]) -> None: | ||
| """Generate the PR description""" | ||
| pr_body = "" | ||
| process = subprocess.run( | ||
| ["git diff --name-only | awk -F'/' '{print $2}' | sort | uniq"], # noqa: S607 | ||
| shell=True, # noqa: S602 | ||
| capture_output=True, | ||
| text=True, | ||
| check=True, | ||
| ) | ||
| succeeded = process.stdout.strip().split() | ||
| if len(succeeded) > 0: | ||
| pr_body += "The following specs were changed:\n -" | ||
| pr_body += "\n -".join(succeeded) | ||
| pr_body += "\n" | ||
| if len(errored) > 0: | ||
| pr_body += "\n\nThe following spec syncs encountered errors:\n -" | ||
| for k, v in errored.items(): | ||
| pr_body += f"\n -{k}\n```{v}\n```" | ||
| pr_body += "\n" | ||
| if len(new) > 0: | ||
| pr_body += "\n\nThe following directories are in the specification repository and not in our test directory:\n -" | ||
| pr_body += "\n -".join(new) | ||
| pr_body += "\n" | ||
| if pr_body != "": | ||
| if filename is None: | ||
| print(f"\n{pr_body}") # noqa: T201 | ||
| else: | ||
| with open(filename, "w") as f: | ||
| # replacements made for proper json | ||
| f.write(pr_body.replace("\n", "\\n").replace("\t", "\\t")) | ||
|
|
||
|
|
||
| def main(args: Namespace): | ||
| directory = pathlib.Path("./test") | ||
| errored: dict[str, str] = {} | ||
| resync_specs(directory, errored) | ||
| apply_patches() | ||
| new = check_new_spec_directories(directory) | ||
| write_summary(errored, new, args.filename) | ||
|
|
||
|
|
||
| if __name__ == "__main__": | ||
| parser = argparse.ArgumentParser( | ||
| description="Python Script to resync all specs and generate summary for PR." | ||
| ) | ||
| parser.add_argument( | ||
| "--filename", help="Name of file for the summary to be written into.", default=None | ||
| ) | ||
| args = parser.parse_args() | ||
| main(args) | ||
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,42 @@ | ||
| #!/usr/bin/env bash | ||
| # Run spec syncing script and create PR | ||
|
|
||
| # SETUP | ||
| SRC_URL="https://github.com/mongodb/specifications.git" | ||
| # needs to be set for resync-specs.sh | ||
| SPEC_SRC="$(realpath "../specifications")" | ||
| SCRIPT="$(realpath "./.evergreen/resync-specs.sh")" | ||
|
|
||
| # Clone the spec repo if the directory does not exist | ||
| if [[ ! -d $SPEC_SRC ]]; then | ||
| git clone $SRC_URL $SPEC_SRC | ||
| if [[ $? -ne 0 ]]; then | ||
| echo "Error: Failed to clone repository." | ||
| exit 1 | ||
| fi | ||
| fi | ||
|
|
||
| # Set environment variable to the cloned spec repo for resync-specs.sh | ||
| export MDB_SPECS="$SPEC_SRC" | ||
|
|
||
| # Check that resync-specs.sh exists and is executable | ||
| if [[ ! -x $SCRIPT ]]; then | ||
| echo "Error: $SCRIPT not found or is not executable." | ||
| exit 1 | ||
| fi | ||
|
|
||
| PR_DESC="spec_sync.txt" | ||
|
|
||
| # run python script that actually does all the resyncing | ||
| if ! [ -n "${CI:-}" ] | ||
| then | ||
| # we're running locally | ||
| python3 ./.evergreen/scripts/resync-all-specs.py | ||
| else | ||
| /opt/devtools/bin/python3.11 ./.evergreen/scripts/resync-all-specs.py "$PR_DESC" | ||
| if [[ -f $PR_DESC ]]; then | ||
| # changes were made -> call scrypt to create PR for us | ||
| .evergreen/scripts/create-spec-pr.sh "$PR_DESC" | ||
| rm "$PR_DESC" | ||
| fi | ||
| fi |
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,12 @@ | ||
| diff --git a/test/bson_corpus/datetime.json b/test/bson_corpus/datetime.json | ||
| index f857afdc..1554341d 100644 | ||
| --- a/test/bson_corpus/datetime.json | ||
| +++ b/test/bson_corpus/datetime.json | ||
| @@ -24,6 +24,7 @@ | ||
| { | ||
| "description" : "Y10K", | ||
| "canonical_bson" : "1000000009610000DC1FD277E6000000", | ||
| + "relaxed_extjson" : "{\"a\":{\"$date\":{\"$numberLong\":\"253402300800000\"}}}", | ||
| "canonical_extjson" : "{\"a\":{\"$date\":{\"$numberLong\":\"253402300800000\"}}}" | ||
| }, | ||
| { |
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,24 @@ | ||
| diff --git a/test/connection_monitoring/pool-create-min-size-error.json b/test/connection_monitoring/pool-create-min-size-error.json | ||
| index 1c744b85..509b2a23 100644 | ||
| --- a/test/connection_monitoring/pool-create-min-size-error.json | ||
| +++ b/test/connection_monitoring/pool-create-min-size-error.json | ||
| @@ -49,15 +49,15 @@ | ||
| "type": "ConnectionCreated", | ||
| "address": 42 | ||
| }, | ||
| + { | ||
| + "type": "ConnectionPoolCleared", | ||
| + "address": 42 | ||
| + }, | ||
| { | ||
| "type": "ConnectionClosed", | ||
| "address": 42, | ||
| "connectionId": 42, | ||
| "reason": "error" | ||
| - }, | ||
| - { | ||
| - "type": "ConnectionPoolCleared", | ||
| - "address": 42 | ||
| } | ||
| ], | ||
| "ignore": [ |
Oops, something went wrong.
Oops, something went wrong.
Add this suggestion to a batch that can be applied as a single commit.
This suggestion is invalid because no changes were made to the code.
Suggestions cannot be applied while the pull request is closed.
Suggestions cannot be applied while viewing a subset of changes.
Only one suggestion per line can be applied in a batch.
Add this suggestion to a batch that can be applied as a single commit.
Applying suggestions on deleted lines is not supported.
You must change the existing code in this line in order to create a valid suggestion.
Outdated suggestions cannot be applied.
This suggestion has been applied or marked resolved.
Suggestions cannot be applied from pending reviews.
Suggestions cannot be applied on multi-line comments.
Suggestions cannot be applied while the pull request is queued to merge.
Suggestion cannot be applied right now. Please check back later.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
CI is always defined in an EVG run and we only want to run these things in a local checkout.