PSMDB-1769 - Adding Github action for Ruff (#401)

* Adding Ruff linter github actions to run against any python file that has changes
* Test fixes
* Change so tests do not run against a Draft PR

Author: keithquinnpercona

.github/workflows/LINT.yml

@@ -0,0 +1,40 @@
+name: LINT
+
+on:
+  workflow_dispatch:
+
+  pull_request:
+    branches:
+      - main
+
+jobs:
+  lint:
+    runs-on: ubuntu-latest
+    timeout-minutes: 5
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+
+      - name: Install dependencies
+        run: pip install ruff uv
+
+      - name: Get changed Python files
+        id: changed-files
+        if: github.event_name == 'pull_request'
+        run: |
+          BASE_SHA="${{ github.event.pull_request.base.sha }}"
+          HEAD_SHA="${{ github.event.pull_request.head.sha }}"
+          FILES="$(git diff --name-only "$BASE_SHA" "$HEAD_SHA" -- '*.py' | tr '\n' ' ' || true)"
+          echo "files=$FILES" >> "$GITHUB_OUTPUT"
+
+      - name: Run ruff only on changed files
+        run: |
+          FILES="${{ steps.changed-files.outputs.files }}"
+          if [ -z "$FILES" ]; then
+            echo "No changed Python files."
+            exit 0
+          fi
+          echo "Linting changed files:"
+          echo "$FILES"
+          uv run ruff check $FILES

.github/workflows/PBM-FULL.yml

@@ -11,13 +11,15 @@ on:
         required: false
 
   pull_request:
+    types: [opened, reopened, synchronize, ready_for_review]
     branches:
       - main
     paths:
       - 'pbm-functional/pytest/**'
 
 jobs:
   test:
+    if: github.event.pull_request.draft == false
     runs-on: ubuntu-latest
     timeout-minutes: 180
     strategy:

.github/workflows/PLM-FULL.yml

@@ -0,0 +1,53 @@
+name: PLM-FULL
+
+on:
+  workflow_dispatch:
+    inputs:
+      plm_branch:
+        description: "plm branch"
+        required: false
+      go_ver:
+        description: "golang version"
+        required: false
+
+  pull_request:
+    types: [opened, reopened, synchronize, ready_for_review]
+    branches:
+      - main
+    paths:
+      - 'plm-pytest/**'
+
+jobs:
+  test:
+    # Will not run if item is a draft
+    if: github.event.pull_request.draft == false
+    runs-on: ubuntu-latest
+    timeout-minutes: 180
+    strategy:
+      fail-fast: false
+      matrix:
+        psmdb: ["6.0", "7.0", "8.0"]
+    env:
+      PLM_BRANCH: ${{ github.event.inputs.plm_branch || 'main' }}
+      GO_VER: ${{ github.event.inputs.go_ver || 'bullseye' }}
+    steps:
+    - uses: actions/checkout@v4
+    - name: Setup environment with PSMDB ${{ matrix.psmdb }} and PLM branch ${{ env.PLM_BRANCH }}
+      run: |
+        PSMDB=perconalab/percona-server-mongodb:${{ matrix.psmdb }} docker compose build easyrsa
+        PSMDB=perconalab/percona-server-mongodb:${{ matrix.psmdb }} docker compose build
+        docker compose up -d
+      working-directory: ./plm-pytest
+    - name: Test ${{ matrix.test }} backup/restore on PSMDB ${{ matrix.psmdb }} and PBM branch ${{ env.PLM_BRANCH }}
+      run: |
+        docker compose run --rm test pytest -s --junitxml=junit.xml
+      working-directory: ./plm-pytest
+    - name: Print PLM logs
+      if: failure()
+      run: |
+        docker logs plink
+    - name: Publish Test Report
+      uses: mikepenz/action-junit-report@v4
+      if: success() || failure()
+      with:
+        report_paths: '**/junit.xml'

plm-pytest/conftest.py

@@ -0,0 +1,3 @@
+pytest_plugins = [
+    "metrics_collector"
+]

plm-pytest/test_basic_sync_rs.py

@@ -134,9 +134,7 @@ def test_rs_plink_PML_T2(reset_state, srcRS, dstRS, plink, metrics_collector):
 def test_rs_plink_PML_T3(reset_state, srcRS, dstRS, plink):
     """
     Test to validate handling of index creation failures during clone and replication phase due to
-    IndexOptionsConflict error (index with the same key spec already exists with a different name).
-    Since PLM temporarily creates unique indexes as non-unique, creation of index with the same options
-    but different name should fail. It's expected behavior, but all other indexes should be created successfully.
+    IndexOptionsConflict error (index with the same key spec already exists with a different name). The failed index will be created during the finalization stage.
     """
     try:
         src = pymongo.MongoClient(srcRS.connection)
@@ -197,8 +195,6 @@ def test_rs_plink_PML_T3(reset_state, srcRS, dstRS, plink):
         name="compound_test_sparse_index", sparse=True
         )
 
-        time.sleep(5)
-
     except Exception:
         raise
     finally:
@@ -219,28 +215,16 @@ def test_rs_plink_PML_T3(reset_state, srcRS, dstRS, plink):
     result = plink.finalize()
     assert result is True, "Failed to finalize plink service"
 
-    expected_mismatches = [
-        ("init_test_db.invalid_index_collection", "compound_test_unique_index"),
-        ("clone_test_db.invalid_index_collection", "compound_test_unique_index"),
-        ("repl_test_db.invalid_index_collection", "compound_test_unique_index")
-    ]
-
     result, summary = compare_data_rs(srcRS, dstRS)
-    assert result is False, "Data mismatch after synchronization"
-
-    missing_mismatches = [index for index in expected_mismatches if index not in summary]
-    unexpected_mismatches = [mismatch for mismatch in summary if mismatch not in expected_mismatches]
-
-    assert not missing_mismatches, f"Expected mismatches missing: {missing_mismatches}"
-    if unexpected_mismatches:
-        pytest.fail("Unexpected mismatches:\n" + "\n".join(unexpected_mismatches))
+    assert result is True, "Data mismatch after synchronization"
 
     plink_error, error_logs = plink.check_plink_errors()
     expected_error = "ERR One or more indexes failed to create"
     if not plink_error:
         unexpected = [line for line in error_logs if expected_error not in line]
         if unexpected:
             pytest.fail("Unexpected error(s) in logs:\n" + "\n".join(unexpected))
+    assert len(error_logs) == 3
 
 @pytest.mark.timeout(300,func_only=True)
 @pytest.mark.usefixtures("start_cluster")
@@ -674,7 +658,7 @@ def test_rs_plink_PML_T30(reset_state, srcRS, dstRS, plink):
     Test to validate handling of concurrent data clone and index build failure
     """
     src = pymongo.MongoClient(srcRS.connection)
-    dst = pymongo.MongoClient(dstRS.connection)
+    pymongo.MongoClient(dstRS.connection)
     normal_docs = [{"a": {"b": 1}, "words": "omnibus"} for _ in range(20000)]
     src["init_test_db"].invalid_text_collection1.insert_many(normal_docs)
     src["init_test_db"].invalid_text_collection1.insert_one({"a": {"b": []}, "words": "omnibus"})
@@ -699,7 +683,7 @@ def invalid_index_creation():
     t2.start()
     t1.join()
     t2.join()
-    assert plink.wait_for_repl_stage(timeout=30) is True, "Failed to start replication stage"
+    assert plink.wait_for_repl_stage(timeout=60) is True, "Failed to start replication stage"
     assert plink.wait_for_zero_lag() is True, "Failed to catch up on replication"
     assert plink.finalize() is True, "Failed to finalize plink service"
     result, _ = compare_data_rs(srcRS, dstRS)

plm-pytest/test_edge_case_id_rs.py

@@ -174,7 +174,7 @@ def test_rs_plink_PML_T34(reset_state, srcRS, dstRS, plink, id_type):
 
     def add_data(target_suffix):
         batch_size = 5000
-        num_batches = 80
+        num_batches = 20
 
         for meta in collections_meta:
             if not meta["name"].endswith(target_suffix):