feat: add validation for zombienet flaky tests

.github/ZOMBIENET_FLAKY_TESTS.md

@@ -45,7 +45,11 @@ If you encounter a flaky test that needs to be temporarily disabled:
    zombienet-<suite>-<test-name>:<issue-number>
    ```
 3. **Commit and push** the change
-4. The test will be automatically skipped in subsequent CI runs
+4. The CI will automatically validate that:
+   - The entry follows the correct format
+   - The referenced GitHub issue exists
+   - (Warning if the issue is closed)
+5. The test will be automatically skipped in subsequent CI runs
 
 ## Re-enabling a Test
 
@@ -57,8 +61,23 @@ Once a flaky test has been fixed:
 4. **Commit and push** the change
 5. The test will be automatically included in subsequent CI runs
 
+## Validation
+
+The `.github/zombienet-flaky-tests` file is automatically validated in CI whenever it's modified. The validation checks:
+
+- **Format**: Each entry must follow the `<test-name>:<issue-number>` format
+- **Issue vs PR**: The referenced number must be a GitHub Issue, not a Pull Request
+- **Issue existence**: The referenced GitHub issue must exist in the repository
+- **Issue state**: A warning is shown if the referenced issue is closed (suggesting the entry might be outdated)
+
+The validation workflow runs on pull requests that modify:
+- `.github/zombienet-flaky-tests`
+- `.github/scripts/check-zombienet-flaky-tests.sh`
+- `.github/workflows/check-zombienet-flaky-tests.yml`
+
 ## Monitoring
 
 - The number of currently disabled tests is displayed in the CI logs during zombienet test runs
 - You can view the current list at: [`.github/zombienet-flaky-tests`](./zombienet-flaky-tests)
 - Each disabled test should have an associated GitHub issue for tracking
+- The validation script can be run locally: `.github/scripts/check-zombienet-flaky-tests.sh .github/zombienet-flaky-tests`

.github/scripts/check-zombienet-flaky-tests.sh

@@ -0,0 +1,93 @@
+#!/usr/bin/env bash
+
+# Validates the .github/zombienet-flaky-tests file to ensure:
+# 1. Each entry has the correct format: <test-name>:<issue-number>
+# 2. The referenced number is a GitHub Issue
+# 3. The GitHub issue exists
+# 4. The issue is OPEN (warns if closed)
+
+set -uo pipefail
+
+FLAKY_TESTS_FILE="${1:-.github/zombienet-flaky-tests}"
+
+if [[ ! -f "$FLAKY_TESTS_FILE" ]]; then
+    echo "Error: File not found: $FLAKY_TESTS_FILE" >&2
+    exit 1
+fi
+
+if ! command -v gh &> /dev/null; then
+    echo "Error: gh CLI is not installed" >&2
+    exit 1
+fi
+
+echo "Validating $FLAKY_TESTS_FILE..."
+echo
+
+has_errors=false
+line_num=0
+
+while IFS= read -r line || [[ -n "$line" ]]; do
+    line_num=$((line_num + 1))
+
+    if [[ -z "$line" ]]; then
+        continue
+    fi
+
+    # Parse format: test-name:issue-number
+    if [[ ! "$line" =~ ^([^:]+):([0-9]+)$ ]]; then
+        echo "❌ Line $line_num: Missing required issue number" >&2
+        echo "   Entry: '$line'" >&2
+        echo "   Expected format: <test-name>:<issue-number>" >&2
+        echo "   Example: zombienet-polkadot-test-name:1234" >&2
+        has_errors=true
+        continue
+    fi
+
+    test_name="${BASH_REMATCH[1]}"
+    issue_number="${BASH_REMATCH[2]}"
+
+    set +e
+    issue_data=$(gh issue view "$issue_number" --json state,title,url 2>&1)
+    gh_exit_code=$?
+    set -e
+
+    if [[ $gh_exit_code -ne 0 ]]; then
+        echo "❌ Line $line_num: Issue #$issue_number does not exist" >&2
+        echo "   Test: $test_name" >&2
+        has_errors=true
+        continue
+    fi
+
+    url=$(echo "$issue_data" | jq -r '.url')
+    state=$(echo "$issue_data" | jq -r '.state')
+    title=$(echo "$issue_data" | jq -r '.title')
+
+    # Check if it's an issue (not a PR) by verifying the URL contains '/issues/'
+    if [[ ! "$url" =~ /issues/ ]]; then
+        echo "❌ Line $line_num: #$issue_number is a Pull Request, not an Issue" >&2
+        echo "   Test: $test_name" >&2
+        echo "   URL: $url" >&2
+        echo "   Please reference a GitHub Issue, not a PR" >&2
+        has_errors=true
+        continue
+    fi
+
+    if [[ "$state" == "OPEN" ]]; then
+        echo "✅ Line $line_num: $test_name -> Issue #$issue_number (open)"
+    else
+        echo "⚠️  Line $line_num: Issue #$issue_number is closed: '$title'" >&2
+        echo "   Test: $test_name" >&2
+        echo "   Consider removing this entry if the issue is resolved." >&2
+    fi
+
+done < "$FLAKY_TESTS_FILE"
+
+echo
+
+if [[ "$has_errors" == "true" ]]; then
+    echo "❌ Validation failed with errors" >&2
+    exit 1
+else
+    echo "✅ All entries are valid"
+    exit 0
+fi

.github/workflows/check-zombienet-flaky-tests.yml

@@ -0,0 +1,39 @@
+name: Check Zombienet Flaky Tests
+
+concurrency:
+  group: check-zombienet-flaky-tests-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
+
+on:
+  pull_request:
+    types: [opened, synchronize, reopened]
+    paths:
+      - '.github/zombienet-flaky-tests'
+      - '.github/scripts/check-zombienet-flaky-tests.sh'
+      - '.github/workflows/check-zombienet-flaky-tests.yml'
+  merge_group:
+
+permissions:
+  contents: read
+
+jobs:
+  check-flaky-tests:
+    runs-on: ubuntu-latest
+    timeout-minutes: 5
+    steps:
+      - name: Checkout repo
+        uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
+
+      - name: Validate zombienet-flaky-tests
+        env:
+          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        run: |
+          .github/scripts/check-zombienet-flaky-tests.sh .github/zombienet-flaky-tests
+
+      - name: Check results
+        if: failure()
+        run: |
+          echo "::error::Validation failed. Please ensure all entries in .github/zombienet-flaky-tests have valid format and reference existing GitHub issues."
+          echo "Format: <test-name>:<issue-number>"
+          echo "See .github/ZOMBIENET_FLAKY_TESTS.md for more information."
+          exit 1

.github/zombienet-flaky-tests

@@ -1,5 +1,4 @@
 zombienet-polkadot-coretime-revenue:8944
-zombienet-polkadot-smoke-0003-deregister-register-validator
 zombienet-polkadot-elastic-scaling-slot-based-12cores:8945
 zombienet-polkadot-functional-0004-parachains-disputes-garbage-candidate:8942
 zombienet-polkadot-disputes-slashing:8946