diff --git a/.github/actions/e2e/run-log-tests/action.yml b/.github/actions/e2e/run-log-tests/action.yml index f2f9a4b236e..037c34a41c8 100644 --- a/.github/actions/e2e/run-log-tests/action.yml +++ b/.github/actions/e2e/run-log-tests/action.yml @@ -1,4 +1,4 @@ -name: 'Run Tests' +name: 'Run E2E Tests with Retry & Upload Logs' description: 'Runs E2E tests with retry & upload logs and screenshots' runs: @@ -10,12 +10,17 @@ runs: shell: /bin/bash +e {0} run: | npm run test:e2e-ci - - if [[ -f "$E2E_RESULT_FILEPATH" ]]; then - E2E_NUM_FAILED_TEST_SUITES=$(cat "$E2E_RESULT_FILEPATH" | jq '.stats["unexpected"]') - echo "FIRST_RUN_FAILED_TEST_SUITES=$(echo $E2E_NUM_FAILED_TEST_SUITES)" >> $GITHUB_OUTPUT - if [[ ${E2E_NUM_FAILED_TEST_SUITES} -gt 0 ]]; then - echo "::notice::${E2E_NUM_FAILED_TEST_SUITES} test suite(s) failed in the first run but we will try (it) them again in the second run." + RESULTS_JSON="$E2E_RESULT_FILEPATH" + if [[ -f "$RESULTS_JSON" ]]; then + # Build unique list of spec files with any failed/unexpected/timedOut/interrupted result and count them. + FAILED_SPECS_COUNT=$( jq -r '[ .. | objects | select(has("specs")) | .specs[] + | select( ( any(.tests[]?; (.status=="unexpected") or (.status=="failed")) ) + or ( any(.tests[]?.results[]?; (.status=="failed") or (.status=="timedOut") or (.status=="interrupted")) ) ) + | .file ] | unique | length' "$RESULTS_JSON" ) + echo "FIRST_RUN_FAILED_TEST_SUITES=$FAILED_SPECS_COUNT" >> $GITHUB_OUTPUT + echo "RESULTS_JSON=$RESULTS_JSON" >> $GITHUB_OUTPUT + if [[ ${FAILED_SPECS_COUNT} -gt 0 ]]; then + echo "::notice::${FAILED_SPECS_COUNT} spec file(s) failed in the first run. We will re-run only the failed specs." exit 0 fi else @@ -27,9 +32,32 @@ runs: - name: Re-try Failed E2E Files if: ${{ steps.first_run_e2e_tests.outputs.FIRST_RUN_FAILED_TEST_SUITES > 0 }} shell: bash - # Filter failed E2E files from the result JSON file, and re-run them. run: | - npm run test:e2e-ci $(cat $E2E_RESULT_FILEPATH | jq -r '[.suites[] | (if has("suites") then .suites[] | .specs[] else .specs[] end) | select(.tests[].status == "unexpected") | .file] | unique | .[]') + set -e + RESULTS_JSON="${{ steps.first_run_e2e_tests.outputs.RESULTS_JSON }}" + echo "Using results file: $RESULTS_JSON" + + # Build a unique list of spec files that had unexpected/failed outcomes. + # This uses a recursive jq search to find any node with a 'specs' array, then selects specs + # where any test result is marked unexpected/failed. + mapfile -t FAILED_SPECS < <( jq -r '[ .. | objects | select(has("specs")) | .specs[] + | select( ( any(.tests[]?; (.status=="unexpected") or (.status=="failed")) ) + or ( any(.tests[]?.results[]?; (.status=="failed") or (.status=="timedOut") or (.status=="interrupted")) ) ) + | .file ] | unique | .[]' "$RESULTS_JSON" ) + + if [[ ${#FAILED_SPECS[@]} -eq 0 ]]; then + echo "::notice::No failed specs found in results file. Re-running full suite instead." + npm run test:e2e-ci + exit 0 + fi + + echo "Retrying failed specs (${#FAILED_SPECS[@]}):" + for f in "${FAILED_SPECS[@]}"; do + echo " - $f" + done + + # Re-run only the failed spec files. + npm run test:e2e-ci "${FAILED_SPECS[@]}" # Archive screenshots if any - name: Archive e2e test screenshots & logs @@ -41,5 +69,6 @@ runs: playwright-report/ tests/e2e/test-results ${{ env.E2E_RESULT_FILEPATH }} + ${{ steps.first_run_e2e_tests.outputs.RESULTS_JSON }} if-no-files-found: ignore retention-days: 14 diff --git a/.github/actions/setup-php/action.yml b/.github/actions/setup-php/action.yml index 44e797aeb6d..f7d78e784ae 100644 --- a/.github/actions/setup-php/action.yml +++ b/.github/actions/setup-php/action.yml @@ -1,5 +1,5 @@ name: "Set up PHP" -description: "Extracts the required PHP version from plugin file and uses it to build PHP." +description: "Extracts the required PHP version from plugin file and uses it to build PHP. Can be overridden with E2E_PHP_VERSION environment variable." runs: using: composite @@ -14,6 +14,6 @@ runs: - name: "Setup PHP" uses: shivammathur/setup-php@v2 with: - php-version: ${{ steps.get_min_php_version.outputs.MIN_PHP_VERSION }} + php-version: ${{ env.E2E_PHP_VERSION || steps.get_min_php_version.outputs.MIN_PHP_VERSION }} tools: composer coverage: none diff --git a/.github/scripts/README.md b/.github/scripts/README.md new file mode 100644 index 00000000000..f29523f09a7 --- /dev/null +++ b/.github/scripts/README.md @@ -0,0 +1,111 @@ +# GitHub Actions Scripts + +This directory contains scripts used by GitHub Actions workflows for dynamic version management and matrix generation. + +## Scripts + +### `generate-wc-matrix.sh` + +Generates the WooCommerce version matrix for E2E tests with dynamic version resolution and optimized PHP version strategy. + +**Usage:** + +```bash +.github/scripts/generate-wc-matrix.sh +``` + +**Output:** +Single JSON object containing versions array and metadata: + +```json +{ + "versions": [ + "7.7.0", + "9.9.5", + "latest", + "10.1.0-rc.2" + ], + "metadata": { + "l1_version": "9.9.5", + "rc_version": "10.1.0-rc.2", + "beta_version": null + } +} +``` + +**Features:** + +- Fetches latest WC version from WordPress.org API +- Dynamically calculates L-1 version (latest stable in previous major branch) +- Includes only L-1 and current major versions (skipping intermediate versions) +- Dynamically resolves beta and RC versions from current major branch +- Outputs structured JSON for easy parsing +- Skips beta versions when not available +- Provides debug output to stderr for troubleshooting + +## Matrix Generation Strategy + +### PHP Version Strategy + +The workflow uses an optimized PHP version strategy to reduce job count while maintaining comprehensive coverage: + +- **WC 7.7.0**: PHP 7.3 (legacy support - minimum required version) +- **WC L-1**: PHP 8.3 (stable) +- **WC latest**: PHP 8.3 (stable) +- **WC beta**: PHP 8.3 (stable) - only when available +- **WC rc**: PHP 8.4 (latest) + +### Version Resolution + +- **L-1 Version**: Extracted from JSON metadata +- **Beta Version**: Extracted from JSON metadata, only included when available +- **RC Version**: Always included - extracted from JSON metadata or falls back to string "rc" + +## How It Works + +### Script Execution + +1. Fetches the latest WooCommerce version from `https://api.wordpress.org/plugins/info/1.0/woocommerce.json` +2. Dynamically calculates the L-1 version by finding the latest stable version in the previous major branch +3. Fetches beta and RC versions from the current major branch only +4. Outputs JSON object to stdout for matrix generation + +### Workflow Integration + +1. Script runs and outputs structured JSON with versions and metadata +2. Workflow extracts specific versions using standard JSON parsing +3. Workflow builds optimized matrix with selective PHP version testing +4. Matrix includes only necessary combinations to reduce job count + +### Version Extraction + +```bash +# Get script result +SCRIPT_RESULT=$( .github/scripts/generate-wc-matrix.sh ) + +# Extract versions and metadata using jq +WC_VERSIONS=$(echo "$SCRIPT_RESULT" | jq -r '.versions') +L1_VERSION=$(echo "$SCRIPT_RESULT" | jq -r '.metadata.l1_version') +RC_VERSION=$(echo "$SCRIPT_RESULT" | jq -r '.metadata.rc_version') +BETA_VERSION=$(echo "$SCRIPT_RESULT" | jq -r '.metadata.beta_version') +``` + +## Dependencies + +- `curl`: For API requests +- `jq`: For JSON parsing and array generation +- `bash`: For script execution + +## Error Handling + +- Scripts use `set -e` to exit on any error +- Version extraction includes validation checks +- Graceful handling of missing beta versions +- If the API is unavailable or returns unexpected data, the workflow will fail gracefully + +## Future Considerations + +- Automatically adapts to new WooCommerce releases +- Will include beta versions when they become available +- Supports L-2 policy implementation if needed +- Maintains business continuity with WC 7.7.0 support diff --git a/.github/scripts/generate-wc-matrix.sh b/.github/scripts/generate-wc-matrix.sh new file mode 100755 index 00000000000..02a6daf7fcd --- /dev/null +++ b/.github/scripts/generate-wc-matrix.sh @@ -0,0 +1,157 @@ +#!/bin/bash + +# Script to dynamically generate WooCommerce version matrix for L-1 policy +# This script fetches the latest WC version and calculates the L-1 version + +set -e + +# Function to get the latest WooCommerce version from WordPress.org API +get_latest_wc_version() { + curl -s https://api.wordpress.org/plugins/info/1.0/woocommerce.json | jq -r '.version' +} + +# Function to get the latest stable version for a specific major version +get_latest_stable_for_major() { + local major_version=$1 + curl -s https://api.wordpress.org/plugins/info/1.0/woocommerce.json | \ + jq -r --arg major "$major_version" '.versions | with_entries(select(.key | startswith($major + ".") and (contains("-") | not))) | keys | sort_by( . | split(".") | map(tonumber) ) | last' +} + +# Function to get the L-1 version (previous major version's latest stable) +get_l1_version() { + local latest_version=$1 + local major_version=$(echo "$latest_version" | cut -d. -f1) + local l1_major=$((major_version - 1)) + get_latest_stable_for_major "$l1_major" +} + +# Function to get specific major versions' latest stable +get_major_versions_latest() { + local latest_version=$1 + local major_version=$(echo "$latest_version" | cut -d. -f1) + local versions=() + + # Dynamically calculate L-1 major version + local l1_major=$((major_version - 1)) + + # Only get L-1 version (previous major) and current major + # Skip intermediate major versions as they don't align with L-1 policy + for ((i=l1_major; i<=major_version; i++)); do + latest_stable=$(get_latest_stable_for_major "$i") + if [[ -n "$latest_stable" && "$latest_stable" != "null" ]]; then + versions+=("$latest_stable") + fi + done + + echo "${versions[@]}" +} + +# Function to get the latest RC version from WordPress.org API +get_latest_rc_version() { + curl -s https://api.wordpress.org/plugins/info/1.0/woocommerce.json | \ + jq -r '.versions | with_entries(select(.key|match("rc";"i"))) | keys | sort_by( . | split("-")[0] | split(".") | map(tonumber) ) | last' +} + +# Function to get the latest beta version from WordPress.org API +get_latest_beta_version() { + local latest_version=$1 + local major_version=$(echo "$latest_version" | cut -d. -f1) + curl -s https://api.wordpress.org/plugins/info/1.0/woocommerce.json | \ + jq -r --arg major "$major_version" '.versions | with_entries(select(.key | startswith($major + ".") and contains("beta"))) | keys | sort_by( . | split("-")[0] | split(".") | map(tonumber) ) | last' +} + +# Get the latest WooCommerce version +echo "Fetching latest WooCommerce version..." >&2 +LATEST_WC_VERSION=$(get_latest_wc_version) +echo "Latest WC version: $LATEST_WC_VERSION" >&2 + +# Get the L-1 version +L1_VERSION=$(get_l1_version "$LATEST_WC_VERSION") +echo "L-1 version: $L1_VERSION" >&2 + +# Get major versions latest stable +MAJOR_VERSIONS=($(get_major_versions_latest "$LATEST_WC_VERSION")) +echo "Major versions latest stable: ${MAJOR_VERSIONS[*]}" >&2 + +# Get latest RC and beta versions +echo "Fetching latest RC and beta versions..." >&2 +LATEST_RC_VERSION=$(get_latest_rc_version) +LATEST_BETA_VERSION=$(get_latest_beta_version "$LATEST_WC_VERSION") +echo "Latest RC version: $LATEST_RC_VERSION" >&2 +echo "Latest beta version: $LATEST_BETA_VERSION" >&2 + +# Build the version array +VERSIONS=("7.7.0") # Keep for business reasons (significant TPV) + +# Add major versions latest stable (excluding current major since we'll use 'latest') +for version in "${MAJOR_VERSIONS[@]}"; do + # Skip the current major version since we'll use 'latest' instead + if [[ "$version" != "$LATEST_WC_VERSION" ]]; then + VERSIONS+=("$version") + fi +done + +# Add latest, beta, rc (with actual versions) +VERSIONS+=("latest") +if [[ -n "$LATEST_BETA_VERSION" && "$LATEST_BETA_VERSION" != "null" ]]; then + VERSIONS+=("$LATEST_BETA_VERSION") + echo "Including beta version: $LATEST_BETA_VERSION" >&2 +else + echo "No beta version available, skipping beta tests" >&2 +fi + +# Decide whether to include RC: only include if RC base version (without suffix) is strictly greater than the latest stable. +INCLUDED_RC_VERSION="" +if [[ -n "$LATEST_RC_VERSION" && "$LATEST_RC_VERSION" != "null" ]]; then + RC_BASE="${LATEST_RC_VERSION%%-*}" + # Compare RC_BASE vs LATEST_WC_VERSION using sort -V + HIGHEST=$(printf '%s\n%s\n' "$RC_BASE" "$LATEST_WC_VERSION" | sort -V | tail -n1) + if [[ "$HIGHEST" == "$RC_BASE" && "$RC_BASE" != "$LATEST_WC_VERSION" ]]; then + INCLUDED_RC_VERSION="$LATEST_RC_VERSION" + VERSIONS+=("$LATEST_RC_VERSION") + echo "Including RC version: $LATEST_RC_VERSION (base $RC_BASE > latest $LATEST_WC_VERSION)" >&2 + else + echo "Skipping RC version $LATEST_RC_VERSION because stable $LATEST_WC_VERSION is already released for this line." >&2 + fi +else + echo "No RC version available, skipping rc tests" >&2 +fi + +# Validate versions before output +if [[ -z "$L1_VERSION" || "$L1_VERSION" == "null" ]]; then + echo "Error: Could not extract L-1 version" >&2 + exit 1 +fi + +if [[ ! "$L1_VERSION" =~ ^[0-9]+\.[0-9]+\.[0-9]+$ ]]; then + echo "Error: Invalid L-1 version: $L1_VERSION" >&2 + exit 1 +fi + +# RC is optional; do not fail if not present or skipped + +# Only validate beta if it's available +if [[ -n "$LATEST_BETA_VERSION" && "$LATEST_BETA_VERSION" != "null" ]]; then + if [[ ! "$LATEST_BETA_VERSION" =~ ^[0-9]+\.[0-9]+\.[0-9]+ ]]; then + echo "Error: Invalid beta version: $LATEST_BETA_VERSION" >&2 + exit 1 + fi +fi + +# Convert to JSON array and output only the JSON (no extra whitespace or newlines) +# Output a single JSON object with both versions and metadata +RESULT=$(jq -n \ + --argjson versions "$(printf '%s\n' "${VERSIONS[@]}" | jq -R . | jq -s .)" \ + --arg l1_version "$L1_VERSION" \ + --arg rc_version "${INCLUDED_RC_VERSION}" \ + --arg beta_version "${LATEST_BETA_VERSION}" \ + '{ + versions: $versions, + metadata: { + l1_version: $l1_version, + rc_version: (if ($rc_version // "") == "" or ($rc_version == "null") then null else $rc_version end), + beta_version: (if ($beta_version // "") == "" or ($beta_version == "null") then null else $beta_version end) + } + }') + +echo "$RESULT" diff --git a/.github/workflows/e2e-pull-request.yml b/.github/workflows/e2e-pull-request.yml index 75080631e7e..1b91a2f977e 100644 --- a/.github/workflows/e2e-pull-request.yml +++ b/.github/workflows/e2e-pull-request.yml @@ -41,21 +41,68 @@ concurrency: cancel-in-progress: true jobs: - wcpay-e2e-tests: + generate-matrix: runs-on: ubuntu-latest + outputs: + matrix: ${{ steps.generate_matrix.outputs.matrix }} + steps: + - name: Checkout WCPay repository + uses: actions/checkout@v4 + with: + ref: ${{ inputs.repo-branch || github.ref }} + + - name: "Generate matrix" + id: generate_matrix + run: | + # Use dynamic script to get WooCommerce versions (L-1 policy) + SCRIPT_RESULT=$( .github/scripts/generate-wc-matrix.sh ) + + # Extract versions and metadata from JSON + L1_VERSION=$(echo "$SCRIPT_RESULT" | jq -r '.metadata.l1_version') + + echo "Using L-1 version: $L1_VERSION" >&2 + # Create PR matrix with L-1 and latest versions + # For PRs, we test against L-1 and latest with PHP 8.3 (stable) + PHP_STABLE="8.3" + TEST_GROUPS_WCPAY="wcpay" + TEST_GROUPS_SUBSCRIPTIONS="subscriptions" + TEST_BRANCHES_MERCHANT="merchant" + TEST_BRANCHES_SHOPPER="shopper" + + # Initialize empty matrix array + MATRIX_ENTRIES=() + + # Add L-1 version with PHP 8.3 + MATRIX_ENTRIES+=("{\"woocommerce\":\"$L1_VERSION\",\"php\":\"$PHP_STABLE\",\"test_groups\":\"$TEST_GROUPS_WCPAY\",\"test_branches\":\"$TEST_BRANCHES_MERCHANT\"}") + MATRIX_ENTRIES+=("{\"woocommerce\":\"$L1_VERSION\",\"php\":\"$PHP_STABLE\",\"test_groups\":\"$TEST_GROUPS_WCPAY\",\"test_branches\":\"$TEST_BRANCHES_SHOPPER\"}") + MATRIX_ENTRIES+=("{\"woocommerce\":\"$L1_VERSION\",\"php\":\"$PHP_STABLE\",\"test_groups\":\"$TEST_GROUPS_SUBSCRIPTIONS\",\"test_branches\":\"$TEST_BRANCHES_MERCHANT\"}") + MATRIX_ENTRIES+=("{\"woocommerce\":\"$L1_VERSION\",\"php\":\"$PHP_STABLE\",\"test_groups\":\"$TEST_GROUPS_SUBSCRIPTIONS\",\"test_branches\":\"$TEST_BRANCHES_SHOPPER\"}") + + # Add latest with PHP 8.3 + MATRIX_ENTRIES+=("{\"woocommerce\":\"latest\",\"php\":\"$PHP_STABLE\",\"test_groups\":\"$TEST_GROUPS_WCPAY\",\"test_branches\":\"$TEST_BRANCHES_MERCHANT\"}") + MATRIX_ENTRIES+=("{\"woocommerce\":\"latest\",\"php\":\"$PHP_STABLE\",\"test_groups\":\"$TEST_GROUPS_WCPAY\",\"test_branches\":\"$TEST_BRANCHES_SHOPPER\"}") + MATRIX_ENTRIES+=("{\"woocommerce\":\"latest\",\"php\":\"$PHP_STABLE\",\"test_groups\":\"$TEST_GROUPS_SUBSCRIPTIONS\",\"test_branches\":\"$TEST_BRANCHES_MERCHANT\"}") + MATRIX_ENTRIES+=("{\"woocommerce\":\"latest\",\"php\":\"$PHP_STABLE\",\"test_groups\":\"$TEST_GROUPS_SUBSCRIPTIONS\",\"test_branches\":\"$TEST_BRANCHES_SHOPPER\"}") + + # Convert array to JSON + MATRIX_INCLUDE=$(printf '%s\n' "${MATRIX_ENTRIES[@]}" | jq -s . -c) + + echo "matrix={\"include\":$MATRIX_INCLUDE}" >> $GITHUB_OUTPUT + + wcpay-e2e-tests: + runs-on: ubuntu-latest + needs: generate-matrix strategy: fail-fast: false - matrix: - wc_version: [ 'latest' ] - test_groups: [ 'wcpay', 'subscriptions' ] # [TODO] Unskip blocks tests after investigating constant failures. - test_branches: [ 'merchant', 'shopper' ] + matrix: ${{ fromJSON(needs.generate-matrix.outputs.matrix) }} - name: WC - ${{ matrix.wc_version }} | ${{ matrix.test_groups }} - ${{ matrix.test_branches }} + name: WC - ${{ matrix.woocommerce }} | PHP - ${{ matrix.php }} | ${{ matrix.test_groups }} - ${{ matrix.test_branches }} env: E2E_WP_VERSION: 'latest' - E2E_WC_VERSION: ${{ matrix.wc_version }} + E2E_WC_VERSION: ${{ matrix.woocommerce }} + E2E_PHP_VERSION: ${{ matrix.php }} E2E_GROUP: ${{ matrix.test_groups }} E2E_BRANCH: ${{ matrix.test_branches }} diff --git a/.github/workflows/e2e-test.yml b/.github/workflows/e2e-test.yml index 9dac946779f..943755cc822 100644 --- a/.github/workflows/e2e-test.yml +++ b/.github/workflows/e2e-test.yml @@ -29,30 +29,105 @@ env: jobs: generate-matrix: - name: "Generate the matrix for subscriptions-tests dynamically" + name: "Generate the test matrix dynamically" runs-on: ubuntu-latest outputs: matrix: ${{ steps.generate_matrix.outputs.matrix }} steps: + - name: Checkout repository + uses: actions/checkout@v4 + - name: "Generate matrix" id: generate_matrix run: | - WC_VERSIONS=$( echo "[\"7.7.0\", \"latest\", \"beta\", \"rc\"]" ) - echo "matrix={\"woocommerce\":$WC_VERSIONS,\"test_groups\":[\"wcpay\", \"subscriptions\"],\"test_branches\":[\"merchant\", \"shopper\"]}" >> $GITHUB_OUTPUT - - # Run WCPay & subscriptions tests against specific WC versions - wcpay-subscriptions-tests: + # Use dynamic script to get WooCommerce versions (L-1 policy) + SCRIPT_RESULT=$( .github/scripts/generate-wc-matrix.sh ) + + # Extract versions and metadata from JSON + L1_VERSION=$(echo "$SCRIPT_RESULT" | jq -r '.metadata.l1_version') + RC_VERSION=$(echo "$SCRIPT_RESULT" | jq -r '.metadata.rc_version') + BETA_VERSION=$(echo "$SCRIPT_RESULT" | jq -r '.metadata.beta_version') + + if [[ "$BETA_VERSION" == "null" ]]; then + BETA_VERSION="" + fi + + echo "Using L-1 version: $L1_VERSION" >&2 + echo "Using RC version: $RC_VERSION" >&2 + if [[ -n "$BETA_VERSION" ]]; then + echo "Using beta version: $BETA_VERSION" >&2 + else + echo "No beta version available" >&2 + fi + + # Create optimized matrix with selective PHP version testing + # Build matrix dynamically based on WC versions + + # Define common values to reduce repetition + PHP_LEGACY="7.3" + PHP_STABLE="8.3" + PHP_LATEST="8.4" + TEST_GROUPS_WCPAY="wcpay" + TEST_GROUPS_SUBSCRIPTIONS="subscriptions" + TEST_BRANCHES_MERCHANT="merchant" + TEST_BRANCHES_SHOPPER="shopper" + + # Initialize empty matrix array + MATRIX_ENTRIES=() + + # Add WC 7.7.0 with PHP 7.3 only (legacy) + MATRIX_ENTRIES+=("{\"woocommerce\":\"7.7.0\",\"php\":\"$PHP_LEGACY\",\"test_groups\":\"$TEST_GROUPS_WCPAY\",\"test_branches\":\"$TEST_BRANCHES_MERCHANT\"}") + MATRIX_ENTRIES+=("{\"woocommerce\":\"7.7.0\",\"php\":\"$PHP_LEGACY\",\"test_groups\":\"$TEST_GROUPS_WCPAY\",\"test_branches\":\"$TEST_BRANCHES_SHOPPER\"}") + MATRIX_ENTRIES+=("{\"woocommerce\":\"7.7.0\",\"php\":\"$PHP_LEGACY\",\"test_groups\":\"$TEST_GROUPS_SUBSCRIPTIONS\",\"test_branches\":\"$TEST_BRANCHES_MERCHANT\"}") + MATRIX_ENTRIES+=("{\"woocommerce\":\"7.7.0\",\"php\":\"$PHP_LEGACY\",\"test_groups\":\"$TEST_GROUPS_SUBSCRIPTIONS\",\"test_branches\":\"$TEST_BRANCHES_SHOPPER\"}") + + # Add L-1 version with PHP 8.3 only + MATRIX_ENTRIES+=("{\"woocommerce\":\"$L1_VERSION\",\"php\":\"$PHP_STABLE\",\"test_groups\":\"$TEST_GROUPS_WCPAY\",\"test_branches\":\"$TEST_BRANCHES_MERCHANT\"}") + MATRIX_ENTRIES+=("{\"woocommerce\":\"$L1_VERSION\",\"php\":\"$PHP_STABLE\",\"test_groups\":\"$TEST_GROUPS_WCPAY\",\"test_branches\":\"$TEST_BRANCHES_SHOPPER\"}") + MATRIX_ENTRIES+=("{\"woocommerce\":\"$L1_VERSION\",\"php\":\"$PHP_STABLE\",\"test_groups\":\"$TEST_GROUPS_SUBSCRIPTIONS\",\"test_branches\":\"$TEST_BRANCHES_MERCHANT\"}") + MATRIX_ENTRIES+=("{\"woocommerce\":\"$L1_VERSION\",\"php\":\"$PHP_STABLE\",\"test_groups\":\"$TEST_GROUPS_SUBSCRIPTIONS\",\"test_branches\":\"$TEST_BRANCHES_SHOPPER\"}") + + # Add latest with PHP 8.3 only + MATRIX_ENTRIES+=("{\"woocommerce\":\"latest\",\"php\":\"$PHP_STABLE\",\"test_groups\":\"$TEST_GROUPS_WCPAY\",\"test_branches\":\"$TEST_BRANCHES_MERCHANT\"}") + MATRIX_ENTRIES+=("{\"woocommerce\":\"latest\",\"php\":\"$PHP_STABLE\",\"test_groups\":\"$TEST_GROUPS_WCPAY\",\"test_branches\":\"$TEST_BRANCHES_SHOPPER\"}") + MATRIX_ENTRIES+=("{\"woocommerce\":\"latest\",\"php\":\"$PHP_STABLE\",\"test_groups\":\"$TEST_GROUPS_SUBSCRIPTIONS\",\"test_branches\":\"$TEST_BRANCHES_MERCHANT\"}") + MATRIX_ENTRIES+=("{\"woocommerce\":\"latest\",\"php\":\"$PHP_STABLE\",\"test_groups\":\"$TEST_GROUPS_SUBSCRIPTIONS\",\"test_branches\":\"$TEST_BRANCHES_SHOPPER\"}") + + # Add beta with PHP 8.3 only (only if beta version is available) + if [[ -n "$BETA_VERSION" ]]; then + MATRIX_ENTRIES+=("{\"woocommerce\":\"$BETA_VERSION\",\"php\":\"$PHP_STABLE\",\"test_groups\":\"$TEST_GROUPS_WCPAY\",\"test_branches\":\"$TEST_BRANCHES_MERCHANT\"}") + MATRIX_ENTRIES+=("{\"woocommerce\":\"$BETA_VERSION\",\"php\":\"$PHP_STABLE\",\"test_groups\":\"$TEST_GROUPS_WCPAY\",\"test_branches\":\"$TEST_BRANCHES_SHOPPER\"}") + MATRIX_ENTRIES+=("{\"woocommerce\":\"$BETA_VERSION\",\"php\":\"$PHP_STABLE\",\"test_groups\":\"$TEST_GROUPS_SUBSCRIPTIONS\",\"test_branches\":\"$TEST_BRANCHES_MERCHANT\"}") + MATRIX_ENTRIES+=("{\"woocommerce\":\"$BETA_VERSION\",\"php\":\"$PHP_STABLE\",\"test_groups\":\"$TEST_GROUPS_SUBSCRIPTIONS\",\"test_branches\":\"$TEST_BRANCHES_SHOPPER\"}") + fi + + # Add rc with PHP 8.4 only (only if RC version is available) + if [[ -n "$RC_VERSION" && "$RC_VERSION" != "null" ]]; then + MATRIX_ENTRIES+=("{\"woocommerce\":\"$RC_VERSION\",\"php\":\"$PHP_LATEST\",\"test_groups\":\"$TEST_GROUPS_WCPAY\",\"test_branches\":\"$TEST_BRANCHES_MERCHANT\"}") + MATRIX_ENTRIES+=("{\"woocommerce\":\"$RC_VERSION\",\"php\":\"$PHP_LATEST\",\"test_groups\":\"$TEST_GROUPS_WCPAY\",\"test_branches\":\"$TEST_BRANCHES_SHOPPER\"}") + MATRIX_ENTRIES+=("{\"woocommerce\":\"$RC_VERSION\",\"php\":\"$PHP_LATEST\",\"test_groups\":\"$TEST_GROUPS_SUBSCRIPTIONS\",\"test_branches\":\"$TEST_BRANCHES_MERCHANT\"}") + MATRIX_ENTRIES+=("{\"woocommerce\":\"$RC_VERSION\",\"php\":\"$PHP_LATEST\",\"test_groups\":\"$TEST_GROUPS_SUBSCRIPTIONS\",\"test_branches\":\"$TEST_BRANCHES_SHOPPER\"}") + else + echo "No RC version available, skipping RC matrix entries" >&2 + fi + + # Convert array to JSON + MATRIX_INCLUDE=$(printf '%s\n' "${MATRIX_ENTRIES[@]}" | jq -s . -c) + + echo "matrix={\"include\":$MATRIX_INCLUDE}" >> $GITHUB_OUTPUT + + # Run WCPay and Subscriptions tests across WooCommerce and PHP matrix + wcpay-matrix-tests: runs-on: ubuntu-latest needs: generate-matrix strategy: fail-fast: false matrix: ${{ fromJSON(needs.generate-matrix.outputs.matrix) }} - - name: WC - ${{ matrix.woocommerce }} | ${{ matrix.test_groups }} - ${{ matrix.test_branches }} - + name: "WC - ${{ matrix.woocommerce }} | PHP - ${{ matrix.php }} | ${{ matrix.test_groups }} - ${{ matrix.test_branches }}" env: E2E_WP_VERSION: 'latest' E2E_WC_VERSION: ${{ matrix.woocommerce }} + E2E_PHP_VERSION: ${{ matrix.php }} E2E_GROUP: ${{ matrix.test_groups }} E2E_BRANCH: ${{ matrix.test_branches }} SKIP_WC_BLOCKS_TESTS: 1 # skip running blocks tests diff --git a/changelog/dev-woopmnt-5249-e2e-ensure-version-coverage-for-woocommerce-and-php b/changelog/dev-woopmnt-5249-e2e-ensure-version-coverage-for-woocommerce-and-php new file mode 100644 index 00000000000..285d33e6c02 --- /dev/null +++ b/changelog/dev-woopmnt-5249-e2e-ensure-version-coverage-for-woocommerce-and-php @@ -0,0 +1,5 @@ +Significance: patch +Type: dev +Comment: Ensure version coverage in E2E tests + + diff --git a/tests/e2e/README.md b/tests/e2e/README.md index 0fc12c134e6..adec3cb8bb8 100644 --- a/tests/e2e/README.md +++ b/tests/e2e/README.md @@ -4,15 +4,37 @@ WooPayments e2e tests can be found in the `./tests/e2e/specs` directory. These t E2E tests can be run locally or in GitHub Actions. Github Actions are already configured and don't require any changes to run the tests. +## Retry strategy + +We don't use Playwright's built-in retries in this repo (retries are `0`). Instead, CI re-runs only the spec files that failed in the first run: + +- First run: Execute the full suite and generate a `results.json` file +- Selective re-run: If some specs failed, CI re-runs just those spec files once +- Local runs: No automatic retries + +This approach is intentional. Playwright’s `--last-failed` flag was evaluated but led to instability, likely due to some tests depending on previous steps within the same file. For example, a spec may first save a card, then process a payment with it, and finally delete the card. Retrying only the “delete the card” step without the earlier context will fail. + +Note: The Playwright config sets `video: on-first-retry`, which applies only if you enable built-in retries locally. + +## Dynamic matrix generation + +- L-1 policy: Tests run against the latest WooCommerce version and the L-1 (previous major) version +- Dynamic version resolution: Automatically fetches latest WC, RC, and beta versions from WordPress.org API +- Optimized PHP strategy: Reduces job count while maintaining comprehensive coverage +- Business continuity: Maintains support for WC 7.7.0 for significant TPV reasons + +CI coverage differences: + +- Pull requests: Run against WC latest and WC L-1 on PHP 8.3 +- Scheduled/full runs: Include WC 7.7.0 (PHP 7.3), WC L-1 (PHP 8.3), WC latest (PHP 8.3), optional WC beta (PHP 8.3), and WC RC (PHP 8.4 when applicable) + ## Setting up & running E2E tests For running E2E tests locally, create a new file named `local.env` under `tests/e2e/config` folder with the following env variables (replace values as required). -
-Required env variables -

+### Required env variables -``` +```bash # WooPayments Dev Tools Repo WCP_DEV_TOOLS_REPO='https://github.com/dev-tools-repo.git or git@github.com:org/dev-tools-repo.git' @@ -20,22 +42,15 @@ WCP_DEV_TOOLS_REPO='https://github.com/dev-tools-repo.git or git@github.com:org/ DEBUG=false ``` -

-
- ---- +### Choose Transact Platform Server instance -
-Choose Transact Platform Server instance -

+It is possible to use the live server or a local docker instance of the Transact Platform Server when testing locally. On GitHub Actions, the live server is used for tests. Add the following env variables to your `local.env` based on your preference (replace values as required). -It is possible to use the live server or a local docker instance of the Transact Platform Server when testing locally. On Github Actions, the live server is used for tests. Add the following env variables to your `local.env` based on your preference (replace values as required). - -**Using Local Server on Docker** +#### Using Local Server on Docker By default, the local E2E environment is configured to use the Transact Platform local server instance. Add the following env variables to configure the local server instance. -``` +```bash # Transact Platform Server Repo TRANSACT_PLATFORM_SERVER_REPO='https://github.com/server-repo.git or git@github.com:org/server-repo.git' @@ -50,10 +65,11 @@ E2E_WCPAY_STRIPE_ACCOUNT_ID= E2E_WOOPAY_BLOG_ID= ``` -**Using Live Server** +#### Using Live Server For using the live server, you'll need to add a Jetpack blog token, user token, & blog id from one of your test sites connected to a WooPayments test account. On a connected test site, you can use the code below to extract the blog id & tokens. -``` + +```php Jetpack_Options::get_option( 'id' ); Jetpack_Options::get_option( 'blog_token' ); Jetpack_Options::get_option( 'user_tokens' ); @@ -62,7 +78,8 @@ Jetpack_Options::get_option( 'user_tokens' ); Set the value of `E2E_USE_LOCAL_SERVER` to `false` to enable live server. Once you have the blog id & tokens, add the following env variables to your `local.env`. -``` + +```bash # Set local server to false for using live server. Default: true. E2E_USE_LOCAL_SERVER=false @@ -71,92 +88,62 @@ E2E_JP_USER_TOKEN='' E2E_JP_SITE_ID='' ``` -

-
- ---- - -
-Installing Plugins -

+### Installing plugins If you wish to run E2E tests for WC Subscriptions, the following env variables need to be added to your `local.env` (replace values as required). For the `E2E_GH_TOKEN`, follow [these instructions to generate a GitHub Personal Access Token](https://docs.github.com/en/github/authenticating-to-github/creating-a-personal-access-token) and assign the `repo` scope to it. -``` +```bash E2E_GH_TOKEN='githubPersonalAccessToken' WC_SUBSCRIPTIONS_REPO='{owner}/{repo}' ``` -

-
- ---- - -
-Skipping Plugins -

+### Skipping plugins If you wish to skip E2E tests for WC Subscriptions, Action Scheduler, or WC Gutenberg Products Blocks, the following env variables need to be added to your `local.env`. -``` + +```bash SKIP_WC_SUBSCRIPTIONS_TESTS=1 SKIP_WC_ACTION_SCHEDULER_TESTS=1 SKIP_WC_BLOCKS_TESTS=1 ``` -

-
- ---- - -
-Using a specific version of WordPress or WooCommerce -

+### Using a specific version of WordPress or WooCommerce To use a specific version of WordPress or WooCommerce for testing, the following env variables need to be added to your `local.env`. -``` + +```bash E2E_WP_VERSION='' E2E_WC_VERSION='' ``` -

-
- ---- +### Initialize E2E docker environment -
-Initialize E2E docker environment -

+1. Make sure to run `npm install`, `composer install` and `npm run build:client` before running the setup script. +2. Run the setup script `npm run test:e2e-setup` to spin up E2E environment in docker containers. - 1. Make sure to run `npm install`, `composer install` and `npm run build:client` before running the setup script. - 2. Run the setup script `npm run test:e2e-setup` to spin up E2E environment in docker containers. +After the E2E environment is up, you can access the containers on: - After the E2E environment is up, you can access the containers on: +- WC E2E Client: +- WC E2E Server: (Available only when using local server) - - WC E2E Client: http://localhost:8084 - - WC E2E Server: http://localhost:8088 (Available only when using local server) +Note: Be aware that the server port may change in the `docker-compose.e2e.yml` configuration, so when you can't access the server, try running `docker port transact_platform_server_wordpress_e2e 80` to find out the bound port of the E2E server container. - **Note:** Be aware that the server port may change in the `docker-compose.e2e.yml` configuration, so when you can't access the server, try running `docker port transact_platform_server_wordpress_e2e 80` to find out the bound port of the E2E server container. - -

-
- ---- - -
-Running tests -

+### Running tests There are two modes for running tests: -1. **Headless mode**: `npm run test:e2e`. In headless mode the test runner executes all or specified specs without launching a Chromium user interface. -2. **UI mode**: `npm run test:e2e-ui`. UI mode is interactive and launches a Chromium user interface. It's useful for developing, debugging, and troubleshooting failing tests. For more information about Playwright UI mode, see the [Playwright UI Mode docs](https://playwright.dev/docs/test-ui-mode#introduction). +1. Headless mode: `npm run test:e2e`. In headless mode the test runner executes all or specified specs without launching a Chromium user interface. +2. UI mode: `npm run test:e2e-ui`. UI mode is interactive and launches a Chromium user interface. It's useful for developing, debugging, and troubleshooting failing tests. For more information about Playwright UI mode, see the [Playwright UI Mode docs](https://playwright.dev/docs/test-ui-mode#introduction). -**Additional options** +Additional options: -- `npm run test:e2e keyword` runs tests only with a specific keyword in the file name, e.g. `dispute` or `checkout`. -- `npm run test:e2e -- --update-snapshots` updates snapshots. This can be combined with a keyword to update a specific set of snapshots, e.g. `npm run test:e2e -- --update-snapshots deposits`. +- Filter by path or glob: `npm run test:e2e tests/e2e/specs/**/checkout*.spec.ts` +- Filter by test title: `npm run test:e2e -- -g "Checkout"` (or `--grep`) +- Update snapshots (optionally with a filter): + - `npm run test:e2e -- --update-snapshots` + - `npm run test:e2e -- --update-snapshots tests/e2e/specs/**/deposits*.spec.ts` #### Running only a single test suite @@ -185,9 +172,6 @@ Handy utility scripts for managing your E2E environment: - `npm run test:e2e-reset` Stops containers and performs cleanup. - `npm run test:e2e-up` Starts containers without setting up again. -

-
- ### Running on Atomic site For running E2E tests on the Atomic site, follow the same guidelines mentioned above, and add `NODE_ENV=atomic` to your `local.env` file. Then bring up your E2E environment. Lastly, run tests using `npm run test:e2e` or `npm run test:e2e-ui`. @@ -203,9 +187,15 @@ Place new spec files in the appropriate directory under `tests/e2e/specs`. The d ## Debugging tests -Currently, the best way to debug tests is to use the Playwright UI mode. This mode allows you to see the browser and interact with it after the test runs. -You can use the locator functionality to help correctly determine the locator syntax to correctly target the HTML element you need. Lastly, you can also use -`console.log()` to assist with debugging tests in UI mode. To run tests in UI mode, use the `npm run test:e2e-ui path/to/test.spec` command. +The best way to debug tests is to use the Playwright UI mode. This mode allows you to see the browser and interact with it after the test runs. You can use the locator functionality to help correctly determine the locator syntax to correctly target the HTML element you need. Lastly, you can also use `console.log()` to assist with debugging tests in UI mode. To run tests in UI mode, use the `npm run test:e2e-ui path/to/test.spec` command. + +### Understanding test failures and re-runs + +In CI, failed spec files are re-run once in a second pass: + +1. First run: Execute the full matrix and produce `results.json` +2. Selective re-run: CI re-runs only the spec files that failed +3. Final results: The workflow finishes after the selective re-run and uploads artifacts ## Slack integration @@ -314,20 +304,30 @@ test.describe( 'Sign in as customer', () => { } ); ``` -**How can I investigate and interact with a test failures?** +**How does the dynamic matrix generation work?** + +The E2E test matrix is dynamically generated using the `.github/scripts/generate-wc-matrix.sh` script: + +- L-1 policy: Automatically tests against the latest WooCommerce version and the L-1 (previous major) version +- Version resolution: Fetches latest WC, RC, and beta versions from WordPress.org API +- PHP strategy: + - WC 7.7.0: PHP 7.3 (legacy support) + - WC L-1 & latest: PHP 8.3 (stable) + - WC RC: PHP 8.4 (latest) +- Business continuity: Maintains WC 7.7.0 support for significant TPV reasons + +This ensures comprehensive test coverage while optimizing CI execution time and resource usage. + +**How can I investigate and interact with a test failure?** -- **Github Action test runs** +- GitHub Action test runs - View GitHub checks in the "Checks" tab of a PR - - There are currently four E2E test workflows: - - E2E Tests - Pull Request / WC - latest | wcpay - merchant (pull_request) - - E2E Tests - Pull Request / WC - latest | wcpay - shopper (pull_request) - - E2E Tests - Pull Request / WC - latest | subscriptions - merchant (pull_request) - - E2E Tests - Pull Request / WC - latest | subscriptions - shopper (pull_request) + - The E2E test matrix is generated dynamically. PRs cover WC latest and L-1 on PHP 8.3; scheduled runs also include 7.7.0 (PHP 7.3), RC (PHP 8.4), and beta (when available) - Click on the details link to the right of the failed job to see the summary - - In the job summary, click on the "Run tests, upload screenshots & logs" section. + - In the job summary, click on the "Run tests, upload screenshots & logs" section - Click on the artifact download link at the end of the section, then extract and copy the `playwright-report` directory to the root of the WooPayments repository - Run `npx playwright show-report` to open the report in a browser - - Alternatively, after extracting the artifact you can open the `playwright-report/index.html` file in a browser. -- **Local test runs**: + - Alternatively, after extracting the artifact you can open the `playwright-report/index.html` file in a browser +- Local test runs - Local test reports will output in the `playwright-report` directory - Run `npx playwright show-report` to open the report in a browser diff --git a/tests/e2e/env/setup.sh b/tests/e2e/env/setup.sh index 0d09983f3d7..0cdf61a9458 100755 --- a/tests/e2e/env/setup.sh +++ b/tests/e2e/env/setup.sh @@ -208,12 +208,6 @@ cli wp plugin install wordpress-importer --activate # Install WooCommerce if [[ -n "$E2E_WC_VERSION" && $E2E_WC_VERSION != 'latest' ]]; then - # If specified version is 'beta' or 'rc', fetch the latest beta version from WordPress.org API - if [[ $E2E_WC_VERSION == 'beta' ]] || [[ $E2E_WC_VERSION == 'rc' ]]; then - # Get the latest non-trunk version number from the .org repo. This will usually be the latest release, beta, or rc. - E2E_WC_VERSION=$(curl https://api.wordpress.org/plugins/info/1.0/woocommerce.json | jq -r '.versions | with_entries(select(.key|match("'$E2E_WC_VERSION'";"i"))) | keys | sort_by( . | split("-")[0] | split(".") | map(tonumber) ) | last' --sort-keys) - fi - echo "Installing and activating specified WooCommerce version..." cli wp plugin install woocommerce --version="$E2E_WC_VERSION" --activate else diff --git a/tests/e2e/playwright.config.ts b/tests/e2e/playwright.config.ts index a603cd12607..e6a5030e44a 100644 --- a/tests/e2e/playwright.config.ts +++ b/tests/e2e/playwright.config.ts @@ -51,8 +51,7 @@ export default defineConfig( { fullyParallel: false, /* Fail the build on CI if you accidentally left test.only in the source code. */ forbidOnly: !! process.env.CI, - /* Retry on CI only */ - retries: process.env.CI ? 2 : 0, + retries: 0, /* Opt out of parallel tests. */ workers: 1, /* Reporters to use. See https://playwright.dev/docs/test-reporters */