Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
47 changes: 38 additions & 9 deletions .github/actions/e2e/run-log-tests/action.yml
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
name: 'Run Tests'
name: 'Run E2E Tests with Retry & Upload Logs'
description: 'Runs E2E tests with retry & upload logs and screenshots'

runs:
Expand All @@ -10,12 +10,17 @@ runs:
shell: /bin/bash +e {0}
run: |
npm run test:e2e-ci

if [[ -f "$E2E_RESULT_FILEPATH" ]]; then
E2E_NUM_FAILED_TEST_SUITES=$(cat "$E2E_RESULT_FILEPATH" | jq '.stats["unexpected"]')
echo "FIRST_RUN_FAILED_TEST_SUITES=$(echo $E2E_NUM_FAILED_TEST_SUITES)" >> $GITHUB_OUTPUT
if [[ ${E2E_NUM_FAILED_TEST_SUITES} -gt 0 ]]; then
echo "::notice::${E2E_NUM_FAILED_TEST_SUITES} test suite(s) failed in the first run but we will try (it) them again in the second run."
RESULTS_JSON="$E2E_RESULT_FILEPATH"
if [[ -f "$RESULTS_JSON" ]]; then
# Build unique list of spec files with any failed/unexpected/timedOut/interrupted result and count them.
FAILED_SPECS_COUNT=$( jq -r '[ .. | objects | select(has("specs")) | .specs[]
| select( ( any(.tests[]?; (.status=="unexpected") or (.status=="failed")) )
or ( any(.tests[]?.results[]?; (.status=="failed") or (.status=="timedOut") or (.status=="interrupted")) ) )
| .file ] | unique | length' "$RESULTS_JSON" )
echo "FIRST_RUN_FAILED_TEST_SUITES=$FAILED_SPECS_COUNT" >> $GITHUB_OUTPUT
echo "RESULTS_JSON=$RESULTS_JSON" >> $GITHUB_OUTPUT
if [[ ${FAILED_SPECS_COUNT} -gt 0 ]]; then
echo "::notice::${FAILED_SPECS_COUNT} spec file(s) failed in the first run. We will re-run only the failed specs."
exit 0
fi
else
Expand All @@ -27,9 +32,32 @@ runs:
- name: Re-try Failed E2E Files
if: ${{ steps.first_run_e2e_tests.outputs.FIRST_RUN_FAILED_TEST_SUITES > 0 }}
shell: bash
# Filter failed E2E files from the result JSON file, and re-run them.
run: |
npm run test:e2e-ci $(cat $E2E_RESULT_FILEPATH | jq -r '[.suites[] | (if has("suites") then .suites[] | .specs[] else .specs[] end) | select(.tests[].status == "unexpected") | .file] | unique | .[]')
set -e
RESULTS_JSON="${{ steps.first_run_e2e_tests.outputs.RESULTS_JSON }}"
echo "Using results file: $RESULTS_JSON"

# Build a unique list of spec files that had unexpected/failed outcomes.
# This uses a recursive jq search to find any node with a 'specs' array, then selects specs
# where any test result is marked unexpected/failed.
mapfile -t FAILED_SPECS < <( jq -r '[ .. | objects | select(has("specs")) | .specs[]
| select( ( any(.tests[]?; (.status=="unexpected") or (.status=="failed")) )
or ( any(.tests[]?.results[]?; (.status=="failed") or (.status=="timedOut") or (.status=="interrupted")) ) )
| .file ] | unique | .[]' "$RESULTS_JSON" )

if [[ ${#FAILED_SPECS[@]} -eq 0 ]]; then
echo "::notice::No failed specs found in results file. Re-running full suite instead."
npm run test:e2e-ci
exit 0
fi

echo "Retrying failed specs (${#FAILED_SPECS[@]}):"
for f in "${FAILED_SPECS[@]}"; do
echo " - $f"
done

# Re-run only the failed spec files.
npm run test:e2e-ci "${FAILED_SPECS[@]}"

# Archive screenshots if any
- name: Archive e2e test screenshots & logs
Expand All @@ -41,5 +69,6 @@ runs:
playwright-report/
tests/e2e/test-results
${{ env.E2E_RESULT_FILEPATH }}
${{ steps.first_run_e2e_tests.outputs.RESULTS_JSON }}
if-no-files-found: ignore
retention-days: 14
4 changes: 2 additions & 2 deletions .github/actions/setup-php/action.yml
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
name: "Set up PHP"
description: "Extracts the required PHP version from plugin file and uses it to build PHP."
description: "Extracts the required PHP version from plugin file and uses it to build PHP. Can be overridden with E2E_PHP_VERSION environment variable."

runs:
using: composite
Expand All @@ -14,6 +14,6 @@ runs:
- name: "Setup PHP"
uses: shivammathur/setup-php@v2
with:
php-version: ${{ steps.get_min_php_version.outputs.MIN_PHP_VERSION }}
php-version: ${{ env.E2E_PHP_VERSION || steps.get_min_php_version.outputs.MIN_PHP_VERSION }}
tools: composer
coverage: none
111 changes: 111 additions & 0 deletions .github/scripts/README.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,111 @@
# GitHub Actions Scripts

This directory contains scripts used by GitHub Actions workflows for dynamic version management and matrix generation.

## Scripts

### `generate-wc-matrix.sh`

Generates the WooCommerce version matrix for E2E tests with dynamic version resolution and optimized PHP version strategy.

**Usage:**

```bash
.github/scripts/generate-wc-matrix.sh
```

**Output:**
Single JSON object containing versions array and metadata:

```json
{
"versions": [
"7.7.0",
"9.9.5",
"latest",
"10.1.0-rc.2"
],
"metadata": {
"l1_version": "9.9.5",
"rc_version": "10.1.0-rc.2",
"beta_version": null
}
}
```

**Features:**

- Fetches latest WC version from WordPress.org API
- Dynamically calculates L-1 version (latest stable in previous major branch)
- Includes only L-1 and current major versions (skipping intermediate versions)
- Dynamically resolves beta and RC versions from current major branch
- Outputs structured JSON for easy parsing
- Skips beta versions when not available
- Provides debug output to stderr for troubleshooting

## Matrix Generation Strategy

### PHP Version Strategy

The workflow uses an optimized PHP version strategy to reduce job count while maintaining comprehensive coverage:

- **WC 7.7.0**: PHP 7.3 (legacy support - minimum required version)
- **WC L-1**: PHP 8.3 (stable)
- **WC latest**: PHP 8.3 (stable)
- **WC beta**: PHP 8.3 (stable) - only when available
- **WC rc**: PHP 8.4 (latest)

### Version Resolution

- **L-1 Version**: Extracted from JSON metadata
- **Beta Version**: Extracted from JSON metadata, only included when available
- **RC Version**: Always included - extracted from JSON metadata or falls back to string "rc"

## How It Works

### Script Execution

1. Fetches the latest WooCommerce version from `https://api.wordpress.org/plugins/info/1.0/woocommerce.json`
2. Dynamically calculates the L-1 version by finding the latest stable version in the previous major branch
3. Fetches beta and RC versions from the current major branch only
4. Outputs JSON object to stdout for matrix generation

### Workflow Integration

1. Script runs and outputs structured JSON with versions and metadata
2. Workflow extracts specific versions using standard JSON parsing
3. Workflow builds optimized matrix with selective PHP version testing
4. Matrix includes only necessary combinations to reduce job count

### Version Extraction

```bash
# Get script result
SCRIPT_RESULT=$( .github/scripts/generate-wc-matrix.sh )

# Extract versions and metadata using jq
WC_VERSIONS=$(echo "$SCRIPT_RESULT" | jq -r '.versions')
L1_VERSION=$(echo "$SCRIPT_RESULT" | jq -r '.metadata.l1_version')
RC_VERSION=$(echo "$SCRIPT_RESULT" | jq -r '.metadata.rc_version')
BETA_VERSION=$(echo "$SCRIPT_RESULT" | jq -r '.metadata.beta_version')
```

## Dependencies

- `curl`: For API requests
- `jq`: For JSON parsing and array generation
- `bash`: For script execution

## Error Handling

- Scripts use `set -e` to exit on any error
- Version extraction includes validation checks
- Graceful handling of missing beta versions
- If the API is unavailable or returns unexpected data, the workflow will fail gracefully

## Future Considerations

- Automatically adapts to new WooCommerce releases
- Will include beta versions when they become available
- Supports L-2 policy implementation if needed
- Maintains business continuity with WC 7.7.0 support
157 changes: 157 additions & 0 deletions .github/scripts/generate-wc-matrix.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,157 @@
#!/bin/bash

# Script to dynamically generate WooCommerce version matrix for L-1 policy
# This script fetches the latest WC version and calculates the L-1 version

set -e

# Function to get the latest WooCommerce version from WordPress.org API
get_latest_wc_version() {
curl -s https://api.wordpress.org/plugins/info/1.0/woocommerce.json | jq -r '.version'
}

# Function to get the latest stable version for a specific major version
get_latest_stable_for_major() {
local major_version=$1
curl -s https://api.wordpress.org/plugins/info/1.0/woocommerce.json | \
jq -r --arg major "$major_version" '.versions | with_entries(select(.key | startswith($major + ".") and (contains("-") | not))) | keys | sort_by( . | split(".") | map(tonumber) ) | last'
}

# Function to get the L-1 version (previous major version's latest stable)
get_l1_version() {
local latest_version=$1
local major_version=$(echo "$latest_version" | cut -d. -f1)
local l1_major=$((major_version - 1))
get_latest_stable_for_major "$l1_major"
}

# Function to get specific major versions' latest stable
get_major_versions_latest() {
local latest_version=$1
local major_version=$(echo "$latest_version" | cut -d. -f1)
local versions=()

# Dynamically calculate L-1 major version
local l1_major=$((major_version - 1))

# Only get L-1 version (previous major) and current major
# Skip intermediate major versions as they don't align with L-1 policy
for ((i=l1_major; i<=major_version; i++)); do
latest_stable=$(get_latest_stable_for_major "$i")
if [[ -n "$latest_stable" && "$latest_stable" != "null" ]]; then
versions+=("$latest_stable")
fi
done

echo "${versions[@]}"
}

# Function to get the latest RC version from WordPress.org API
get_latest_rc_version() {
curl -s https://api.wordpress.org/plugins/info/1.0/woocommerce.json | \
jq -r '.versions | with_entries(select(.key|match("rc";"i"))) | keys | sort_by( . | split("-")[0] | split(".") | map(tonumber) ) | last'
}

# Function to get the latest beta version from WordPress.org API
get_latest_beta_version() {
local latest_version=$1
local major_version=$(echo "$latest_version" | cut -d. -f1)
curl -s https://api.wordpress.org/plugins/info/1.0/woocommerce.json | \
jq -r --arg major "$major_version" '.versions | with_entries(select(.key | startswith($major + ".") and contains("beta"))) | keys | sort_by( . | split("-")[0] | split(".") | map(tonumber) ) | last'
}

# Get the latest WooCommerce version
echo "Fetching latest WooCommerce version..." >&2
LATEST_WC_VERSION=$(get_latest_wc_version)
echo "Latest WC version: $LATEST_WC_VERSION" >&2

# Get the L-1 version
L1_VERSION=$(get_l1_version "$LATEST_WC_VERSION")
echo "L-1 version: $L1_VERSION" >&2

# Get major versions latest stable
MAJOR_VERSIONS=($(get_major_versions_latest "$LATEST_WC_VERSION"))
echo "Major versions latest stable: ${MAJOR_VERSIONS[*]}" >&2

# Get latest RC and beta versions
echo "Fetching latest RC and beta versions..." >&2
LATEST_RC_VERSION=$(get_latest_rc_version)
LATEST_BETA_VERSION=$(get_latest_beta_version "$LATEST_WC_VERSION")
echo "Latest RC version: $LATEST_RC_VERSION" >&2
echo "Latest beta version: $LATEST_BETA_VERSION" >&2

# Build the version array
VERSIONS=("7.7.0") # Keep for business reasons (significant TPV)

# Add major versions latest stable (excluding current major since we'll use 'latest')
for version in "${MAJOR_VERSIONS[@]}"; do
# Skip the current major version since we'll use 'latest' instead
if [[ "$version" != "$LATEST_WC_VERSION" ]]; then
VERSIONS+=("$version")
fi
done

# Add latest, beta, rc (with actual versions)
VERSIONS+=("latest")
if [[ -n "$LATEST_BETA_VERSION" && "$LATEST_BETA_VERSION" != "null" ]]; then
VERSIONS+=("$LATEST_BETA_VERSION")
echo "Including beta version: $LATEST_BETA_VERSION" >&2
else
echo "No beta version available, skipping beta tests" >&2
fi

# Decide whether to include RC: only include if RC base version (without suffix) is strictly greater than the latest stable.
INCLUDED_RC_VERSION=""
if [[ -n "$LATEST_RC_VERSION" && "$LATEST_RC_VERSION" != "null" ]]; then
RC_BASE="${LATEST_RC_VERSION%%-*}"
# Compare RC_BASE vs LATEST_WC_VERSION using sort -V
HIGHEST=$(printf '%s\n%s\n' "$RC_BASE" "$LATEST_WC_VERSION" | sort -V | tail -n1)
if [[ "$HIGHEST" == "$RC_BASE" && "$RC_BASE" != "$LATEST_WC_VERSION" ]]; then
INCLUDED_RC_VERSION="$LATEST_RC_VERSION"
VERSIONS+=("$LATEST_RC_VERSION")
echo "Including RC version: $LATEST_RC_VERSION (base $RC_BASE > latest $LATEST_WC_VERSION)" >&2
else
echo "Skipping RC version $LATEST_RC_VERSION because stable $LATEST_WC_VERSION is already released for this line." >&2
fi
else
echo "No RC version available, skipping rc tests" >&2
fi

# Validate versions before output
if [[ -z "$L1_VERSION" || "$L1_VERSION" == "null" ]]; then
echo "Error: Could not extract L-1 version" >&2
exit 1
fi

if [[ ! "$L1_VERSION" =~ ^[0-9]+\.[0-9]+\.[0-9]+$ ]]; then
echo "Error: Invalid L-1 version: $L1_VERSION" >&2
exit 1
fi

# RC is optional; do not fail if not present or skipped

# Only validate beta if it's available
if [[ -n "$LATEST_BETA_VERSION" && "$LATEST_BETA_VERSION" != "null" ]]; then
if [[ ! "$LATEST_BETA_VERSION" =~ ^[0-9]+\.[0-9]+\.[0-9]+ ]]; then
echo "Error: Invalid beta version: $LATEST_BETA_VERSION" >&2
exit 1
fi
fi

# Convert to JSON array and output only the JSON (no extra whitespace or newlines)
# Output a single JSON object with both versions and metadata
RESULT=$(jq -n \
--argjson versions "$(printf '%s\n' "${VERSIONS[@]}" | jq -R . | jq -s .)" \
--arg l1_version "$L1_VERSION" \
--arg rc_version "${INCLUDED_RC_VERSION}" \
--arg beta_version "${LATEST_BETA_VERSION}" \
'{
versions: $versions,
metadata: {
l1_version: $l1_version,
rc_version: (if ($rc_version // "") == "" or ($rc_version == "null") then null else $rc_version end),
beta_version: (if ($beta_version // "") == "" or ($beta_version == "null") then null else $beta_version end)
}
}')

echo "$RESULT"
Loading
Loading