Hardware and Wokwi tests #20
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Hardware and Wokwi tests | |
| on: | |
| workflow_run: | |
| workflows: ["Runtime Tests"] | |
| types: | |
| - completed | |
| # No permissions by default | |
| permissions: | |
| contents: read | |
| jobs: | |
| get-artifacts: | |
| name: Get required artifacts | |
| runs-on: ubuntu-latest | |
| permissions: | |
| actions: read | |
| statuses: write | |
| outputs: | |
| pr_num: ${{ steps.set-ref.outputs.pr_num }} | |
| ref: ${{ steps.set-ref.outputs.ref }} | |
| base: ${{ steps.set-ref.outputs.base }} | |
| targets: ${{ steps.set-ref.outputs.targets }} | |
| types: ${{ steps.set-ref.outputs.types }} | |
| hw_tests_enabled: ${{ steps.set-ref.outputs.hw_tests_enabled }} | |
| push_time: ${{ steps.set-ref.outputs.push_time }} | |
| steps: | |
| - name: Report pending | |
| uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea # v7.0.1 | |
| with: | |
| script: | | |
| const owner = '${{ github.repository_owner }}'; | |
| const repo = '${{ github.repository }}'.split('/')[1]; | |
| const sha = '${{ github.event.workflow_run.head_sha }}'; | |
| core.debug(`owner: ${owner}`); | |
| core.debug(`repo: ${repo}`); | |
| core.debug(`sha: ${sha}`); | |
| const { context: name, state } = (await github.rest.repos.createCommitStatus({ | |
| context: 'Runtime Tests / Wokwi (Get artifacts) (${{ github.event.workflow_run.event }} -> workflow_run)', | |
| owner: owner, | |
| repo: repo, | |
| sha: sha, | |
| state: 'pending', | |
| target_url: 'https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}' | |
| })).data; | |
| core.info(`${name} is ${state}`); | |
| - name: Download and extract event file | |
| uses: actions/download-artifact@95815c38cf2ff2164869cbab79da8d1f422bc89e # v4.2.1 | |
| with: | |
| github-token: ${{ secrets.GITHUB_TOKEN }} | |
| run-id: ${{ github.event.workflow_run.id }} | |
| name: event_file | |
| path: artifacts/event_file | |
| - name: Download and extract matrix info | |
| uses: actions/download-artifact@95815c38cf2ff2164869cbab79da8d1f422bc89e # v4.2.1 | |
| with: | |
| github-token: ${{ secrets.GITHUB_TOKEN }} | |
| run-id: ${{ github.event.workflow_run.id }} | |
| name: matrix_info | |
| path: artifacts/matrix_info | |
| - name: Try to read PR number | |
| id: set-ref | |
| run: | | |
| pr_num=$(jq -r '.pull_request.number' artifacts/event_file/event.json | tr -cd "[:digit:]") | |
| if [ -z "$pr_num" ] || [ "$pr_num" == "null" ]; then | |
| pr_num="" | |
| fi | |
| ref=$pr_num | |
| if [ -z "$ref" ] || [ "$ref" == "null" ]; then | |
| ref=${{ github.ref }} | |
| fi | |
| action=$(jq -r '.action' artifacts/event_file/event.json | tr -cd "[:alpha:]_") | |
| if [ "$action" == "null" ]; then | |
| action="" | |
| fi | |
| base=$(jq -r '.pull_request.base.ref' artifacts/event_file/event.json | tr -cd "[:alnum:]/_.-") | |
| if [ -z "$base" ] || [ "$base" == "null" ]; then | |
| base=${{ github.ref }} | |
| fi | |
| hw_tests_enabled="true" | |
| if [[ -n "$pr_num" ]]; then | |
| # This is a PR, check for hil_test label | |
| has_hil_label=$(jq -r '.pull_request.labels[]?.name' artifacts/event_file/event.json 2>/dev/null | grep -q "hil_test" && echo "true" || echo "false") | |
| echo "Has hil_test label: $has_hil_label" | |
| if [[ "$has_hil_label" != "true" ]]; then | |
| echo "PR does not have hil_test label, hardware tests will be disabled" | |
| hw_tests_enabled="false" | |
| fi | |
| fi | |
| push_time=$(jq -r '.repository.pushed_at' artifacts/event_file/event.json | tr -cd "[:alnum:]:-") | |
| if [ -z "$push_time" ] || [ "$push_time" == "null" ]; then | |
| push_time="" | |
| fi | |
| types=$(cat artifacts/matrix_info/wokwi_types.txt | tr -cd "[:alpha:],[]'") | |
| targets=$(cat artifacts/matrix_info/targets.txt | tr -cd "[:alnum:],[]'") | |
| echo "base = $base" | |
| echo "targets = $targets" | |
| echo "types = $types" | |
| echo "pr_num = $pr_num" | |
| echo "hw_tests_enabled = $hw_tests_enabled" | |
| echo "push_time = $push_time" | |
| printf "$ref" >> artifacts/ref.txt | |
| printf "Ref = " | |
| cat artifacts/ref.txt | |
| printf "${{ github.event.workflow_run.event }}" >> artifacts/event.txt | |
| printf "\nEvent name = " | |
| cat artifacts/event.txt | |
| printf "${{ github.event.workflow_run.head_sha || github.sha }}" >> artifacts/sha.txt | |
| printf "\nHead SHA = " | |
| cat artifacts/sha.txt | |
| printf "$action" >> artifacts/action.txt | |
| printf "\nAction = " | |
| cat artifacts/action.txt | |
| printf "${{ github.event.workflow_run.id }}" >> artifacts/run_id.txt | |
| printf "\nRun ID = " | |
| cat artifacts/run_id.txt | |
| if [ -z "$ref" ] || [ "$ref" == "null" ]; then | |
| echo "Failed to get PR number or ref" | |
| exit 1 | |
| fi | |
| conclusion="${{ github.event.workflow_run.conclusion }}" | |
| printf "$conclusion" >> artifacts/conclusion.txt | |
| printf "\nConclusion = " | |
| cat artifacts/conclusion.txt | |
| echo "pr_num=$pr_num" >> $GITHUB_OUTPUT | |
| echo "base=$base" >> $GITHUB_OUTPUT | |
| echo "targets=$targets" >> $GITHUB_OUTPUT | |
| echo "types=$types" >> $GITHUB_OUTPUT | |
| echo "ref=$ref" >> $GITHUB_OUTPUT | |
| echo "hw_tests_enabled=$hw_tests_enabled" >> $GITHUB_OUTPUT | |
| echo "push_time=$push_time" >> $GITHUB_OUTPUT | |
| - name: Download and extract parent QEMU results | |
| uses: actions/download-artifact@95815c38cf2ff2164869cbab79da8d1f422bc89e # v4.2.1 | |
| continue-on-error: true | |
| with: | |
| github-token: ${{ secrets.GITHUB_TOKEN }} | |
| run-id: ${{ github.event.workflow_run.id }} | |
| pattern: test-results-qemu-* | |
| merge-multiple: true | |
| path: artifacts/results/qemu | |
| - name: Upload parent artifacts | |
| uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 | |
| with: | |
| name: parent-artifacts | |
| path: artifacts | |
| if-no-files-found: error | |
| - name: Report conclusion | |
| uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea # v7.0.1 | |
| if: always() | |
| with: | |
| script: | | |
| const owner = '${{ github.repository_owner }}'; | |
| const repo = '${{ github.repository }}'.split('/')[1]; | |
| const sha = '${{ github.event.workflow_run.head_sha }}'; | |
| core.debug(`owner: ${owner}`); | |
| core.debug(`repo: ${repo}`); | |
| core.debug(`sha: ${sha}`); | |
| const { context: name, state } = (await github.rest.repos.createCommitStatus({ | |
| context: 'Runtime Tests / Wokwi (Get artifacts) (${{ github.event.workflow_run.event }} -> workflow_run)', | |
| owner: owner, | |
| repo: repo, | |
| sha: sha, | |
| state: '${{ job.status }}', | |
| target_url: 'https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}' | |
| })).data; | |
| core.info(`${name} is ${state}`); | |
| hardware-test: | |
| name: Internal Hardware Tests | |
| if: | | |
| (github.event.workflow_run.conclusion == 'success' || | |
| github.event.workflow_run.conclusion == 'failure' || | |
| github.event.workflow_run.conclusion == 'timed_out') && | |
| needs.get-artifacts.outputs.hw_tests_enabled == 'true' | |
| runs-on: ubuntu-latest | |
| needs: get-artifacts | |
| env: | |
| id: ${{ needs.get-artifacts.outputs.ref }}-${{ github.event.workflow_run.head_sha || github.sha }} | |
| permissions: | |
| actions: read | |
| statuses: write | |
| steps: | |
| - name: Report pending | |
| uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea # v7.0.1 | |
| with: | |
| script: | | |
| const owner = '${{ github.repository_owner }}'; | |
| const repo = '${{ github.repository }}'.split('/')[1]; | |
| const sha = '${{ github.event.workflow_run.head_sha }}'; | |
| core.debug(`owner: ${owner}`); | |
| core.debug(`repo: ${repo}`); | |
| core.debug(`sha: ${sha}`); | |
| const { context: name, state } = (await github.rest.repos.createCommitStatus({ | |
| context: 'Runtime Tests / Internal Hardware Tests (${{ github.event.workflow_run.event }} -> workflow_run)', | |
| owner: owner, | |
| repo: repo, | |
| sha: sha, | |
| state: 'pending', | |
| target_url: 'https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}' | |
| })).data; | |
| core.info(`${name} is ${state}`); | |
| - name: Check if already passed | |
| id: get-cache-results | |
| if: needs.get-artifacts.outputs.pr_num | |
| uses: actions/cache/restore@5a3ec84eff668545956fd18022155c47e93e2684 # v4.2.3 | |
| with: | |
| key: test-${{ env.id }}-results-hw | |
| path: | | |
| tests/**/*.xml | |
| tests/**/result_*.json | |
| - name: Evaluate if tests should be run | |
| id: check-tests | |
| run: | | |
| cache_exists=${{ steps.get-cache-results.outputs.cache-hit == 'true' }} | |
| enabled=true | |
| # Check cache first | |
| if [[ $cache_exists == 'true' ]]; then | |
| echo "Already ran, skipping GitLab pipeline trigger" | |
| enabled=false | |
| else | |
| echo "Cache miss, hardware tests will run" | |
| fi | |
| echo "enabled=$enabled" >> $GITHUB_OUTPUT | |
| - name: Wait for GitLab sync | |
| if: ${{ steps.check-tests.outputs.enabled == 'true' }} | |
| run: | | |
| # A webhook to sync the repository is sent to GitLab when a commit is pushed to GitHub | |
| # We wait for 10 minutes after the push to GitHub to be safe | |
| echo "Ensuring GitLab sync has completed before triggering pipeline..." | |
| # Use push time determined in get-artifacts job | |
| push_time="${{ needs.get-artifacts.outputs.push_time }}" | |
| if [ -n "$push_time" ]; then | |
| echo "Push time: $push_time" | |
| # Convert push time to epoch | |
| push_epoch=$(date -d "$push_time" +%s 2>/dev/null || echo "") | |
| if [ -n "$push_epoch" ]; then | |
| current_epoch=$(date +%s) | |
| elapsed_minutes=$(( (current_epoch - push_epoch) / 60 )) | |
| echo "Elapsed time since push: ${elapsed_minutes} minutes" | |
| if [ $elapsed_minutes -lt 10 ]; then | |
| wait_time=$(( (10 - elapsed_minutes) * 60 )) | |
| echo "Waiting ${wait_time} seconds for GitLab sync to complete..." | |
| sleep $wait_time | |
| else | |
| echo "GitLab sync should be complete (${elapsed_minutes} minutes elapsed)" | |
| fi | |
| else | |
| echo "Could not parse push timestamp, waiting 60 seconds as fallback..." | |
| sleep 60 | |
| fi | |
| else | |
| echo "Could not determine push time, waiting 60 seconds as fallback..." | |
| sleep 60 | |
| fi | |
| echo "Proceeding with GitLab pipeline trigger..." | |
| - name: Checkout repository | |
| if: ${{ steps.check-tests.outputs.enabled == 'true' }} | |
| uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 | |
| with: | |
| ref: ${{ needs.get-artifacts.outputs.base || github.ref }} | |
| - name: Trigger GitLab Pipeline and Download Artifacts | |
| if: ${{ steps.check-tests.outputs.enabled == 'true' }} | |
| uses: digital-blueprint/[email protected] | |
| id: gitlab-trigger | |
| with: | |
| host: ${{ secrets.GITLAB_URL }} | |
| id: ${{ secrets.GITLAB_PROJECT_ID }} | |
| ref: 'ci/hw_gitlab' | |
| #ref: 'master' | |
| trigger_token: ${{ secrets.GITLAB_TRIGGER_TOKEN }} | |
| access_token: ${{ secrets.GITLAB_ACCESS_TOKEN }} | |
| download_artifacts: 'true' | |
| download_artifacts_on_failure: 'true' | |
| download_path: './gitlab-artifacts' | |
| variables: '{"TEST_TYPES":"${{ needs.get-artifacts.outputs.types }}","TEST_CHIPS":"${{ needs.get-artifacts.outputs.targets }}","PIPELINE_ID":"${{ env.id }}","BINARIES_RUN_ID":"${{ github.event.workflow_run.id }}","GITHUB_REPOSITORY":"${{ github.repository }}"}' | |
| - name: Process Downloaded Artifacts | |
| if: ${{ always() && steps.check-tests.outputs.enabled == 'true' }} | |
| run: | | |
| echo "GitLab Pipeline Status: ${{ steps.gitlab-trigger.outputs.status }}" | |
| echo "Artifacts Downloaded: ${{ steps.gitlab-trigger.outputs.artifacts_downloaded }}" | |
| # Create tests directory structure expected by GitHub caching | |
| mkdir -p tests | |
| # Process downloaded GitLab artifacts | |
| if [ "${{ steps.gitlab-trigger.outputs.artifacts_downloaded }}" = "true" ]; then | |
| echo "Processing downloaded GitLab artifacts..." | |
| # Find and copy test result files while preserving directory structure | |
| # The GitLab artifacts have the structure: gitlab-artifacts/job_*/artifacts/tests/... | |
| # We want to preserve the tests/... part of the structure | |
| for job_dir in ./gitlab-artifacts/job_*; do | |
| if [ -d "$job_dir/artifacts/tests" ]; then | |
| mv "$job_dir/artifacts/tests/"* tests/ | |
| fi | |
| done | |
| echo "Test results found:" | |
| ls -laR tests/ || echo "No test results found" | |
| else | |
| echo "No artifacts were downloaded from GitLab" | |
| fi | |
| - name: Upload hardware results as cache | |
| uses: actions/cache/save@5a3ec84eff668545956fd18022155c47e93e2684 # v4.2.3 | |
| if: steps.check-tests.outputs.enabled == 'true' && needs.get-artifacts.outputs.pr_num | |
| with: | |
| key: test-${{ env.id }}-results-hw | |
| path: | | |
| tests/**/*.xml | |
| tests/**/result_*.json | |
| - name: Upload hardware results as artifacts | |
| uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 | |
| if: always() | |
| with: | |
| name: test-results-hw | |
| overwrite: true | |
| path: | | |
| tests/**/*.xml | |
| tests/**/result_*.json | |
| - name: Report conclusion | |
| uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea # v7.0.1 | |
| if: always() | |
| with: | |
| script: | | |
| const owner = '${{ github.repository_owner }}'; | |
| const repo = '${{ github.repository }}'.split('/')[1]; | |
| const sha = '${{ github.event.workflow_run.head_sha }}'; | |
| core.debug(`owner: ${owner}`); | |
| core.debug(`repo: ${repo}`); | |
| core.debug(`sha: ${sha}`); | |
| const { context: name, state } = (await github.rest.repos.createCommitStatus({ | |
| context: 'Runtime Tests / Internal Hardware Tests (${{ github.event.workflow_run.event }} -> workflow_run)', | |
| owner: owner, | |
| repo: repo, | |
| sha: sha, | |
| state: '${{ job.status }}', | |
| target_url: 'https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}' | |
| })).data; | |
| core.info(`${name} is ${state}`); | |
| wokwi-test: | |
| name: Wokwi ${{ matrix.chip }} ${{ matrix.type }} tests | |
| if: | | |
| github.event.workflow_run.conclusion == 'success' || | |
| github.event.workflow_run.conclusion == 'failure' || | |
| github.event.workflow_run.conclusion == 'timed_out' | |
| runs-on: ubuntu-latest | |
| needs: get-artifacts | |
| env: | |
| id: ${{ needs.get-artifacts.outputs.ref }}-${{ github.event.workflow_run.head_sha || github.sha }}-${{ matrix.chip }}-${{ matrix.type }} | |
| permissions: | |
| actions: read | |
| statuses: write | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| type: ${{ fromJson(needs.get-artifacts.outputs.types) }} | |
| chip: ${{ fromJson(needs.get-artifacts.outputs.targets) }} | |
| steps: | |
| - name: Report pending | |
| uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea # v7.0.1 | |
| with: | |
| script: | | |
| const owner = '${{ github.repository_owner }}'; | |
| const repo = '${{ github.repository }}'.split('/')[1]; | |
| const sha = '${{ github.event.workflow_run.head_sha }}'; | |
| core.debug(`owner: ${owner}`); | |
| core.debug(`repo: ${repo}`); | |
| core.debug(`sha: ${sha}`); | |
| const { context: name, state } = (await github.rest.repos.createCommitStatus({ | |
| context: 'Runtime Tests / Wokwi (${{ matrix.type }}, ${{ matrix.chip }}) / Wokwi ${{ matrix.chip }} ${{ matrix.type }} tests (${{ github.event.workflow_run.event }} -> workflow_run)', | |
| owner: owner, | |
| repo: repo, | |
| sha: sha, | |
| state: 'pending', | |
| target_url: 'https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}' | |
| })).data; | |
| core.info(`${name} is ${state}`); | |
| - name: Check if already passed | |
| id: get-cache-results | |
| if: needs.get-artifacts.outputs.pr_num | |
| uses: actions/cache/restore@5a3ec84eff668545956fd18022155c47e93e2684 # v4.2.3 | |
| with: | |
| key: test-${{ env.id }}-results-wokwi | |
| path: | | |
| tests/**/*.xml | |
| tests/**/result_*.json | |
| - name: Evaluate if tests should be run | |
| id: check-tests | |
| run: | | |
| cache_exists=${{ steps.get-cache-results.outputs.cache-hit == 'true' }} | |
| enabled=true | |
| if [[ $cache_exists == 'true' ]]; then | |
| echo "Already ran, skipping" | |
| enabled=false | |
| fi | |
| echo "enabled=$enabled" >> $GITHUB_OUTPUT | |
| # Note that changes to the workflows and tests will only be picked up after the PR is merged | |
| # DO NOT CHECKOUT THE USER'S REPOSITORY IN THIS WORKFLOW. IT HAS HIGH SECURITY RISKS. | |
| - name: Checkout repository | |
| if: ${{ steps.check-tests.outputs.enabled == 'true' }} | |
| uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 | |
| with: | |
| ref: ${{ needs.get-artifacts.outputs.base || github.ref }} | |
| - uses: actions/setup-python@42375524e23c412d93fb67b49958b491fce71c38 # v5.0.4 | |
| if: ${{ steps.check-tests.outputs.enabled == 'true' }} | |
| with: | |
| cache-dependency-path: tests/requirements.txt | |
| cache: "pip" | |
| python-version: "3.x" | |
| - name: Install dependencies | |
| if: ${{ steps.check-tests.outputs.enabled == 'true' }} | |
| run: | | |
| pip install -U pip | |
| pip install -r tests/requirements.txt --extra-index-url https://dl.espressif.com/pypi | |
| - name: Wokwi CI Server | |
| if: ${{ steps.check-tests.outputs.enabled == 'true' }} | |
| uses: wokwi/wokwi-ci-server-action@a6fabb5a49e080158c7a1d121ea5b789536a82c3 # v1 | |
| - name: Get binaries | |
| if: ${{ steps.check-tests.outputs.enabled == 'true' }} | |
| uses: actions/download-artifact@95815c38cf2ff2164869cbab79da8d1f422bc89e # v4.2.1 | |
| with: | |
| github-token: ${{ secrets.GITHUB_TOKEN }} | |
| run-id: ${{ github.event.workflow_run.id }} | |
| name: test-bin-${{ matrix.chip }}-${{ matrix.type }} | |
| path: | | |
| ~/.arduino/tests/${{ matrix.chip }} | |
| - name: Run Tests | |
| if: ${{ steps.check-tests.outputs.enabled == 'true' }} | |
| env: | |
| WOKWI_CLI_TOKEN: ${{ secrets.WOKWI_CLI_TOKEN }} | |
| run: | | |
| bash .github/scripts/tests_run.sh -c -type ${{ matrix.type }} -t ${{ matrix.chip }} -i 0 -m 1 -W | |
| - name: Upload ${{ matrix.chip }} ${{ matrix.type }} Wokwi results as cache | |
| uses: actions/cache/save@5a3ec84eff668545956fd18022155c47e93e2684 # v4.2.3 | |
| if: steps.check-tests.outputs.enabled == 'true' && needs.get-artifacts.outputs.pr_num | |
| with: | |
| key: test-${{ env.id }}-results-wokwi | |
| path: | | |
| tests/**/*.xml | |
| tests/**/result_*.json | |
| - name: Upload ${{ matrix.chip }} ${{ matrix.type }} Wokwi results as artifacts | |
| uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 | |
| if: always() | |
| with: | |
| name: test-results-wokwi-${{ matrix.chip }}-${{ matrix.type }} | |
| overwrite: true | |
| path: | | |
| tests/**/*.xml | |
| tests/**/result_*.json | |
| - name: Report conclusion | |
| uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea # v7.0.1 | |
| if: always() | |
| with: | |
| script: | | |
| const owner = '${{ github.repository_owner }}'; | |
| const repo = '${{ github.repository }}'.split('/')[1]; | |
| const sha = '${{ github.event.workflow_run.head_sha }}'; | |
| core.debug(`owner: ${owner}`); | |
| core.debug(`repo: ${repo}`); | |
| core.debug(`sha: ${sha}`); | |
| const { context: name, state } = (await github.rest.repos.createCommitStatus({ | |
| context: 'Runtime Tests / Wokwi (${{ matrix.type }}, ${{ matrix.chip }}) / Wokwi ${{ matrix.chip }} ${{ matrix.type }} tests (${{ github.event.workflow_run.event }} -> workflow_run)', | |
| owner: owner, | |
| repo: repo, | |
| sha: sha, | |
| state: '${{ job.status }}', | |
| target_url: 'https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}' | |
| })).data; | |
| core.info(`${name} is ${state}`); |