Merge pull request #4224 from jandubois/test-retries

Add global `LIMA_BATS_ALL_TESTS_RETRIES` variable

Author: AkihiroSuda

.github/workflows/test.yml

@@ -362,14 +362,9 @@ jobs:
       with:
         template: templates/k8s.yaml
     - name: "Run BATS k8s tests"
-      # Wish we could use BATS_TEST_RETRIES=3 as an environment variable here,
-      # but bats does not seem to support it.
-      uses: nick-fields/retry@ce71cc2ab81d554ebbe88c79ab5975992d79ba08  # v3.0.2
-      with:
-        timeout_minutes: 30
-        retry_on: error
-        max_attempts: 3
-        command: ./hack/bats/lib/bats-core/bin/bats --timing ./hack/bats/extras/k8s.bats
+      run: ./hack/bats/lib/bats-core/bin/bats --timing ./hack/bats/extras/k8s.bats
+      env:
+        LIMA_BATS_ALL_TESTS_RETRIES: 3
 
   colima:
     name: "Colima tests (QEMU, Linux host)"

hack/bats/helpers/load.bash

@@ -8,6 +8,16 @@ set -o errexit -o nounset -o pipefail
 # The upstream PR https://github.com/bats-core/bats-core/pull/1118 is still open, so our submodule points to the PR commit.
 export BATS_RUN_ERREXIT=1
 
+# BATS_TEST_RETRIES must be set for the individual test and cannot be imported from the
+# parent environment because the BATS test runner sets it to 0 before running the test.
+BATS_TEST_RETRIES=${LIMA_BATS_ALL_TESTS_RETRIES:-0}
+
+# Known flaky tests should call `flaky` inside the @test to allow retries up to
+# LIMA_BATS_FLAKY_TESTS_RETRIES even when the LIMA_BATS_ALL_TESTS_RETRIES is lower.
+flaky() {
+    BATS_TEST_RETRIES=${LIMA_BATS_FLAKY_TESTS_RETRIES:-$BATS_TEST_RETRIES}
+}
+
 # Don't run the tests in ~/.lima because they may destroy _config, _templates etc.
 export LIMA_HOME=${LIMA_BATS_LIMA_HOME:-$HOME/.lima-bats}
 
@@ -71,4 +81,4 @@ assert_output_lines_count() {
 declare -A -g TEST_CONTAINER_IMAGES=(
     ["nginx"]="ghcr.io/stargz-containers/nginx:1.19-alpine-org"
     ["coredns"]="public.ecr.aws/eks-distro/coredns/coredns:v1.12.2-eks-1-31-latest"
-)
+)