From 70056f462fa3e4f901defb61f2223619e6029675 Mon Sep 17 00:00:00 2001 From: Andrew Thorp Date: Thu, 16 Oct 2025 16:30:54 -0400 Subject: [PATCH] ci: Ensure e2e setup errors fail tests, add retries during e2e setup Setting up the correct configuration values like `enable-api-fields` is not optional since many tests depend on these values at runtime. If a test requires non-default config values which failed to be set, the failure may not cause an e2e test to fail for some time and the test failure message will cause confusion; the configuration being correct during setup is a given. By ensuring a failure in any of the configuration setup functions results in the test suite failing, we ensure the tests will only run against the expected configuration. This change ensures that if the e2e test configuration fails, the tests will not continue. Fixing that improves the clarity of some flaky failures, but it doesn't reduce the flakiness itself. To reduce failures during e2e environment configuration, this change also implements a retry mechanism to each of the set configuration functions. This reduces flakiness due to network connectivity or race conditions during environment setup. If all config retries are exhausted with failure, then the tests will fail as above. --- test/e2e-common.sh | 20 ++++++++++++++++++++ test/e2e-tests.sh | 33 ++++++++++++++++----------------- 2 files changed, 36 insertions(+), 17 deletions(-) diff --git a/test/e2e-common.sh b/test/e2e-common.sh index b908ed38adb..226a3b0ea93 100755 --- a/test/e2e-common.sh +++ b/test/e2e-common.sh @@ -18,6 +18,26 @@ source $(git rev-parse --show-toplevel)/vendor/github.com/tektoncd/plumbing/scripts/e2e-tests.sh +# Run the given five times or until it succeeds. +# Sleeps 5 seconds after earch retry. +# example usage: `with_retries ping fakeserver.com` +function with_retries() ( + set +eo pipefail + + success="" + for retry in 1 2 3 4 5; do + "$@" + success="$?" + if [ "${success}" -eq "0" ]; then + break + fi + sleep 5 + [[ "${retry}" != "5" ]] && echo "Retrying..." + done + + return "${success}" +) + function install_pipeline_crd() { echo ">> Deploying Tekton Pipelines" local ko_target="$(mktemp)" diff --git a/test/e2e-tests.sh b/test/e2e-tests.sh index 4ddd15ac88d..d97d27bb350 100755 --- a/test/e2e-tests.sh +++ b/test/e2e-tests.sh @@ -34,7 +34,6 @@ ENABLE_PARAM_ENUM=${ENABLE_PARAM_ENUM:="false"} ENABLE_ARTIFACTS=${ENABLE_ARTIFACTS:="false"} ENABLE_CONCISE_RESOLVER_SYNTAX=${ENABLE_CONCISE_RESOLVER_SYNTAX:="false"} ENABLE_KUBERNETES_SIDECAR=${ENABLE_KUBERNETES_SIDECAR:="false"} -failed=0 # Script entry point. @@ -49,7 +48,6 @@ install_pipeline_crd export SYSTEM_NAMESPACE=tekton-pipelines set +x -failed=0 function add_spire() { local gate="$1" @@ -59,7 +57,6 @@ function add_spire() { install_spire patch_pipeline_spire kubectl apply -n tekton-pipelines -f "$DIR"/testdata/spire/config-spire.yaml - failed=0 fi } @@ -72,7 +69,7 @@ function set_feature_gate() { printf "Setting feature gate to %s\n", ${gate} jsonpatch=$(printf "{\"data\": {\"enable-api-fields\": \"%s\"}}" $1) echo "feature-flags ConfigMap patch: ${jsonpatch}" - kubectl patch configmap feature-flags -n tekton-pipelines -p "$jsonpatch" + with_retries kubectl patch configmap feature-flags -n tekton-pipelines -p "$jsonpatch" } function set_result_extraction_method() { @@ -84,7 +81,7 @@ function set_result_extraction_method() { printf "Setting results-from to %s\n", ${method} jsonpatch=$(printf "{\"data\": {\"results-from\": \"%s\"}}" $1) echo "feature-flags ConfigMap patch: ${jsonpatch}" - kubectl patch configmap feature-flags -n tekton-pipelines -p "$jsonpatch" + with_retries kubectl patch configmap feature-flags -n tekton-pipelines -p "$jsonpatch" } function set_keep_pod_on_cancel() { @@ -96,7 +93,7 @@ function set_keep_pod_on_cancel() { printf "Setting keep-pod-on-cancel to %s\n", ${method} jsonpatch=$(printf "{\"data\": {\"keep-pod-on-cancel\": \"%s\"}}" $1) echo "feature-flags ConfigMap patch: ${jsonpatch}" - kubectl patch configmap feature-flags -n tekton-pipelines -p "$jsonpatch" + with_retries kubectl patch configmap feature-flags -n tekton-pipelines -p "$jsonpatch" } function set_cel_in_whenexpression() { @@ -107,7 +104,7 @@ function set_cel_in_whenexpression() { fi jsonpatch=$(printf "{\"data\": {\"enable-cel-in-whenexpression\": \"%s\"}}" $1) echo "feature-flags ConfigMap patch: ${jsonpatch}" - kubectl patch configmap feature-flags -n tekton-pipelines -p "$jsonpatch" + with_retries kubectl patch configmap feature-flags -n tekton-pipelines -p "$jsonpatch" } function set_enable_param_enum() { @@ -119,7 +116,7 @@ function set_enable_param_enum() { printf "Setting enable-param-enum to %s\n", ${method} jsonpatch=$(printf "{\"data\": {\"enable-param-enum\": \"%s\"}}" $1) echo "feature-flags ConfigMap patch: ${jsonpatch}" - kubectl patch configmap feature-flags -n tekton-pipelines -p "$jsonpatch" + with_retries kubectl patch configmap feature-flags -n tekton-pipelines -p "$jsonpatch" } function set_enable_artifacts() { @@ -131,7 +128,7 @@ function set_enable_artifacts() { printf "Setting enable-artifacts to %s\n", ${method} jsonpatch=$(printf "{\"data\": {\"enable-artifacts\": \"%s\"}}" $1) echo "feature-flags ConfigMap patch: ${jsonpatch}" - kubectl patch configmap feature-flags -n tekton-pipelines -p "$jsonpatch" + with_retries kubectl patch configmap feature-flags -n tekton-pipelines -p "$jsonpatch" } function set_enable_concise_resolver_syntax() { @@ -143,7 +140,7 @@ function set_enable_concise_resolver_syntax() { printf "Setting enable-concise-resolver-syntax to %s\n", ${method} jsonpatch=$(printf "{\"data\": {\"enable-concise-resolver-syntax\": \"%s\"}}" $1) echo "feature-flags ConfigMap patch: ${jsonpatch}" - kubectl patch configmap feature-flags -n tekton-pipelines -p "$jsonpatch" + with_retries kubectl patch configmap feature-flags -n tekton-pipelines -p "$jsonpatch" } function set_enable_kubernetes_sidecar() { @@ -155,14 +152,14 @@ function set_enable_kubernetes_sidecar() { printf "Setting enable-kubernetes-sidecar to %s\n", ${method} jsonpatch=$(printf "{\"data\": {\"enable-kubernetes-sidecar\": \"%s\"}}" $1) echo "feature-flags ConfigMap patch: ${jsonpatch}" - kubectl patch configmap feature-flags -n tekton-pipelines -p "$jsonpatch" + with_retries kubectl patch configmap feature-flags -n tekton-pipelines -p "$jsonpatch" } function set_default_sidecar_log_polling_interval() { # Sets the default-sidecar-log-polling-interval in the config-defaults ConfigMap to 0ms for e2e tests echo "Patching config-defaults ConfigMap: setting default-sidecar-log-polling-interval to 0ms" jsonpatch='{"data": {"default-sidecar-log-polling-interval": "0ms"}}' - kubectl patch configmap config-defaults -n tekton-pipelines -p "$jsonpatch" + with_retries kubectl patch configmap config-defaults -n tekton-pipelines -p "$jsonpatch" } function run_e2e() { @@ -170,25 +167,28 @@ function run_e2e() { header "Running Go e2e tests" # Skip ./test/*.go tests if SKIP_GO_E2E_TESTS == true if [ "${SKIP_GO_E2E_TESTS}" != "true" ]; then - go_test_e2e -timeout=${E2E_GO_TEST_TIMEOUT} ./test/... || failed=1 + go_test_e2e -timeout=${E2E_GO_TEST_TIMEOUT} ./test/... fi # Run these _after_ the integration tests b/c they don't quite work all the way # and they cause a lot of noise in the logs, making it harder to debug integration # test failures. if [ "${RUN_YAML_TESTS}" == "true" ]; then - go_test_e2e -mod=readonly -tags=examples -timeout=${E2E_GO_TEST_TIMEOUT} ./test/ || failed=1 + go_test_e2e -mod=readonly -tags=examples -timeout=${E2E_GO_TEST_TIMEOUT} ./test/ fi if [ "${RUN_FEATUREFLAG_TESTS}" == "true" ]; then - go_test_e2e -mod=readonly -tags=featureflags -timeout=${E2E_GO_TEST_TIMEOUT} ./test/ || failed=1 + go_test_e2e -mod=readonly -tags=featureflags -timeout=${E2E_GO_TEST_TIMEOUT} ./test/ fi } +set -eo pipefail +trap '[[ "$?" == "0" ]] || fail_test' EXIT + add_spire "$PIPELINE_FEATURE_GATE" set_feature_gate "$PIPELINE_FEATURE_GATE" set_result_extraction_method "$RESULTS_FROM" -set_keep_pod_on_cancel "$KEEP_POD_ON_CANCEL" +set_keep_pod_on_cancel "$KEEP_POD_ON_CANCEL" set_cel_in_whenexpression "$ENABLE_CEL_IN_WHENEXPRESSION" set_enable_param_enum "$ENABLE_PARAM_ENUM" set_enable_artifacts "$ENABLE_ARTIFACTS" @@ -197,5 +197,4 @@ set_enable_kubernetes_sidecar "$ENABLE_KUBERNETES_SIDECAR" set_default_sidecar_log_polling_interval run_e2e -(( failed )) && fail_test success