Workaround for clusters hibernated more than 24 hours

Author: rhopp

integration-tests/pipelines/rhtap-cli-e2e_hive.yaml

@@ -119,6 +119,93 @@ spec:
               admin_pass_secret=$(oc get clusterdeployment -n $cp_namespace  $cp_namespace -o jsonpath={.spec.clusterMetadata.adminPasswordSecretRef.name})
               kubeadminpass=$(oc get secret $admin_pass_secret -n $cp_namespace -o jsonpath={.data.password} |base64 -d)
               echo "oc login --insecure-skip-tls-verify=true $api_url -u kubeadmin -p $kubeadminpass" > $(step.results.ocp-login-command.path)
+
+              export KUBECONFIG=/tmp/kube.config
+              kubeconfig_secret=$(oc get clusterdeployment -n $cp_namespace  $cp_namespace -o jsonpath={.spec.clusterMetadata.adminKubeconfigSecretRef.name})
+              oc get secret -n $cp_namespace $kubeconfig_secret -o jsonpath={.data.kubeconfig} |base64 -d > $KUBECONFIG
+
+              csr_max_retries=5
+              csr_sleep_duration=10
+              approved_csrs=false
+
+              console_max_retries=30
+              console_sleep_duration=10
+              console_connect_timeout=10
+              console_accessible=false
+
+              echo "--- Starting CSR Approval Process ---"
+              for ((i=1; i<=csr_max_retries; i++)); do
+                echo "CSR Attempt $i of $csr_max_retries: Checking for pending CSRs..."
+                if ! oc get csr 2>/dev/null | grep -i Pending; then
+                  echo "No pending CSRs found. Continuing"
+                  approved_csrs=true
+                  break
+                else
+                  echo "There are pending CSRs. That probably means cluster was hibernated for more than 24 hours. Need to approve them (until OCPBUGS-55339 is resolved)"
+                  if oc get csr -oname | xargs oc adm certificate approve; then
+                    echo "Successfully submitted approval for CSRs on attempt $i."
+                    sleep 2 # Small delay for changes to propagate
+                    if ! oc get csr 2>/dev/null | grep -i Pending; then
+                      echo "Confirmed no pending CSRs after approval."
+                      approved_csrs=true
+                      break
+                    else
+                      echo "Pending CSRs still exist after approval attempt $i."
+                    fi
+                  else
+                    echo "Failed to run approval command for CSRs on attempt $i."
+                  fi
+                fi
+
+                if [[ "$i" -lt "$csr_max_retries" ]]; then
+                  echo "Sleeping for $csr_sleep_duration seconds before next CSR retry..."
+                  sleep "$csr_sleep_duration"
+                fi
+              done
+
+              if [[ "$approved_csrs" == "true" ]]; then
+                echo "CSR check and approval process completed successfully."
+              else
+                echo "Failed to ensure all pending CSRs were approved after $csr_max_retries attempts."
+                exit 1
+              fi
+              echo "--- CSR Approval Process Finished ---"
+
+              # --- Console URL Accessibility Check ---
+              echo "--- Starting Console Accessibility Check ---"
+
+              console_url="https://$(oc get route console -n openshift-console -o jsonpath='{.spec.host}' 2>/dev/null)"
+
+              if [[ -z "$console_url" ]]; then
+                echo "Error: Could not retrieve OpenShift console URL."
+                exit 1
+              else
+                echo "Console URL found: $console_url"
+                for ((j=1; j<=console_max_retries; j++)); do
+                  echo "Console Check Attempt $j of $console_max_retries: Checking console URL accessibility..."
+                  if curl -k --silent --output /dev/null --head --fail --connect-timeout "$console_connect_timeout" "$console_url"; then
+                    echo "Console URL $console_url is accessible (HTTP 2xx)."
+                    console_accessible=true
+                    break
+                  else
+                    curl_exit_code=$?
+                    echo "Console URL $console_url not accessible on attempt $j (curl exit code: $curl_exit_code)."
+                  fi
+
+                  if [[ "$j" -lt "$console_max_retries" ]]; then
+                    echo "Sleeping for $console_sleep_duration seconds before next console check retry..."
+                    sleep "$console_sleep_duration"
+                  fi
+                done
+
+                if [[ "$console_accessible" == "true" ]]; then
+                  echo "Console is ready. Continuing."
+                else
+                  echo "Failed to access console URL $console_url after $console_max_retries attempts."
+                  exit 1
+                fi
+              fi
+              echo "--- Console Accessibility Check Finished ---"
     - name: rhtap-install
       runAfter:
         - provision-cluster