diff --git a/ci-operator/config/openshift/sandboxed-containers-operator/openshift-sandboxed-containers-operator-devel__downstream-candidate.yaml b/ci-operator/config/openshift/sandboxed-containers-operator/openshift-sandboxed-containers-operator-devel__downstream-candidate.yaml index 8c050347dbc92..568d9d5269c35 100644 --- a/ci-operator/config/openshift/sandboxed-containers-operator/openshift-sandboxed-containers-operator-devel__downstream-candidate.yaml +++ b/ci-operator/config/openshift/sandboxed-containers-operator/openshift-sandboxed-containers-operator-devel__downstream-candidate.yaml @@ -4,7 +4,7 @@ base_images: namespace: ci tag: "4.21" upi-installer: - name: "4.19" + name: "4.17" namespace: ocp tag: upi-installer releases: @@ -12,7 +12,7 @@ releases: release: architecture: amd64 channel: stable - version: "4.19" + version: "4.17" resources: '*': requests: @@ -31,7 +31,7 @@ tests: ENABLE_MUST_GATHER: "true" EXPECTED_OPERATOR_VERSION: 1.11.0 INSTALL_KATA_RPM: "true" - KATA_RPM_VERSION: 3.21.0-3.rhaos4.19.el9 + KATA_RPM_VERSION: 3.21.0-3.rhaos4.17.el9 MUST_GATHER_IMAGE: registry.redhat.io/openshift-sandboxed-containers/osc-must-gather-rhel9:latest MUST_GATHER_ON_FAILURE_ONLY: "true" SLEEP_DURATION: 0h @@ -97,6 +97,28 @@ tests: WORKLOAD_TO_TEST: coco workflow: sandboxed-containers-operator-e2e-azure timeout: 24h0m0s +- as: aro-ipi-peerpods + cron: 0 0 31 2 1 + steps: + cluster_profile: azure-qe + env: + ARO_CLUSTER_VERSION: "4.17" + CATALOG_SOURCE_IMAGE: quay.io/redhat-user-workloads/ose-osc-tenant/osc-test-fbc:1.11.0-1764610958 + CATALOG_SOURCE_NAME: brew-catalog + ENABLE_MUST_GATHER: "true" + ENABLEPEERPODS: "true" + EXPECTED_OPERATOR_VERSION: 1.11.0 + INSTALL_KATA_RPM: "true" + KATA_RPM_VERSION: 3.21.0-3.rhaos4.17.el9 + RUNTIMECLASS: kata-remote + SLEEP_DURATION: 0h + TEST_FILTERS: ~DisconnectedOnly&;~Disruptive& + TEST_RELEASE_TYPE: Pre-GA + TEST_SCENARIOS: sig-kata.*Kata Author + TEST_TIMEOUT: "75" + WORKLOAD_TO_TEST: peer-pods + workflow: sandboxed-containers-operator-e2e-aro + timeout: 24h0m0s - as: aws-ipi-peerpods cron: 0 0 31 2 1 steps: diff --git a/ci-operator/config/openshift/sandboxed-containers-operator/openshift-sandboxed-containers-operator-devel__downstream-release.yaml b/ci-operator/config/openshift/sandboxed-containers-operator/openshift-sandboxed-containers-operator-devel__downstream-release.yaml index b84f5d31f707b..199ad3d750390 100644 --- a/ci-operator/config/openshift/sandboxed-containers-operator/openshift-sandboxed-containers-operator-devel__downstream-release.yaml +++ b/ci-operator/config/openshift/sandboxed-containers-operator/openshift-sandboxed-containers-operator-devel__downstream-release.yaml @@ -97,6 +97,32 @@ tests: WORKLOAD_TO_TEST: coco workflow: sandboxed-containers-operator-e2e-azure timeout: 24h0m0s +- as: aro-ipi-peerpods + cron: 0 0 31 2 1 + steps: + cluster_profile: azure-qe + env: + ARO_CLUSTER_VERSION: "4.16" + CATALOG_SOURCE_IMAGE: "" + CATALOG_SOURCE_NAME: redhat-operators + ENABLE_MUST_GATHER: "true" + ENABLEPEERPODS: "true" + EXPECTED_OPERATOR_VERSION: 1.10.1 + INSTALL_KATA_RPM: "false" + KATA_RPM_VERSION: "" + MUST_GATHER_IMAGE: registry.redhat.io/openshift-sandboxed-containers/osc-must-gather-rhel9:latest + MUST_GATHER_ON_FAILURE_ONLY: "true" + RUNTIMECLASS: kata-remote + SLEEP_DURATION: 0h + TEST_FILTERS: ~DisconnectedOnly&;~Disruptive& + TEST_RELEASE_TYPE: GA + TEST_SCENARIOS: sig-kata.*Kata Author + TEST_TIMEOUT: "90" + TRUSTEE_CATALOG_SOURCE_IMAGE: "" + TRUSTEE_CATALOG_SOURCE_NAME: redhat-operators + WORKLOAD_TO_TEST: peer-pods + workflow: sandboxed-containers-operator-e2e-aro + timeout: 24h0m0s - as: aws-ipi-peerpods cron: 0 0 31 2 1 steps: diff --git a/ci-operator/jobs/openshift/sandboxed-containers-operator/openshift-sandboxed-containers-operator-devel-periodics.yaml b/ci-operator/jobs/openshift/sandboxed-containers-operator/openshift-sandboxed-containers-operator-devel-periodics.yaml index 8bd7039c58cc5..30e074e1115ba 100644 --- a/ci-operator/jobs/openshift/sandboxed-containers-operator/openshift-sandboxed-containers-operator-devel-periodics.yaml +++ b/ci-operator/jobs/openshift/sandboxed-containers-operator/openshift-sandboxed-containers-operator-devel-periodics.yaml @@ -1,4 +1,79 @@ periodics: +- agent: kubernetes + cluster: build03 + cron: 0 0 31 2 1 + decorate: true + decoration_config: + skip_cloning: true + timeout: 24h0m0s + extra_refs: + - base_ref: devel + org: openshift + repo: sandboxed-containers-operator + labels: + ci-operator.openshift.io/cloud: azure4 + ci-operator.openshift.io/cloud-cluster-profile: azure-qe + ci-operator.openshift.io/variant: downstream-candidate + ci.openshift.io/generator: prowgen + pj-rehearse.openshift.io/can-be-rehearsed: "true" + name: periodic-ci-openshift-sandboxed-containers-operator-devel-downstream-candidate-aro-ipi-peerpods + spec: + containers: + - args: + - --gcs-upload-secret=/secrets/gcs/service-account.json + - --image-import-pull-secret=/etc/pull-secret/.dockerconfigjson + - --lease-server-credentials-file=/etc/boskos/credentials + - --report-credentials-file=/etc/report/credentials + - --secret-dir=/secrets/ci-pull-credentials + - --target=aro-ipi-peerpods + - --variant=downstream-candidate + command: + - ci-operator + image: quay-proxy.ci.openshift.org/openshift/ci:ci_ci-operator_latest + imagePullPolicy: Always + name: "" + resources: + requests: + cpu: 10m + volumeMounts: + - mountPath: /etc/boskos + name: boskos + readOnly: true + - mountPath: /secrets/ci-pull-credentials + name: ci-pull-credentials + readOnly: true + - mountPath: /secrets/gcs + name: gcs-credentials + readOnly: true + - mountPath: /secrets/manifest-tool + name: manifest-tool-local-pusher + readOnly: true + - mountPath: /etc/pull-secret + name: pull-secret + readOnly: true + - mountPath: /etc/report + name: result-aggregator + readOnly: true + serviceAccountName: ci-operator + volumes: + - name: boskos + secret: + items: + - key: credentials + path: credentials + secretName: boskos-credentials + - name: ci-pull-credentials + secret: + secretName: ci-pull-credentials + - name: manifest-tool-local-pusher + secret: + secretName: manifest-tool-local-pusher + - name: pull-secret + secret: + secretName: registry-pull-credentials + - name: result-aggregator + secret: + secretName: result-aggregator - agent: kubernetes cluster: build09 cron: 0 0 31 2 1 @@ -429,6 +504,81 @@ periodics: - name: result-aggregator secret: secretName: result-aggregator +- agent: kubernetes + cluster: build03 + cron: 0 0 31 2 1 + decorate: true + decoration_config: + skip_cloning: true + timeout: 24h0m0s + extra_refs: + - base_ref: devel + org: openshift + repo: sandboxed-containers-operator + labels: + ci-operator.openshift.io/cloud: azure4 + ci-operator.openshift.io/cloud-cluster-profile: azure-qe + ci-operator.openshift.io/variant: downstream-release + ci.openshift.io/generator: prowgen + pj-rehearse.openshift.io/can-be-rehearsed: "true" + name: periodic-ci-openshift-sandboxed-containers-operator-devel-downstream-release-aro-ipi-peerpods + spec: + containers: + - args: + - --gcs-upload-secret=/secrets/gcs/service-account.json + - --image-import-pull-secret=/etc/pull-secret/.dockerconfigjson + - --lease-server-credentials-file=/etc/boskos/credentials + - --report-credentials-file=/etc/report/credentials + - --secret-dir=/secrets/ci-pull-credentials + - --target=aro-ipi-peerpods + - --variant=downstream-release + command: + - ci-operator + image: quay-proxy.ci.openshift.org/openshift/ci:ci_ci-operator_latest + imagePullPolicy: Always + name: "" + resources: + requests: + cpu: 10m + volumeMounts: + - mountPath: /etc/boskos + name: boskos + readOnly: true + - mountPath: /secrets/ci-pull-credentials + name: ci-pull-credentials + readOnly: true + - mountPath: /secrets/gcs + name: gcs-credentials + readOnly: true + - mountPath: /secrets/manifest-tool + name: manifest-tool-local-pusher + readOnly: true + - mountPath: /etc/pull-secret + name: pull-secret + readOnly: true + - mountPath: /etc/report + name: result-aggregator + readOnly: true + serviceAccountName: ci-operator + volumes: + - name: boskos + secret: + items: + - key: credentials + path: credentials + secretName: boskos-credentials + - name: ci-pull-credentials + secret: + secretName: ci-pull-credentials + - name: manifest-tool-local-pusher + secret: + secretName: manifest-tool-local-pusher + - name: pull-secret + secret: + secretName: registry-pull-credentials + - name: result-aggregator + secret: + secretName: result-aggregator - agent: kubernetes cluster: build09 cron: 0 0 31 2 1 diff --git a/ci-operator/step-registry/sandboxed-containers-operator/e2e/aro/OWNERS b/ci-operator/step-registry/sandboxed-containers-operator/e2e/aro/OWNERS new file mode 100644 index 0000000000000..5c31fe0ceccfc --- /dev/null +++ b/ci-operator/step-registry/sandboxed-containers-operator/e2e/aro/OWNERS @@ -0,0 +1,10 @@ +reviewers: + - ldoktor + - tbuskey + - vvoronko + - wainersm +approvers: + - ldoktor + - tbuskey + - vvoronko + - wainersm diff --git a/ci-operator/step-registry/sandboxed-containers-operator/e2e/aro/sandboxed-containers-operator-e2e-aro-workflow.metadata.json b/ci-operator/step-registry/sandboxed-containers-operator/e2e/aro/sandboxed-containers-operator-e2e-aro-workflow.metadata.json new file mode 100644 index 0000000000000..cda8dd442b320 --- /dev/null +++ b/ci-operator/step-registry/sandboxed-containers-operator/e2e/aro/sandboxed-containers-operator-e2e-aro-workflow.metadata.json @@ -0,0 +1,17 @@ +{ + "path": "sandboxed-containers-operator/e2e/aro/sandboxed-containers-operator-e2e-aro-workflow.yaml", + "owners": { + "approvers": [ + "ldoktor", + "tbuskey", + "vvoronko", + "wainersm" + ], + "reviewers": [ + "ldoktor", + "tbuskey", + "vvoronko", + "wainersm" + ] + } +} \ No newline at end of file diff --git a/ci-operator/step-registry/sandboxed-containers-operator/e2e/aro/sandboxed-containers-operator-e2e-aro-workflow.yaml b/ci-operator/step-registry/sandboxed-containers-operator/e2e/aro/sandboxed-containers-operator-e2e-aro-workflow.yaml new file mode 100644 index 0000000000000..84233174cc9cc --- /dev/null +++ b/ci-operator/step-registry/sandboxed-containers-operator/e2e/aro/sandboxed-containers-operator-e2e-aro-workflow.yaml @@ -0,0 +1,30 @@ +workflow: + as: sandboxed-containers-operator-e2e-aro + steps: + env: + TEST_PARALLEL: 1 + FORCE_SUCCESS_EXIT: yes + MUST_GATHER_IMAGE: "registry.redhat.io/openshift-sandboxed-containers/osc-must-gather-rhel9:latest" + MUST_GATHER_TIMEOUT: "35m" + ENABLE_MUST_GATHER: "true" + MUST_GATHER_ON_FAILURE_ONLY: "false" + ARO_WORKER_COUNT: "3" + ARO_WORKER_VM_SIZE: Standard_D8s_v3 + pre: + - ref: azure-provision-resourcegroup + - ref: aro-provision-vnet + - ref: aro-provision-cluster + - ref: ipi-install-rbac + - ref: openshift-cluster-bot-rbac + - chain: sandboxed-containers-operator-pre + post: + - ref: cucushift-installer-wait + timeout: 12h10m0s + - ref: sandboxed-containers-operator-gather-must-gather + - ref: sandboxed-containers-operator-post + - ref: aro-deprovision + test: + - ref: openshift-extended-test + documentation: |- + This workflow run openshift-extented-test on an Azure cluster with + the sandboxed containers deployed. diff --git a/ci-operator/step-registry/sandboxed-containers-operator/peerpods/param-cm/sandboxed-containers-operator-peerpods-param-cm-commands.sh b/ci-operator/step-registry/sandboxed-containers-operator/peerpods/param-cm/sandboxed-containers-operator-peerpods-param-cm-commands.sh index cbeebec2bd059..bc7dfe60ed5f9 100755 --- a/ci-operator/step-registry/sandboxed-containers-operator/peerpods/param-cm/sandboxed-containers-operator-peerpods-param-cm-commands.sh +++ b/ci-operator/step-registry/sandboxed-containers-operator/peerpods/param-cm/sandboxed-containers-operator-peerpods-param-cm-commands.sh @@ -99,12 +99,12 @@ create_ssh_key() { } handle_azure() { + local IS_ARO local AZURE_RESOURCE_GROUP local AZURE_AUTH_LOCATION local AZURE_CLIENT_SECRET local AZURE_TENANT_ID local AZURE_CLIENT_ID - local AZURE_VNET_ID local AZURE_VNET_NAME local AZURE_SUBNET_ID local AZURE_SUBNET_NAME @@ -112,14 +112,13 @@ handle_azure() { local AZURE_REGION local PP_REGION local PP_RESOURCE_GROUP - local PP_VNET_ID local PP_VNET_NAME local PP_SUBNET_NAME local PP_SUBNET_ID local PP_RESOURCE_GROUP local PP_NSG_ID - local PP_NSG_NAME + IS_ARO=$(oc get crd clusters.aro.openshift.io &>/dev/null && echo true || echo false) # Note: Keep the following commands in sync with https://raw.githubusercontent.com/kata-containers/kata-containers/refs/heads/main/ci/openshift-ci/peer-pods-azure.sh # as much as possible. @@ -146,12 +145,33 @@ handle_azure() { fi AZURE_SUBSCRIPTION_ID="$(jq -r .data.azure_subscription_id azure_credentials.json|base64 -d)" rm -f azure_credentials.json - - AZURE_RESOURCE_GROUP=$(oc get infrastructure/cluster -o jsonpath='{.status.platformStatus.azure.resourceGroupName}') + # Login to Azure for NAT gateway creation az login --service-principal --username "${AZURE_CLIENT_ID}" --password "${AZURE_CLIENT_SECRET}" --tenant "${AZURE_TENANT_ID}" - # Recommended on az sites to refresh the subscription az account set --subscription "${AZURE_SUBSCRIPTION_ID}" - # This command still sometimes fails directly after login + + if [[ "${IS_ARO}" == "true" ]]; then + # On ARO we have to assign extra roles and add NSG using upper-cloud + # credentials + local AZURE_NSG_NAME + local CLUSTER_NAME + local RG_SCOPE_ID + local SP_CLIENT_ID + AZURE_RESOURCE_GROUP="$(cat "${SHARED_DIR}/resourcegroup")" + CLUSTER_NAME="$(cat "${SHARED_DIR}/cluster-name")" + AZURE_REGION="${LEASED_RESOURCE}" + SP_CLIENT_ID="$(az aro show --name "${CLUSTER_NAME}" --resource-group "${AZURE_RESOURCE_GROUP}" --query "servicePrincipalProfile.clientId" -o tsv)" + RG_SCOPE_ID="$(az group show --name "${AZURE_RESOURCE_GROUP}" --query "id" -o tsv)" + az role assignment create --assignee "${SP_CLIENT_ID}" --role "Network Contributor" --scope "${RG_SCOPE_ID}" + az role assignment create --assignee "${SP_CLIENT_ID}" --role "Contributor" --scope "${RG_SCOPE_ID}" + AZURE_NSG_NAME="$(oc get configmap cloud-conf -n openshift-cloud-controller-manager -o json | jq -r '.data."cloud.conf" | fromjson' | jq -r '.securityGroupName')" + az network nsg create -g "${AZURE_RESOURCE_GROUP}" -n "${AZURE_NSG_NAME}" -l "${AZURE_REGION}" || echo "::warning:: NSG create failed, probably already existing" + else + # On normal cluster wait for it to be Available + echo "Waiting for OpenShift infrastructure to be ready..." + oc wait --for=condition=Available --timeout=600s infrastructure/cluster + AZURE_RESOURCE_GROUP=$(oc get infrastructure/cluster -o jsonpath='{.status.platformStatus.azure.resourceGroupName}') + AZURE_REGION=$(az group show --resource-group "${AZURE_RESOURCE_GROUP}" --query "{Location:location}" --output tsv) + fi for I in {1..30}; do AZURE_VNET_NAME=$(az network vnet list --resource-group "${AZURE_RESOURCE_GROUP}" --query "[].{Name:name}" --output tsv ||:) if [[ -z "${AZURE_VNET_NAME}" ]]; then @@ -168,33 +188,15 @@ handle_azure() { AZURE_SUBNET_NAME=$(az network vnet subnet list --resource-group "${AZURE_RESOURCE_GROUP}" --vnet-name "${AZURE_VNET_NAME}" --query "[].{Id:name} | [? contains(Id, 'worker')]" --output tsv) AZURE_SUBNET_ID=$(az network vnet subnet list --resource-group "${AZURE_RESOURCE_GROUP}" --vnet-name "${AZURE_VNET_NAME}" --query "[].{Id:id} | [? contains(Id, 'worker')]" --output tsv) AZURE_NSG_ID=$(az network nsg list --resource-group "${AZURE_RESOURCE_GROUP}" --query "[].{Id:id}" --output tsv) - AZURE_REGION=$(az group show --resource-group "${AZURE_RESOURCE_GROUP}" --query "{Location:location}" --output tsv) - PP_REGION=eastus - if [[ "${AZURE_REGION}" == "${PP_REGION}" ]]; then - echo "Using the current region ${AZURE_REGION}" - PP_RESOURCE_GROUP="${AZURE_RESOURCE_GROUP}" - PP_VNET_NAME="${AZURE_VNET_NAME}" - PP_SUBNET_NAME="${AZURE_SUBNET_NAME}" - PP_SUBNET_ID="${AZURE_SUBNET_ID}" - PP_NSG_ID="${AZURE_NSG_ID}" - else - echo "Creating peering between ${AZURE_REGION} and ${PP_REGION}" - PP_RESOURCE_GROUP="${AZURE_RESOURCE_GROUP}-eastus" - PP_VNET_NAME="${AZURE_VNET_NAME}-eastus" - PP_SUBNET_NAME="${AZURE_SUBNET_NAME}-eastus" - PP_NSG_NAME="${AZURE_VNET_NAME}-nsg-eastus" - az group create --name "${PP_RESOURCE_GROUP}" --location "${PP_REGION}" - az network vnet create --resource-group "${PP_RESOURCE_GROUP}" --name "${PP_VNET_NAME}" --location "${PP_REGION}" --address-prefixes 10.2.0.0/16 --subnet-name "${PP_SUBNET_NAME}" --subnet-prefixes 10.2.1.0/24 - az network nsg create --resource-group "${PP_RESOURCE_GROUP}" --name "${PP_NSG_NAME}" --location "${PP_REGION}" - az network vnet subnet update --resource-group "${PP_RESOURCE_GROUP}" --vnet-name "${PP_VNET_NAME}" --name "${PP_SUBNET_NAME}" --network-security-group "${PP_NSG_NAME}" - AZURE_VNET_ID=$(az network vnet show --resource-group "${AZURE_RESOURCE_GROUP}" --name "${AZURE_VNET_NAME}" --query id --output tsv) - PP_VNET_ID=$(az network vnet show --resource-group "${PP_RESOURCE_GROUP}" --name "${PP_VNET_NAME}" --query id --output tsv) - az network vnet peering create --name westus-to-eastus --resource-group "${AZURE_RESOURCE_GROUP}" --vnet-name "${AZURE_VNET_NAME}" --remote-vnet "${PP_VNET_ID}" --allow-vnet-access - az network vnet peering create --name eastus-to-westus --resource-group "${PP_RESOURCE_GROUP}" --vnet-name "${PP_VNET_NAME}" --remote-vnet "${AZURE_VNET_ID}" --allow-vnet-access - PP_SUBNET_ID=$(az network vnet subnet list --resource-group "${PP_RESOURCE_GROUP}" --vnet-name "${PP_VNET_NAME}" --query "[].{Id:id} | [? contains(Id, 'worker')]" --output tsv) - PP_NSG_ID=$(az network nsg list --resource-group "${PP_RESOURCE_GROUP}" --query "[].{Id:id}" --output tsv) -fi + # Downstream version generates podvm, no need to peer to eastus + # (keeping the PP_* variables to be close to upstream setup) + PP_REGION="${AZURE_REGION}" + PP_RESOURCE_GROUP="${AZURE_RESOURCE_GROUP}" + PP_VNET_NAME="${AZURE_VNET_NAME}" + PP_SUBNET_NAME="${AZURE_SUBNET_NAME}" + PP_SUBNET_ID="${AZURE_SUBNET_ID}" + PP_NSG_ID="${AZURE_NSG_ID}" # Peer-pod requires gateway az network public-ip create \ @@ -213,8 +215,6 @@ fi --name "${PP_SUBNET_NAME}" \ --nat-gateway MyNatGateway - # Start the downstream-only commands - create_ssh_key # Creating peerpods-param-cm config map with all the cloud params needed for test case execution diff --git a/ci-operator/step-registry/sandboxed-containers-operator/post/sandboxed-containers-operator-post-commands.sh b/ci-operator/step-registry/sandboxed-containers-operator/post/sandboxed-containers-operator-post-commands.sh index ba0a2b1a0e771..020298207a9fd 100755 --- a/ci-operator/step-registry/sandboxed-containers-operator/post/sandboxed-containers-operator-post-commands.sh +++ b/ci-operator/step-registry/sandboxed-containers-operator/post/sandboxed-containers-operator-post-commands.sh @@ -92,6 +92,44 @@ cleanup_aws() { exit 0 } +# Remove our NSG on ARO+peer-pods (do nothing on plain azure) +cleanup_azure() { + local IS_ARO + IS_ARO=$(oc get crd clusters.aro.openshift.io &>/dev/null && echo true || echo false) + if [[ "${IS_ARO}" != "true" ]]; then + echo "We are not on ARO" + return + fi + if [[ "${ENABLEPEERPODS:-no}" != "true" ]]; then + echo "Peer-pods not enabled" + return + fi + + RESOURCEGROUP=${RESOURCEGROUP:=$(cat "${SHARED_DIR}/resourcegroup")} + AZURE_AUTH_LOCATION="${CLUSTER_PROFILE_DIR}/osServicePrincipal.json" + AZURE_AUTH_CLIENT_ID="$(<"${AZURE_AUTH_LOCATION}" jq -r .clientId)" + AZURE_AUTH_CLIENT_SECRET="$(<"${AZURE_AUTH_LOCATION}" jq -r .clientSecret)" + AZURE_AUTH_TENANT_ID="$(<"${AZURE_AUTH_LOCATION}" jq -r .tenantId)" + az login --service-principal -u "${AZURE_AUTH_CLIENT_ID}" -p "${AZURE_AUTH_CLIENT_SECRET}" --tenant "${AZURE_AUTH_TENANT_ID}" --output none + + # delete kataconfig (to prevent starting new podvms) + oc delete kataconfigs.kataconfiguration.openshift.io --all --wait=false || echo "::warning:: Failed to delete kata-config" + + # Delete potentially left-behind VMs + az vm list --resource-group "${RESOURCEGROUP}" --query "[?starts_with(name, 'podvm-')].name" --output tsv | xargs -I {} az vm delete --resource-group "${RESOURCEGROUP}" --name {} --yes --force-deletion 1 --no-wait + echo "Deletion initiated. Waiting up to 60 seconds for all VMs to be removed..." + SECONDS=0 + while [[ "${SECONDS}" -lt 60 ]] && az vm list --resource-group "${RESOURCEGROUP}" --query "[?starts_with(name, 'podvm-')].name" --output tsv | grep -q .; do + sleep 5 + done + [[ "${SECONDS}" -ge 60 ]] && echo "::error:: Failed to delete all vms in 60s" && az vm list --resource-group "${RESOURCEGROUP}" --query "[?starts_with(name, 'podvm-')].name" --output tsv + + # Now delete the NSG + AZURE_NSG_NAME="$(oc get configmap cloud-conf -n openshift-cloud-controller-manager -o json | jq -r '.data."cloud.conf" | fromjson' | jq -r '.securityGroupName')" + # Ignore failures (perhaps not created) + az network nsg delete -g "${RESOURCEGROUP}" -n "${AZURE_NSG_NAME}" || echo "::warning:: Failed to delete nsg: az network nsg delete -g '${RESOURCEGROUP}' -n '${AZURE_NSG_NAME}'" +} + # First check if PODVM_IMAGE was provided or generated if [[ "${PODVM_IMAGE_URL}" ]]; then echo "Skipping cleanup, custom PODVM_IMAGE_URL=${PODVM_IMAGE_URL} specified" @@ -104,6 +142,8 @@ provider="$(oc get infrastructure -n cluster -o json | jq '.items[].status.platf case ${provider} in aws) cleanup_aws ;; + azure) + cleanup_azure ;; *) echo "No post defined for provider ${provider}, skipping cleanup" exit 0;; diff --git a/ci-operator/step-registry/sandboxed-containers-operator/post/sandboxed-containers-operator-post-ref.yaml b/ci-operator/step-registry/sandboxed-containers-operator/post/sandboxed-containers-operator-post-ref.yaml index ecb45fc7b38a6..09c8c53354902 100644 --- a/ci-operator/step-registry/sandboxed-containers-operator/post/sandboxed-containers-operator-post-ref.yaml +++ b/ci-operator/step-registry/sandboxed-containers-operator/post/sandboxed-containers-operator-post-ref.yaml @@ -17,6 +17,10 @@ ref: Variable used by openshift-tests-private and OSC tests to use pre-built image; here we use it to skip post-cleanup as we don't want to delete the custom provided image. + - name: ENABLEPEERPODS + default: "false" + documentation: |- + Whether peer-pods support is enabled (to perform additional cleanup) documentation: |- A post-job to cleanup extra resources that might have been reserved on cloud during OSC testing.