fix(setup): Add retry logic to post-setup

Troubleshooting the lab build failures, post setup is being executed
within seconds after the nova install completes, which may be too fast.

If I can confirm it's a race condition, we will want a more intelligent
way to determine when cluster is ready before proceeding to post-setup.

For now, just adding dumb retry logic to the openstack commands to see
if that helps.

Author: LukeRepko

scripts/hyperconverged-lab.sh

@@ -1261,32 +1261,38 @@ set -e
 sudo bash <<HERE
 sudo /opt/genestack/bin/setup-openstack-rc.sh
 source /opt/genestack/scripts/genestack.rc
-if ! openstack --os-cloud default flavor show ${LAB_NAME_PREFIX}-test; then
-  openstack --os-cloud default flavor create ${LAB_NAME_PREFIX}-test \
-            --public \
-            --ram 2048 \
-            --disk 10 \
-            --vcpus 2
-fi
-if ! openstack --os-cloud default network show flat; then
-  openstack --os-cloud default network create \
-            --share \
-            --availability-zone-hint az1 \
-            --external \
-            --provider-network-type flat \
-            --provider-physical-network physnet1 \
-            flat
-fi
-if ! openstack --os-cloud default subnet show flat_subnet; then
-  openstack --os-cloud default subnet create \
-            --subnet-range 192.168.102.0/24 \
-            --gateway 192.168.102.1 \
-            --dns-nameserver 1.1.1.1 \
-            --allocation-pool start=192.168.102.100,end=192.168.102.109 \
-            --dhcp \
-            --network flat \
-            flat_subnet
-fi
+
+# Function to retry openstack commands with backoff
+retry_openstack_command() {
+  local cmd="\\\$1"
+  local description="\\\$2"
+  local retry_count=0
+  local max_retries=30  # 30 retries * 10 seconds = 5 minutes
+  
+  while [ \\\$retry_count -lt \\\$max_retries ]; do
+    if eval "\\\$cmd" 2>/dev/null; then
+      echo "\\\$description succeeded"
+      return 0
+    else
+      retry_count=\\\$((retry_count + 1))
+      echo "\\\$description failed (attempt \\\$retry_count/\\\$max_retries). Retrying in 10 seconds..."
+      if [ \\\$retry_count -eq \\\$max_retries ]; then
+        echo "\\\$description failed after \\\$max_retries attempts. Continuing anyway..."
+        return 1
+      fi
+      sleep 10
+    fi
+  done
+}
+
+# Create flavor with retry
+retry_openstack_command "if ! openstack --os-cloud default flavor show ${LAB_NAME_PREFIX}-test; then openstack --os-cloud default flavor create ${LAB_NAME_PREFIX}-test --public --ram 2048 --disk 10 --vcpus 2; fi" "Flavor setup"
+
+# Create network with retry  
+retry_openstack_command "if ! openstack --os-cloud default network show flat; then openstack --os-cloud default network create --share --availability-zone-hint az1 --external --provider-network-type flat --provider-physical-network physnet1 flat; fi" "Network setup"
+
+# Create subnet with retry
+retry_openstack_command "if ! openstack --os-cloud default subnet show flat_subnet; then openstack --os-cloud default subnet create --subnet-range 192.168.102.0/24 --gateway 192.168.102.1 --dns-nameserver 1.1.1.1 --allocation-pool start=192.168.102.100,end=192.168.102.109 --dhcp --network flat flat_subnet; fi" "Subnet setup"
 HERE
 EOC