chore: raise pod probes and test timeouts (#905)

razvan · web-flow · commit 2189c1d7fdf7 · 2025-10-28T14:15:32.000Z
* test: raise kraft timeouts

* remove junk line

* raise controller pod probes timeout to 1 minute

* re-enable downscale tests and ensure clean ns termination
diff --git a/rust/operator-binary/src/resource/statefulset.rs b/rust/operator-binary/src/resource/statefulset.rs
@@ -693,17 +693,19 @@ pub fn build_controller_rolegroup_statefulset(
                 port: IntOrString::Int(kafka_security.client_port().into()),
                 ..Default::default()
             }),
-            timeout_seconds: Some(5),
-            period_seconds: Some(5),
+            timeout_seconds: Some(10),
+            period_seconds: Some(10),
+            failure_threshold: Some(6),
             ..Probe::default()
         })
         .readiness_probe(Probe {
             tcp_socket: Some(TCPSocketAction {
                 port: IntOrString::Int(kafka_security.client_port().into()),
                 ..Default::default()
             }),
-            timeout_seconds: Some(5),
-            period_seconds: Some(5),
+            timeout_seconds: Some(10),
+            period_seconds: Some(10),
+            failure_threshold: Some(6),
             ..Probe::default()
         });
 
diff --git a/tests/templates/kuttl/operations-kraft/20-install-kafka.yaml.j2 b/tests/templates/kuttl/operations-kraft/20-install-kafka.yaml.j2
@@ -1,7 +1,7 @@
 ---
 apiVersion: kuttl.dev/v1beta1
 kind: TestStep
-timeout: 300
+timeout: 600
 ---
 apiVersion: kafka.stackable.tech/v1alpha1
 kind: KafkaCluster
diff --git a/tests/templates/kuttl/operations-kraft/25-assert.yaml.j2 b/tests/templates/kuttl/operations-kraft/25-assert.yaml.j2
@@ -1,7 +1,7 @@
 ---
 apiVersion: kuttl.dev/v1beta1
 kind: TestAssert
-timeout: 300
+timeout: 600
 commands:
   - script: kubectl -n $NAMESPACE wait --for=condition=reconciliationPaused kafkaclusters.kafka.stackable.tech/test-kafka --timeout 301s
 ---
diff --git a/tests/templates/kuttl/operations-kraft/25-pause-kafka.yaml.j2 b/tests/templates/kuttl/operations-kraft/25-pause-kafka.yaml.j2
@@ -1,7 +1,7 @@
 ---
 apiVersion: kuttl.dev/v1beta1
 kind: TestStep
-timeout: 300
+timeout: 600
 ---
 apiVersion: kafka.stackable.tech/v1alpha1
 kind: KafkaCluster
diff --git a/tests/templates/kuttl/operations-kraft/30-stop-kafka.yaml.j2 b/tests/templates/kuttl/operations-kraft/30-stop-kafka.yaml.j2
@@ -1,7 +1,7 @@
 ---
 apiVersion: kuttl.dev/v1beta1
 kind: TestStep
-timeout: 300
+timeout: 600
 ---
 apiVersion: kafka.stackable.tech/v1alpha1
 kind: KafkaCluster
diff --git a/tests/templates/kuttl/operations-kraft/50-assert.yaml.j2 b/tests/templates/kuttl/operations-kraft/50-assert.yaml.j2
@@ -1,7 +1,7 @@
 ---
 apiVersion: kuttl.dev/v1beta1
 kind: TestAssert
-timeout: 300
+timeout: 600
 commands:
   - script: kubectl -n $NAMESPACE wait --for=condition=available kafkaclusters.kafka.stackable.tech/test-kafka --timeout 301s
 ---
diff --git a/tests/templates/kuttl/operations-kraft/50-restart-kafka.yaml.j2 b/tests/templates/kuttl/operations-kraft/50-restart-kafka.yaml.j2
@@ -1,7 +1,7 @@
 ---
 apiVersion: kuttl.dev/v1beta1
 kind: TestStep
-timeout: 300
+timeout: 600
 ---
 apiVersion: kafka.stackable.tech/v1alpha1
 kind: KafkaCluster
diff --git a/tests/templates/kuttl/operations-kraft/60-assert.yaml.j2 b/tests/templates/kuttl/operations-kraft/60-assert.yaml.j2
@@ -2,7 +2,7 @@
 ---
 apiVersion: kuttl.dev/v1beta1
 kind: TestAssert
-timeout: 300
+timeout: 600
 commands:
   - script: kubectl -n $NAMESPACE wait --for=condition=available kafkaclusters.kafka.stackable.tech/test-kafka --timeout 301s
 ---
diff --git a/tests/templates/kuttl/operations-kraft/60-scale-controller-up.yaml.j2 b/tests/templates/kuttl/operations-kraft/60-scale-controller-up.yaml.j2
@@ -2,7 +2,7 @@
 ---
 apiVersion: kuttl.dev/v1beta1
 kind: TestStep
-timeout: 300
+timeout: 600
 ---
 apiVersion: kafka.stackable.tech/v1alpha1
 kind: KafkaCluster
diff --git a/tests/templates/kuttl/operations-kraft/70-assert.yaml.j2 b/tests/templates/kuttl/operations-kraft/70-assert.yaml.j2
@@ -2,7 +2,7 @@
 ---
 apiVersion: kuttl.dev/v1beta1
 kind: TestAssert
-timeout: 300
+timeout: 600
 commands:
   - script: kubectl -n $NAMESPACE wait --for=condition=available kafkaclusters.kafka.stackable.tech/test-kafka --timeout 301s
 ---
diff --git a/tests/templates/kuttl/operations-kraft/70-scale-controller-down.yaml.j2 b/tests/templates/kuttl/operations-kraft/70-scale-controller-down.yaml.j2
@@ -1,12 +1,8 @@
-#
-# This test step is disabled becaus Kraft controller do scale down reliably.
-# This leads to flaky tests.
-#
 {% if not test_scenario['values']['kafka-kraft'].startswith("3.7") %}
 ---
 apiVersion: kuttl.dev/v1beta1
 kind: TestStep
-timeout: 300
+timeout: 600
 ---
 apiVersion: kafka.stackable.tech/v1alpha1
 kind: KafkaCluster
diff --git a/tests/templates/kuttl/operations-kraft/80-assert.yaml.j2 b/tests/templates/kuttl/operations-kraft/80-assert.yaml.j2
@@ -0,0 +1,21 @@
+{% if not test_scenario['values']['kafka-kraft'].startswith("3.7") %}
+---
+apiVersion: kuttl.dev/v1beta1
+kind: TestAssert
+timeout: 600
+---
+apiVersion: apps/v1
+kind: StatefulSet
+metadata:
+  name: test-kafka-broker-default
+status:
+  replicas: 0
+---
+apiVersion: apps/v1
+kind: StatefulSet
+metadata:
+  name: test-kafka-controller-default
+status:
+  readyReplicas: 3
+  replicas: 3
+{% endif %}
diff --git a/tests/templates/kuttl/operations-kraft/80-scale-broker-down.yaml.j2 b/tests/templates/kuttl/operations-kraft/80-scale-broker-down.yaml.j2
@@ -0,0 +1,46 @@
+#
+# This is a test helper to ensure that all broker pods are deleted before
+# the test namespace is terminated.
+# The brokers must be deleted because otherwise they are left dangling until
+# the test timeouts and fails.
+#
+{% if not test_scenario['values']['kafka-kraft'].startswith("3.7") %}
+---
+apiVersion: kuttl.dev/v1beta1
+kind: TestStep
+timeout: 600
+---
+apiVersion: kafka.stackable.tech/v1alpha1
+kind: KafkaCluster
+metadata:
+  name: test-kafka
+spec:
+  image:
+{% if test_scenario['values']['kafka-kraft'].find(",") > 0 %}
+    custom: "{{ test_scenario['values']['kafka-kraft'].split(',')[1] }}"
+    productVersion: "{{ test_scenario['values']['kafka-kraft'].split(',')[0] }}"
+{% else %}
+    productVersion: "{{ test_scenario['values']['kafka-kraft'] }}"
+{% endif %}
+{% if lookup('env', 'VECTOR_AGGREGATOR') %}
+  clusterConfig:
+    vectorAggregatorConfigMapName: vector-aggregator-discovery
+{% endif %}
+  controllers:
+    config:
+      logging:
+        enableVectorAgent: {{ lookup('env', 'VECTOR_AGGREGATOR') | length > 0 }}
+    roleGroups:
+      default:
+        replicas: 3
+  brokers:
+    config:
+      logging:
+        enableVectorAgent: {{ lookup('env', 'VECTOR_AGGREGATOR') | length > 0 }}
+    roleGroups:
+      default:
+        replicas: 0
+  clusterOperation:
+    stopped: false
+    reconciliationPaused: false
+{% endif %}
