|
| 1 | +#!/usr/bin/env bash |
| 2 | +# |
| 3 | +# Test script for Wave's global rate limiting feature. |
| 4 | +# |
| 5 | +# Usage: ./hack/test-update-throttling.sh [kubernetes-version] [update-rate] [update-burst] |
| 6 | +# |
| 7 | +# Arguments: |
| 8 | +# kubernetes-version: Kubernetes version for minikube (default: v1.21) |
| 9 | +# update-rate: Updates per second globally (default: 0.2 = 1 update per 5 seconds) |
| 10 | +# update-burst: Maximum burst size (default: 2) |
| 11 | +# |
| 12 | +# Examples: |
| 13 | +# ./hack/test-update-throttling.sh # Use defaults |
| 14 | +# ./hack/test-update-throttling.sh v1.21 0.5 5 # 0.5 updates/sec, burst of 5 |
| 15 | +# ./hack/test-update-throttling.sh v1.21 1.0 10 # 1 update/sec, burst of 10 (production defaults) |
| 16 | + |
| 17 | +set -eu |
| 18 | +set -o pipefail |
| 19 | + |
| 20 | +helm --help > /dev/null 2>&1 || { |
| 21 | + echo "helm is not installed" |
| 22 | + exit 1 |
| 23 | +} |
| 24 | + |
| 25 | +kubectl --help > /dev/null 2>&1 || { |
| 26 | + echo "kubectl is not installed" |
| 27 | + exit 1 |
| 28 | +} |
| 29 | + |
| 30 | +MINIKUBE_ALREADY_RUNNING=0 |
| 31 | +kubectl get node minikube >/dev/null 2>&1 && MINIKUBE_ALREADY_RUNNING=1 |
| 32 | + |
| 33 | +minikube --help > /dev/null 2>&1 || { |
| 34 | + echo "minikube is not installed" |
| 35 | + exit 1 |
| 36 | +} |
| 37 | + |
| 38 | +KUBERNETES_VERSION=${1:-v1.21} |
| 39 | +UPDATE_RATE=${2:-0.2} |
| 40 | +UPDATE_BURST=${3:-2} |
| 41 | + |
| 42 | +[ $MINIKUBE_ALREADY_RUNNING -eq 0 ] && { |
| 43 | + echo Starting minikube... |
| 44 | + minikube start --kubernetes-version "$KUBERNETES_VERSION" |
| 45 | + trap "minikube delete" EXIT |
| 46 | +} |
| 47 | + |
| 48 | +eval $(minikube -p minikube docker-env) |
| 49 | +docker build -f ./Dockerfile -t wave-local:local . |
| 50 | + |
| 51 | +echo Installing wave with updateRate=${UPDATE_RATE} updateBurst=${UPDATE_BURST}... |
| 52 | +helm install wave charts/wave --set image.name=wave-local --set image.tag=local --set updateRate=${UPDATE_RATE} --set updateBurst=${UPDATE_BURST} |
| 53 | + |
| 54 | +while [ "$(kubectl get pods -n default | grep -cE 'wave-wave')" -gt 1 ]; do echo Waiting for \"wave\" to be scheduled; sleep 10; done |
| 55 | + |
| 56 | +while [ "$(kubectl get pods -A | grep -cEv 'Running|Completed')" -gt 1 ]; do echo Waiting for \"cluster\" to start; sleep 10; done |
| 57 | + |
| 58 | +echo Creating test resources... |
| 59 | +kubectl apply -f - <<'EOF' |
| 60 | +apiVersion: v1 |
| 61 | +kind: ConfigMap |
| 62 | +metadata: |
| 63 | + name: throttle-test |
| 64 | +data: |
| 65 | + counter: "0" |
| 66 | + timestamp: "0" |
| 67 | +EOF |
| 68 | + |
| 69 | +kubectl apply -f - <<'EOF' |
| 70 | +apiVersion: apps/v1 |
| 71 | +kind: Deployment |
| 72 | +metadata: |
| 73 | + name: throttle-test |
| 74 | + annotations: |
| 75 | + wave.pusher.com/update-on-config-change: "true" |
| 76 | +spec: |
| 77 | + replicas: 1 |
| 78 | + selector: |
| 79 | + matchLabels: |
| 80 | + app: throttle-test |
| 81 | + template: |
| 82 | + metadata: |
| 83 | + labels: |
| 84 | + app: throttle-test |
| 85 | + spec: |
| 86 | + containers: |
| 87 | + - name: test |
| 88 | + image: nixery.dev/shell/bash |
| 89 | + command: |
| 90 | + - /bin/bash |
| 91 | + - -c |
| 92 | + - | |
| 93 | + echo "Pod started at $(date +%s)" |
| 94 | + echo "Counter: $(cat /etc/config/counter)" |
| 95 | + sleep infinity |
| 96 | + volumeMounts: |
| 97 | + - name: config |
| 98 | + mountPath: /etc/config |
| 99 | + volumes: |
| 100 | + - name: config |
| 101 | + configMap: |
| 102 | + name: throttle-test |
| 103 | +EOF |
| 104 | + |
| 105 | +echo Waiting for initial deployment to be ready... |
| 106 | +kubectl wait --for=condition=available --timeout=60s deployment/throttle-test |
| 107 | + |
| 108 | +# Get the initial pod name and creation time |
| 109 | +INITIAL_POD=$(kubectl get pods -l app=throttle-test -o jsonpath='{.items[0].metadata.name}') |
| 110 | +INITIAL_CREATION=$(kubectl get pod $INITIAL_POD -o jsonpath='{.metadata.creationTimestamp}') |
| 111 | +echo "Initial pod: $INITIAL_POD created at $INITIAL_CREATION" |
| 112 | + |
| 113 | +# Record start time |
| 114 | +START_TIME=$(date +%s) |
| 115 | + |
| 116 | +echo "" |
| 117 | +echo "Testing update throttling by rapidly updating the ConfigMap..." |
| 118 | +echo "Expected behavior with updateRate=${UPDATE_RATE} updateBurst=${UPDATE_BURST}:" |
| 119 | +echo " - First ${UPDATE_BURST} updates will happen immediately (burst tokens)" |
| 120 | +echo " - Subsequent updates will be rate-limited to ${UPDATE_RATE} per second" |
| 121 | +echo "" |
| 122 | + |
| 123 | +# Rapidly update the ConfigMap 10 times |
| 124 | +for i in 1 2 3 4 5 6 7 8 9 10; do |
| 125 | + echo "Update $i at $(date +%H:%M:%S)" |
| 126 | + kubectl patch configmap throttle-test --type merge -p "{\"data\":{\"counter\":\"$i\",\"timestamp\":\"$(date +%s)\"}}" |
| 127 | + sleep 1 |
| 128 | +done |
| 129 | + |
| 130 | +echo "" |
| 131 | +echo "Waiting 20 seconds to observe throttling behavior..." |
| 132 | +sleep 20 |
| 133 | + |
| 134 | +# Check how many pod restarts/updates occurred |
| 135 | +echo "" |
| 136 | +echo "Checking deployment update history..." |
| 137 | +kubectl get replicasets -l app=throttle-test -o wide |
| 138 | + |
| 139 | +# Get all pods that were created (including terminated ones) |
| 140 | +echo "" |
| 141 | +echo "Checking pod creation times..." |
| 142 | +POD_COUNT=$(kubectl get pods -l app=throttle-test --show-all 2>/dev/null | grep -c throttle-test || kubectl get pods -l app=throttle-test | grep -c throttle-test) |
| 143 | +echo "Total pods created: $POD_COUNT" |
| 144 | + |
| 145 | +# Get the deployment's pod template hash changes |
| 146 | +HASH_CHANGES=$(kubectl get replicasets -l app=throttle-test -o jsonpath='{range .items[*]}{.metadata.creationTimestamp}{"\t"}{.spec.template.spec.containers[0].image}{"\t"}{.spec.replicas}{"\n"}{end}') |
| 147 | +echo "" |
| 148 | +echo "ReplicaSet history (creation time, image, replicas):" |
| 149 | +echo "$HASH_CHANGES" |
| 150 | + |
| 151 | +# Count how many distinct replicasets were created |
| 152 | +RS_COUNT=$(kubectl get replicasets -l app=throttle-test --no-headers | wc -l) |
| 153 | +echo "" |
| 154 | +echo "Number of ReplicaSets created: $RS_COUNT" |
| 155 | + |
| 156 | +# Check wave controller logs for throttling messages |
| 157 | +echo "" |
| 158 | +echo "Wave controller logs (throttling messages):" |
| 159 | +kubectl logs -l app=wave --tail=50 | grep -i "throttl\|delayed" || echo "No throttling messages found in logs" |
| 160 | + |
| 161 | +# Verify throttling worked |
| 162 | +# Calculate expected updates: burst + (rate * time) |
| 163 | +# With rate=0.2, burst=2, and ~30 seconds total time: |
| 164 | +# Expected: 2 (burst) + (0.2 * 30) = 2 + 6 = 8 updates max |
| 165 | +# We'll allow some margin and expect ≤9 to account for timing variations |
| 166 | +echo "" |
| 167 | +echo "=== Test Results ===" |
| 168 | +EXPECTED_MAX=$((UPDATE_BURST + 7)) |
| 169 | +if [ "$RS_COUNT" -le "$EXPECTED_MAX" ]; then |
| 170 | + echo "✓ PASS: Rate limiting is working correctly" |
| 171 | + echo " - ConfigMap was updated 10 times rapidly" |
| 172 | + echo " - $RS_COUNT deployment updates occurred (expected ≤${EXPECTED_MAX} with rate=${UPDATE_RATE}/sec, burst=${UPDATE_BURST})" |
| 173 | + echo " - Burst allowed ${UPDATE_BURST} immediate updates, then rate-limited to ${UPDATE_RATE} per second" |
| 174 | + exit 0 |
| 175 | +else |
| 176 | + echo "✗ FAIL: Rate limiting may not be working correctly" |
| 177 | + echo " - ConfigMap was updated 10 times rapidly" |
| 178 | + echo " - $RS_COUNT deployment updates occurred (expected ≤${EXPECTED_MAX} with rate=${UPDATE_RATE}/sec, burst=${UPDATE_BURST})" |
| 179 | + exit 1 |
| 180 | +fi |
0 commit comments