Merge pull request #8430 from vitanovs/feat/vpa-updater-failed-evictions-counter-metric

feat(vpa/updater): Add a new counter metric to measure the total number of failed Pods evictions attempts

Author: k8s-ci-robot

vertical-pod-autoscaler/pkg/updater/logic/updater.go

@@ -316,6 +316,7 @@ func (u *updater) RunOnce(ctx context.Context) {
 			evictErr := evictionLimiter.Evict(pod, vpa, u.eventRecorder)
 			if evictErr != nil {
 				klog.V(0).InfoS("Eviction failed", "error", evictErr, "pod", klog.KObj(pod))
+				metrics_updater.RecordFailedEviction(vpaSize, updateMode, "EvictionError")
 			} else {
 				withEvicted = true
 				metrics_updater.AddEvictedPod(vpaSize, updateMode)

vertical-pod-autoscaler/pkg/utils/metrics/updater/updater.go

@@ -84,6 +84,14 @@ var (
 		}, []string{"vpa_size_log2", "update_mode"},
 	)
 
+	failedEvictionAttempts = prometheus.NewCounterVec(
+		prometheus.CounterOpts{
+			Namespace: metricsNamespace,
+			Name:      "failed_eviction_attempts_total",
+			Help:      "Number of failed attempts to update Pods by eviction",
+		}, []string{"vpa_size_log2", "update_mode", "reason"},
+	)
+
 	inPlaceUpdatableCount = prometheus.NewGaugeVec(
 		prometheus.GaugeOpts{
 			Namespace: metricsNamespace,
@@ -130,7 +138,21 @@ var (
 
 // Register initializes all metrics for VPA Updater
 func Register() {
-	prometheus.MustRegister(controlledCount, evictableCount, evictedCount, vpasWithEvictablePodsCount, vpasWithEvictedPodsCount, inPlaceUpdatableCount, inPlaceUpdatedCount, vpasWithInPlaceUpdatablePodsCount, vpasWithInPlaceUpdatedPodsCount, failedInPlaceUpdateAttempts, functionLatency)
+	collectors := []prometheus.Collector{
+		controlledCount,
+		evictableCount,
+		evictedCount,
+		vpasWithEvictablePodsCount,
+		vpasWithEvictedPodsCount,
+		failedEvictionAttempts,
+		inPlaceUpdatableCount,
+		inPlaceUpdatedCount,
+		vpasWithInPlaceUpdatablePodsCount,
+		vpasWithInPlaceUpdatedPodsCount,
+		failedInPlaceUpdateAttempts,
+		functionLatency,
+	}
+	prometheus.MustRegister(collectors...)
 }
 
 // NewExecutionTimer provides a timer for Updater's RunOnce execution
@@ -183,6 +205,12 @@ func AddEvictedPod(vpaSize int, mode vpa_types.UpdateMode) {
 	evictedCount.WithLabelValues(strconv.Itoa(log2), string(mode)).Inc()
 }
 
+// RecordFailedEviction increases the counter of failed eviction attempts by given VPA size, update mode and reason
+func RecordFailedEviction(vpaSize int, mode vpa_types.UpdateMode, reason string) {
+	log2 := metrics.GetVpaSizeLog2(vpaSize)
+	failedEvictionAttempts.WithLabelValues(strconv.Itoa(log2), string(mode), reason).Inc()
+}
+
 // NewInPlaceUpdatablePodsCounter returns a wrapper for counting Pods which are matching in-place update criteria
 func NewInPlaceUpdatablePodsCounter() *SizeBasedGauge {
 	return newSizeBasedGauge(inPlaceUpdatableCount)

vertical-pod-autoscaler/pkg/utils/metrics/updater/updater_test.go

@@ -58,6 +58,40 @@ func TestAddEvictedPod(t *testing.T) {
 	}
 }
 
+func TestRecordFailedEviction(t *testing.T) {
+	testCases := []struct {
+		desc    string
+		vpaSize int
+		mode    vpa_types.UpdateMode
+		reason  string
+		log2    string
+	}{
+		{
+			desc:    "VPA size 2, some reason",
+			vpaSize: 2,
+			reason:  "some_reason",
+			log2:    "1",
+		},
+		{
+			desc:    "VPA size 20, another reason",
+			vpaSize: 20,
+			reason:  "another_reason",
+			log2:    "4",
+		},
+	}
+
+	for _, tc := range testCases {
+		t.Run(tc.desc, func(t *testing.T) {
+			t.Cleanup(failedEvictionAttempts.Reset)
+			RecordFailedEviction(tc.vpaSize, tc.mode, tc.reason)
+			val := testutil.ToFloat64(failedEvictionAttempts.WithLabelValues(tc.log2, string(tc.mode), tc.reason))
+			if val != 1 {
+				t.Errorf("Unexpected value for FailedEviction metric with labels (%s, %s): got %v, want 1", tc.log2, tc.reason, val)
+			}
+		})
+	}
+}
+
 func TestAddInPlaceUpdatedPod(t *testing.T) {
 	testCases := []struct {
 		desc    string