fix(observability): virtualization virt metrics state (#1291)

Remove alert D8InternalVirtualizationVirtHandlerTargetAbsent and D8InternalVirtualizationVirtHandlerTargetDown

Add new metrics fo virtualization virt metrics state:
- D8InternalVirtualizationVirtMetricsTargetAbsent
- D8InternalVirtualizationVirtMetricsTargetDown
---------

Signed-off-by: Nikita Korolev <[email protected]>
Co-authored-by: Ivan Mikheykin <[email protected]>

Author: universal-itengineer

monitoring/prometheus-rules/internal-virtualization-virt-hander.yaml

@@ -1,41 +1,5 @@
 - name: kubernetes.internal.virtualization.virt_handler_state
   rules:
-    - alert: D8InternalVirtualizationVirtHandlerTargetDown
-      expr: max by (job) (up{job="internal-virtualization-kubevirt-prometheus-metrics"}) == 0
-      for: 1m
-      labels:
-        severity_level: "6"
-        tier: cluster
-      annotations:
-        plk_markup_format: "markdown"
-        plk_protocol_version: "1"
-        plk_create_group_if_not_exists__d8_internal_virtualization_virt_handler_health: "D8InternalVirtualizationVirtHandlerlHealth,tier=~tier,prometheus=deckhouse,kubernetes=~kubernetes"
-        plk_grouped_by__d8_internal_virtualization_virt_handler_health: "D8InternalVirtualizationVirtHandlerlHealth,tier=~tier,prometheus=deckhouse,kubernetes=~kubernetes"
-        plk_ignore_labels: "job"
-        summary: Prometheus cannot scrape the virtualization-controller metrics.
-        description: |
-          The recommended course of action:
-          1. Check the Pod status: `kubectl -n d8-virtualization get pod -l kubevirt.internal.virtualization.deckhouse.io=virt-handler`
-          2. Or check the Pod logs: `kubectl -n d8-virtualization logs daemonsets/virt-handler`
-
-    - alert: D8InternalVirtualizationVirtHandlerTargetAbsent
-      expr: absent(up{job="internal-virtualization-kubevirt-prometheus-metrics"}) == 1
-      labels:
-        severity_level: "6"
-        tier: cluster
-      for: 15m
-      annotations:
-        plk_protocol_version: "1"
-        plk_markup_format: "markdown"
-        plk_ignore_labels: "job"
-        plk_create_group_if_not_exists__d8_internal_virtualization_virt_handler_health: "D8InternalVirtualizationVirtHandlerHealth,tier=~tier,prometheus=deckhouse,kubernetes=~kubernetes"
-        plk_grouped_by__d8_internal_virtualization_virt_handler_health: "D8InternalVirtualizationVirtHandlerHealth,tier=~tier,prometheus=deckhouse,kubernetes=~kubernetes"
-        summary: There is no `virtualization-controller` target in Prometheus.
-        description: |
-          The recommended course of action:
-          1. Check the Pod status: `kubectl -n d8-virtualization get pod -l kubevirt.internal.virtualization.deckhouse.io=virt-handler`
-          2. Or check the Pod logs: `kubectl -n d8-virtualization logs daemonsets/virt-handler`
-
     - alert: D8InternalVirtualizationVirtHandlerPodIsNotReady
       expr: min by (pod) (avg by(node,pod,namespace)(kube_pod_info{}) * on(pod, namespace) group_right(node) kube_pod_status_ready{condition="true", namespace="d8-virtualization", pod=~"virt-handler-.*"}) != 1
       for: 10m

monitoring/prometheus-rules/virt-metrics.yaml

@@ -0,0 +1,36 @@
+- name: kubernetes.internal.virtualization.virt_metrics_state
+  rules:
+    - alert: D8InternalVirtualizationVirtMetricsTargetDown
+      expr: max by (job) (up{job="kubevirt-prometheus-metrics"}) == 0
+      for: 1m
+      labels:
+        severity_level: "6"
+        tier: cluster
+      annotations:
+        plk_markup_format: "markdown"
+        plk_protocol_version: "1"
+        plk_create_group_if_not_exists__d8_internal_virtualization_virt_metrics_health: "D8InternalVirtualizationVirtMetricslHealth,tier=~tier,prometheus=deckhouse,kubernetes=~kubernetes"
+        plk_grouped_by__d8_internal_virtualization_virt_handler_health: "D8InternalVirtualizationVirtMetricslHealth,tier=~tier,prometheus=deckhouse,kubernetes=~kubernetes"
+        plk_ignore_labels: "job"
+        summary: Prometheus cannot scrape the virt-handler, virt-api, and virt-controller metrics.
+        description: |
+          The recommended course of action:
+          1. Check the service: `kubectl -n d8-virtualization get service kubevirt-prometheus-metrics`
+
+    - alert: D8InternalVirtualizationVirtMetricsTargetAbsent
+      expr: absent(up{job="kubevirt-prometheus-metrics"}) == 1
+      labels:
+        severity_level: "6"
+        tier: cluster
+      for: 1m
+      annotations:
+        plk_protocol_version: "1"
+        plk_markup_format: "markdown"
+        plk_ignore_labels: "job"
+        plk_create_group_if_not_exists__d8_internal_virtualization_virt_metrics_health: "D8InternalVirtualizationVirtMetricsHealth,tier=~tier,prometheus=deckhouse,kubernetes=~kubernetes"
+        plk_grouped_by__d8_internal_virtualization_virt_handler_health: "D8InternalVirtualizationVirtMetricsHealth,tier=~tier,prometheus=deckhouse,kubernetes=~kubernetes"
+        summary: Prometheus can't discover targets for virt-handler, virt-api, and virt-controller metrics.
+        description: |
+          The recommended course of action:
+          1. Check that service exists `kubectl -n d8-virtualization get service kubevirt-prometheus-metrics`
+          2. Check that virt-handler, virt-api, and virt-controller exist: `kubectl -n d8-virtualization get po -l prometheus.kubevirt.internal.virtualization.deckhouse.io=true`