Merge pull request #905 from gcpsoares/pipelineruns_alert

feat(SPRE-1859): Include pipelinerun increase alert.

Author: mftb

rhobs/alerting/data_plane/prometheus.pipelineruns_increase.yaml

@@ -0,0 +1,25 @@
+apiVersion: monitoring.coreos.com/v1
+kind: PrometheusRule
+metadata:
+  name: rhtap-pipelineruns-increase-alerting
+  labels:
+    tenant: rhtap
+spec:
+  groups:
+    - name: pipelineruns_increase_alerts
+      interval: 30m
+      rules:
+
+      - alert: PipelineRunsIncrease
+        expr: |
+          sum by (source_cluster,namespace) (increase(pipelinerun_duration_scheduled_seconds_count{namespace=~".*-tenant"}[3h])) > 2000
+        for: 25m
+        labels:
+          severity: high
+        annotations:
+          summary: >-
+             Increase number of Tekton pipelineruns in one namespace.
+          description: >-
+             Increase number of Tekton pipelineruns in one namespace in {{ $labels.source_cluster }}.
+          alert_team_handle: <!subteam^S04PYECHCCU>
+          team: pipelines

test/promql/tests/data_plane/pipelineruns_increase_test.yaml

@@ -0,0 +1,29 @@
+evaluation_interval: 1m
+
+rule_files:
+  - prometheus.pipelineruns_increase.yaml
+
+tests:
+
+  - interval: 1m
+    input_series:
+
+      - series: 'pipelinerun_duration_scheduled_seconds_count{source_cluster="cluster01",namespace="webapp-tenant"}'
+        values: '1000+12x210'
+
+    alert_rule_test:
+      # Evaluate at 205m (180m for 3h window + 25m )
+      - eval_time: 205m
+        alertname: PipelineRunsIncrease
+        exp_alerts:
+          - exp_labels:
+              severity: high
+              source_cluster: cluster01
+              namespace: webapp-tenant
+            exp_annotations:
+              summary: >-
+                Increase number of Tekton pipelineruns in one namespace.
+              description: >-
+                Increase number of Tekton pipelineruns in one namespace in cluster01.
+              alert_team_handle: <!subteam^S04PYECHCCU>
+              team: pipelines