imp: cache to 3 TB gp2, mod loki coredump alert, mod perf vault role

Author: johnalotoski

nix/cloud/alerts.nix

@@ -2,6 +2,25 @@
   inputs,
   cell,
 }: {
+  ci-world-loki = {
+    datasource = "loki";
+    rules = [
+      {
+        alert = "CoredumpDetected";
+        expr = ''sum(rate({syslog_identifier="systemd-coredump", host=~"core.*"}[1h] != "sshd" |= "dumped core")) by (host) > 0'';
+        for = "1m";
+        labels.severity = "critical";
+        annotations = {
+          description = ''
+            Detected a coredump on {{ $labels.host }}.
+             This usually requires attention and most likely manual intervention.
+             To analyze a coredump, run `coredumpctl list` on the affected machine, and run `coredump debug $id` in a nix shell with gdb.'';
+          summary = "Detected a coredump on {{ $labels.host }}";
+        };
+      }
+    ];
+  };
+
   ci-world-darwin = {
     datasource = "vm";
     rules = [

nix/cloud/hydrationProfile.nix

@@ -133,8 +133,8 @@ in {
             "auth/token/lookup-self" = [r];
             "auth/token/renew-self" = [u];
             "sys/capabilities-self" = [u];
-            "kv/data/perf/*" = [r l];
-            "kv/metadata/perf/*" = [r l];
+            "kv/data/postgrest/*" = [r l];
+            "kv/metadata/postgrest/*" = [r l];
             "nomad/creds/perf" = [r u];
             "consul/creds/developer" = [r u];
             "sops/keys/dev" = [r l];
@@ -265,6 +265,7 @@ in {
           # Cell Block local declared dashboards
           inherit
             (cell.alerts)
+            ci-world-loki
             ci-world-darwin
             ci-world-spongix
             ci-world-nomad-follower
@@ -276,7 +277,6 @@ in {
             (inputs.bitte-cells.bitte.alerts)
             bitte-consul
             bitte-deadmanssnitch
-            bitte-loki
             bitte-system
             bitte-vault
             bitte-vm-health

nix/metal/bitteProfile/default.nix

@@ -347,7 +347,7 @@ in {
           instanceType = "m5n.4xlarge";
           privateIP = "172.16.0.52";
           subnet = cluster.vpc.subnets.core-1;
-          volumeSize = 2000;
+          volumeSize = 3000;
 
           modules = [
             (bitte + /profiles/auxiliaries/telegraf.nix)