Configure alerts for disk/filesystem usage, memory, CPU

site-cookbooks/kosmos_prometheus/attributes/default.rb

@@ -15,7 +15,7 @@ node.default["kosmos_prometheus"]["node_exporter"]["checksum"] = "9f5ea48e5bc7b6
 
 node.default["kosmos_prometheus"]["alertmanager"]["route"] = {
   "receiver" => "xmpp",
-  "group_by" => ["alertname"],
+  "group_by" => ["alertname", "severity"],
   "group_wait" => "20s",
   "group_interval" => "30s",
   "repeat_interval" => "3h",

site-cookbooks/kosmos_prometheus/files/default/rules/node_exporter.yml

@@ -8,3 +8,87 @@ groups:
           severity: warning
         annotations:
           summary: "Node is down"
+
+      - alert: NodeFilesystemAlmostOutOfSpace
+        expr: |
+          (node_filesystem_avail_bytes{fstype!~"tmpfs|fuse.lxcfs|overlay|squashfs|ramfs",mountpoint!~"/run.*|/var/lib/docker/.*"}
+            / node_filesystem_size_bytes{fstype!~"tmpfs|fuse.lxcfs|overlay|squashfs|ramfs",mountpoint!~"/run.*|/var/lib/docker/.*"}) * 100 < 10
+        for: 5m
+        labels:
+          severity: warning
+        annotations:
+          summary: "Filesystem has < 10% free space"
+
+      - alert: NodeFilesystemAlmostOutOfSpace
+        expr: |
+          (node_filesystem_avail_bytes{fstype!~"tmpfs|fuse.lxcfs|overlay|squashfs|ramfs",mountpoint!~"/run.*|/var/lib/docker/.*"}
+            / node_filesystem_size_bytes{fstype!~"tmpfs|fuse.lxcfs|overlay|squashfs|ramfs",mountpoint!~"/run.*|/var/lib/docker/.*"}) * 100 < 5
+        for: 5m
+        labels:
+          severity: critical
+        annotations:
+          summary: "Filesystem has < 5% free space"
+
+      - alert: NodeFilesystemAlmostOutOfInodes
+        expr: (node_filesystem_files_free / node_filesystem_files) * 100 < 10
+        for: 5m
+        labels:
+          severity: warning
+        annotations:
+          summary: "Filesystem has < 10% free inodes"
+
+      - alert: NodeFilesystemReadOnly
+        expr: node_filesystem_readonly == 1
+        for: 2m
+        labels:
+          severity: critical
+        annotations:
+          summary: "Filesystem remounted read-only"
+
+      - alert: NodeMemoryHigh
+        expr: (1 - node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes) * 100 > 90
+        for: 10m
+        labels:
+          severity: warning
+        annotations:
+          summary: "Memory usage > 90%"
+
+      - alert: NodeMemoryHigh
+        expr: (1 - node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes) * 100 > 95
+        for: 5m
+        labels:
+          severity: critical
+        annotations:
+          summary: "Memory usage > 95%"
+
+      - alert: NodeSwapHigh
+        expr: (node_memory_SwapTotal_bytes > 0) and ((1 - node_memory_SwapFree_bytes / node_memory_SwapTotal_bytes) * 100 > 50)
+        for: 10m
+        labels:
+          severity: warning
+        annotations:
+          summary: "Swap usage > 50%"
+
+      - alert: NodeCpuHigh
+        expr: 100 - (avg by (instance) (rate(node_cpu_seconds_total{mode="idle"}[5m])) * 100) > 80
+        for: 10m
+        labels:
+          severity: warning
+        annotations:
+          summary: "CPU usage > 80% for 10m"
+
+      - alert: NodeCpuHigh
+        expr: 100 - (avg by (instance) (rate(node_cpu_seconds_total{mode="idle"}[5m])) * 100) > 95
+        for: 5m
+        labels:
+          severity: critical
+        annotations:
+          summary: "CPU usage > 95% for 5m"
+
+      - alert: NodeLoadHigh
+        expr: node_load1 / count by (instance) (node_cpu_seconds_total{mode="idle"}) > 1.5
+        for: 10m
+        labels:
+          severity: warning
+        annotations:
+          summary: "System load1 > 1.5x CPU cores"