From 076a52cdda59ad95dfa1569d4e28b000162f2637 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A2u=20Cao?= Date: Thu, 27 Aug 2026 12:10:35 -0600 Subject: [PATCH] Adjust NodeFilesystemAlmostOutOfSpace for VM hosts The hosts have much larger drives, so 10% and 5% aren't ideal for alerts --- nodes | 2 +- .../files/default/rules/node_exporter.yml | 30 +++++++++++++++---- 2 files changed, 25 insertions(+), 7 deletions(-) diff --git a/nodes b/nodes index 5020b10..2b6479d 160000 --- a/nodes +++ b/nodes @@ -1 +1 @@ -Subproject commit 5020b10aa8db2dcae2d65fc40f0a9ea8a46e6f9f +Subproject commit 2b6479d2de17bb8abd2308a295db46ac32c535b2 diff --git a/site-cookbooks/kosmos_prometheus/files/default/rules/node_exporter.yml b/site-cookbooks/kosmos_prometheus/files/default/rules/node_exporter.yml index 854e0f5..a693f4f 100644 --- a/site-cookbooks/kosmos_prometheus/files/default/rules/node_exporter.yml +++ b/site-cookbooks/kosmos_prometheus/files/default/rules/node_exporter.yml @@ -11,23 +11,41 @@ groups: - alert: NodeFilesystemAlmostOutOfSpace expr: | - (node_filesystem_avail_bytes{fstype!~"tmpfs|fuse.lxcfs|overlay|squashfs|ramfs",mountpoint!~"/run.*|/var/lib/docker/.*"} - / node_filesystem_size_bytes{fstype!~"tmpfs|fuse.lxcfs|overlay|squashfs|ramfs",mountpoint!~"/run.*|/var/lib/docker/.*"}) * 100 < 10 + (node_filesystem_avail_bytes{fstype!~"tmpfs|fuse.lxcfs|overlay|squashfs|ramfs",mountpoint!~"/run.*|/var/lib/docker/.*",vm_host=~".+"} + / node_filesystem_size_bytes{fstype!~"tmpfs|fuse.lxcfs|overlay|squashfs|ramfs",mountpoint!~"/run.*|/var/lib/docker/.*",vm_host=~".+"}) * 100 < 10 for: 5m labels: severity: warning annotations: - summary: "Filesystem has < 10% free space" + summary: "VM filesystem has < 10% free space" - alert: NodeFilesystemAlmostOutOfSpace expr: | - (node_filesystem_avail_bytes{fstype!~"tmpfs|fuse.lxcfs|overlay|squashfs|ramfs",mountpoint!~"/run.*|/var/lib/docker/.*"} - / node_filesystem_size_bytes{fstype!~"tmpfs|fuse.lxcfs|overlay|squashfs|ramfs",mountpoint!~"/run.*|/var/lib/docker/.*"}) * 100 < 5 + (node_filesystem_avail_bytes{fstype!~"tmpfs|fuse.lxcfs|overlay|squashfs|ramfs",mountpoint!~"/run.*|/var/lib/docker/.*",vm_host=~".+"} + / node_filesystem_size_bytes{fstype!~"tmpfs|fuse.lxcfs|overlay|squashfs|ramfs",mountpoint!~"/run.*|/var/lib/docker/.*",vm_host=~".+"}) * 100 < 5 for: 5m labels: severity: critical annotations: - summary: "Filesystem has < 5% free space" + summary: "VM filesystem has < 5% free space" + + - alert: NodeFilesystemAlmostOutOfSpace + expr: | + node_filesystem_avail_bytes{fstype!~"tmpfs|fuse.lxcfs|overlay|squashfs|ramfs",mountpoint!~"/run.*|/var/lib/docker/.*",vm_host!~".+"} < 53687091200 + for: 5m + labels: + severity: warning + annotations: + summary: "Host filesystem has < 50GB free space" + + - alert: NodeFilesystemAlmostOutOfSpace + expr: | + node_filesystem_avail_bytes{fstype!~"tmpfs|fuse.lxcfs|overlay|squashfs|ramfs",mountpoint!~"/run.*|/var/lib/docker/.*",vm_host!~".+"} < 21474836480 + for: 5m + labels: + severity: critical + annotations: + summary: "Host filesystem has < 20GB free space" - alert: NodeFilesystemAlmostOutOfInodes expr: (node_filesystem_files_free / node_filesystem_files) * 100 < 10