Upgrade Sentry to 26.6.0

Update node configs
Increase padding for NodeDown rule
2026-07-08 16:44:09 +02:00 · 2026-07-07 16:58:36 +02:00 · 2026-07-07 16:57:22 +02:00 · 2026-07-07 15:20:48 +02:00 · 2026-07-07 15:20:33 +02:00 · 2026-07-07 15:05:25 +02:00
70 changed files with 1133 additions and 453 deletions
@@ -0,0 +1,40 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+SCRIPT="$(git rev-parse --show-toplevel)/scripts/util/sanitize_node.sh"
+
+FILES=$(git diff --cached --name-only --diff-filter=ACM -- 'nodes/*.json' || true)
+
+[[ -z "$FILES" ]] && exit 0
+
+DIRTY=0
+
+while IFS= read -r file; do
+  staged_tmp=$(mktemp)
+  sanitized_tmp=$(mktemp)
+
+  # 1. get staged version
+  git show ":$file" > "$staged_tmp"
+
+  # 2. sanitize IN PLACE (on temp copy)
+  cp "$staged_tmp" "$sanitized_tmp"
+  "$SCRIPT" "$sanitized_tmp"
+
+  # 3. if sanitizer changed file, update working tree
+  if ! diff -q "$staged_tmp" "$sanitized_tmp" >/dev/null; then
+    cp "$sanitized_tmp" "$file"
+    echo "Sanitized: $file"
+    DIRTY=1
+  fi
+
+  rm -f "$staged_tmp" "$sanitized_tmp"
+
+done <<< "$FILES"
+
+if [[ "$DIRTY" -ne 0 ]]; then
+  echo ""
+  echo "Commit paused, sanitized files must be re-staged."
+  exit 1
+fi
+
+exit 0
@@ -28,6 +28,10 @@ Clone this repository, `cd` into it, and run:

    bundle install

+### Enable Git hooks
+
+    git config core.hooksPath .githooks
+
 ## Common tasks

 ### Bootstrap a new host server
@@ -54,6 +58,10 @@ Clone this repository, `cd` into it, and run:

    knife zero converge name:dev.kosmos.org --client-version 15.3.14

+### Upgrade Chef Client on a node:
+
+    knife ssh -a knife_zero.host 'name:foo-1' 'curl -L https://chef.io/chef/install.sh | sudo bash -s -- -v 18.10.17'
+
 ## Managing cookbooks

 Cookbooks are managed via Berkshelf. Run `berks --help` for command help.
@@ -1,4 +0,0 @@
-{
-  "name": "garage-10",
-  "public_key": "-----BEGIN PUBLIC KEY-----\nMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAw2+3Wo+KkXVJCOX1SxT9\nSdwKXgPbCDM3EI9uwoxhMxQfRyN53dxIsBDsQUVOIe1Z8yqm4FenMQlNmeDR+QLE\nvNFf1fisinW+D9VVRm+CjcJy96i/Dyt786Z6YRrDlB860HxCbfTL2Zv5BRtbyIKg\nhz5gO+9PMEpPVR2ij9iue4K6jbM1AAL2ia/P6zDWLJqeIzUocCeHV5N0Z3jXH6qr\nf444v78x35MMJ+3tg5h95SU1/PDCpdSTct4uHEuKIosiN7p4DlYMoM5iSyvVoujr\nflRQPEpGzS9qEt3rDo/F4ltzYMx6bf1tB/0QaBKD+zwPZWTTwf61tSBo5/NkGvJc\nFQIDAQAB\n-----END PUBLIC KEY-----\n"
-}
@@ -1,4 +0,0 @@
-{
-  "name": "garage-12",
-  "public_key": "-----BEGIN PUBLIC KEY-----\nMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEA9GtHHi298BjiIqpZ3WkT\nkYAPfWD60hFe/8icYcq/F/6cHLYKZQ4chek9X/hDCMq4tHEN6Oh58T5x/nuNdPrK\nIAMGyVAGk6ekWlmD4jwdEf6TGb/J3ffJTRDvwX/I8xD/DW3wtXsN+X24T59ByGTm\nrnwRmmmwHF3otRx9wnCsIgDQ0AjiUujsfNNv1FcLXD/WJLys9lEeU5aJ4XtHTwDv\ntJM8YyVEFhEnuvgdKmzn5+F5k9VGdUwForlFOBfvzbCnTZMDMmDVeiUtAUv/7xWQ\nQl2mLUGCtgWuYJYXsQacAJ6pa3h+7cQyshC6w3dwUG+1fS9lNO0Yp1GGX1AGYKpp\nPQIDAQAB\n-----END PUBLIC KEY-----\n"
-}
@@ -1,4 +0,0 @@
-{
-  "name": "garage-13",
-  "public_key": "-----BEGIN PUBLIC KEY-----\nMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAvbqWc6OwRxgHfsQuTNL4\naxeVvNen5d9srYpZSHjuBB/k9NHB+9P6vU5qF37XHkw1lVUGeYbPHzhYsx3O0/kZ\nH5f4+4SMy/P9jc6SE7AJF4qtYKgJ88koZdqCww07c6K9g+BnEGFFZui/h3hUBxWj\nTfhBHEWPyQ2bl/lr9sIJwsEz+EN0isGn/eIXkmw9J6LdLJ5Q0LLks33K28FNOU7q\nfeAN4MiBVMUtgCGyT2Voe6WrOXwQLSDXQONOp3sfSfFExsIJ1s24xdd7AMD7/9a7\n4sFDZ4swhqAWgWmW2giR7Kb8wTvGQLO/O/uUbmKz3DZXgkOKXHdHCEB/PZx1mRNM\nEwIDAQAB\n-----END PUBLIC KEY-----\n"
-}
@@ -1,4 +0,0 @@
-{
-  "name": "garage-9",
-  "public_key": "-----BEGIN PUBLIC KEY-----\nMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAnMHzKE8JBrsQkmRDeMjX\n71mBzvRzNM90cwA8xtvIkXesdTyGqohX9k/PJbCY5ySGK9PpMaYDPVAnwnUP8LFQ\n3G98aSbLxUjqU/PBzRsnWpihehr05uz9zYcNFzr4LTNvGQZsq47nN9Tk+LG3zHP7\nAZViv2mJ4ZRnukXf6KHlyoVvhuTu+tiBM8QzjTF97iP/aguNPzYHmrecy9Uf5bSA\nZrbNZT+ayxtgswC2OclhRucx7XLSuHXtpwFqsQzSAhiX1aQ3wwCyH9WJtVwpfUsE\nlxTjcQiSM9aPZ8iSC0shpBaKD1j3iF/2K2Jk+88++zMhJJPLermvaJxzsdePgvyk\nKQIDAQAB\n-----END PUBLIC KEY-----\n"
-}
@@ -0,0 +1,4 @@
+{
+  "name": "prometheus-1",
+  "public_key": "-----BEGIN PUBLIC KEY-----\nMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAp7T/OBo/TZm3YqnN4+ok\nHwcJ0kW9w2rl9UfrOlWUvoPHBd2LrqpEv3Az3a150IylQ1H/UozmQA7DtjIoTA7d\nV3oLY970vYrYiURcojOo8qAZBy8EH7dfAHxuZryUeELr+3vdcHF5WrrfSt2FdFVX\nPTY95ikafAnOO0Nt8jvnlPoDn7REV8TOE6KOiUzcHKa2xGlfaIe0oRC21LD86uQm\nR09xY1YaJkVgZfeN/opoRjZawkU3FFs3jlUEVBF8k153oOw9W3bgsFFjSOtRtRRg\nDwyQ7oDeMH83kXnaCdpkNZd59wjzPcpxYAL4LRN52ZXA4Btr4DTi+GxHz98Dr0kU\nUQIDAQAB\n-----END PUBLIC KEY-----\n"
+}
@@ -1,4 +0,0 @@
-{
-  "name": "uploads-1",
-  "public_key": "-----BEGIN PUBLIC KEY-----\nMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAwJwWaz8TeGv3SFlKzLMx\nqN8GTL/c0N9ppBvv8xNSS/yF9Y40SbL418uxYzm9hIhOXgIygIgLT2EKIXX32t+R\neOJCdYycQFM3At2fhMkjhuUW0gmDRcYBcBJLC5hLh2EZ+A8V7k4qgrBpPLOjEv48\nhQY0vuAw2DGndWr4QLh5NLUmQiOrfuzcZSSNCBOTIgUZgNmRd9QcCHDq4WDH3poa\nosJo4a9JGEGUL1irOivvEdyJPwEd2f++nYAdWwj8pjCYgpRshQlLhxOlylMx7MxB\nQt2bgJC9sahfbfJCOqdlCU3DMJL0bRUiuxK77WeSsxWBJmrsiF3+Ljs2Ix+s7fnS\nywIDAQAB\n-----END PUBLIC KEY-----\n"
-}
@@ -0,0 +1,10 @@
+{
+  "id": "sentry",
+  "launchpad_rpc_shared_secret": {
+    "encrypted_data": "Y8BFYM4YZjz/nv4zHEAkyheAVB18HdNi1KzvqJrRXXnnYI5bC8fVO9g=\n",
+    "iv": "1txuL+Uy+HIIs6CF\n",
+    "auth_tag": "aVHnUaVRo/xXeZRh0Z1pbw==\n",
+    "version": 3,
+    "cipher": "aes-256-gcm"
+  }
+}
@@ -4,7 +4,8 @@
  "normal": {
    "knife_zero": {
      "host": "10.1.1.144"
-    }
+    },
+    "vm_host": "draco"
  },
  "automatic": {
    "fqdn": "akkounts-1",
@@ -24,6 +25,7 @@
    "recipes": [
      "kosmos-base",
      "kosmos-base::default",
+      "kosmos_prometheus::node_exporter",
      "kosmos_kvm::guest",
      "kosmos-dirsrv::hostsfile",
      "kosmos_sentry::client",
@@ -3,7 +3,8 @@
  "normal": {
    "knife_zero": {
      "host": "10.1.1.163"
-    }
+    },
+    "vm_host": "draco"
  },
  "automatic": {
    "fqdn": "bitcoin-2",
@@ -24,6 +25,7 @@
    "recipes": [
      "kosmos-base",
      "kosmos-base::default",
+      "kosmos_prometheus::node_exporter",
      "kosmos_kvm::guest",
      "kosmos_sentry::client",
      "tor-full",
@@ -4,15 +4,17 @@
  "normal": {
    "knife_zero": {
      "host": "10.1.1.35"
-    }
+    },
+    "vm_host": "fornax"
  },
  "automatic": {
    "fqdn": "discourse-2",
    "os": "linux",
-    "os_version": "5.4.0-1087-kvm",
+    "os_version": "5.4.0-1133-kvm",
    "hostname": "discourse-2",
    "ipaddress": "192.168.122.104",
    "roles": [
+      "base",
      "kvm_guest",
      "garage_gateway",
      "kosmos_discourse"
@@ -20,6 +22,7 @@
    "recipes": [
      "kosmos-base",
      "kosmos-base::default",
+      "kosmos_prometheus::node_exporter",
      "kosmos_kvm::guest",
      "kosmos_garage",
      "kosmos_garage::default",
@@ -32,6 +35,7 @@
      "timezone_iii::debian",
      "ntp::default",
      "ntp::apparmor",
+      "kosmos-base::journald_conf",
      "kosmos-base::systemd_emails",
      "apt::unattended-upgrades",
      "kosmos-base::firewall",
@@ -42,7 +46,6 @@
      "postfix::sasl_auth",
      "hostname::default",
      "firewall::default",
-      "chef-sugar::default",
      "discourse::default"
    ],
    "platform": "ubuntu",
@@ -50,18 +53,18 @@
    "cloud": null,
    "chef_packages": {
      "chef": {
-        "version": "17.9.52",
-        "chef_root": "/opt/chef/embedded/lib/ruby/gems/3.0.0/gems/chef-17.9.52/lib",
+        "version": "18.10.17",
+        "chef_root": "/opt/chef/embedded/lib/ruby/gems/3.1.0/gems/chef-18.10.17/lib",
        "chef_effortless": null
      },
      "ohai": {
-        "version": "17.9.0",
-        "ohai_root": "/opt/chef/embedded/lib/ruby/gems/3.0.0/gems/ohai-17.9.0/lib/ohai"
+        "version": "18.2.13",
+        "ohai_root": "/opt/chef/embedded/lib/ruby/gems/3.1.0/gems/ohai-18.2.13/lib/ohai"
      }
    }
  },
  "run_list": [
-    "recipe[kosmos-base]",
+    "role[base]",
    "role[kvm_guest]",
    "role[garage_gateway]",
    "role[kosmos_discourse]"
@@ -36,6 +36,7 @@
    "recipes": [
      "kosmos-base",
      "kosmos-base::default",
+      "kosmos_prometheus::node_exporter",
      "kosmos_kvm::host",
      "kosmos_kvm::backup",
      "kosmos_email::firewall",
@@ -3,7 +3,8 @@
  "normal": {
    "knife_zero": {
      "host": "10.1.1.128"
-    }
+    },
+    "vm_host": "fornax"
  },
  "automatic": {
    "fqdn": "drone-1",
@@ -12,6 +13,7 @@
    "hostname": "drone-1",
    "ipaddress": "192.168.122.200",
    "roles": [
+      "base",
      "kvm_guest",
      "drone",
      "postgresql_client"
@@ -19,6 +21,7 @@
    "recipes": [
      "kosmos-base",
      "kosmos-base::default",
+      "kosmos_prometheus::node_exporter",
      "kosmos_kvm::guest",
      "kosmos_postgresql::hostsfile",
      "kosmos_drone",
@@ -37,7 +40,8 @@
      "postfix::_common",
      "postfix::_attributes",
      "postfix::sasl_auth",
-      "hostname::default"
+      "hostname::default",
+      "firewall::default"
    ],
    "platform": "ubuntu",
    "platform_version": "20.04",
@@ -55,7 +59,7 @@
    }
  },
  "run_list": [
-    "recipe[kosmos-base]",
+    "role[base]",
    "role[kvm_guest]",
    "role[drone]"
  ]
@@ -4,7 +4,8 @@
  "normal": {
    "knife_zero": {
      "host": "10.1.1.113"
-    }
+    },
+    "vm_host": "draco"
  },
  "automatic": {
    "fqdn": "ejabberd-4",
@@ -23,6 +24,7 @@
    "recipes": [
      "kosmos-base",
      "kosmos-base::default",
+      "kosmos_prometheus::node_exporter",
      "kosmos_kvm::guest",
      "kosmos-dirsrv::hostsfile",
      "kosmos_postgresql::hostsfile",
@@ -72,4 +74,4 @@
    "role[ldap_client]",
    "role[ejabberd]"
  ]
-}
+}
@@ -4,7 +4,8 @@
  "normal": {
    "knife_zero": {
      "host": "10.1.1.123"
-    }
+    },
+    "vm_host": "fornax"
  },
  "automatic": {
    "fqdn": "ejabberd-8",
@@ -23,6 +24,7 @@
    "recipes": [
      "kosmos-base",
      "kosmos-base::default",
+      "kosmos_prometheus::node_exporter",
      "kosmos_kvm::guest",
      "kosmos-dirsrv::hostsfile",
      "kosmos_postgresql::hostsfile",
@@ -72,4 +74,4 @@
    "role[ldap_client]",
    "role[ejabberd]"
  ]
-}
+}
@@ -30,6 +30,7 @@
    "recipes": [
      "kosmos-base",
      "kosmos-base::default",
+      "kosmos_prometheus::node_exporter",
      "kosmos_kvm::host",
      "kosmos_kvm::backup",
      "kosmos_openresty",
@@ -1,64 +0,0 @@
-{
-  "name": "garage-10",
-  "chef_environment": "production",
-  "normal": {
-    "knife_zero": {
-      "host": "10.1.1.27"
-    }
-  },
-  "automatic": {
-    "fqdn": "garage-10",
-    "os": "linux",
-    "os_version": "5.4.0-1090-kvm",
-    "hostname": "garage-10",
-    "ipaddress": "192.168.122.70",
-    "roles": [
-      "base",
-      "kvm_guest",
-      "garage_node"
-    ],
-    "recipes": [
-      "kosmos-base",
-      "kosmos-base::default",
-      "kosmos_kvm::guest",
-      "kosmos_garage",
-      "kosmos_garage::default",
-      "kosmos_garage::firewall_rpc",
-      "kosmos_garage::firewall_apis",
-      "apt::default",
-      "timezone_iii::default",
-      "timezone_iii::debian",
-      "ntp::default",
-      "ntp::apparmor",
-      "kosmos-base::systemd_emails",
-      "apt::unattended-upgrades",
-      "kosmos-base::firewall",
-      "kosmos-postfix::default",
-      "postfix::default",
-      "postfix::_common",
-      "postfix::_attributes",
-      "postfix::sasl_auth",
-      "hostname::default",
-      "firewall::default"
-    ],
-    "platform": "ubuntu",
-    "platform_version": "20.04",
-    "cloud": null,
-    "chef_packages": {
-      "chef": {
-        "version": "18.5.0",
-        "chef_root": "/opt/chef/embedded/lib/ruby/gems/3.1.0/gems/chef-18.5.0/lib",
-        "chef_effortless": null
-      },
-      "ohai": {
-        "version": "18.1.11",
-        "ohai_root": "/opt/chef/embedded/lib/ruby/gems/3.1.0/gems/ohai-18.1.11/lib/ohai"
-      }
-    }
-  },
-  "run_list": [
-    "role[base]",
-    "role[kvm_guest]",
-    "role[garage_node]"
-  ]
-}
@@ -4,7 +4,8 @@
  "normal": {
    "knife_zero": {
      "host": "10.1.1.165"
-    }
+    },
+    "vm_host": "fornax"
  },
  "automatic": {
    "fqdn": "garage-11",
@@ -20,6 +21,7 @@
    "recipes": [
      "kosmos-base",
      "kosmos-base::default",
+      "kosmos_prometheus::node_exporter",
      "kosmos_kvm::guest",
      "kosmos_garage",
      "kosmos_garage::default",
@@ -30,6 +32,7 @@
      "timezone_iii::debian",
      "ntp::default",
      "ntp::apparmor",
+      "kosmos-base::journald_conf",
      "kosmos-base::systemd_emails",
      "apt::unattended-upgrades",
      "kosmos-base::firewall",
@@ -61,4 +64,4 @@
    "role[kvm_guest]",
    "role[garage_node]"
  ]
-}
+}
@@ -1,65 +0,0 @@
-{
-  "name": "garage-12",
-  "chef_environment": "production",
-  "normal": {
-    "knife_zero": {
-      "host": "10.1.1.224"
-    }
-  },
-  "automatic": {
-    "fqdn": "garage-12",
-    "os": "linux",
-    "os_version": "5.15.0-1059-kvm",
-    "hostname": "garage-12",
-    "ipaddress": "192.168.122.173",
-    "roles": [
-      "base",
-      "kvm_guest",
-      "garage_node"
-    ],
-    "recipes": [
-      "kosmos-base",
-      "kosmos-base::default",
-      "kosmos_kvm::guest",
-      "kosmos_garage",
-      "kosmos_garage::default",
-      "kosmos_garage::firewall_rpc",
-      "kosmos_garage::firewall_apis",
-      "apt::default",
-      "timezone_iii::default",
-      "timezone_iii::debian",
-      "ntp::default",
-      "ntp::apparmor",
-      "kosmos-base::journald_conf",
-      "kosmos-base::systemd_emails",
-      "apt::unattended-upgrades",
-      "kosmos-base::firewall",
-      "kosmos-postfix::default",
-      "postfix::default",
-      "postfix::_common",
-      "postfix::_attributes",
-      "postfix::sasl_auth",
-      "hostname::default",
-      "firewall::default"
-    ],
-    "platform": "ubuntu",
-    "platform_version": "22.04",
-    "cloud": null,
-    "chef_packages": {
-      "chef": {
-        "version": "18.7.10",
-        "chef_root": "/opt/chef/embedded/lib/ruby/gems/3.1.0/gems/chef-18.7.10/lib",
-        "chef_effortless": null
-      },
-      "ohai": {
-        "version": "18.2.5",
-        "ohai_root": "/opt/chef/embedded/lib/ruby/gems/3.1.0/gems/ohai-18.2.5/lib/ohai"
-      }
-    }
-  },
-  "run_list": [
-    "role[base]",
-    "role[kvm_guest]",
-    "role[garage_node]"
-  ]
-}
@@ -4,7 +4,8 @@
  "normal": {
    "knife_zero": {
      "host": "10.1.1.151"
-    }
+    },
+    "vm_host": "leo"
  },
  "automatic": {
    "fqdn": "garage-14",
@@ -20,6 +21,7 @@
    "recipes": [
      "kosmos-base",
      "kosmos-base::default",
+      "kosmos_prometheus::node_exporter",
      "kosmos_kvm::guest",
      "kosmos_garage",
      "kosmos_garage::default",
@@ -62,4 +64,4 @@
    "role[kvm_guest]",
    "role[garage_node]"
  ]
-}
+}
@@ -4,7 +4,8 @@
  "normal": {
    "knife_zero": {
      "host": "10.1.1.82"
-    }
+    },
+    "vm_host": "leo"
  },
  "automatic": {
    "fqdn": "garage-15",
@@ -20,6 +21,7 @@
    "recipes": [
      "kosmos-base",
      "kosmos-base::default",
+      "kosmos_prometheus::node_exporter",
      "kosmos_kvm::guest",
      "kosmos_garage",
      "kosmos_garage::default",
@@ -62,4 +64,4 @@
    "role[kvm_guest]",
    "role[garage_node]"
  ]
-}
+}
@@ -4,7 +4,8 @@
  "normal": {
    "knife_zero": {
      "host": "10.1.1.153"
-    }
+    },
+    "vm_host": "leo"
  },
  "automatic": {
    "fqdn": "garage-16",
@@ -20,6 +21,7 @@
    "recipes": [
      "kosmos-base",
      "kosmos-base::default",
+      "kosmos_prometheus::node_exporter",
      "kosmos_kvm::guest",
      "kosmos_garage",
      "kosmos_garage::default",
@@ -60,4 +62,4 @@
    "role[kvm_guest]",
    "role[garage_node]"
  ]
-}
+}
@@ -4,7 +4,8 @@
  "normal": {
    "knife_zero": {
      "host": "10.1.1.40"
-    }
+    },
+    "vm_host": "draco"
  },
  "automatic": {
    "fqdn": "garage-2",
@@ -20,6 +21,7 @@
    "recipes": [
      "kosmos-base",
      "kosmos-base::default",
+      "kosmos_prometheus::node_exporter",
      "kosmos_kvm::guest",
      "kosmos_garage",
      "kosmos_garage::default",
@@ -30,6 +32,7 @@
      "timezone_iii::debian",
      "ntp::default",
      "ntp::apparmor",
+      "kosmos-base::journald_conf",
      "kosmos-base::systemd_emails",
      "apt::unattended-upgrades",
      "kosmos-base::firewall",
@@ -46,13 +49,13 @@
    "cloud": null,
    "chef_packages": {
      "chef": {
-        "version": "18.3.0",
-        "chef_root": "/opt/chef/embedded/lib/ruby/gems/3.1.0/gems/chef-18.3.0/lib",
+        "version": "18.10.17",
+        "chef_root": "/opt/chef/embedded/lib/ruby/gems/3.1.0/gems/chef-18.10.17/lib",
        "chef_effortless": null
      },
      "ohai": {
-        "version": "18.1.4",
-        "ohai_root": "/opt/chef/embedded/lib/ruby/gems/3.1.0/gems/ohai-18.1.4/lib/ohai"
+        "version": "18.2.13",
+        "ohai_root": "/opt/chef/embedded/lib/ruby/gems/3.1.0/gems/ohai-18.2.13/lib/ohai"
      }
    }
  },
@@ -4,7 +4,8 @@
  "normal": {
    "knife_zero": {
      "host": "10.1.1.182"
-    }
+    },
+    "vm_host": "draco"
  },
  "automatic": {
    "fqdn": "garage-7",
@@ -20,6 +21,7 @@
    "recipes": [
      "kosmos-base",
      "kosmos-base::default",
+      "kosmos_prometheus::node_exporter",
      "kosmos_kvm::guest",
      "kosmos_garage",
      "kosmos_garage::default",
@@ -30,6 +32,7 @@
      "timezone_iii::debian",
      "ntp::default",
      "ntp::apparmor",
+      "kosmos-base::journald_conf",
      "kosmos-base::systemd_emails",
      "apt::unattended-upgrades",
      "kosmos-base::firewall",
@@ -61,4 +64,4 @@
    "role[kvm_guest]",
    "role[garage_node]"
  ]
-}
+}
@@ -4,7 +4,8 @@
  "normal": {
    "knife_zero": {
      "host": "10.1.1.61"
-    }
+    },
+    "vm_host": "fornax"
  },
  "automatic": {
    "fqdn": "garage-8",
@@ -20,6 +21,7 @@
    "recipes": [
      "kosmos-base",
      "kosmos-base::default",
+      "kosmos_prometheus::node_exporter",
      "kosmos_kvm::guest",
      "kosmos_garage",
      "kosmos_garage::default",
@@ -30,6 +32,7 @@
      "timezone_iii::debian",
      "ntp::default",
      "ntp::apparmor",
+      "kosmos-base::journald_conf",
      "kosmos-base::systemd_emails",
      "apt::unattended-upgrades",
      "kosmos-base::firewall",
@@ -61,4 +64,4 @@
    "role[kvm_guest]",
    "role[garage_node]"
  ]
-}
+}
@@ -1,64 +0,0 @@
-{
-  "name": "garage-9",
-  "chef_environment": "production",
-  "normal": {
-    "knife_zero": {
-      "host": "10.1.1.223"
-    }
-  },
-  "automatic": {
-    "fqdn": "garage-9",
-    "os": "linux",
-    "os_version": "5.4.0-1090-kvm",
-    "hostname": "garage-9",
-    "ipaddress": "192.168.122.21",
-    "roles": [
-      "base",
-      "kvm_guest",
-      "garage_node"
-    ],
-    "recipes": [
-      "kosmos-base",
-      "kosmos-base::default",
-      "kosmos_kvm::guest",
-      "kosmos_garage",
-      "kosmos_garage::default",
-      "kosmos_garage::firewall_rpc",
-      "kosmos_garage::firewall_apis",
-      "apt::default",
-      "timezone_iii::default",
-      "timezone_iii::debian",
-      "ntp::default",
-      "ntp::apparmor",
-      "kosmos-base::systemd_emails",
-      "apt::unattended-upgrades",
-      "kosmos-base::firewall",
-      "kosmos-postfix::default",
-      "postfix::default",
-      "postfix::_common",
-      "postfix::_attributes",
-      "postfix::sasl_auth",
-      "hostname::default",
-      "firewall::default"
-    ],
-    "platform": "ubuntu",
-    "platform_version": "20.04",
-    "cloud": null,
-    "chef_packages": {
-      "chef": {
-        "version": "18.5.0",
-        "chef_root": "/opt/chef/embedded/lib/ruby/gems/3.1.0/gems/chef-18.5.0/lib",
-        "chef_effortless": null
-      },
-      "ohai": {
-        "version": "18.1.11",
-        "ohai_root": "/opt/chef/embedded/lib/ruby/gems/3.1.0/gems/ohai-18.1.11/lib/ohai"
-      }
-    }
-  },
-  "run_list": [
-    "role[base]",
-    "role[kvm_guest]",
-    "role[garage_node]"
-  ]
-}
@@ -4,7 +4,8 @@
  "normal": {
    "knife_zero": {
      "host": "10.1.1.21"
-    }
+    },
+    "vm_host": "fornax"
  },
  "automatic": {
    "fqdn": "gitea-2",
@@ -24,6 +25,7 @@
    "recipes": [
      "kosmos-base",
      "kosmos-base::default",
+      "kosmos_prometheus::node_exporter",
      "kosmos_kvm::guest",
      "kosmos-dirsrv::hostsfile",
      "kosmos_garage",
@@ -58,13 +60,13 @@
    "cloud": null,
    "chef_packages": {
      "chef": {
-        "version": "18.2.7",
-        "chef_root": "/opt/chef/embedded/lib/ruby/gems/3.1.0/gems/chef-18.2.7/lib",
+        "version": "18.10.17",
+        "chef_root": "/opt/chef/embedded/lib/ruby/gems/3.1.0/gems/chef-18.10.17/lib",
        "chef_effortless": null
      },
      "ohai": {
-        "version": "18.1.4",
-        "ohai_root": "/opt/chef/embedded/lib/ruby/gems/3.1.0/gems/ohai-18.1.4/lib/ohai"
+        "version": "18.2.13",
+        "ohai_root": "/opt/chef/embedded/lib/ruby/gems/3.1.0/gems/ohai-18.2.13/lib/ohai"
      }
    }
  },
@@ -76,4 +78,4 @@
    "role[gitea]",
    "role[gitea_actions_runner]"
  ]
-}
+}
@@ -9,9 +9,9 @@
  "automatic": {
    "fqdn": "her",
    "os": "linux",
-    "os_version": "5.15.0-84-generic",
+    "os_version": "5.15.0-101-generic",
    "hostname": "her",
-    "ipaddress": "192.168.30.172",
+    "ipaddress": "192.168.30.173",
    "roles": [
      "base",
      "kvm_host"
@@ -19,6 +19,7 @@
    "recipes": [
      "kosmos-base",
      "kosmos-base::default",
+      "kosmos_prometheus::node_exporter",
      "kosmos_kvm::host",
      "kosmos_kvm::backup",
      "apt::default",
@@ -26,6 +27,7 @@
      "timezone_iii::debian",
      "ntp::default",
      "ntp::apparmor",
+      "kosmos-base::journald_conf",
      "kosmos-base::systemd_emails",
      "apt::unattended-upgrades",
      "kosmos-base::firewall",
@@ -55,4 +57,4 @@
    "role[base]",
    "role[kvm_host]"
  ]
-}
+}
@@ -3,7 +3,8 @@
  "normal": {
    "knife_zero": {
      "host": "10.1.1.198"
-    }
+    },
+    "vm_host": "draco"
  },
  "automatic": {
    "fqdn": "ipfs-1",
@@ -12,12 +13,14 @@
    "hostname": "ipfs-1",
    "ipaddress": "192.168.122.195",
    "roles": [
+      "base",
      "kvm_guest",
      "ipfs_gateway"
    ],
    "recipes": [
      "kosmos-base",
      "kosmos-base::default",
+      "kosmos_prometheus::node_exporter",
      "kosmos_kvm::guest",
      "kosmos-ipfs",
      "kosmos-ipfs::default",
@@ -27,6 +30,7 @@
      "timezone_iii::debian",
      "ntp::default",
      "ntp::apparmor",
+      "kosmos-base::journald_conf",
      "kosmos-base::systemd_emails",
      "apt::unattended-upgrades",
      "kosmos-base::firewall",
@@ -60,7 +64,7 @@
    }
  },
  "run_list": [
-    "recipe[kosmos-base]",
+    "role[base]",
    "role[kvm_guest]",
    "role[ipfs_gateway]"
  ]
@@ -3,7 +3,8 @@
  "normal": {
    "knife_zero": {
      "host": "10.1.1.6"
-    }
+    },
+    "vm_host": "draco"
  },
  "automatic": {
    "fqdn": "ldap-3.kosmos.org",
@@ -12,12 +13,14 @@
    "hostname": "ldap-3",
    "ipaddress": "192.168.122.34",
    "roles": [
+      "base",
      "kvm_guest",
-      "dirsrv_primary"
+      "dirsrv_supplier"
    ],
    "recipes": [
      "kosmos-base",
      "kosmos-base::default",
+      "kosmos_prometheus::node_exporter",
      "kosmos_kvm::guest",
      "kosmos-dirsrv",
      "kosmos-dirsrv::default",
@@ -26,6 +29,7 @@
      "timezone_iii::debian",
      "ntp::default",
      "ntp::apparmor",
+      "kosmos-base::journald_conf",
      "kosmos-base::systemd_emails",
      "apt::unattended-upgrades",
      "kosmos-base::firewall",
@@ -35,7 +39,7 @@
      "postfix::_attributes",
      "postfix::sasl_auth",
      "hostname::default",
-      "kosmos-dirsrv::hostsfile",
+      "firewall::default",
      "kosmos-dirsrv::firewall",
      "backup::default",
      "logrotate::default",
@@ -46,19 +50,19 @@
    "cloud": null,
    "chef_packages": {
      "chef": {
-        "version": "17.10.3",
-        "chef_root": "/opt/chef/embedded/lib/ruby/gems/3.0.0/gems/chef-17.10.3/lib",
+        "version": "18.10.17",
+        "chef_root": "/opt/chef/embedded/lib/ruby/gems/3.1.0/gems/chef-18.10.17/lib",
        "chef_effortless": null
      },
      "ohai": {
-        "version": "17.9.0",
-        "ohai_root": "/opt/chef/embedded/lib/ruby/gems/3.0.0/gems/ohai-17.9.0/lib/ohai"
+        "version": "18.2.13",
+        "ohai_root": "/opt/chef/embedded/lib/ruby/gems/3.1.0/gems/ohai-18.2.13/lib/ohai"
      }
    }
  },
  "run_list": [
-    "recipe[kosmos-base]",
+    "role[base]",
    "role[kvm_guest]",
    "role[dirsrv_supplier]"
  ]
-}
+}
@@ -4,7 +4,8 @@
  "normal": {
    "knife_zero": {
      "host": "10.1.1.106"
-    }
+    },
+    "vm_host": "fornax"
  },
  "automatic": {
    "fqdn": "ldap-4.kosmos.org",
@@ -14,17 +15,22 @@
    "ipaddress": "192.168.122.73",
    "roles": [
      "base",
-      "kvm_guest"
+      "kvm_guest",
+      "dirsrv_supplier"
    ],
    "recipes": [
      "kosmos-base",
      "kosmos-base::default",
+      "kosmos_prometheus::node_exporter",
      "kosmos_kvm::guest",
+      "kosmos-dirsrv",
+      "kosmos-dirsrv::default",
      "apt::default",
      "timezone_iii::default",
      "timezone_iii::debian",
      "ntp::default",
      "ntp::apparmor",
+      "kosmos-base::journald_conf",
      "kosmos-base::systemd_emails",
      "apt::unattended-upgrades",
      "kosmos-base::firewall",
@@ -33,20 +39,25 @@
      "postfix::_common",
      "postfix::_attributes",
      "postfix::sasl_auth",
-      "hostname::default"
+      "hostname::default",
+      "firewall::default",
+      "kosmos-dirsrv::firewall",
+      "backup::default",
+      "logrotate::default",
+      "ulimit::default"
    ],
    "platform": "ubuntu",
    "platform_version": "20.04",
    "cloud": null,
    "chef_packages": {
      "chef": {
-        "version": "17.10.3",
-        "chef_root": "/opt/chef/embedded/lib/ruby/gems/3.0.0/gems/chef-17.10.3/lib",
+        "version": "18.10.17",
+        "chef_root": "/opt/chef/embedded/lib/ruby/gems/3.1.0/gems/chef-18.10.17/lib",
        "chef_effortless": null
      },
      "ohai": {
-        "version": "17.9.0",
-        "ohai_root": "/opt/chef/embedded/lib/ruby/gems/3.0.0/gems/ohai-17.9.0/lib/ohai"
+        "version": "18.2.13",
+        "ohai_root": "/opt/chef/embedded/lib/ruby/gems/3.1.0/gems/ohai-18.2.13/lib/ohai"
      }
    }
  },
@@ -55,4 +66,4 @@
    "role[kvm_guest]",
    "role[dirsrv_supplier]"
  ]
-}
+}
@@ -1,12 +1,13 @@
 {
-  "name": "leo",
+  "name": "leo.kosmos.org",
+  "chef_environment": "production",
  "normal": {
    "knife_zero": {
-      "host": "leo.kosmos.org"
+      "host": "10.1.1.204"
    }
  },
  "automatic": {
-    "fqdn": "leo",
+    "fqdn": "leo.kosmos.org",
    "os": "linux",
    "os_version": "5.15.0-173-generic",
    "hostname": "leo",
@@ -17,6 +18,7 @@
    "recipes": [
      "kosmos-base",
      "kosmos-base::default",
+      "kosmos_prometheus::node_exporter",
      "kosmos_kvm::host",
      "apt::default",
      "timezone_iii::default",
@@ -32,7 +34,8 @@
      "postfix::_common",
      "postfix::_attributes",
      "postfix::sasl_auth",
-      "hostname::default"
+      "hostname::default",
+      "firewall::default"
    ],
    "platform": "ubuntu",
    "platform_version": "22.04",
@@ -53,4 +56,4 @@
    "role[base]",
    "recipe[kosmos_kvm::host]"
  ]
-}
+}
@@ -4,7 +4,8 @@
  "normal": {
    "knife_zero": {
      "host": "10.1.1.87"
-    }
+    },
+    "vm_host": "draco"
  },
  "automatic": {
    "fqdn": "lq-1",
@@ -21,6 +22,7 @@
    "recipes": [
      "kosmos-base",
      "kosmos-base::default",
+      "kosmos_prometheus::node_exporter",
      "kosmos_kvm::guest",
      "kosmos_garage",
      "kosmos_garage::default",
@@ -32,6 +34,7 @@
      "timezone_iii::debian",
      "ntp::default",
      "ntp::apparmor",
+      "kosmos-base::journald_conf",
      "kosmos-base::systemd_emails",
      "apt::unattended-upgrades",
      "kosmos-base::firewall",
@@ -49,13 +52,13 @@
    "cloud": null,
    "chef_packages": {
      "chef": {
-        "version": "18.2.7",
-        "chef_root": "/opt/chef/embedded/lib/ruby/gems/3.1.0/gems/chef-18.2.7/lib",
+        "version": "18.10.17",
+        "chef_root": "/opt/chef/embedded/lib/ruby/gems/3.1.0/gems/chef-18.10.17/lib",
        "chef_effortless": null
      },
      "ohai": {
-        "version": "18.1.4",
-        "ohai_root": "/opt/chef/embedded/lib/ruby/gems/3.1.0/gems/ohai-18.1.4/lib/ohai"
+        "version": "18.2.13",
+        "ohai_root": "/opt/chef/embedded/lib/ruby/gems/3.1.0/gems/ohai-18.2.13/lib/ohai"
      }
    }
  },
@@ -4,7 +4,8 @@
  "normal": {
    "knife_zero": {
      "host": "10.1.1.188"
-    }
+    },
+    "vm_host": "fornax"
  },
  "automatic": {
    "fqdn": "lq-2",
@@ -21,6 +22,7 @@
    "recipes": [
      "kosmos-base",
      "kosmos-base::default",
+      "kosmos_prometheus::node_exporter",
      "kosmos_kvm::guest",
      "kosmos_garage",
      "kosmos_garage::default",
@@ -32,6 +34,7 @@
      "timezone_iii::debian",
      "ntp::default",
      "ntp::apparmor",
+      "kosmos-base::journald_conf",
      "kosmos-base::systemd_emails",
      "apt::unattended-upgrades",
      "kosmos-base::firewall",
@@ -49,13 +52,13 @@
    "cloud": null,
    "chef_packages": {
      "chef": {
-        "version": "18.2.7",
-        "chef_root": "/opt/chef/embedded/lib/ruby/gems/3.1.0/gems/chef-18.2.7/lib",
+        "version": "18.10.17",
+        "chef_root": "/opt/chef/embedded/lib/ruby/gems/3.1.0/gems/chef-18.10.17/lib",
        "chef_effortless": null
      },
      "ohai": {
-        "version": "18.1.4",
-        "ohai_root": "/opt/chef/embedded/lib/ruby/gems/3.1.0/gems/ohai-18.1.4/lib/ohai"
+        "version": "18.2.13",
+        "ohai_root": "/opt/chef/embedded/lib/ruby/gems/3.1.0/gems/ohai-18.2.13/lib/ohai"
      }
    }
  },
@@ -4,13 +4,14 @@
  "normal": {
    "knife_zero": {
      "host": "10.1.1.95"
-    }
+    },
+    "vm_host": "draco"
  },
  "automatic": {
    "fqdn": "mail.kosmos.org",
    "os": "linux",
    "os_version": "5.15.0-1048-kvm",
-    "hostname": "mail.kosmos.org",
+    "hostname": "mail",
    "ipaddress": "192.168.122.131",
    "roles": [
      "base",
@@ -21,6 +22,7 @@
    "recipes": [
      "kosmos-base",
      "kosmos-base::default",
+      "kosmos_prometheus::node_exporter",
      "kosmos_kvm::guest",
      "kosmos-dirsrv::hostsfile",
      "kosmos_email",
@@ -68,4 +70,4 @@
    "role[kvm_guest]",
    "role[email_server]"
  ]
-}
+}
@@ -4,7 +4,8 @@
  "normal": {
    "knife_zero": {
      "host": "10.1.1.30"
-    }
+    },
+    "vm_host": "fornax"
  },
  "automatic": {
    "fqdn": "mastodon-3",
@@ -83,7 +84,7 @@
    }
  },
  "run_list": [
-    "recipe[kosmos-base]",
+    "role[base]",
    "role[kvm_guest]",
    "role[ldap_client]",
    "role[garage_gateway]",
@@ -3,7 +3,8 @@
  "normal": {
    "knife_zero": {
      "host": "10.1.1.138"
-    }
+    },
+    "vm_host": "fornax"
  },
  "automatic": {
    "fqdn": "nodejs-4",
@@ -21,6 +22,7 @@
    "recipes": [
      "kosmos-base",
      "kosmos-base::default",
+      "kosmos_prometheus::node_exporter",
      "kosmos_kvm::guest",
      "kosmos-hubot::botka_irc-libera-chat",
      "kosmos-hubot::hal8000_xmpp",
@@ -35,6 +37,7 @@
      "timezone_iii::debian",
      "ntp::default",
      "ntp::apparmor",
+      "kosmos-base::journald_conf",
      "kosmos-base::systemd_emails",
      "apt::unattended-upgrades",
      "kosmos-base::firewall",
@@ -4,7 +4,8 @@
  "normal": {
    "knife_zero": {
      "host": "10.1.1.91"
-    }
+    },
+    "vm_host": "leo"
  },
  "automatic": {
    "fqdn": "postgres-11",
@@ -20,6 +21,7 @@
    "recipes": [
      "kosmos-base",
      "kosmos-base::default",
+      "kosmos_prometheus::node_exporter",
      "kosmos_kvm::guest",
      "kosmos_postgresql::primary",
      "kosmos-akkounts::pg_db",
@@ -66,4 +68,4 @@
    "role[kvm_guest]",
    "role[postgresql_primary]"
  ]
-}
+}
@@ -4,7 +4,8 @@
  "normal": {
    "knife_zero": {
      "host": "10.1.1.134"
-    }
+    },
+    "vm_host": "her"
  },
  "automatic": {
    "fqdn": "postgres-12",
@@ -20,6 +21,7 @@
    "recipes": [
      "kosmos-base",
      "kosmos-base::default",
+      "kosmos_prometheus::node_exporter",
      "kosmos_kvm::guest",
      "kosmos_postgresql::hostsfile",
      "kosmos_postgresql::replica",
@@ -61,4 +63,4 @@
    "role[kvm_guest]",
    "role[postgresql_replica]"
  ]
-}
+}
@@ -1,35 +1,33 @@
 {
-  "name": "garage-13",
+  "name": "prometheus-1",
  "chef_environment": "production",
  "normal": {
    "knife_zero": {
-      "host": "10.1.1.179"
-    }
+      "host": "10.1.1.146"
+    },
+    "vm_host": "leo"
  },
  "automatic": {
-    "fqdn": "garage-13",
+    "fqdn": "prometheus-1",
    "os": "linux",
-    "os_version": "5.15.0-1059-kvm",
-    "hostname": "garage-13",
-    "ipaddress": "192.168.122.27",
+    "os_version": "6.8.0-134-generic",
+    "hostname": "prometheus-1",
+    "ipaddress": "192.168.122.166",
    "roles": [
      "base",
      "kvm_guest",
-      "garage_node"
+      "prometheus_server"
    ],
    "recipes": [
      "kosmos-base",
      "kosmos-base::default",
+      "kosmos_prometheus::node_exporter",
      "kosmos_kvm::guest",
-      "kosmos_garage",
-      "kosmos_garage::default",
-      "kosmos_garage::firewall_rpc",
-      "kosmos_garage::firewall_apis",
+      "kosmos_prometheus::server",
+      "kosmos_prometheus::alertmanager",
      "apt::default",
      "timezone_iii::default",
      "timezone_iii::debian",
-      "ntp::default",
-      "ntp::apparmor",
      "kosmos-base::journald_conf",
      "kosmos-base::systemd_emails",
      "apt::unattended-upgrades",
@@ -43,23 +41,23 @@
      "firewall::default"
    ],
    "platform": "ubuntu",
-    "platform_version": "22.04",
+    "platform_version": "24.04",
    "cloud": null,
    "chef_packages": {
      "chef": {
-        "version": "18.7.10",
-        "chef_root": "/opt/chef/embedded/lib/ruby/gems/3.1.0/gems/chef-18.7.10/lib",
+        "version": "18.10.17",
+        "chef_root": "/opt/chef/embedded/lib/ruby/gems/3.1.0/gems/chef-18.10.17/lib",
        "chef_effortless": null
      },
      "ohai": {
-        "version": "18.2.5",
-        "ohai_root": "/opt/chef/embedded/lib/ruby/gems/3.1.0/gems/ohai-18.2.5/lib/ohai"
+        "version": "18.2.13",
+        "ohai_root": "/opt/chef/embedded/lib/ruby/gems/3.1.0/gems/ohai-18.2.13/lib/ohai"
      }
    }
  },
  "run_list": [
    "role[base]",
    "role[kvm_guest]",
-    "role[garage_node]"
+    "role[prometheus_server]"
  ]
-}
+}
@@ -4,7 +4,8 @@
  "normal": {
    "knife_zero": {
      "host": "10.1.1.225"
-    }
+    },
+    "vm_host": "draco"
  },
  "automatic": {
    "fqdn": "redis-1",
@@ -20,6 +21,7 @@
    "recipes": [
      "kosmos-base",
      "kosmos-base::default",
+      "kosmos_prometheus::node_exporter",
      "kosmos_kvm::guest",
      "kosmos_redis",
      "kosmos_redis::default",
@@ -30,6 +32,7 @@
      "timezone_iii::debian",
      "ntp::default",
      "ntp::apparmor",
+      "kosmos-base::journald_conf",
      "kosmos-base::systemd_emails",
      "apt::unattended-upgrades",
      "kosmos-base::firewall",
@@ -39,6 +42,7 @@
      "postfix::_attributes",
      "postfix::sasl_auth",
      "hostname::default",
+      "firewall::default",
      "redisio::default",
      "redisio::_install_prereqs",
      "redisio::install",
@@ -4,7 +4,8 @@
  "normal": {
    "knife_zero": {
      "host": "10.1.1.208"
-    }
+    },
+    "vm_host": "fornax"
  },
  "automatic": {
    "fqdn": "redis-2",
@@ -20,6 +21,7 @@
    "recipes": [
      "kosmos-base",
      "kosmos-base::default",
+      "kosmos_prometheus::node_exporter",
      "kosmos_kvm::guest",
      "kosmos_redis::replica",
      "kosmos_redis::firewall",
@@ -28,6 +30,7 @@
      "timezone_iii::debian",
      "ntp::default",
      "ntp::apparmor",
+      "kosmos-base::journald_conf",
      "kosmos-base::systemd_emails",
      "apt::unattended-upgrades",
      "kosmos-base::firewall",
@@ -37,6 +40,7 @@
      "postfix::_attributes",
      "postfix::sasl_auth",
      "hostname::default",
+      "firewall::default",
      "redisio::default",
      "redisio::_install_prereqs",
      "redisio::install",
@@ -3,29 +3,33 @@
  "normal": {
    "knife_zero": {
      "host": "10.1.1.187"
-    }
+    },
+    "vm_host": "fornax"
  },
  "automatic": {
    "fqdn": "rs-discourse-1",
    "os": "linux",
-    "os_version": "5.4.0-1076-kvm",
+    "os_version": "5.4.0-1118-kvm",
    "hostname": "rs-discourse-1",
    "ipaddress": "192.168.122.30",
    "roles": [
-      "remotestorage_discourse",
-      "kvm_guest"
+      "base",
+      "kvm_guest",
+      "remotestorage_discourse"
    ],
    "recipes": [
      "kosmos-base",
      "kosmos-base::default",
+      "kosmos_prometheus::node_exporter",
+      "kosmos_kvm::guest",
      "remotestorage_discourse",
      "remotestorage_discourse::default",
-      "kosmos_kvm::guest",
      "apt::default",
      "timezone_iii::default",
      "timezone_iii::debian",
      "ntp::default",
      "ntp::apparmor",
+      "kosmos-base::journald_conf",
      "kosmos-base::systemd_emails",
      "apt::unattended-upgrades",
      "kosmos-base::firewall",
@@ -35,28 +39,27 @@
      "postfix::_attributes",
      "postfix::sasl_auth",
      "hostname::default",
-      "discourse::default",
      "firewall::default",
-      "chef-sugar::default"
+      "discourse::default"
    ],
    "platform": "ubuntu",
    "platform_version": "20.04",
    "cloud": null,
    "chef_packages": {
      "chef": {
-        "version": "17.10.3",
-        "chef_root": "/opt/chef/embedded/lib/ruby/gems/3.0.0/gems/chef-17.10.3/lib",
+        "version": "18.10.17",
+        "chef_root": "/opt/chef/embedded/lib/ruby/gems/3.1.0/gems/chef-18.10.17/lib",
        "chef_effortless": null
      },
      "ohai": {
-        "version": "17.9.0",
-        "ohai_root": "/opt/chef/embedded/lib/ruby/gems/3.0.0/gems/ohai-17.9.0/lib/ohai"
+        "version": "18.2.13",
+        "ohai_root": "/opt/chef/embedded/lib/ruby/gems/3.1.0/gems/ohai-18.2.13/lib/ohai"
      }
    }
  },
  "run_list": [
-    "recipe[kosmos-base]",
+    "role[base]",
    "role[kvm_guest]",
    "role[remotestorage_discourse]"
  ]
-}
+}
@@ -3,7 +3,8 @@
  "normal": {
    "knife_zero": {
      "host": "10.1.1.150"
-    }
+    },
+    "vm_host": "her"
  },
  "automatic": {
    "fqdn": "rsk-mainnet-3",
@@ -58,4 +59,4 @@
    "role[kvm_guest]",
    "role[rskj_mainnet]"
  ]
-}
+}
@@ -3,7 +3,8 @@
  "normal": {
    "knife_zero": {
      "host": "10.1.1.126"
-    }
+    },
+    "vm_host": "her"
  },
  "automatic": {
    "fqdn": "rsk-testnet-4",
@@ -57,4 +58,4 @@
    "role[base]",
    "role[kvm_guest]"
  ]
-}
+}
@@ -3,7 +3,8 @@
  "normal": {
    "knife_zero": {
      "host": "10.1.1.20"
-    }
+    },
+    "vm_host": "leo"
  },
  "automatic": {
    "fqdn": "rsk-testnet-6",
@@ -19,6 +20,7 @@
    "recipes": [
      "kosmos-base",
      "kosmos-base::default",
+      "kosmos_prometheus::node_exporter",
      "kosmos_kvm::guest",
      "kosmos_rsk::rskj",
      "apt::default",
@@ -34,8 +36,8 @@
      "postfix::_attributes",
      "postfix::sasl_auth",
      "hostname::default",
-      "kosmos_rsk::firewall",
-      "firewall::default"
+      "firewall::default",
+      "kosmos_rsk::firewall"
    ],
    "platform": "ubuntu",
    "platform_version": "24.04",
@@ -57,4 +59,4 @@
    "role[kvm_guest]",
    "role[rskj_testnet]"
  ]
-}
+}
@@ -4,12 +4,13 @@
  "normal": {
    "knife_zero": {
      "host": "10.1.1.132"
-    }
+    },
+    "vm_host": "her"
  },
  "automatic": {
    "fqdn": "sentry-1",
    "os": "linux",
-    "os_version": "5.4.0-1087-kvm",
+    "os_version": "6.8.0-134-generic",
    "hostname": "sentry-1",
    "ipaddress": "192.168.122.251",
    "roles": [
@@ -20,14 +21,14 @@
    "recipes": [
      "kosmos-base",
      "kosmos-base::default",
+      "kosmos_prometheus::node_exporter",
      "kosmos_kvm::guest",
      "kosmos_sentry",
      "kosmos_sentry::default",
      "apt::default",
      "timezone_iii::default",
      "timezone_iii::debian",
-      "ntp::default",
-      "ntp::apparmor",
+      "kosmos-base::journald_conf",
      "kosmos-base::systemd_emails",
      "apt::unattended-upgrades",
      "kosmos-base::firewall",
@@ -37,21 +38,20 @@
      "postfix::_attributes",
      "postfix::sasl_auth",
      "hostname::default",
-      "firewall::default",
-      "chef-sugar::default"
+      "firewall::default"
    ],
    "platform": "ubuntu",
-    "platform_version": "20.04",
+    "platform_version": "24.04",
    "cloud": null,
    "chef_packages": {
      "chef": {
-        "version": "17.10.3",
-        "chef_root": "/opt/chef/embedded/lib/ruby/gems/3.0.0/gems/chef-17.10.3/lib",
+        "version": "18.7.10",
+        "chef_root": "/opt/chef/embedded/lib/ruby/gems/3.1.0/gems/chef-18.7.10/lib",
        "chef_effortless": null
      },
      "ohai": {
-        "version": "17.9.0",
-        "ohai_root": "/opt/chef/embedded/lib/ruby/gems/3.0.0/gems/ohai-17.9.0/lib/ohai"
+        "version": "18.2.5",
+        "ohai_root": "/opt/chef/embedded/lib/ruby/gems/3.1.0/gems/ohai-18.2.5/lib/ohai"
      }
    }
  },
@@ -4,7 +4,8 @@
  "normal": {
    "knife_zero": {
      "host": "10.1.1.164"
-    }
+    },
+    "vm_host": "draco"
  },
  "automatic": {
    "fqdn": "strfry-1",
@@ -22,6 +23,7 @@
    "recipes": [
      "kosmos-base",
      "kosmos-base::default",
+      "kosmos_prometheus::node_exporter",
      "kosmos_kvm::guest",
      "kosmos-dirsrv::hostsfile",
      "strfry",
@@ -70,4 +72,4 @@
    "role[strfry]",
    "role[blossom]"
  ]
-}
+}
@@ -1,67 +0,0 @@
-{
-  "name": "uploads-1",
-  "normal": {
-    "knife_zero": {
-      "host": "10.1.1.98"
-    }
-  },
-  "automatic": {
-    "fqdn": "uploads-1",
-    "os": "linux",
-    "os_version": "5.4.0-128-generic",
-    "hostname": "uploads-1",
-    "ipaddress": "192.168.122.230",
-    "roles": [
-      "kvm_guest"
-    ],
-    "recipes": [
-      "kosmos-base",
-      "kosmos-base::default",
-      "kosmos_kvm::guest",
-      "kosmos-ejabberd::upload_service",
-      "apt::default",
-      "timezone_iii::default",
-      "timezone_iii::debian",
-      "ntp::default",
-      "ntp::apparmor",
-      "kosmos-base::systemd_emails",
-      "apt::unattended-upgrades",
-      "kosmos-base::firewall",
-      "kosmos-postfix::default",
-      "postfix::default",
-      "postfix::_common",
-      "postfix::_attributes",
-      "postfix::sasl_auth",
-      "hostname::default",
-      "kosmos-nginx::with_perl",
-      "nginx::default",
-      "nginx::package",
-      "nginx::ohai_plugin",
-      "nginx::repo",
-      "nginx::commons",
-      "nginx::commons_dir",
-      "nginx::commons_script",
-      "nginx::commons_conf",
-      "kosmos-nginx::firewall",
-      "kosmos-base::letsencrypt"
-    ],
-    "platform": "ubuntu",
-    "platform_version": "20.04",
-    "cloud": null,
-    "chef_packages": {
-      "chef": {
-        "version": "15.14.0",
-        "chef_root": "/opt/chef/embedded/lib/ruby/gems/2.6.0/gems/chef-15.14.0/lib"
-      },
-      "ohai": {
-        "version": "15.12.0",
-        "ohai_root": "/opt/chef/embedded/lib/ruby/gems/2.6.0/gems/ohai-15.12.0/lib/ohai"
-      }
-    }
-  },
-  "run_list": [
-    "recipe[kosmos-base]",
-    "role[kvm_guest]",
-    "recipe[kosmos-ejabberd::upload_service]"
-  ]
-}
@@ -3,7 +3,8 @@
  "normal": {
    "knife_zero": {
      "host": "10.1.1.206"
-    }
+    },
+    "vm_host": "draco"
  },
  "automatic": {
    "fqdn": "wiki-1",
@@ -19,6 +20,7 @@
    "recipes": [
      "kosmos-base",
      "kosmos-base::default",
+      "kosmos_prometheus::node_exporter",
      "kosmos_kvm::guest",
      "kosmos-dirsrv::hostsfile",
      "kosmos-mediawiki",
@@ -2,4 +2,5 @@ name "base"

 run_list %w(
  kosmos-base::default
+  kosmos_prometheus::node_exporter
 )
@@ -0,0 +1,12 @@
+name "prometheus_server"
+
+default_run_list = [
+  "kosmos_prometheus::server",
+  "kosmos_prometheus::alertmanager"
+]
+
+env_run_lists(
+  "_default" => default_run_list,
+  "development" => default_run_list,
+  "production" => default_run_list
+)
@@ -0,0 +1,69 @@
+#!/usr/bin/env bash
+# Strip the top-level "override" and "default" attributes from Chef node JSON files.
+#
+# Usage:
+#   ./scripts/util/strip_node_attrs.sh <node> [<node> ...]
+#   ./scripts/util/strip_node_attrs.sh --all
+#
+# A <node> may be a bare node name (e.g. "prometheus-1"), a path relative to the
+# repository root (e.g. "nodes/prometheus-1.json"), or an absolute path.
+#
+# Exit codes:
+#   0 - all target files processed successfully
+#   1 - one or more targets could not be processed (missing args, missing jq,
+#       file not found, invalid JSON, or write failure)
+
+set -uo pipefail
+
+command -v jq >/dev/null 2>&1 || exit 1
+
+root="$(git rev-parse --show-toplevel 2>/dev/null)" || root="$(pwd)"
+
+targets=()
+if [ "$#" -eq 0 ]; then
+  exit 1
+fi
+
+for arg in "$@"; do
+  if [ "$arg" = "--all" ]; then
+    shopt -s nullglob
+    for f in "$root"/nodes/*.json; do
+      targets+=("$f")
+    done
+    shopt -u nullglob
+    continue
+  fi
+
+  if [[ "$arg" == */* ]]; then
+    if [[ "$arg" = /* ]]; then
+      targets+=("$arg")
+    else
+      targets+=("$root/$arg")
+    fi
+  else
+    targets+=("$root/nodes/$arg.json")
+  fi
+done
+
+if [ "${#targets[@]}" -eq 0 ]; then
+  exit 1
+fi
+
+rc=0
+for file in "${targets[@]}"; do
+  [ -f "$file" ] || { rc=1; continue; }
+
+  jq -e . "$file" >/dev/null 2>&1 || { rc=1; continue; }
+
+  tmp="$(mktemp "${file}.XXXXXX")" || { rc=1; continue; }
+  if jq --indent 2 'del(.override, .default)' "$file" > "$tmp" 2>/dev/null; then
+    truncate -s -1 "$tmp" 2>/dev/null || true
+    mv "$tmp" "$file" || { rm -f "$tmp"; rc=1; continue; }
+  else
+    rm -f "$tmp"
+    rc=1
+    continue
+  fi
+done
+
+exit "$rc"
@@ -21,5 +21,6 @@ node.default["postfix"]["sasl"] = {
 node.default["postfix"]["main"]["relayhost"]                  = smtp_credentials["relayhost"]
 node.default["postfix"]["main"]["smtp_sasl_auth_enable"]      = "yes"
 node.default["postfix"]["main"]["smtp_sasl_security_options"] = "noanonymous"
+node.default["postfix"]["main"]["smtpd_relay_restrictions"] = "permit_mynetworks, reject"

 include_recipe "postfix::default"
@@ -0,0 +1,25 @@
+.vagrant
+*~
+*#
+.#*
+\#*#
+.*.sw[a-z]
+*.un~
+
+# Bundler
+Gemfile.lock
+gems.locked
+bin/*
+.bundle/*
+
+# test kitchen
+.kitchen/
+kitchen.local.yml
+
+# Chef Infra
+Berksfile.lock
+.zero-knife.rb
+Policyfile.lock.json
+
+.idea/
+
@@ -0,0 +1,7 @@
+# kosmos_prometheus CHANGELOG
+
+This file is used to list changes made in each version of the kosmos_prometheus cookbook.
+
+## 0.1.0
+
+Initial release.
@@ -0,0 +1,20 @@
+Copyright (c) 2026 Kosmos Developers
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
@@ -0,0 +1,4 @@
+# kosmos_prometheus
+
+TODO: Enter the cookbook description here.
+
@@ -0,0 +1,82 @@
+node.default["kosmos_prometheus"]["version"] = "3.13.0"
+node.default["kosmos_prometheus"]["checksum"] = "744d93324cc024d82089921737bd797474d7f1e5dbbfd1c6b387bad258538cb9"
+
+node.default["kosmos_prometheus"]["data_dir"] = "/var/lib/prometheus"
+node.default["kosmos_prometheus"]["config_dir"] = "/etc/prometheus"
+node.default["kosmos_prometheus"]["rules_dir"] = "#{node["kosmos_prometheus"]["config_dir"]}/rules"
+
+node.default["kosmos_prometheus"]["web"]["port"] = 9090
+
+node.default["kosmos_prometheus"]["alertmanager"]["version"] = "0.33.0"
+node.default["kosmos_prometheus"]["alertmanager"]["checksum"] = "8ce11c42e8a6dfbbf93a59c0b193cb1329210b36d0c7ef3df7b745608675a1d1"
+
+node.default["kosmos_prometheus"]["node_exporter"]["version"] = "1.11.1"
+node.default["kosmos_prometheus"]["node_exporter"]["checksum"] = "9f5ea48e5bc7b656f8a91a32e7d7deb89f70f73dabd0d974418aca15f37d6810"
+
+node.default["kosmos_prometheus"]["alertmanager"]["route"] = {
+  "receiver" => "xmpp",
+  "group_by" => ["alertname"],
+  "group_wait" => "20s",
+  "group_interval" => "30s",
+  "repeat_interval" => "3h",
+  "routes" => [
+    {
+      "receiver" => "xmpp",
+      "continue" => true,
+    },
+    {
+      "receiver" => "email",
+      "matchers" => ['severity="critical"'],
+      "continue" => true,
+    }
+  ],
+}
+
+node.default["kosmos_prometheus"]["alertmanager"]["receivers"] = [
+  {
+    "name" => "xmpp",
+    "webhook_configs" => [
+      {
+        "send_resolved" => true,
+        "room" => "info@kosmos.chat",
+      }
+    ]
+  },
+  {
+    "name" => "email",
+    "email_configs" => [
+      {
+        "send_resolved" => true,
+        "to" => "ops@kosmos.org",
+        "from" => "ops@kosmos.org",
+        "smarthost" => "localhost:25",
+        "require_tls" => false,
+      }
+    ]
+  }
+]
+
+node.default["kosmos_prometheus"]["global"] = {
+  "scrape_interval" => "15s",
+  "evaluation_interval" => "15s",
+}
+
+node.default["kosmos_prometheus"]["alerting"] = {
+  "alertmanagers" => [
+    "static_configs" => [
+      { "targets" => ["127.0.0.1:9093"] }
+    ]
+  ]
+}
+
+node.default["kosmos_prometheus"]["jobs"] = {
+  "prometheus" => { "targets" => [{ "target" => "localhost:#{node["kosmos_prometheus"]["web"]["port"]}", "instance" => "localhost" }] },
+}
+
+node.default["kosmos_prometheus"]["rule_files"] = [
+  "#{node["kosmos_prometheus"]["rules_dir"]}/*.yml"
+]
+
+node.default["kosmos_prometheus"]["rules"] = [
+  "node_exporter"
+]
@@ -0,0 +1,115 @@
+# Put files/directories that should be ignored in this file when uploading
+# to a Chef Infra Server or Supermarket.
+# Lines that start with '# ' are comments.
+
+# OS generated files #
+######################
+.DS_Store
+ehthumbs.db
+Icon?
+nohup.out
+Thumbs.db
+.envrc
+
+# EDITORS #
+###########
+.#*
+.project
+.settings
+*_flymake
+*_flymake.*
+*.bak
+*.sw[a-z]
+*.tmproj
+*~
+\#*
+REVISION
+TAGS*
+tmtags
+.vscode
+.editorconfig
+
+## COMPILED ##
+##############
+*.class
+*.com
+*.dll
+*.exe
+*.o
+*.pyc
+*.so
+*/rdoc/
+a.out
+mkmf.log
+
+# Testing #
+###########
+.circleci/*
+.codeclimate.yml
+.delivery/*
+.foodcritic
+.kitchen*
+.mdlrc
+.overcommit.yml
+.rspec
+.rubocop.yml
+.travis.yml
+.watchr
+.yamllint
+azure-pipelines.yml
+Dangerfile
+examples/*
+features/*
+Guardfile
+kitchen.yml*
+mlc_config.json
+Procfile
+Rakefile
+spec/*
+test/*
+
+# SCM #
+#######
+.git
+.gitattributes
+.gitconfig
+.github/*
+.gitignore
+.gitkeep
+.gitmodules
+.svn
+*/.bzr/*
+*/.git
+*/.hg/*
+*/.svn/*
+
+# Berkshelf #
+#############
+Berksfile
+Berksfile.lock
+cookbooks/*
+tmp
+
+# Bundler #
+###########
+vendor/*
+Gemfile
+Gemfile.lock
+
+# Policyfile #
+##############
+Policyfile.rb
+Policyfile.lock.json
+
+# Documentation #
+#############
+CODE_OF_CONDUCT*
+CONTRIBUTING*
+documentation/*
+TESTING*
+UPGRADING*
+
+# Vagrant #
+###########
+.vagrant
+Vagrantfile
@@ -0,0 +1,10 @@
+groups:
+  - name: NodeExporter
+    rules:
+      - alert: NodeDown
+        expr: up{job="node"} == 0
+        for: 20s
+        labels:
+          severity: warning
+        annotations:
+          summary: "Node is down"
@@ -0,0 +1,21 @@
+name 'kosmos_prometheus'
+maintainer 'Kosmos Developers'
+maintainer_email 'mail@kosmos.org'
+license 'MIT'
+description 'Installs/Configures prometheus'
+version '0.1.0'
+chef_version '>= 16.0'
+
+depends "firewall"
+
+# The `issues_url` points to the location where issues for this cookbook are
+# tracked.  A `View Issues` link will be displayed on this cookbook's page when
+# uploaded to a Supermarket.
+#
+# issues_url 'https://github.com/<insert_org_here>/kosmos_prometheus/issues'
+
+# The `source_url` points to the development repository for this cookbook.  A
+# `View Source` link will be displayed on this cookbook's page when uploaded to
+# a Supermarket.
+#
+# source_url 'https://github.com/<insert_org_here>/kosmos_prometheus'
@@ -0,0 +1,105 @@
+#
+# Cookbook:: kosmos_prometheus
+# Recipe:: alertmanager
+#
+
+include_recipe "firewall"
+
+version = node["kosmos_prometheus"]["alertmanager"]["version"]
+checksum = node["kosmos_prometheus"]["alertmanager"]["checksum"]
+tarball = "#{Chef::Config[:file_cache_path]}/alertmanager-#{version}.linux-amd64.tar.gz"
+binary_url = "https://github.com/prometheus/alertmanager/releases/download/v#{version}/alertmanager-#{version}.linux-amd64.tar.gz"
+
+group "alertmanager"
+
+user "alertmanager" do
+  gid "alertmanager"
+  system true
+  shell "/bin/false"
+  home "/nonexistent"
+end
+
+directory "/var/lib/alertmanager" do
+  owner "alertmanager"
+  group "alertmanager"
+  mode "0755"
+  recursive true
+end
+
+directory "/etc/prometheus" do
+  owner "root"
+  group "root"
+  mode "0755"
+  recursive true
+end
+
+package %w(tar bzip2)
+
+remote_file tarball do
+  source binary_url
+  checksum checksum
+  action :create
+  notifies :run, "execute[install_alertmanager]", :immediately
+end
+
+execute "install_alertmanager" do
+  command "tar -xzf #{tarball} -C /usr/local/bin --strip-components=1 alertmanager-#{version}.linux-amd64/alertmanager"
+  action :nothing
+  notifies :restart, "service[alertmanager]", :delayed
+end
+
+file "/usr/local/bin/alertmanager" do
+  owner "root"
+  group "root"
+  mode "0755"
+  notifies :restart, "service[alertmanager]", :delayed
+end
+
+# TODO use Auth header once supported by hubot-incoming-webhook
+credentials = data_bag_item("credentials", "hal8000_xmpp")
+webhook_url = "https://hal8000.kosmos.chat/incoming/#{credentials['webhook_token']}"
+
+template "/etc/prometheus/alertmanager.yml" do
+  source "alertmanager.yml.erb"
+  owner "root"
+  group "alertmanager"
+  mode "0644"
+  variables route: node["kosmos_prometheus"]["alertmanager"]["route"],
+            receivers: node["kosmos_prometheus"]["alertmanager"]["receivers"],
+            webhook_url: webhook_url
+  notifies :restart, "service[alertmanager]", :delayed
+  sensitive true
+end
+
+systemd_unit "alertmanager.service" do
+  content({
+    Unit: {
+      Description: "Prometheus Alertmanager",
+      After: "network.target",
+    },
+    Service: {
+      Type: "simple",
+      User: "alertmanager",
+      Group: "alertmanager",
+      ExecStart: "/usr/local/bin/alertmanager --config.file=/etc/prometheus/alertmanager.yml --storage.path=/var/lib/alertmanager --web.listen-address=:9093",
+      Restart: "on-failure",
+      RestartSec: "5",
+    },
+    Install: {
+      WantedBy: "multi-user.target",
+    },
+  })
+  triggers_reload true
+  action :create
+end
+
+service "alertmanager" do
+  action [:enable, :start]
+end
+
+firewall_rule "prometheus alertmanager" do
+  port 9093
+  source "10.1.1.0/24"
+  protocol :tcp
+  command :allow
+end
@@ -0,0 +1,83 @@
+#
+# Cookbook:: kosmos_prometheus
+# Recipe:: node_exporter
+#
+
+version = node["kosmos_prometheus"]["node_exporter"]["version"]
+checksum = node["kosmos_prometheus"]["node_exporter"]["checksum"]
+tarball = "#{Chef::Config[:file_cache_path]}/node_exporter-#{version}.linux-amd64.tar.gz"
+binary_url = "https://github.com/prometheus/node_exporter/releases/download/v#{version}/node_exporter-#{version}.linux-amd64.tar.gz"
+
+group "node_exporter"
+
+user "node_exporter" do
+  gid "node_exporter"
+  system true
+  shell "/bin/false"
+  home "/nonexistent"
+end
+
+directory "/var/lib/node_exporter/textfile" do
+  owner "node_exporter"
+  group "node_exporter"
+  mode "0755"
+  recursive true
+end
+
+package %w(tar bzip2)
+
+remote_file tarball do
+  source binary_url
+  checksum checksum
+  action :create
+  notifies :run, "execute[install_node_exporter]", :immediately
+end
+
+execute "install_node_exporter" do
+  command "tar -xzf #{tarball} -C /usr/local/bin --strip-components=1 node_exporter-#{version}.linux-amd64/node_exporter"
+  action :nothing
+  notifies :restart, "service[node_exporter]", :delayed
+end
+
+file "/usr/local/bin/node_exporter" do
+  owner "root"
+  group "root"
+  mode "0755"
+  notifies :restart, "service[node_exporter]", :delayed
+end
+
+systemd_unit "node_exporter.service" do
+  content({
+    Unit: {
+      Description: "Prometheus node exporter",
+      Documentation: ["https://github.com/prometheus/node_exporter"],
+    },
+    Service: {
+      Type: "simple",
+      User: "node_exporter",
+      Group: "node_exporter",
+      ExecStart: "/usr/local/bin/node_exporter --web.listen-address=:9100 --collector.textfile.directory=/var/lib/node_exporter/textfile",
+      Restart: "on-failure",
+      RestartSec: "5",
+      NoNewPrivileges: "yes",
+      ProtectSystem: "full",
+      ProtectHome: "yes",
+    },
+    Install: {
+      WantedBy: "multi-user.target",
+    },
+  })
+  triggers_reload true
+  action :create
+end
+
+service "node_exporter" do
+  action [:enable, :start]
+end
+
+firewall_rule "node_exporter" do
+  port 9100
+  source "10.1.1.0/24"
+  protocol :tcp
+  command :allow
+end
@@ -0,0 +1,150 @@
+#
+# Cookbook:: kosmos_prometheus
+# Recipe:: server
+#
+
+include_recipe "firewall"
+
+version = node["kosmos_prometheus"]["version"]
+checksum = node["kosmos_prometheus"]["checksum"]
+tarball = "#{Chef::Config[:file_cache_path]}/prometheus-#{version}.linux-amd64.tar.gz"
+binary_url = "https://github.com/prometheus/prometheus/releases/download/v#{version}/prometheus-#{version}.linux-amd64.tar.gz"
+
+data_dir   = node["kosmos_prometheus"]["data_dir"]
+config_dir = node["kosmos_prometheus"]["config_dir"]
+rules_dir  = node["kosmos_prometheus"]["rules_dir"]
+web_port   = node["kosmos_prometheus"]["web"]["port"]
+
+group "prometheus"
+
+user "prometheus" do
+  gid "prometheus"
+  system true
+  shell "/bin/false"
+  home "/nonexistent"
+end
+
+directory data_dir do
+  owner "prometheus"
+  group "prometheus"
+  mode "0750"
+  recursive true
+end
+
+directory config_dir do
+  owner "root"
+  group "root"
+  mode "0755"
+  recursive true
+end
+
+directory rules_dir do
+  owner "root"
+  group "prometheus"
+  mode "0755"
+  recursive true
+end
+
+node["kosmos_prometheus"]["rules"].each do |rule_name|
+  cookbook_file "#{rules_dir}/#{rule_name}.yml" do
+    source "rules/#{rule_name}.yml"
+    owner "root"
+    group "prometheus"
+    mode "0644"
+    notifies :reload, "service[prometheus]", :delayed
+  end
+end
+
+package %w(tar bzip2)
+
+remote_file tarball do
+  source binary_url
+  checksum checksum
+  action :create
+  notifies :run, "execute[install_prometheus]", :immediately
+end
+
+execute "install_prometheus" do
+  command "tar -xzf #{tarball} -C /usr/local/bin --strip-components=1 prometheus-#{version}.linux-amd64/prometheus"
+  action :nothing
+  notifies :restart, "service[prometheus]", :delayed
+end
+
+file "/usr/local/bin/prometheus" do
+  owner "root"
+  group "root"
+  mode "0755"
+  notifies :restart, "service[prometheus]", :delayed
+end
+
+jobs = node["kosmos_prometheus"]["jobs"].merge(
+{
+  # node exporter
+  "node" => {
+    "query" => "role:base",
+    "port" => 9100
+  },
+  # garage metrics
+  "garage" => {
+    "query" => "role:garage_node",
+    "port" => 3903
+  },
+}.transform_values do |config|
+    {
+      "targets" => search(:node, config["query"]).map do |n|
+        target = { "target" => "#{n['knife_zero']['host']}:#{config['port']}", "instance" => n.name }
+        target["env"] = n.chef_environment if n.chef_environment
+        target["vm_host"] = n["vm_host"] if n["vm_host"]
+        target
+      end.compact.sort_by { |t| t["instance"] },
+    }
+  end
+)
+
+template "#{config_dir}/prometheus.yml" do
+  source "prometheus.yml.erb"
+  owner "root"
+  group "prometheus"
+  mode "0644"
+  variables(
+    global_config: node["kosmos_prometheus"]["global"],
+    alerting: node["kosmos_prometheus"]["alerting"],
+    jobs: jobs,
+    rule_files: node["kosmos_prometheus"]["rule_files"]
+  )
+  notifies :reload, "service[prometheus]", :delayed
+end
+
+systemd_unit "prometheus.service" do
+  content({
+    Unit: {
+      Description: "Prometheus",
+      After: "network.target",
+    },
+    Service: {
+      Type: "simple",
+      User: "prometheus",
+      Group: "prometheus",
+      ExecStart: "/usr/local/bin/prometheus --config.file=/etc/prometheus/prometheus.yml --storage.tsdb.path=/var/lib/prometheus --storage.tsdb.retention.time=15d --web.listen-address=:#{web_port} --web.enable-lifecycle",
+      ExecReload: "/bin/kill -HUP $MAINPID",
+      Restart: "on-failure",
+      RestartSec: "5",
+    },
+    Install: {
+      WantedBy: "multi-user.target",
+    },
+  })
+  triggers_reload true
+  action :create
+end
+
+service "prometheus" do
+  action [:enable, :start]
+end
+
+firewall_rule "prometheus web" do
+  port web_port
+  source "10.1.1.0/24"
+  protocol :tcp
+  command :allow
+end
@@ -0,0 +1,47 @@
+global:
+  resolve_timeout: 5m
+
+route:
+<% @route.each do |k, v| %>
+<% if k == "routes" && v.is_a?(Array) && !v.empty? %>
+  routes:
+<% v.each do |child| %>
+  - receiver: <%= child["receiver"] %>
+<% child.each do |ck, cv| %>
+<% next if ck == "receiver" %>
+    <%= ck %>: <%= cv.is_a?(Array) ? cv.inspect : cv %>
+<% end %>
+<% end %>
+<% else %>
+  <%= k %>: <%= v.is_a?(Array) ? v.inspect : v %>
+<% end %>
+<% end %>
+
+receivers:
+<% @receivers.each do |receiver| %>
+- name: <%= receiver["name"] %>
+<% if receiver["webhook_configs"] %>
+<% receiver["webhook_configs"].each do |wc| %>
+  webhook_configs:
+  - url: "<%= @webhook_url %>"
+    send_resolved: <%= wc["send_resolved"] %>
+    payload:
+      room: "<%= wc["room"] %>"
+      message: '{{ if eq .Status "resolved" }}✅{{ else if eq .CommonLabels.severity "warning" }}⚠️{{ else }}🚨{{ end }} {{ if eq .Status "resolved" }}RESOLVED{{ else if .CommonLabels.severity }}{{ .CommonLabels.severity | toUpper }}{{ else }}ALERT{{ end }}: {{ .CommonLabels.alertname }} {{ if eq .Status "resolved" }}({{ len .Alerts }} resolved){{ else if eq (len .Alerts.Firing) (len .Alerts) }}({{ len .Alerts }} firing){{ else }}({{ len .Alerts.Firing }} firing, {{ len .Alerts.Resolved }} resolved){{ end }}{{ range $i, $a := .Alerts }}{{ if lt $i 5 }}{{ "\n" }}  - {{ $a.Labels.instance }}{{ if $a.Labels.vm_host }} ({{ $a.Labels.vm_host }}){{ end }}: {{ if eq $a.Status "resolved" }}resolved{{ else }}{{ $a.Annotations.summary }}{{ end }}{{ end }}{{ end }}{{ if gt (len .Alerts) 5 }}{{ "\n" }}  ... and more{{ end }}'
+<% end %>
+<% end %>
+<% if receiver["email_configs"] %>
+<% receiver["email_configs"].each do |ec| %>
+  email_configs:
+  - to: "<%= ec["to"] %>"
+    from: "<%= ec["from"] %>"
+    smarthost: "<%= ec["smarthost"] %>"
+    require_tls: <%= ec["require_tls"] %>
+    send_resolved: <%= ec["send_resolved"] %>
+    text: '{{ if eq .Status "resolved" }}RESOLVED{{ else if .CommonLabels.severity }}{{ .CommonLabels.severity | toUpper }}{{ else }}ALERT{{ end }}: {{ .CommonLabels.alertname }} {{ if eq .Status "resolved" }}({{ len .Alerts }} resolved){{ else if eq (len .Alerts.Firing) (len .Alerts) }}({{ len .Alerts }} firing){{ else }}({{ len .Alerts.Firing }} firing, {{ len .Alerts.Resolved }} resolved){{ end }}{{ range .Alerts }}
+---{{ range .Labels.SortedPairs }}
+  {{ .Name }}: {{ .Value }}{{ end }}{{ range .Annotations.SortedPairs }}
+  {{ .Name }}: {{ .Value }}{{ end }}{{ end }}'
+<% end %>
+<% end %>
+<% end %>
@@ -0,0 +1,50 @@
+global:
+<% @global_config.each do |k, v| %>
+  <%= k %>: "<%= v %>"
+<% end %>
+
+<% if @alerting %>
+alerting:
+  alertmanagers:
+<% @alerting['alertmanagers'].each do |am| %>
+    - static_configs:
+<% am['static_configs'].each do |sc| %>
+        - targets:
+<% sc['targets'].each do |target| %>
+            - <%= target %>
+<% end %>
+<% end %>
+<% end %>
+<% end %>
+
+scrape_configs:
+<% @jobs.each do |name, job| %>
+- job_name: "<%= name %>"
+<% if job['scrape_interval'] %>
+  scrape_interval: "<%= job['scrape_interval'] %>"
+<% end %>
+<% if job['scrape_timeout'] %>
+  scrape_timeout: "<%= job['scrape_timeout'] %>"
+<% end %>
+  metrics_path: "<%= job.fetch('metrics_path', '/metrics') %>"
+  static_configs:
+<% job['targets'].each do |t| %>
+    - targets:
+        - <%= t['target'] %>
+      labels:
+        instance: <%= t['instance'] %>
+<% if t['env'] %>
+        env: <%= t['env'] %>
+<% end %>
+<% if t['vm_host'] %>
+        vm_host: <%= t['vm_host'] %>
+<% end %>
+<% end %>
+<% end %>
+
+<% if @rule_files && !@rule_files.empty? %>
+rule_files:
+<% @rule_files.each do |filename| %>
+  - <%= filename %>
+<% end %>
+<% end %>
@@ -1,9 +1,8 @@
 node.default["sentry"]["repo"] = "https://github.com/getsentry/self-hosted"
-node.default["sentry"]["revision"] = "23.3.1"
+# Make sure to first upgrade to hard stop version if required:
+# https://develop.sentry.dev/self-hosted/releases/
+node.default["sentry"]["revision"] = "26.6.0"
+
 node.default["sentry"]["port"] = 80
 node.default["sentry"]["retention_days"] = 90
 node.default["sentry"]["allowed_ips"] = nil
-# The Sentry setup requires docker-compose >= 1.28, which is newer than the
-# latest stable version for Ubuntu 20.04
-node.default["sentry"]["docker-compose"]["version"] = "2.17.0"
-node.default["sentry"]["docker-compose"]["checksum"] = "65edee934d988471c40ef31305731dbb4381d3cb0aeea13342119b61772f85e2"
@@ -3,12 +3,14 @@
 # Recipe:: default
 #

-package "docker"
+apt_repository "docker" do
+  uri "https://download.docker.com/linux/ubuntu"
+  key "https://download.docker.com/linux/ubuntu/gpg"
+  components ["stable"]
+end

-remote_file "/usr/local/bin/docker-compose" do
-  source "https://github.com/docker/compose/releases/download/v#{node["sentry"]["docker-compose"]["version"]}/docker-compose-linux-x86_64"
-  checksum node["sentry"]["docker-compose"]["checksum"]
-  mode '0755'
+%w[docker-ce docker-ce-cli containerd.io docker-buildx-plugin docker-compose-plugin].each do |pkg|
+  package pkg
 end

 deploy_path = "/opt/sentry"
@@ -16,24 +18,22 @@ deploy_path = "/opt/sentry"
 git deploy_path do
  repository node["sentry"]["repo"]
  revision node["sentry"]["revision"]
+  # TODO test and uncomment
+  # notifies :run, "execute[run_install_script]", :immediately
 end

-# TODO
-# Automatically run install script if sentry/sentry.conf.py does not exist yet
+execute "run_install_script" do
+  cwd deploy_path
+  command "./install.sh --apply-automatic-config-updates --no-report-self-hosted-issues"
+  action :nothing
+end
+
+credentials = data_bag_item("credentials", "sentry")

 env_config = {
  sentry_event_retention_days: node["sentry"]["retention_days"],
  sentry_bind: node["sentry"]["bind"] || node["sentry"]["port"],
-  # Default values from upstream
-  compose_project_name: "sentry-self-hosted",
-  sentry_image: "getsentry/sentry:nightly",
-  snuba_image: "getsentry/snuba:nightly",
-  relay_image: "getsentry/relay:nightly",
-  symbolicator_image: "getsentry/symbolicator:nightly",
-  wal2json_version: "latest",
-  healthcheck_interval: "30s",
-  healthcheck_timeout: "60s",
-  healthcheck_retries: "10"
+  launchpad_rpc_shared_secret: credentials["launchpad_rpc_shared_secret"]
 }

 template "#{deploy_path}/.env.custom" do
@@ -41,7 +41,8 @@ template "#{deploy_path}/.env.custom" do
  mode 0600
  sensitive true
  variables config: env_config
-  notifies :restart, "service[sentry]", :delayed
+  # TODO
+  # notifies :restart, "service[sentry]", :delayed
 end

 systemd_unit "sentry.service" do
@@ -52,11 +53,13 @@ systemd_unit "sentry.service" do
           },
           Service: {
             Type: "oneshot",
-             RemainAfterExit: "true",
+             RemainAfterExit: "yes",
             WorkingDirectory: deploy_path,
-             ExecStart: "docker-compose --env-file #{deploy_path}/.env.custom up -d --remove-orphans",
-             ExecStop: "docker-compose stop",
-             StandardOutput: "syslog"
+             ExecStart: "docker compose --env-file .env --env-file .env.custom up -d --wait",
+             ExecStop: "docker compose down --remove-orphans",
+             StandardOutput: "syslog",
+             TimeoutStartSec: 600,
+             TimeoutStopSec: 300
           },
           Install: {
             WantedBy: "multi-user.target"
@@ -67,7 +70,7 @@ systemd_unit "sentry.service" do
 end

 service "sentry" do
-  action [:enable, :start]
+  action [:enable]
 end

 include_recipe 'firewall'
Author	SHA1	Message	Date
raucao	88a709352d	Upgrade Sentry to 26.6.0	2026-07-08 16:44:09 +02:00
raucao	6f93820936	Update node configs	2026-07-07 16:58:36 +02:00
raucao	9fca5386f3	Increase padding for NodeDown rule	2026-07-07 16:57:22 +02:00
raucao	b6ba5ba679	Update node configs	2026-07-07 15:20:48 +02:00
raucao	4976888358	Remove unnecessary include	2026-07-07 15:20:33 +02:00
raucao	e4ad52d413	Update dirsrv node configs	2026-07-07 15:05:25 +02:00
raucao	94a3c41964	Show severity for new alerts, number of firing vs resolved	2026-07-07 14:26:21 +02:00
raucao	704d0bca0e	Document upgrading Chef client	2026-07-06 18:50:07 +02:00
raucao	02dc95b7de	Update node configs	2026-07-06 18:47:51 +02:00
raucao	2a14837f64	WIP Add prometheus rules, improve alerts	2026-07-06 18:01:22 +02:00
raucao	1ae5210813	Send critical alerts via email (in addition to XMPP)	2026-07-06 14:52:17 +02:00
raucao	20f67d7261	Change prometheus scrape/eval intervals	2026-07-06 14:52:10 +02:00
raucao	4ce77e1a6e	Fix postfix relay sending (on newer Ubuntu versions) Requires relay restrictions to be configured explicitly	2026-07-06 14:50:49 +02:00
raucao	21513e6d10	Critical icon only for critical severity	2026-07-06 13:53:52 +02:00
raucao	33987402e3	Fix port attribute	2026-07-06 13:16:10 +02:00
raucao	81aae3d1bb	Add icons to visualize alert severity	2026-07-06 12:38:07 +02:00
raucao	65c1933a99	WIP Add default alertmanager route and receiver	2026-07-06 12:38:07 +02:00
greg	5e7a0c07b0	Move web port to an attribute	2026-07-05 18:16:18 +02:00
greg	8601e33764	Fix year in license file	2026-07-05 18:03:12 +02:00
raucao	c736468259	Merge branch 'master' into feature/prometheus_cookbook	2026-07-05 17:39:08 +02:00
raucao	6370069e9d	Add new vm_host attribute/value to all VM nodes	2026-07-05 17:32:43 +02:00
raucao	0e9bcca72d	Add vm_host label to prometheus targets	2026-07-05 17:32:14 +02:00
raucao	c762dfe8c1	Merge pull request 'Sanitize node configs via script, and do it automatically in a Git hook' (#635 ) from dev/sanitize_node_configs into feature/prometheus_cookbook Reviewed-on: #635	2026-07-05 15:26:30 +00:00
raucao	53cf95c4e7	Remove decommissioned nodes	2026-07-05 17:22:48 +02:00
raucao	959d506cfb	Remove newline from all node files Was actually added by manually editing the files, but Chef writes them without newline, so our sanitization script also does it that way	2026-07-05 17:22:39 +02:00
raucao	8c71261fc9	Add pre-commit hook for sanitizing node configs	2026-07-05 17:22:34 +02:00
raucao	6c3da8a711	Add util script for sanitizing nodes Deletes the "override" and "default" properties, which may contain sensitive data	2026-07-05 17:22:26 +02:00
raucao	53de8ea119	Merge branch 'master' into feature/prometheus_cookbook	2026-07-05 17:20:48 +02:00
raucao	639800a838	Remove decommissioned node	2026-07-05 17:19:16 +02:00
raucao	171db0fb7c	Add altermanager section to prometheus config	2026-07-05 13:19:18 +02:00
raucao	821b56e6ab	Set node name as prometheus instance name And DRY up the recipe so it's easy and expressive to add more metrics/targets	2026-07-05 12:41:46 +02:00
raucao	e29d90d3d0	Use full domain as fqdn for leo	2026-07-05 12:40:50 +02:00
raucao	f2ebda4a1a	Update node configs	2026-07-04 16:30:15 +02:00
raucao	67f62ebd6c	Scrape garage metrics	2026-07-04 16:14:36 +02:00
raucao	7dc4895da3	Remove old garage nodes	2026-07-04 15:51:39 +02:00
raucao	153b1e77c5	Add all nodes with base role as node exporter targets	2026-07-04 15:46:41 +02:00
raucao	ea69c7cec6	Add prometheus node exporter to base role	2026-07-04 15:45:57 +02:00
raucao	5813a45987	Use base role instead of recipe in all runlists	2026-07-04 15:45:23 +02:00
raucao	63534e1cf5	Remove upstream prometheus cookbook, migrate to our own	2026-07-04 15:27:18 +02:00
raucao	2d835335b5	Re-add global config, change values	2026-07-04 14:15:55 +02:00
raucao	e21797b402	Apply changed configs to prometheus and alertmanager	2026-07-04 14:15:38 +02:00
raucao	7396af5ca4	WIP Add node exporter	2026-07-04 14:15:33 +02:00
raucao	df8c8d1742	Remove obsolete CLI option Was left over from trying to overwrite the defaults before	2026-07-04 13:31:20 +02:00
greg	765d0b080e	WIP Initial kosmos_prometheus wrapper cookbook	2026-07-03 17:47:13 +02:00
greg	4cd6c41254	Add community prometheus cookbook	2026-07-03 17:46:18 +02:00
raucao	ec73dd5b57	Set Chef environment for node	2026-07-03 15:52:46 +02:00
raucao	850db344b7	Add prometheus node	2026-07-03 15:50:15 +02:00