Merge pull request 'Set up Prometheus and Alertmanager' (#632) from feature/prometheus_cookbook into master

Reviewed-on: #632

.githooks/pre-commit

@@ -0,0 +1,40 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+SCRIPT="$(git rev-parse --show-toplevel)/scripts/util/sanitize_node.sh"
+
+FILES=$(git diff --cached --name-only --diff-filter=ACM -- 'nodes/*.json' || true)
+
+[[ -z "$FILES" ]] && exit 0
+
+DIRTY=0
+
+while IFS= read -r file; do
+  staged_tmp=$(mktemp)
+  sanitized_tmp=$(mktemp)
+
+  # 1. get staged version
+  git show ":$file" > "$staged_tmp"
+
+  # 2. sanitize IN PLACE (on temp copy)
+  cp "$staged_tmp" "$sanitized_tmp"
+  "$SCRIPT" "$sanitized_tmp"
+
+  # 3. if sanitizer changed file, update working tree
+  if ! diff -q "$staged_tmp" "$sanitized_tmp" >/dev/null; then
+    cp "$sanitized_tmp" "$file"
+    echo "Sanitized: $file"
+    DIRTY=1
+  fi
+
+  rm -f "$staged_tmp" "$sanitized_tmp"
+
+done <<< "$FILES"
+
+if [[ "$DIRTY" -ne 0 ]]; then
+  echo ""
+  echo "Commit paused, sanitized files must be re-staged."
+  exit 1
+fi
+
+exit 0

README.md

@@ -28,6 +28,10 @@ Clone this repository, `cd` into it, and run:
 
     bundle install
 
+### Enable Git hooks
+
+    git config core.hooksPath .githooks
+
 ## Common tasks
 
 ### Bootstrap a new host server
@@ -54,6 +58,10 @@ Clone this repository, `cd` into it, and run:
 
     knife zero converge name:dev.kosmos.org --client-version 15.3.14
 
+### Upgrade Chef Client on a node:
+
+    knife ssh -a knife_zero.host 'name:foo-1' 'curl -L https://chef.io/chef/install.sh | sudo bash -s -- -v 18.10.17'
+
 ## Managing cookbooks
 
 Cookbooks are managed via Berkshelf. Run `berks --help` for command help.

clients/garage-10.json

@@ -1,4 +0,0 @@
-{
-  "name": "garage-10",
-  "public_key": "-----BEGIN PUBLIC KEY-----\nMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAw2+3Wo+KkXVJCOX1SxT9\nSdwKXgPbCDM3EI9uwoxhMxQfRyN53dxIsBDsQUVOIe1Z8yqm4FenMQlNmeDR+QLE\nvNFf1fisinW+D9VVRm+CjcJy96i/Dyt786Z6YRrDlB860HxCbfTL2Zv5BRtbyIKg\nhz5gO+9PMEpPVR2ij9iue4K6jbM1AAL2ia/P6zDWLJqeIzUocCeHV5N0Z3jXH6qr\nf444v78x35MMJ+3tg5h95SU1/PDCpdSTct4uHEuKIosiN7p4DlYMoM5iSyvVoujr\nflRQPEpGzS9qEt3rDo/F4ltzYMx6bf1tB/0QaBKD+zwPZWTTwf61tSBo5/NkGvJc\nFQIDAQAB\n-----END PUBLIC KEY-----\n"
-}

clients/garage-12.json

@@ -1,4 +0,0 @@
-{
-  "name": "garage-12",
-  "public_key": "-----BEGIN PUBLIC KEY-----\nMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEA9GtHHi298BjiIqpZ3WkT\nkYAPfWD60hFe/8icYcq/F/6cHLYKZQ4chek9X/hDCMq4tHEN6Oh58T5x/nuNdPrK\nIAMGyVAGk6ekWlmD4jwdEf6TGb/J3ffJTRDvwX/I8xD/DW3wtXsN+X24T59ByGTm\nrnwRmmmwHF3otRx9wnCsIgDQ0AjiUujsfNNv1FcLXD/WJLys9lEeU5aJ4XtHTwDv\ntJM8YyVEFhEnuvgdKmzn5+F5k9VGdUwForlFOBfvzbCnTZMDMmDVeiUtAUv/7xWQ\nQl2mLUGCtgWuYJYXsQacAJ6pa3h+7cQyshC6w3dwUG+1fS9lNO0Yp1GGX1AGYKpp\nPQIDAQAB\n-----END PUBLIC KEY-----\n"
-}

clients/garage-13.json

@@ -1,4 +0,0 @@
-{
-  "name": "garage-13",
-  "public_key": "-----BEGIN PUBLIC KEY-----\nMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAvbqWc6OwRxgHfsQuTNL4\naxeVvNen5d9srYpZSHjuBB/k9NHB+9P6vU5qF37XHkw1lVUGeYbPHzhYsx3O0/kZ\nH5f4+4SMy/P9jc6SE7AJF4qtYKgJ88koZdqCww07c6K9g+BnEGFFZui/h3hUBxWj\nTfhBHEWPyQ2bl/lr9sIJwsEz+EN0isGn/eIXkmw9J6LdLJ5Q0LLks33K28FNOU7q\nfeAN4MiBVMUtgCGyT2Voe6WrOXwQLSDXQONOp3sfSfFExsIJ1s24xdd7AMD7/9a7\n4sFDZ4swhqAWgWmW2giR7Kb8wTvGQLO/O/uUbmKz3DZXgkOKXHdHCEB/PZx1mRNM\nEwIDAQAB\n-----END PUBLIC KEY-----\n"
-}

clients/garage-9.json

@@ -1,4 +0,0 @@
-{
-  "name": "garage-9",
-  "public_key": "-----BEGIN PUBLIC KEY-----\nMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAnMHzKE8JBrsQkmRDeMjX\n71mBzvRzNM90cwA8xtvIkXesdTyGqohX9k/PJbCY5ySGK9PpMaYDPVAnwnUP8LFQ\n3G98aSbLxUjqU/PBzRsnWpihehr05uz9zYcNFzr4LTNvGQZsq47nN9Tk+LG3zHP7\nAZViv2mJ4ZRnukXf6KHlyoVvhuTu+tiBM8QzjTF97iP/aguNPzYHmrecy9Uf5bSA\nZrbNZT+ayxtgswC2OclhRucx7XLSuHXtpwFqsQzSAhiX1aQ3wwCyH9WJtVwpfUsE\nlxTjcQiSM9aPZ8iSC0shpBaKD1j3iF/2K2Jk+88++zMhJJPLermvaJxzsdePgvyk\nKQIDAQAB\n-----END PUBLIC KEY-----\n"
-}

clients/prometheus-1.json

@@ -0,0 +1,4 @@
+{
+  "name": "prometheus-1",
+  "public_key": "-----BEGIN PUBLIC KEY-----\nMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAp7T/OBo/TZm3YqnN4+ok\nHwcJ0kW9w2rl9UfrOlWUvoPHBd2LrqpEv3Az3a150IylQ1H/UozmQA7DtjIoTA7d\nV3oLY970vYrYiURcojOo8qAZBy8EH7dfAHxuZryUeELr+3vdcHF5WrrfSt2FdFVX\nPTY95ikafAnOO0Nt8jvnlPoDn7REV8TOE6KOiUzcHKa2xGlfaIe0oRC21LD86uQm\nR09xY1YaJkVgZfeN/opoRjZawkU3FFs3jlUEVBF8k153oOw9W3bgsFFjSOtRtRRg\nDwyQ7oDeMH83kXnaCdpkNZd59wjzPcpxYAL4LRN52ZXA4Btr4DTi+GxHz98Dr0kU\nUQIDAQAB\n-----END PUBLIC KEY-----\n"
+}

clients/uploads-1.json

@@ -1,4 +0,0 @@
-{
-  "name": "uploads-1",
-  "public_key": "-----BEGIN PUBLIC KEY-----\nMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAwJwWaz8TeGv3SFlKzLMx\nqN8GTL/c0N9ppBvv8xNSS/yF9Y40SbL418uxYzm9hIhOXgIygIgLT2EKIXX32t+R\neOJCdYycQFM3At2fhMkjhuUW0gmDRcYBcBJLC5hLh2EZ+A8V7k4qgrBpPLOjEv48\nhQY0vuAw2DGndWr4QLh5NLUmQiOrfuzcZSSNCBOTIgUZgNmRd9QcCHDq4WDH3poa\nosJo4a9JGEGUL1irOivvEdyJPwEd2f++nYAdWwj8pjCYgpRshQlLhxOlylMx7MxB\nQt2bgJC9sahfbfJCOqdlCU3DMJL0bRUiuxK77WeSsxWBJmrsiF3+Ljs2Ix+s7fnS\nywIDAQAB\n-----END PUBLIC KEY-----\n"
-}

nodes/akkounts-1.json

@@ -4,7 +4,8 @@
   "normal": {
     "knife_zero": {
       "host": "10.1.1.144"
-    }
+    },
+    "vm_host": "draco"
   },
   "automatic": {
     "fqdn": "akkounts-1",
@@ -24,6 +25,7 @@
     "recipes": [
       "kosmos-base",
       "kosmos-base::default",
+      "kosmos_prometheus::node_exporter",
       "kosmos_kvm::guest",
       "kosmos-dirsrv::hostsfile",
       "kosmos_sentry::client",

nodes/bitcoin-2.json

@@ -3,7 +3,8 @@
   "normal": {
     "knife_zero": {
       "host": "10.1.1.163"
-    }
+    },
+    "vm_host": "draco"
   },
   "automatic": {
     "fqdn": "bitcoin-2",
@@ -24,6 +25,7 @@
     "recipes": [
       "kosmos-base",
       "kosmos-base::default",
+      "kosmos_prometheus::node_exporter",
       "kosmos_kvm::guest",
       "kosmos_sentry::client",
       "tor-full",

nodes/discourse-2.json

@@ -4,15 +4,17 @@
   "normal": {
     "knife_zero": {
       "host": "10.1.1.35"
-    }
+    },
+    "vm_host": "fornax"
   },
   "automatic": {
     "fqdn": "discourse-2",
     "os": "linux",
-    "os_version": "5.4.0-1087-kvm",
+    "os_version": "5.4.0-1133-kvm",
     "hostname": "discourse-2",
     "ipaddress": "192.168.122.104",
     "roles": [
+      "base",
       "kvm_guest",
       "garage_gateway",
       "kosmos_discourse"
@@ -20,6 +22,7 @@
     "recipes": [
       "kosmos-base",
       "kosmos-base::default",
+      "kosmos_prometheus::node_exporter",
       "kosmos_kvm::guest",
       "kosmos_garage",
       "kosmos_garage::default",
@@ -32,6 +35,7 @@
       "timezone_iii::debian",
       "ntp::default",
       "ntp::apparmor",
+      "kosmos-base::journald_conf",
       "kosmos-base::systemd_emails",
       "apt::unattended-upgrades",
       "kosmos-base::firewall",
@@ -42,7 +46,6 @@
       "postfix::sasl_auth",
       "hostname::default",
       "firewall::default",
-      "chef-sugar::default",
       "discourse::default"
     ],
     "platform": "ubuntu",
@@ -50,18 +53,18 @@
     "cloud": null,
     "chef_packages": {
       "chef": {
-        "version": "17.9.52",
+        "version": "18.10.17",
-        "chef_root": "/opt/chef/embedded/lib/ruby/gems/3.0.0/gems/chef-17.9.52/lib",
+        "chef_root": "/opt/chef/embedded/lib/ruby/gems/3.1.0/gems/chef-18.10.17/lib",
         "chef_effortless": null
       },
       "ohai": {
-        "version": "17.9.0",
+        "version": "18.2.13",
-        "ohai_root": "/opt/chef/embedded/lib/ruby/gems/3.0.0/gems/ohai-17.9.0/lib/ohai"
+        "ohai_root": "/opt/chef/embedded/lib/ruby/gems/3.1.0/gems/ohai-18.2.13/lib/ohai"
       }
     }
   },
   "run_list": [
-    "recipe[kosmos-base]",
+    "role[base]",
     "role[kvm_guest]",
     "role[garage_gateway]",
     "role[kosmos_discourse]"

nodes/draco.kosmos.org.json

@@ -36,6 +36,7 @@
     "recipes": [
       "kosmos-base",
       "kosmos-base::default",
+      "kosmos_prometheus::node_exporter",
       "kosmos_kvm::host",
       "kosmos_kvm::backup",
       "kosmos_email::firewall",

nodes/drone-1.json

@@ -3,7 +3,8 @@
   "normal": {
     "knife_zero": {
       "host": "10.1.1.128"
-    }
+    },
+    "vm_host": "fornax"
   },
   "automatic": {
     "fqdn": "drone-1",
@@ -12,6 +13,7 @@
     "hostname": "drone-1",
     "ipaddress": "192.168.122.200",
     "roles": [
+      "base",
       "kvm_guest",
       "drone",
       "postgresql_client"
@@ -19,6 +21,7 @@
     "recipes": [
       "kosmos-base",
       "kosmos-base::default",
+      "kosmos_prometheus::node_exporter",
       "kosmos_kvm::guest",
       "kosmos_postgresql::hostsfile",
       "kosmos_drone",
@@ -37,7 +40,8 @@
       "postfix::_common",
       "postfix::_attributes",
       "postfix::sasl_auth",
-      "hostname::default"
+      "hostname::default",
+      "firewall::default"
     ],
     "platform": "ubuntu",
     "platform_version": "20.04",
@@ -55,7 +59,7 @@
     }
   },
   "run_list": [
-    "recipe[kosmos-base]",
+    "role[base]",
     "role[kvm_guest]",
     "role[drone]"
   ]

nodes/ejabberd-4.json

@@ -4,7 +4,8 @@
   "normal": {
     "knife_zero": {
       "host": "10.1.1.113"
-    }
+    },
+    "vm_host": "draco"
   },
   "automatic": {
     "fqdn": "ejabberd-4",
@@ -23,6 +24,7 @@
     "recipes": [
       "kosmos-base",
       "kosmos-base::default",
+      "kosmos_prometheus::node_exporter",
       "kosmos_kvm::guest",
       "kosmos-dirsrv::hostsfile",
       "kosmos_postgresql::hostsfile",
@@ -72,4 +74,4 @@
     "role[ldap_client]",
     "role[ejabberd]"
   ]
-}
+}

nodes/ejabberd-8.json

@@ -4,7 +4,8 @@
   "normal": {
     "knife_zero": {
       "host": "10.1.1.123"
-    }
+    },
+    "vm_host": "fornax"
   },
   "automatic": {
     "fqdn": "ejabberd-8",
@@ -23,6 +24,7 @@
     "recipes": [
       "kosmos-base",
       "kosmos-base::default",
+      "kosmos_prometheus::node_exporter",
       "kosmos_kvm::guest",
       "kosmos-dirsrv::hostsfile",
       "kosmos_postgresql::hostsfile",
@@ -72,4 +74,4 @@
     "role[ldap_client]",
     "role[ejabberd]"
   ]
-}
+}

nodes/fornax.kosmos.org.json

@@ -30,6 +30,7 @@
     "recipes": [
       "kosmos-base",
       "kosmos-base::default",
+      "kosmos_prometheus::node_exporter",
       "kosmos_kvm::host",
       "kosmos_kvm::backup",
       "kosmos_openresty",

nodes/garage-10.json

@@ -1,64 +0,0 @@
-{
-  "name": "garage-10",
-  "chef_environment": "production",
-  "normal": {
-    "knife_zero": {
-      "host": "10.1.1.27"
-    }
-  },
-  "automatic": {
-    "fqdn": "garage-10",
-    "os": "linux",
-    "os_version": "5.4.0-1090-kvm",
-    "hostname": "garage-10",
-    "ipaddress": "192.168.122.70",
-    "roles": [
-      "base",
-      "kvm_guest",
-      "garage_node"
-    ],
-    "recipes": [
-      "kosmos-base",
-      "kosmos-base::default",
-      "kosmos_kvm::guest",
-      "kosmos_garage",
-      "kosmos_garage::default",
-      "kosmos_garage::firewall_rpc",
-      "kosmos_garage::firewall_apis",
-      "apt::default",
-      "timezone_iii::default",
-      "timezone_iii::debian",
-      "ntp::default",
-      "ntp::apparmor",
-      "kosmos-base::systemd_emails",
-      "apt::unattended-upgrades",
-      "kosmos-base::firewall",
-      "kosmos-postfix::default",
-      "postfix::default",
-      "postfix::_common",
-      "postfix::_attributes",
-      "postfix::sasl_auth",
-      "hostname::default",
-      "firewall::default"
-    ],
-    "platform": "ubuntu",
-    "platform_version": "20.04",
-    "cloud": null,
-    "chef_packages": {
-      "chef": {
-        "version": "18.5.0",
-        "chef_root": "/opt/chef/embedded/lib/ruby/gems/3.1.0/gems/chef-18.5.0/lib",
-        "chef_effortless": null
-      },
-      "ohai": {
-        "version": "18.1.11",
-        "ohai_root": "/opt/chef/embedded/lib/ruby/gems/3.1.0/gems/ohai-18.1.11/lib/ohai"
-      }
-    }
-  },
-  "run_list": [
-    "role[base]",
-    "role[kvm_guest]",
-    "role[garage_node]"
-  ]
-}

nodes/garage-11.json

@@ -4,7 +4,8 @@
   "normal": {
     "knife_zero": {
       "host": "10.1.1.165"
-    }
+    },
+    "vm_host": "fornax"
   },
   "automatic": {
     "fqdn": "garage-11",
@@ -20,6 +21,7 @@
     "recipes": [
       "kosmos-base",
       "kosmos-base::default",
+      "kosmos_prometheus::node_exporter",
       "kosmos_kvm::guest",
       "kosmos_garage",
       "kosmos_garage::default",
@@ -30,6 +32,7 @@
       "timezone_iii::debian",
       "ntp::default",
       "ntp::apparmor",
+      "kosmos-base::journald_conf",
       "kosmos-base::systemd_emails",
       "apt::unattended-upgrades",
       "kosmos-base::firewall",
@@ -61,4 +64,4 @@
     "role[kvm_guest]",
     "role[garage_node]"
   ]
-}
+}

nodes/garage-12.json

@@ -1,65 +0,0 @@
-{
-  "name": "garage-12",
-  "chef_environment": "production",
-  "normal": {
-    "knife_zero": {
-      "host": "10.1.1.224"
-    }
-  },
-  "automatic": {
-    "fqdn": "garage-12",
-    "os": "linux",
-    "os_version": "5.15.0-1059-kvm",
-    "hostname": "garage-12",
-    "ipaddress": "192.168.122.173",
-    "roles": [
-      "base",
-      "kvm_guest",
-      "garage_node"
-    ],
-    "recipes": [
-      "kosmos-base",
-      "kosmos-base::default",
-      "kosmos_kvm::guest",
-      "kosmos_garage",
-      "kosmos_garage::default",
-      "kosmos_garage::firewall_rpc",
-      "kosmos_garage::firewall_apis",
-      "apt::default",
-      "timezone_iii::default",
-      "timezone_iii::debian",
-      "ntp::default",
-      "ntp::apparmor",
-      "kosmos-base::journald_conf",
-      "kosmos-base::systemd_emails",
-      "apt::unattended-upgrades",
-      "kosmos-base::firewall",
-      "kosmos-postfix::default",
-      "postfix::default",
-      "postfix::_common",
-      "postfix::_attributes",
-      "postfix::sasl_auth",
-      "hostname::default",
-      "firewall::default"
-    ],
-    "platform": "ubuntu",
-    "platform_version": "22.04",
-    "cloud": null,
-    "chef_packages": {
-      "chef": {
-        "version": "18.7.10",
-        "chef_root": "/opt/chef/embedded/lib/ruby/gems/3.1.0/gems/chef-18.7.10/lib",
-        "chef_effortless": null
-      },
-      "ohai": {
-        "version": "18.2.5",
-        "ohai_root": "/opt/chef/embedded/lib/ruby/gems/3.1.0/gems/ohai-18.2.5/lib/ohai"
-      }
-    }
-  },
-  "run_list": [
-    "role[base]",
-    "role[kvm_guest]",
-    "role[garage_node]"
-  ]
-}

nodes/garage-14.json

@@ -4,7 +4,8 @@
   "normal": {
     "knife_zero": {
       "host": "10.1.1.151"
-    }
+    },
+    "vm_host": "leo"
   },
   "automatic": {
     "fqdn": "garage-14",
@@ -20,6 +21,7 @@
     "recipes": [
       "kosmos-base",
       "kosmos-base::default",
+      "kosmos_prometheus::node_exporter",
       "kosmos_kvm::guest",
       "kosmos_garage",
       "kosmos_garage::default",
@@ -62,4 +64,4 @@
     "role[kvm_guest]",
     "role[garage_node]"
   ]
-}
+}

nodes/garage-15.json

@@ -4,7 +4,8 @@
   "normal": {
     "knife_zero": {
       "host": "10.1.1.82"
-    }
+    },
+    "vm_host": "leo"
   },
   "automatic": {
     "fqdn": "garage-15",
@@ -20,6 +21,7 @@
     "recipes": [
       "kosmos-base",
       "kosmos-base::default",
+      "kosmos_prometheus::node_exporter",
       "kosmos_kvm::guest",
       "kosmos_garage",
       "kosmos_garage::default",
@@ -62,4 +64,4 @@
     "role[kvm_guest]",
     "role[garage_node]"
   ]
-}
+}

nodes/garage-16.json

@@ -4,7 +4,8 @@
   "normal": {
     "knife_zero": {
       "host": "10.1.1.153"
-    }
+    },
+    "vm_host": "leo"
   },
   "automatic": {
     "fqdn": "garage-16",
@@ -20,6 +21,7 @@
     "recipes": [
       "kosmos-base",
       "kosmos-base::default",
+      "kosmos_prometheus::node_exporter",
       "kosmos_kvm::guest",
       "kosmos_garage",
       "kosmos_garage::default",
@@ -60,4 +62,4 @@
     "role[kvm_guest]",
     "role[garage_node]"
   ]
-}
+}

nodes/garage-2.json

@@ -4,7 +4,8 @@
   "normal": {
     "knife_zero": {
       "host": "10.1.1.40"
-    }
+    },
+    "vm_host": "draco"
   },
   "automatic": {
     "fqdn": "garage-2",
@@ -20,6 +21,7 @@
     "recipes": [
       "kosmos-base",
       "kosmos-base::default",
+      "kosmos_prometheus::node_exporter",
       "kosmos_kvm::guest",
       "kosmos_garage",
       "kosmos_garage::default",
@@ -30,6 +32,7 @@
       "timezone_iii::debian",
       "ntp::default",
       "ntp::apparmor",
+      "kosmos-base::journald_conf",
       "kosmos-base::systemd_emails",
       "apt::unattended-upgrades",
       "kosmos-base::firewall",
@@ -46,13 +49,13 @@
     "cloud": null,
     "chef_packages": {
       "chef": {
-        "version": "18.3.0",
+        "version": "18.10.17",
-        "chef_root": "/opt/chef/embedded/lib/ruby/gems/3.1.0/gems/chef-18.3.0/lib",
+        "chef_root": "/opt/chef/embedded/lib/ruby/gems/3.1.0/gems/chef-18.10.17/lib",
         "chef_effortless": null
       },
       "ohai": {
-        "version": "18.1.4",
+        "version": "18.2.13",
-        "ohai_root": "/opt/chef/embedded/lib/ruby/gems/3.1.0/gems/ohai-18.1.4/lib/ohai"
+        "ohai_root": "/opt/chef/embedded/lib/ruby/gems/3.1.0/gems/ohai-18.2.13/lib/ohai"
       }
     }
   },

nodes/garage-7.json

@@ -4,7 +4,8 @@
   "normal": {
     "knife_zero": {
       "host": "10.1.1.182"
-    }
+    },
+    "vm_host": "draco"
   },
   "automatic": {
     "fqdn": "garage-7",
@@ -20,6 +21,7 @@
     "recipes": [
       "kosmos-base",
       "kosmos-base::default",
+      "kosmos_prometheus::node_exporter",
       "kosmos_kvm::guest",
       "kosmos_garage",
       "kosmos_garage::default",
@@ -30,6 +32,7 @@
       "timezone_iii::debian",
       "ntp::default",
       "ntp::apparmor",
+      "kosmos-base::journald_conf",
       "kosmos-base::systemd_emails",
       "apt::unattended-upgrades",
       "kosmos-base::firewall",
@@ -61,4 +64,4 @@
     "role[kvm_guest]",
     "role[garage_node]"
   ]
-}
+}

nodes/garage-8.json

@@ -4,7 +4,8 @@
   "normal": {
     "knife_zero": {
       "host": "10.1.1.61"
-    }
+    },
+    "vm_host": "fornax"
   },
   "automatic": {
     "fqdn": "garage-8",
@@ -20,6 +21,7 @@
     "recipes": [
       "kosmos-base",
       "kosmos-base::default",
+      "kosmos_prometheus::node_exporter",
       "kosmos_kvm::guest",
       "kosmos_garage",
       "kosmos_garage::default",
@@ -30,6 +32,7 @@
       "timezone_iii::debian",
       "ntp::default",
       "ntp::apparmor",
+      "kosmos-base::journald_conf",
       "kosmos-base::systemd_emails",
       "apt::unattended-upgrades",
       "kosmos-base::firewall",
@@ -61,4 +64,4 @@
     "role[kvm_guest]",
     "role[garage_node]"
   ]
-}
+}

nodes/garage-9.json

@@ -1,64 +0,0 @@
-{
-  "name": "garage-9",
-  "chef_environment": "production",
-  "normal": {
-    "knife_zero": {
-      "host": "10.1.1.223"
-    }
-  },
-  "automatic": {
-    "fqdn": "garage-9",
-    "os": "linux",
-    "os_version": "5.4.0-1090-kvm",
-    "hostname": "garage-9",
-    "ipaddress": "192.168.122.21",
-    "roles": [
-      "base",
-      "kvm_guest",
-      "garage_node"
-    ],
-    "recipes": [
-      "kosmos-base",
-      "kosmos-base::default",
-      "kosmos_kvm::guest",
-      "kosmos_garage",
-      "kosmos_garage::default",
-      "kosmos_garage::firewall_rpc",
-      "kosmos_garage::firewall_apis",
-      "apt::default",
-      "timezone_iii::default",
-      "timezone_iii::debian",
-      "ntp::default",
-      "ntp::apparmor",
-      "kosmos-base::systemd_emails",
-      "apt::unattended-upgrades",
-      "kosmos-base::firewall",
-      "kosmos-postfix::default",
-      "postfix::default",
-      "postfix::_common",
-      "postfix::_attributes",
-      "postfix::sasl_auth",
-      "hostname::default",
-      "firewall::default"
-    ],
-    "platform": "ubuntu",
-    "platform_version": "20.04",
-    "cloud": null,
-    "chef_packages": {
-      "chef": {
-        "version": "18.5.0",
-        "chef_root": "/opt/chef/embedded/lib/ruby/gems/3.1.0/gems/chef-18.5.0/lib",
-        "chef_effortless": null
-      },
-      "ohai": {
-        "version": "18.1.11",
-        "ohai_root": "/opt/chef/embedded/lib/ruby/gems/3.1.0/gems/ohai-18.1.11/lib/ohai"
-      }
-    }
-  },
-  "run_list": [
-    "role[base]",
-    "role[kvm_guest]",
-    "role[garage_node]"
-  ]
-}

nodes/gitea-2.json

@@ -4,7 +4,8 @@
   "normal": {
     "knife_zero": {
       "host": "10.1.1.21"
-    }
+    },
+    "vm_host": "fornax"
   },
   "automatic": {
     "fqdn": "gitea-2",
@@ -24,6 +25,7 @@
     "recipes": [
       "kosmos-base",
       "kosmos-base::default",
+      "kosmos_prometheus::node_exporter",
       "kosmos_kvm::guest",
       "kosmos-dirsrv::hostsfile",
       "kosmos_garage",
@@ -58,13 +60,13 @@
     "cloud": null,
     "chef_packages": {
       "chef": {
-        "version": "18.2.7",
+        "version": "18.10.17",
-        "chef_root": "/opt/chef/embedded/lib/ruby/gems/3.1.0/gems/chef-18.2.7/lib",
+        "chef_root": "/opt/chef/embedded/lib/ruby/gems/3.1.0/gems/chef-18.10.17/lib",
         "chef_effortless": null
       },
       "ohai": {
-        "version": "18.1.4",
+        "version": "18.2.13",
-        "ohai_root": "/opt/chef/embedded/lib/ruby/gems/3.1.0/gems/ohai-18.1.4/lib/ohai"
+        "ohai_root": "/opt/chef/embedded/lib/ruby/gems/3.1.0/gems/ohai-18.2.13/lib/ohai"
       }
     }
   },
@@ -76,4 +78,4 @@
     "role[gitea]",
     "role[gitea_actions_runner]"
   ]
-}
+}

nodes/her.json

@@ -9,9 +9,9 @@
   "automatic": {
     "fqdn": "her",
     "os": "linux",
-    "os_version": "5.15.0-84-generic",
+    "os_version": "5.15.0-101-generic",
     "hostname": "her",
-    "ipaddress": "192.168.30.172",
+    "ipaddress": "192.168.30.173",
     "roles": [
       "base",
       "kvm_host"
@@ -19,6 +19,7 @@
     "recipes": [
       "kosmos-base",
       "kosmos-base::default",
+      "kosmos_prometheus::node_exporter",
       "kosmos_kvm::host",
       "kosmos_kvm::backup",
       "apt::default",
@@ -26,6 +27,7 @@
       "timezone_iii::debian",
       "ntp::default",
       "ntp::apparmor",
+      "kosmos-base::journald_conf",
       "kosmos-base::systemd_emails",
       "apt::unattended-upgrades",
       "kosmos-base::firewall",
@@ -55,4 +57,4 @@
     "role[base]",
     "role[kvm_host]"
   ]
-}
+}

nodes/ipfs-1.json

@@ -3,7 +3,8 @@
   "normal": {
     "knife_zero": {
       "host": "10.1.1.198"
-    }
+    },
+    "vm_host": "draco"
   },
   "automatic": {
     "fqdn": "ipfs-1",
@@ -12,12 +13,14 @@
     "hostname": "ipfs-1",
     "ipaddress": "192.168.122.195",
     "roles": [
+      "base",
       "kvm_guest",
       "ipfs_gateway"
     ],
     "recipes": [
       "kosmos-base",
       "kosmos-base::default",
+      "kosmos_prometheus::node_exporter",
       "kosmos_kvm::guest",
       "kosmos-ipfs",
       "kosmos-ipfs::default",
@@ -27,6 +30,7 @@
       "timezone_iii::debian",
       "ntp::default",
       "ntp::apparmor",
+      "kosmos-base::journald_conf",
       "kosmos-base::systemd_emails",
       "apt::unattended-upgrades",
       "kosmos-base::firewall",
@@ -60,7 +64,7 @@
     }
   },
   "run_list": [
-    "recipe[kosmos-base]",
+    "role[base]",
     "role[kvm_guest]",
     "role[ipfs_gateway]"
   ]

nodes/ldap-3.kosmos.org.json

@@ -3,7 +3,8 @@
   "normal": {
     "knife_zero": {
       "host": "10.1.1.6"
-    }
+    },
+    "vm_host": "draco"
   },
   "automatic": {
     "fqdn": "ldap-3.kosmos.org",
@@ -12,12 +13,14 @@
     "hostname": "ldap-3",
     "ipaddress": "192.168.122.34",
     "roles": [
+      "base",
       "kvm_guest",
-      "dirsrv_primary"
+      "dirsrv_supplier"
     ],
     "recipes": [
       "kosmos-base",
       "kosmos-base::default",
+      "kosmos_prometheus::node_exporter",
       "kosmos_kvm::guest",
       "kosmos-dirsrv",
       "kosmos-dirsrv::default",
@@ -26,6 +29,7 @@
       "timezone_iii::debian",
       "ntp::default",
       "ntp::apparmor",
+      "kosmos-base::journald_conf",
       "kosmos-base::systemd_emails",
       "apt::unattended-upgrades",
       "kosmos-base::firewall",
@@ -35,7 +39,7 @@
       "postfix::_attributes",
       "postfix::sasl_auth",
       "hostname::default",
-      "kosmos-dirsrv::hostsfile",
+      "firewall::default",
       "kosmos-dirsrv::firewall",
       "backup::default",
       "logrotate::default",
@@ -46,19 +50,19 @@
     "cloud": null,
     "chef_packages": {
       "chef": {
-        "version": "17.10.3",
+        "version": "18.10.17",
-        "chef_root": "/opt/chef/embedded/lib/ruby/gems/3.0.0/gems/chef-17.10.3/lib",
+        "chef_root": "/opt/chef/embedded/lib/ruby/gems/3.1.0/gems/chef-18.10.17/lib",
         "chef_effortless": null
       },
       "ohai": {
-        "version": "17.9.0",
+        "version": "18.2.13",
-        "ohai_root": "/opt/chef/embedded/lib/ruby/gems/3.0.0/gems/ohai-17.9.0/lib/ohai"
+        "ohai_root": "/opt/chef/embedded/lib/ruby/gems/3.1.0/gems/ohai-18.2.13/lib/ohai"
       }
     }
   },
   "run_list": [
-    "recipe[kosmos-base]",
+    "role[base]",
     "role[kvm_guest]",
     "role[dirsrv_supplier]"
   ]
-}
+}

nodes/ldap-4.kosmos.org.json

@@ -4,7 +4,8 @@
   "normal": {
     "knife_zero": {
       "host": "10.1.1.106"
-    }
+    },
+    "vm_host": "fornax"
   },
   "automatic": {
     "fqdn": "ldap-4.kosmos.org",
@@ -14,17 +15,22 @@
     "ipaddress": "192.168.122.73",
     "roles": [
       "base",
-      "kvm_guest"
+      "kvm_guest",
+      "dirsrv_supplier"
     ],
     "recipes": [
       "kosmos-base",
       "kosmos-base::default",
+      "kosmos_prometheus::node_exporter",
       "kosmos_kvm::guest",
+      "kosmos-dirsrv",
+      "kosmos-dirsrv::default",
       "apt::default",
       "timezone_iii::default",
       "timezone_iii::debian",
       "ntp::default",
       "ntp::apparmor",
+      "kosmos-base::journald_conf",
       "kosmos-base::systemd_emails",
       "apt::unattended-upgrades",
       "kosmos-base::firewall",
@@ -33,20 +39,25 @@
       "postfix::_common",
       "postfix::_attributes",
       "postfix::sasl_auth",
-      "hostname::default"
+      "hostname::default",
+      "firewall::default",
+      "kosmos-dirsrv::firewall",
+      "backup::default",
+      "logrotate::default",
+      "ulimit::default"
     ],
     "platform": "ubuntu",
     "platform_version": "20.04",
     "cloud": null,
     "chef_packages": {
       "chef": {
-        "version": "17.10.3",
+        "version": "18.10.17",
-        "chef_root": "/opt/chef/embedded/lib/ruby/gems/3.0.0/gems/chef-17.10.3/lib",
+        "chef_root": "/opt/chef/embedded/lib/ruby/gems/3.1.0/gems/chef-18.10.17/lib",
         "chef_effortless": null
       },
       "ohai": {
-        "version": "17.9.0",
+        "version": "18.2.13",
-        "ohai_root": "/opt/chef/embedded/lib/ruby/gems/3.0.0/gems/ohai-17.9.0/lib/ohai"
+        "ohai_root": "/opt/chef/embedded/lib/ruby/gems/3.1.0/gems/ohai-18.2.13/lib/ohai"
       }
     }
   },
@@ -55,4 +66,4 @@
     "role[kvm_guest]",
     "role[dirsrv_supplier]"
   ]
-}
+}

nodes/leo.kosmos.org.json

@@ -1,12 +1,13 @@
 {
-  "name": "leo",
+  "name": "leo.kosmos.org",
+  "chef_environment": "production",
   "normal": {
     "knife_zero": {
-      "host": "leo.kosmos.org"
+      "host": "10.1.1.204"
     }
   },
   "automatic": {
-    "fqdn": "leo",
+    "fqdn": "leo.kosmos.org",
     "os": "linux",
     "os_version": "5.15.0-173-generic",
     "hostname": "leo",
@@ -17,6 +18,7 @@
     "recipes": [
       "kosmos-base",
       "kosmos-base::default",
+      "kosmos_prometheus::node_exporter",
       "kosmos_kvm::host",
       "apt::default",
       "timezone_iii::default",
@@ -32,7 +34,8 @@
       "postfix::_common",
       "postfix::_attributes",
       "postfix::sasl_auth",
-      "hostname::default"
+      "hostname::default",
+      "firewall::default"
     ],
     "platform": "ubuntu",
     "platform_version": "22.04",
@@ -53,4 +56,4 @@
     "role[base]",
     "recipe[kosmos_kvm::host]"
   ]
-}
+}

nodes/lq-1.json

@@ -4,7 +4,8 @@
   "normal": {
     "knife_zero": {
       "host": "10.1.1.87"
-    }
+    },
+    "vm_host": "draco"
   },
   "automatic": {
     "fqdn": "lq-1",
@@ -21,6 +22,7 @@
     "recipes": [
       "kosmos-base",
       "kosmos-base::default",
+      "kosmos_prometheus::node_exporter",
       "kosmos_kvm::guest",
       "kosmos_garage",
       "kosmos_garage::default",
@@ -32,6 +34,7 @@
       "timezone_iii::debian",
       "ntp::default",
       "ntp::apparmor",
+      "kosmos-base::journald_conf",
       "kosmos-base::systemd_emails",
       "apt::unattended-upgrades",
       "kosmos-base::firewall",
@@ -49,13 +52,13 @@
     "cloud": null,
     "chef_packages": {
       "chef": {
-        "version": "18.2.7",
+        "version": "18.10.17",
-        "chef_root": "/opt/chef/embedded/lib/ruby/gems/3.1.0/gems/chef-18.2.7/lib",
+        "chef_root": "/opt/chef/embedded/lib/ruby/gems/3.1.0/gems/chef-18.10.17/lib",
         "chef_effortless": null
       },
       "ohai": {
-        "version": "18.1.4",
+        "version": "18.2.13",
-        "ohai_root": "/opt/chef/embedded/lib/ruby/gems/3.1.0/gems/ohai-18.1.4/lib/ohai"
+        "ohai_root": "/opt/chef/embedded/lib/ruby/gems/3.1.0/gems/ohai-18.2.13/lib/ohai"
       }
     }
   },

nodes/lq-2.json

@@ -4,7 +4,8 @@
   "normal": {
     "knife_zero": {
       "host": "10.1.1.188"
-    }
+    },
+    "vm_host": "fornax"
   },
   "automatic": {
     "fqdn": "lq-2",
@@ -21,6 +22,7 @@
     "recipes": [
       "kosmos-base",
       "kosmos-base::default",
+      "kosmos_prometheus::node_exporter",
       "kosmos_kvm::guest",
       "kosmos_garage",
       "kosmos_garage::default",
@@ -32,6 +34,7 @@
       "timezone_iii::debian",
       "ntp::default",
       "ntp::apparmor",
+      "kosmos-base::journald_conf",
       "kosmos-base::systemd_emails",
       "apt::unattended-upgrades",
       "kosmos-base::firewall",
@@ -49,13 +52,13 @@
     "cloud": null,
     "chef_packages": {
       "chef": {
-        "version": "18.2.7",
+        "version": "18.10.17",
-        "chef_root": "/opt/chef/embedded/lib/ruby/gems/3.1.0/gems/chef-18.2.7/lib",
+        "chef_root": "/opt/chef/embedded/lib/ruby/gems/3.1.0/gems/chef-18.10.17/lib",
         "chef_effortless": null
       },
       "ohai": {
-        "version": "18.1.4",
+        "version": "18.2.13",
-        "ohai_root": "/opt/chef/embedded/lib/ruby/gems/3.1.0/gems/ohai-18.1.4/lib/ohai"
+        "ohai_root": "/opt/chef/embedded/lib/ruby/gems/3.1.0/gems/ohai-18.2.13/lib/ohai"
       }
     }
   },

nodes/mail.kosmos.org.json

@@ -4,13 +4,14 @@
   "normal": {
     "knife_zero": {
       "host": "10.1.1.95"
-    }
+    },
+    "vm_host": "draco"
   },
   "automatic": {
     "fqdn": "mail.kosmos.org",
     "os": "linux",
     "os_version": "5.15.0-1048-kvm",
-    "hostname": "mail.kosmos.org",
+    "hostname": "mail",
     "ipaddress": "192.168.122.131",
     "roles": [
       "base",
@@ -21,6 +22,7 @@
     "recipes": [
       "kosmos-base",
       "kosmos-base::default",
+      "kosmos_prometheus::node_exporter",
       "kosmos_kvm::guest",
       "kosmos-dirsrv::hostsfile",
       "kosmos_email",
@@ -68,4 +70,4 @@
     "role[kvm_guest]",
     "role[email_server]"
   ]
-}
+}

nodes/mastodon-3.json

@@ -4,7 +4,8 @@
   "normal": {
     "knife_zero": {
       "host": "10.1.1.30"
-    }
+    },
+    "vm_host": "fornax"
   },
   "automatic": {
     "fqdn": "mastodon-3",
@@ -83,7 +84,7 @@
     }
   },
   "run_list": [
-    "recipe[kosmos-base]",
+    "role[base]",
     "role[kvm_guest]",
     "role[ldap_client]",
     "role[garage_gateway]",

nodes/nodejs-4.json

@@ -3,7 +3,8 @@
   "normal": {
     "knife_zero": {
       "host": "10.1.1.138"
-    }
+    },
+    "vm_host": "fornax"
   },
   "automatic": {
     "fqdn": "nodejs-4",
@@ -21,6 +22,7 @@
     "recipes": [
       "kosmos-base",
       "kosmos-base::default",
+      "kosmos_prometheus::node_exporter",
       "kosmos_kvm::guest",
       "kosmos-hubot::botka_irc-libera-chat",
       "kosmos-hubot::hal8000_xmpp",
@@ -35,6 +37,7 @@
       "timezone_iii::debian",
       "ntp::default",
       "ntp::apparmor",
+      "kosmos-base::journald_conf",
       "kosmos-base::systemd_emails",
       "apt::unattended-upgrades",
       "kosmos-base::firewall",

nodes/postgres-11.json

@@ -4,7 +4,8 @@
   "normal": {
     "knife_zero": {
       "host": "10.1.1.91"
-    }
+    },
+    "vm_host": "leo"
   },
   "automatic": {
     "fqdn": "postgres-11",
@@ -20,6 +21,7 @@
     "recipes": [
       "kosmos-base",
       "kosmos-base::default",
+      "kosmos_prometheus::node_exporter",
       "kosmos_kvm::guest",
       "kosmos_postgresql::primary",
       "kosmos-akkounts::pg_db",
@@ -66,4 +68,4 @@
     "role[kvm_guest]",
     "role[postgresql_primary]"
   ]
-}
+}

nodes/postgres-12.json

@@ -4,7 +4,8 @@
   "normal": {
     "knife_zero": {
       "host": "10.1.1.134"
-    }
+    },
+    "vm_host": "her"
   },
   "automatic": {
     "fqdn": "postgres-12",
@@ -20,6 +21,7 @@
     "recipes": [
       "kosmos-base",
       "kosmos-base::default",
+      "kosmos_prometheus::node_exporter",
       "kosmos_kvm::guest",
       "kosmos_postgresql::hostsfile",
       "kosmos_postgresql::replica",
@@ -61,4 +63,4 @@
     "role[kvm_guest]",
     "role[postgresql_replica]"
   ]
-}
+}

nodes/prometheus-1.json

@@ -1,35 +1,33 @@
 {
-  "name": "garage-13",
+  "name": "prometheus-1",
   "chef_environment": "production",
   "normal": {
     "knife_zero": {
-      "host": "10.1.1.179"
+      "host": "10.1.1.146"
-    }
+    },
+    "vm_host": "leo"
   },
   "automatic": {
-    "fqdn": "garage-13",
+    "fqdn": "prometheus-1",
     "os": "linux",
-    "os_version": "5.15.0-1059-kvm",
+    "os_version": "6.8.0-134-generic",
-    "hostname": "garage-13",
+    "hostname": "prometheus-1",
-    "ipaddress": "192.168.122.27",
+    "ipaddress": "192.168.122.166",
     "roles": [
       "base",
       "kvm_guest",
-      "garage_node"
+      "prometheus_server"
     ],
     "recipes": [
       "kosmos-base",
       "kosmos-base::default",
+      "kosmos_prometheus::node_exporter",
       "kosmos_kvm::guest",
-      "kosmos_garage",
+      "kosmos_prometheus::server",
-      "kosmos_garage::default",
+      "kosmos_prometheus::alertmanager",
-      "kosmos_garage::firewall_rpc",
-      "kosmos_garage::firewall_apis",
       "apt::default",
       "timezone_iii::default",
       "timezone_iii::debian",
-      "ntp::default",
-      "ntp::apparmor",
       "kosmos-base::journald_conf",
       "kosmos-base::systemd_emails",
       "apt::unattended-upgrades",
@@ -43,23 +41,23 @@
       "firewall::default"
     ],
     "platform": "ubuntu",
-    "platform_version": "22.04",
+    "platform_version": "24.04",
     "cloud": null,
     "chef_packages": {
       "chef": {
-        "version": "18.7.10",
+        "version": "18.10.17",
-        "chef_root": "/opt/chef/embedded/lib/ruby/gems/3.1.0/gems/chef-18.7.10/lib",
+        "chef_root": "/opt/chef/embedded/lib/ruby/gems/3.1.0/gems/chef-18.10.17/lib",
         "chef_effortless": null
       },
       "ohai": {
-        "version": "18.2.5",
+        "version": "18.2.13",
-        "ohai_root": "/opt/chef/embedded/lib/ruby/gems/3.1.0/gems/ohai-18.2.5/lib/ohai"
+        "ohai_root": "/opt/chef/embedded/lib/ruby/gems/3.1.0/gems/ohai-18.2.13/lib/ohai"
       }
     }
   },
   "run_list": [
     "role[base]",
     "role[kvm_guest]",
-    "role[garage_node]"
+    "role[prometheus_server]"
   ]
-}
+}

nodes/redis-1.json

@@ -4,7 +4,8 @@
   "normal": {
     "knife_zero": {
       "host": "10.1.1.225"
-    }
+    },
+    "vm_host": "draco"
   },
   "automatic": {
     "fqdn": "redis-1",
@@ -20,6 +21,7 @@
     "recipes": [
       "kosmos-base",
       "kosmos-base::default",
+      "kosmos_prometheus::node_exporter",
       "kosmos_kvm::guest",
       "kosmos_redis",
       "kosmos_redis::default",
@@ -30,6 +32,7 @@
       "timezone_iii::debian",
       "ntp::default",
       "ntp::apparmor",
+      "kosmos-base::journald_conf",
       "kosmos-base::systemd_emails",
       "apt::unattended-upgrades",
       "kosmos-base::firewall",
@@ -39,6 +42,7 @@
       "postfix::_attributes",
       "postfix::sasl_auth",
       "hostname::default",
+      "firewall::default",
       "redisio::default",
       "redisio::_install_prereqs",
       "redisio::install",

nodes/redis-2.json

@@ -4,7 +4,8 @@
   "normal": {
     "knife_zero": {
       "host": "10.1.1.208"
-    }
+    },
+    "vm_host": "fornax"
   },
   "automatic": {
     "fqdn": "redis-2",
@@ -20,6 +21,7 @@
     "recipes": [
       "kosmos-base",
       "kosmos-base::default",
+      "kosmos_prometheus::node_exporter",
       "kosmos_kvm::guest",
       "kosmos_redis::replica",
       "kosmos_redis::firewall",
@@ -28,6 +30,7 @@
       "timezone_iii::debian",
       "ntp::default",
       "ntp::apparmor",
+      "kosmos-base::journald_conf",
       "kosmos-base::systemd_emails",
       "apt::unattended-upgrades",
       "kosmos-base::firewall",
@@ -37,6 +40,7 @@
       "postfix::_attributes",
       "postfix::sasl_auth",
       "hostname::default",
+      "firewall::default",
       "redisio::default",
       "redisio::_install_prereqs",
       "redisio::install",

nodes/rs-discourse-1.json

@@ -3,29 +3,33 @@
   "normal": {
     "knife_zero": {
       "host": "10.1.1.187"
-    }
+    },
+    "vm_host": "fornax"
   },
   "automatic": {
     "fqdn": "rs-discourse-1",
     "os": "linux",
-    "os_version": "5.4.0-1076-kvm",
+    "os_version": "5.4.0-1118-kvm",
     "hostname": "rs-discourse-1",
     "ipaddress": "192.168.122.30",
     "roles": [
-      "remotestorage_discourse",
+      "base",
-      "kvm_guest"
+      "kvm_guest",
+      "remotestorage_discourse"
     ],
     "recipes": [
       "kosmos-base",
       "kosmos-base::default",
+      "kosmos_prometheus::node_exporter",
+      "kosmos_kvm::guest",
       "remotestorage_discourse",
       "remotestorage_discourse::default",
-      "kosmos_kvm::guest",
       "apt::default",
       "timezone_iii::default",
       "timezone_iii::debian",
       "ntp::default",
       "ntp::apparmor",
+      "kosmos-base::journald_conf",
       "kosmos-base::systemd_emails",
       "apt::unattended-upgrades",
       "kosmos-base::firewall",
@@ -35,28 +39,27 @@
       "postfix::_attributes",
       "postfix::sasl_auth",
       "hostname::default",
-      "discourse::default",
       "firewall::default",
-      "chef-sugar::default"
+      "discourse::default"
     ],
     "platform": "ubuntu",
     "platform_version": "20.04",
     "cloud": null,
     "chef_packages": {
       "chef": {
-        "version": "17.10.3",
+        "version": "18.10.17",
-        "chef_root": "/opt/chef/embedded/lib/ruby/gems/3.0.0/gems/chef-17.10.3/lib",
+        "chef_root": "/opt/chef/embedded/lib/ruby/gems/3.1.0/gems/chef-18.10.17/lib",
         "chef_effortless": null
       },
       "ohai": {
-        "version": "17.9.0",
+        "version": "18.2.13",
-        "ohai_root": "/opt/chef/embedded/lib/ruby/gems/3.0.0/gems/ohai-17.9.0/lib/ohai"
+        "ohai_root": "/opt/chef/embedded/lib/ruby/gems/3.1.0/gems/ohai-18.2.13/lib/ohai"
       }
     }
   },
   "run_list": [
-    "recipe[kosmos-base]",
+    "role[base]",
     "role[kvm_guest]",
     "role[remotestorage_discourse]"
   ]
-}
+}

nodes/rsk-mainnet-3.json

@@ -3,7 +3,8 @@
   "normal": {
     "knife_zero": {
       "host": "10.1.1.150"
-    }
+    },
+    "vm_host": "her"
   },
   "automatic": {
     "fqdn": "rsk-mainnet-3",
@@ -58,4 +59,4 @@
     "role[kvm_guest]",
     "role[rskj_mainnet]"
   ]
-}
+}

nodes/rsk-testnet-4.json

@@ -3,7 +3,8 @@
   "normal": {
     "knife_zero": {
       "host": "10.1.1.126"
-    }
+    },
+    "vm_host": "her"
   },
   "automatic": {
     "fqdn": "rsk-testnet-4",
@@ -57,4 +58,4 @@
     "role[base]",
     "role[kvm_guest]"
   ]
-}
+}

nodes/rsk-testnet-6.json

@@ -3,7 +3,8 @@
   "normal": {
     "knife_zero": {
       "host": "10.1.1.20"
-    }
+    },
+    "vm_host": "leo"
   },
   "automatic": {
     "fqdn": "rsk-testnet-6",
@@ -19,6 +20,7 @@
     "recipes": [
       "kosmos-base",
       "kosmos-base::default",
+      "kosmos_prometheus::node_exporter",
       "kosmos_kvm::guest",
       "kosmos_rsk::rskj",
       "apt::default",
@@ -34,8 +36,8 @@
       "postfix::_attributes",
       "postfix::sasl_auth",
       "hostname::default",
-      "kosmos_rsk::firewall",
+      "firewall::default",
-      "firewall::default"
+      "kosmos_rsk::firewall"
     ],
     "platform": "ubuntu",
     "platform_version": "24.04",
@@ -57,4 +59,4 @@
     "role[kvm_guest]",
     "role[rskj_testnet]"
   ]
-}
+}

nodes/sentry-1.json

@@ -4,7 +4,8 @@
   "normal": {
     "knife_zero": {
       "host": "10.1.1.132"
-    }
+    },
+    "vm_host": "her"
   },
   "automatic": {
     "fqdn": "sentry-1",

nodes/strfry-1.json

@@ -4,7 +4,8 @@
   "normal": {
     "knife_zero": {
       "host": "10.1.1.164"
-    }
+    },
+    "vm_host": "draco"
   },
   "automatic": {
     "fqdn": "strfry-1",
@@ -22,6 +23,7 @@
     "recipes": [
       "kosmos-base",
       "kosmos-base::default",
+      "kosmos_prometheus::node_exporter",
       "kosmos_kvm::guest",
       "kosmos-dirsrv::hostsfile",
       "strfry",
@@ -70,4 +72,4 @@
     "role[strfry]",
     "role[blossom]"
   ]
-}
+}

nodes/uploads-1.json

@@ -1,67 +0,0 @@
-{
-  "name": "uploads-1",
-  "normal": {
-    "knife_zero": {
-      "host": "10.1.1.98"
-    }
-  },
-  "automatic": {
-    "fqdn": "uploads-1",
-    "os": "linux",
-    "os_version": "5.4.0-128-generic",
-    "hostname": "uploads-1",
-    "ipaddress": "192.168.122.230",
-    "roles": [
-      "kvm_guest"
-    ],
-    "recipes": [
-      "kosmos-base",
-      "kosmos-base::default",
-      "kosmos_kvm::guest",
-      "kosmos-ejabberd::upload_service",
-      "apt::default",
-      "timezone_iii::default",
-      "timezone_iii::debian",
-      "ntp::default",
-      "ntp::apparmor",
-      "kosmos-base::systemd_emails",
-      "apt::unattended-upgrades",
-      "kosmos-base::firewall",
-      "kosmos-postfix::default",
-      "postfix::default",
-      "postfix::_common",
-      "postfix::_attributes",
-      "postfix::sasl_auth",
-      "hostname::default",
-      "kosmos-nginx::with_perl",
-      "nginx::default",
-      "nginx::package",
-      "nginx::ohai_plugin",
-      "nginx::repo",
-      "nginx::commons",
-      "nginx::commons_dir",
-      "nginx::commons_script",
-      "nginx::commons_conf",
-      "kosmos-nginx::firewall",
-      "kosmos-base::letsencrypt"
-    ],
-    "platform": "ubuntu",
-    "platform_version": "20.04",
-    "cloud": null,
-    "chef_packages": {
-      "chef": {
-        "version": "15.14.0",
-        "chef_root": "/opt/chef/embedded/lib/ruby/gems/2.6.0/gems/chef-15.14.0/lib"
-      },
-      "ohai": {
-        "version": "15.12.0",
-        "ohai_root": "/opt/chef/embedded/lib/ruby/gems/2.6.0/gems/ohai-15.12.0/lib/ohai"
-      }
-    }
-  },
-  "run_list": [
-    "recipe[kosmos-base]",
-    "role[kvm_guest]",
-    "recipe[kosmos-ejabberd::upload_service]"
-  ]
-}

nodes/wiki-1.json

@@ -3,7 +3,8 @@
   "normal": {
     "knife_zero": {
       "host": "10.1.1.206"
-    }
+    },
+    "vm_host": "draco"
   },
   "automatic": {
     "fqdn": "wiki-1",
@@ -19,6 +20,7 @@
     "recipes": [
       "kosmos-base",
       "kosmos-base::default",
+      "kosmos_prometheus::node_exporter",
       "kosmos_kvm::guest",
       "kosmos-dirsrv::hostsfile",
       "kosmos-mediawiki",

roles/base.rb

@@ -2,4 +2,5 @@ name "base"
 
 run_list %w(
   kosmos-base::default
+  kosmos_prometheus::node_exporter
 )

roles/prometheus_server.rb

@@ -0,0 +1,12 @@
+name "prometheus_server"
+
+default_run_list = [
+  "kosmos_prometheus::server",
+  "kosmos_prometheus::alertmanager"
+]
+
+env_run_lists(
+  "_default" => default_run_list,
+  "development" => default_run_list,
+  "production" => default_run_list
+)

scripts/util/sanitize_node.sh

@@ -0,0 +1,69 @@
+#!/usr/bin/env bash
+# Strip the top-level "override" and "default" attributes from Chef node JSON files.
+#
+# Usage:
+#   ./scripts/util/strip_node_attrs.sh <node> [<node> ...]
+#   ./scripts/util/strip_node_attrs.sh --all
+#
+# A <node> may be a bare node name (e.g. "prometheus-1"), a path relative to the
+# repository root (e.g. "nodes/prometheus-1.json"), or an absolute path.
+#
+# Exit codes:
+#   0 - all target files processed successfully
+#   1 - one or more targets could not be processed (missing args, missing jq,
+#       file not found, invalid JSON, or write failure)
+
+set -uo pipefail
+
+command -v jq >/dev/null 2>&1 || exit 1
+
+root="$(git rev-parse --show-toplevel 2>/dev/null)" || root="$(pwd)"
+
+targets=()
+if [ "$#" -eq 0 ]; then
+  exit 1
+fi
+
+for arg in "$@"; do
+  if [ "$arg" = "--all" ]; then
+    shopt -s nullglob
+    for f in "$root"/nodes/*.json; do
+      targets+=("$f")
+    done
+    shopt -u nullglob
+    continue
+  fi
+
+  if [[ "$arg" == */* ]]; then
+    if [[ "$arg" = /* ]]; then
+      targets+=("$arg")
+    else
+      targets+=("$root/$arg")
+    fi
+  else
+    targets+=("$root/nodes/$arg.json")
+  fi
+done
+
+if [ "${#targets[@]}" -eq 0 ]; then
+  exit 1
+fi
+
+rc=0
+for file in "${targets[@]}"; do
+  [ -f "$file" ] || { rc=1; continue; }
+
+  jq -e . "$file" >/dev/null 2>&1 || { rc=1; continue; }
+
+  tmp="$(mktemp "${file}.XXXXXX")" || { rc=1; continue; }
+  if jq --indent 2 'del(.override, .default)' "$file" > "$tmp" 2>/dev/null; then
+    truncate -s -1 "$tmp" 2>/dev/null || true
+    mv "$tmp" "$file" || { rm -f "$tmp"; rc=1; continue; }
+  else
+    rm -f "$tmp"
+    rc=1
+    continue
+  fi
+done
+
+exit "$rc"

site-cookbooks/kosmos-postfix/recipes/default.rb

@@ -21,5 +21,6 @@ node.default["postfix"]["sasl"] = {
 node.default["postfix"]["main"]["relayhost"]                  = smtp_credentials["relayhost"]
 node.default["postfix"]["main"]["smtp_sasl_auth_enable"]      = "yes"
 node.default["postfix"]["main"]["smtp_sasl_security_options"] = "noanonymous"
+node.default["postfix"]["main"]["smtpd_relay_restrictions"] = "permit_mynetworks, reject"
 
 include_recipe "postfix::default"

site-cookbooks/kosmos_prometheus/.gitignore

@@ -0,0 +1,25 @@
+.vagrant
+*~
+*#
+.#*
+\#*#
+.*.sw[a-z]
+*.un~
+
+# Bundler
+Gemfile.lock
+gems.locked
+bin/*
+.bundle/*
+
+# test kitchen
+.kitchen/
+kitchen.local.yml
+
+# Chef Infra
+Berksfile.lock
+.zero-knife.rb
+Policyfile.lock.json
+
+.idea/
+

site-cookbooks/kosmos_prometheus/CHANGELOG.md

@@ -0,0 +1,7 @@
+# kosmos_prometheus CHANGELOG
+
+This file is used to list changes made in each version of the kosmos_prometheus cookbook.
+
+## 0.1.0
+
+Initial release.

site-cookbooks/kosmos_prometheus/LICENSE

@@ -0,0 +1,20 @@
+Copyright (c) 2026 Kosmos Developers
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

site-cookbooks/kosmos_prometheus/README.md

@@ -0,0 +1,4 @@
+# kosmos_prometheus
+
+TODO: Enter the cookbook description here.
+

site-cookbooks/kosmos_prometheus/attributes/default.rb

@@ -0,0 +1,85 @@
+node.default["kosmos_prometheus"]["version"] = "3.13.0"
+node.default["kosmos_prometheus"]["checksum"] = "744d93324cc024d82089921737bd797474d7f1e5dbbfd1c6b387bad258538cb9"
+
+node.default["kosmos_prometheus"]["data_dir"] = "/var/lib/prometheus"
+node.default["kosmos_prometheus"]["config_dir"] = "/etc/prometheus"
+node.default["kosmos_prometheus"]["rules_dir"] = "#{node["kosmos_prometheus"]["config_dir"]}/rules"
+
+node.default["kosmos_prometheus"]["web"]["port"] = 9090
+
+node.default["kosmos_prometheus"]["alertmanager"]["version"] = "0.33.0"
+node.default["kosmos_prometheus"]["alertmanager"]["checksum"] = "8ce11c42e8a6dfbbf93a59c0b193cb1329210b36d0c7ef3df7b745608675a1d1"
+
+node.default["kosmos_prometheus"]["node_exporter"]["version"] = "1.11.1"
+node.default["kosmos_prometheus"]["node_exporter"]["checksum"] = "9f5ea48e5bc7b656f8a91a32e7d7deb89f70f73dabd0d974418aca15f37d6810"
+node.default["kosmos_prometheus"]["node_exporter"]["port"] = 9100
+
+node.default["kosmos_prometheus"]["alertmanager"]["route"] = {
+  "receiver" => "xmpp",
+  "group_by" => ["alertname", "severity"],
+  "group_wait" => "20s",
+  "group_interval" => "30s",
+  "repeat_interval" => "3h",
+  "routes" => [
+    {
+      "receiver" => "xmpp",
+      "continue" => true,
+    },
+    {
+      "receiver" => "email",
+      "matchers" => ['severity="critical"'],
+      "continue" => true,
+    }
+  ],
+}
+
+node.default["kosmos_prometheus"]["alertmanager"]["port"] = 9093
+
+node.default["kosmos_prometheus"]["alertmanager"]["receivers"] = [
+  {
+    "name" => "xmpp",
+    "webhook_configs" => [
+      {
+        "send_resolved" => true,
+        "room" => "info@kosmos.chat",
+      }
+    ]
+  },
+  {
+    "name" => "email",
+    "email_configs" => [
+      {
+        "send_resolved" => true,
+        "to" => "ops@kosmos.org",
+        "from" => "ops@kosmos.org",
+        "smarthost" => "localhost:25",
+        "require_tls" => false,
+      }
+    ]
+  }
+]
+
+node.default["kosmos_prometheus"]["global"] = {
+  "scrape_interval" => "15s",
+  "evaluation_interval" => "15s",
+}
+
+node.default["kosmos_prometheus"]["alerting"] = {
+  "alertmanagers" => [
+    "static_configs" => [
+      { "targets" => ["127.0.0.1:#{node["kosmos_prometheus"]["alertmanager"]["port"]}"] }
+    ]
+  ]
+}
+
+node.default["kosmos_prometheus"]["jobs"] = {
+  "prometheus" => { "targets" => [{ "target" => "localhost:#{node["kosmos_prometheus"]["web"]["port"]}", "instance" => "localhost" }] },
+}
+
+node.default["kosmos_prometheus"]["rule_files"] = [
+  "#{node["kosmos_prometheus"]["rules_dir"]}/*.yml"
+]
+
+node.default["kosmos_prometheus"]["rules"] = [
+  "node_exporter"
+]

site-cookbooks/kosmos_prometheus/chefignore

@@ -0,0 +1,115 @@
+# Put files/directories that should be ignored in this file when uploading
+# to a Chef Infra Server or Supermarket.
+# Lines that start with '# ' are comments.
+
+# OS generated files #
+######################
+.DS_Store
+ehthumbs.db
+Icon?
+nohup.out
+Thumbs.db
+.envrc
+
+# EDITORS #
+###########
+.#*
+.project
+.settings
+*_flymake
+*_flymake.*
+*.bak
+*.sw[a-z]
+*.tmproj
+*~
+\#*
+REVISION
+TAGS*
+tmtags
+.vscode
+.editorconfig
+
+## COMPILED ##
+##############
+*.class
+*.com
+*.dll
+*.exe
+*.o
+*.pyc
+*.so
+*/rdoc/
+a.out
+mkmf.log
+
+# Testing #
+###########
+.circleci/*
+.codeclimate.yml
+.delivery/*
+.foodcritic
+.kitchen*
+.mdlrc
+.overcommit.yml
+.rspec
+.rubocop.yml
+.travis.yml
+.watchr
+.yamllint
+azure-pipelines.yml
+Dangerfile
+examples/*
+features/*
+Guardfile
+kitchen.yml*
+mlc_config.json
+Procfile
+Rakefile
+spec/*
+test/*
+
+# SCM #
+#######
+.git
+.gitattributes
+.gitconfig
+.github/*
+.gitignore
+.gitkeep
+.gitmodules
+.svn
+*/.bzr/*
+*/.git
+*/.hg/*
+*/.svn/*
+
+# Berkshelf #
+#############
+Berksfile
+Berksfile.lock
+cookbooks/*
+tmp
+
+# Bundler #
+###########
+vendor/*
+Gemfile
+Gemfile.lock
+
+# Policyfile #
+##############
+Policyfile.rb
+Policyfile.lock.json
+
+# Documentation #
+#############
+CODE_OF_CONDUCT*
+CONTRIBUTING*
+documentation/*
+TESTING*
+UPGRADING*
+
+# Vagrant #
+###########
+.vagrant
+Vagrantfile

site-cookbooks/kosmos_prometheus/files/default/rules/node_exporter.yml

@@ -0,0 +1,94 @@
+groups:
+  - name: NodeExporter
+    rules:
+      - alert: NodeDown
+        expr: up{job="node"} == 0
+        for: 20s
+        labels:
+          severity: warning
+        annotations:
+          summary: "Node is down"
+
+      - alert: NodeFilesystemAlmostOutOfSpace
+        expr: |
+          (node_filesystem_avail_bytes{fstype!~"tmpfs|fuse.lxcfs|overlay|squashfs|ramfs",mountpoint!~"/run.*|/var/lib/docker/.*"}
+            / node_filesystem_size_bytes{fstype!~"tmpfs|fuse.lxcfs|overlay|squashfs|ramfs",mountpoint!~"/run.*|/var/lib/docker/.*"}) * 100 < 10
+        for: 5m
+        labels:
+          severity: warning
+        annotations:
+          summary: "Filesystem has < 10% free space"
+
+      - alert: NodeFilesystemAlmostOutOfSpace
+        expr: |
+          (node_filesystem_avail_bytes{fstype!~"tmpfs|fuse.lxcfs|overlay|squashfs|ramfs",mountpoint!~"/run.*|/var/lib/docker/.*"}
+            / node_filesystem_size_bytes{fstype!~"tmpfs|fuse.lxcfs|overlay|squashfs|ramfs",mountpoint!~"/run.*|/var/lib/docker/.*"}) * 100 < 5
+        for: 5m
+        labels:
+          severity: critical
+        annotations:
+          summary: "Filesystem has < 5% free space"
+
+      - alert: NodeFilesystemAlmostOutOfInodes
+        expr: (node_filesystem_files_free / node_filesystem_files) * 100 < 10
+        for: 5m
+        labels:
+          severity: warning
+        annotations:
+          summary: "Filesystem has < 10% free inodes"
+
+      - alert: NodeFilesystemReadOnly
+        expr: node_filesystem_readonly == 1
+        for: 2m
+        labels:
+          severity: critical
+        annotations:
+          summary: "Filesystem remounted read-only"
+
+      - alert: NodeMemoryHigh
+        expr: (1 - node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes) * 100 > 90
+        for: 10m
+        labels:
+          severity: warning
+        annotations:
+          summary: "Memory usage > 90%"
+
+      - alert: NodeMemoryHigh
+        expr: (1 - node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes) * 100 > 95
+        for: 5m
+        labels:
+          severity: critical
+        annotations:
+          summary: "Memory usage > 95%"
+
+      - alert: NodeSwapHigh
+        expr: (node_memory_SwapTotal_bytes > 0) and ((1 - node_memory_SwapFree_bytes / node_memory_SwapTotal_bytes) * 100 > 50)
+        for: 10m
+        labels:
+          severity: warning
+        annotations:
+          summary: "Swap usage > 50%"
+
+      - alert: NodeCpuHigh
+        expr: 100 - (avg by (instance) (rate(node_cpu_seconds_total{mode="idle"}[5m])) * 100) > 80
+        for: 10m
+        labels:
+          severity: warning
+        annotations:
+          summary: "CPU usage > 80% for 10m"
+
+      - alert: NodeCpuHigh
+        expr: 100 - (avg by (instance) (rate(node_cpu_seconds_total{mode="idle"}[5m])) * 100) > 95
+        for: 5m
+        labels:
+          severity: critical
+        annotations:
+          summary: "CPU usage > 95% for 5m"
+
+      - alert: NodeLoadHigh
+        expr: node_load1 / count by (instance) (node_cpu_seconds_total{mode="idle"}) > 1.5
+        for: 10m
+        labels:
+          severity: warning
+        annotations:
+          summary: "System load1 > 1.5x CPU cores"

site-cookbooks/kosmos_prometheus/metadata.rb

@@ -0,0 +1,21 @@
+name 'kosmos_prometheus'
+maintainer 'Kosmos Developers'
+maintainer_email 'mail@kosmos.org'
+license 'MIT'
+description 'Installs/Configures prometheus'
+version '0.1.0'
+chef_version '>= 16.0'
+
+depends "firewall"
+
+# The `issues_url` points to the location where issues for this cookbook are
+# tracked.  A `View Issues` link will be displayed on this cookbook's page when
+# uploaded to a Supermarket.
+#
+# issues_url 'https://github.com/<insert_org_here>/kosmos_prometheus/issues'
+
+# The `source_url` points to the development repository for this cookbook.  A
+# `View Source` link will be displayed on this cookbook's page when uploaded to
+# a Supermarket.
+#
+# source_url 'https://github.com/<insert_org_here>/kosmos_prometheus'

site-cookbooks/kosmos_prometheus/recipes/alertmanager.rb

@@ -0,0 +1,107 @@
+#
+# Cookbook:: kosmos_prometheus
+# Recipe:: alertmanager
+#
+
+include_recipe "firewall"
+
+version = node["kosmos_prometheus"]["alertmanager"]["version"]
+checksum = node["kosmos_prometheus"]["alertmanager"]["checksum"]
+tarball = "#{Chef::Config[:file_cache_path]}/alertmanager-#{version}.linux-amd64.tar.gz"
+binary_url = "https://github.com/prometheus/alertmanager/releases/download/v#{version}/alertmanager-#{version}.linux-amd64.tar.gz"
+
+alertmanager_port = node["kosmos_prometheus"]["alertmanager"]["port"]
+
+group "alertmanager"
+
+user "alertmanager" do
+  gid "alertmanager"
+  system true
+  shell "/bin/false"
+  home "/nonexistent"
+end
+
+directory "/var/lib/alertmanager" do
+  owner "alertmanager"
+  group "alertmanager"
+  mode "0755"
+  recursive true
+end
+
+directory "/etc/prometheus" do
+  owner "root"
+  group "root"
+  mode "0755"
+  recursive true
+end
+
+package %w(tar bzip2)
+
+remote_file tarball do
+  source binary_url
+  checksum checksum
+  action :create
+  notifies :run, "execute[install_alertmanager]", :immediately
+end
+
+execute "install_alertmanager" do
+  command "tar -xzf #{tarball} -C /usr/local/bin --strip-components=1 alertmanager-#{version}.linux-amd64/alertmanager"
+  action :nothing
+  notifies :restart, "service[alertmanager]", :delayed
+end
+
+file "/usr/local/bin/alertmanager" do
+  owner "root"
+  group "root"
+  mode "0755"
+  notifies :restart, "service[alertmanager]", :delayed
+end
+
+# TODO use Auth header once supported by hubot-incoming-webhook
+credentials = data_bag_item("credentials", "hal8000_xmpp")
+webhook_url = "https://hal8000.kosmos.chat/incoming/#{credentials['webhook_token']}"
+
+template "/etc/prometheus/alertmanager.yml" do
+  source "alertmanager.yml.erb"
+  owner "root"
+  group "alertmanager"
+  mode "0644"
+  variables route: node["kosmos_prometheus"]["alertmanager"]["route"],
+            receivers: node["kosmos_prometheus"]["alertmanager"]["receivers"],
+            webhook_url: webhook_url
+  notifies :restart, "service[alertmanager]", :delayed
+  sensitive true
+end
+
+systemd_unit "alertmanager.service" do
+  content({
+    Unit: {
+      Description: "Prometheus Alertmanager",
+      After: "network.target",
+    },
+    Service: {
+      Type: "simple",
+      User: "alertmanager",
+      Group: "alertmanager",
+      ExecStart: "/usr/local/bin/alertmanager --config.file=/etc/prometheus/alertmanager.yml --storage.path=/var/lib/alertmanager --web.listen-address=:#{alertmanager_port}",
+      Restart: "on-failure",
+      RestartSec: "5",
+    },
+    Install: {
+      WantedBy: "multi-user.target",
+    },
+  })
+  triggers_reload true
+  action :create
+end
+
+service "alertmanager" do
+  action [:enable, :start]
+end
+
+firewall_rule "prometheus alertmanager" do
+  port alertmanager_port
+  source "10.1.1.0/24"
+  protocol :tcp
+  command :allow
+end

site-cookbooks/kosmos_prometheus/recipes/node_exporter.rb

@@ -0,0 +1,85 @@
+#
+# Cookbook:: kosmos_prometheus
+# Recipe:: node_exporter
+#
+
+version = node["kosmos_prometheus"]["node_exporter"]["version"]
+checksum = node["kosmos_prometheus"]["node_exporter"]["checksum"]
+tarball = "#{Chef::Config[:file_cache_path]}/node_exporter-#{version}.linux-amd64.tar.gz"
+binary_url = "https://github.com/prometheus/node_exporter/releases/download/v#{version}/node_exporter-#{version}.linux-amd64.tar.gz"
+
+node_exporter_port = node["kosmos_prometheus"]["node_exporter"]["port"]
+
+group "node_exporter"
+
+user "node_exporter" do
+  gid "node_exporter"
+  system true
+  shell "/bin/false"
+  home "/nonexistent"
+end
+
+directory "/var/lib/node_exporter/textfile" do
+  owner "node_exporter"
+  group "node_exporter"
+  mode "0755"
+  recursive true
+end
+
+package %w(tar bzip2)
+
+remote_file tarball do
+  source binary_url
+  checksum checksum
+  action :create
+  notifies :run, "execute[install_node_exporter]", :immediately
+end
+
+execute "install_node_exporter" do
+  command "tar -xzf #{tarball} -C /usr/local/bin --strip-components=1 node_exporter-#{version}.linux-amd64/node_exporter"
+  action :nothing
+  notifies :restart, "service[node_exporter]", :delayed
+end
+
+file "/usr/local/bin/node_exporter" do
+  owner "root"
+  group "root"
+  mode "0755"
+  notifies :restart, "service[node_exporter]", :delayed
+end
+
+systemd_unit "node_exporter.service" do
+  content({
+    Unit: {
+      Description: "Prometheus node exporter",
+      Documentation: ["https://github.com/prometheus/node_exporter"],
+    },
+    Service: {
+      Type: "simple",
+      User: "node_exporter",
+      Group: "node_exporter",
+      ExecStart: "/usr/local/bin/node_exporter --web.listen-address=:#{node_exporter_port} --collector.textfile.directory=/var/lib/node_exporter/textfile",
+      Restart: "on-failure",
+      RestartSec: "5",
+      NoNewPrivileges: "yes",
+      ProtectSystem: "full",
+      ProtectHome: "yes",
+    },
+    Install: {
+      WantedBy: "multi-user.target",
+    },
+  })
+  triggers_reload true
+  action :create
+end
+
+service "node_exporter" do
+  action [:enable, :start]
+end
+
+firewall_rule "node_exporter" do
+  port node_exporter_port
+  source "10.1.1.0/24"
+  protocol :tcp
+  command :allow
+end

site-cookbooks/kosmos_prometheus/recipes/server.rb

@@ -0,0 +1,150 @@
+#
+# Cookbook:: kosmos_prometheus
+# Recipe:: server
+#
+
+include_recipe "firewall"
+
+version = node["kosmos_prometheus"]["version"]
+checksum = node["kosmos_prometheus"]["checksum"]
+tarball = "#{Chef::Config[:file_cache_path]}/prometheus-#{version}.linux-amd64.tar.gz"
+binary_url = "https://github.com/prometheus/prometheus/releases/download/v#{version}/prometheus-#{version}.linux-amd64.tar.gz"
+
+data_dir   = node["kosmos_prometheus"]["data_dir"]
+config_dir = node["kosmos_prometheus"]["config_dir"]
+rules_dir  = node["kosmos_prometheus"]["rules_dir"]
+web_port   = node["kosmos_prometheus"]["web"]["port"]
+
+group "prometheus"
+
+user "prometheus" do
+  gid "prometheus"
+  system true
+  shell "/bin/false"
+  home "/nonexistent"
+end
+
+directory data_dir do
+  owner "prometheus"
+  group "prometheus"
+  mode "0750"
+  recursive true
+end
+
+directory config_dir do
+  owner "root"
+  group "root"
+  mode "0755"
+  recursive true
+end
+
+directory rules_dir do
+  owner "root"
+  group "prometheus"
+  mode "0755"
+  recursive true
+end
+
+node["kosmos_prometheus"]["rules"].each do |rule_name|
+  cookbook_file "#{rules_dir}/#{rule_name}.yml" do
+    source "rules/#{rule_name}.yml"
+    owner "root"
+    group "prometheus"
+    mode "0644"
+    notifies :reload, "service[prometheus]", :delayed
+  end
+end
+
+package %w(tar bzip2)
+
+remote_file tarball do
+  source binary_url
+  checksum checksum
+  action :create
+  notifies :run, "execute[install_prometheus]", :immediately
+end
+
+execute "install_prometheus" do
+  command "tar -xzf #{tarball} -C /usr/local/bin --strip-components=1 prometheus-#{version}.linux-amd64/prometheus"
+  action :nothing
+  notifies :restart, "service[prometheus]", :delayed
+end
+
+file "/usr/local/bin/prometheus" do
+  owner "root"
+  group "root"
+  mode "0755"
+  notifies :restart, "service[prometheus]", :delayed
+end
+
+jobs = node["kosmos_prometheus"]["jobs"].merge(
+{
+  # node exporter
+  "node" => {
+    "query" => "role:base",
+    "port" => node["kosmos_prometheus"]["node_exporter"]["port"]
+  },
+  # garage metrics
+  "garage" => {
+    "query" => "role:garage_node",
+    "port" => 3903
+  },
+}.transform_values do |config|
+    {
+      "targets" => search(:node, config["query"]).map do |n|
+        target = { "target" => "#{n['knife_zero']['host']}:#{config['port']}", "instance" => n.name }
+        target["env"] = n.chef_environment if n.chef_environment
+        target["vm_host"] = n["vm_host"] if n["vm_host"]
+        target
+      end.compact.sort_by { |t| t["instance"] },
+    }
+  end
+)
+
+template "#{config_dir}/prometheus.yml" do
+  source "prometheus.yml.erb"
+  owner "root"
+  group "prometheus"
+  mode "0644"
+  variables(
+    global_config: node["kosmos_prometheus"]["global"],
+    alerting: node["kosmos_prometheus"]["alerting"],
+    jobs: jobs,
+    rule_files: node["kosmos_prometheus"]["rule_files"]
+  )
+  notifies :reload, "service[prometheus]", :delayed
+end
+
+systemd_unit "prometheus.service" do
+  content({
+    Unit: {
+      Description: "Prometheus",
+      After: "network.target",
+    },
+    Service: {
+      Type: "simple",
+      User: "prometheus",
+      Group: "prometheus",
+      ExecStart: "/usr/local/bin/prometheus --config.file=/etc/prometheus/prometheus.yml --storage.tsdb.path=/var/lib/prometheus --storage.tsdb.retention.time=15d --web.listen-address=:#{web_port} --web.enable-lifecycle",
+      ExecReload: "/bin/kill -HUP $MAINPID",
+      Restart: "on-failure",
+      RestartSec: "5",
+    },
+    Install: {
+      WantedBy: "multi-user.target",
+    },
+  })
+  triggers_reload true
+  action :create
+end
+
+service "prometheus" do
+  action [:enable, :start]
+end
+
+firewall_rule "prometheus web" do
+  port web_port
+  source "10.1.1.0/24"
+  protocol :tcp
+  command :allow
+end

site-cookbooks/kosmos_prometheus/templates/default/alertmanager.yml.erb

@@ -0,0 +1,47 @@
+global:
+  resolve_timeout: 5m
+
+route:
+<% @route.each do |k, v| %>
+<% if k == "routes" && v.is_a?(Array) && !v.empty? %>
+  routes:
+<% v.each do |child| %>
+  - receiver: <%= child["receiver"] %>
+<% child.each do |ck, cv| %>
+<% next if ck == "receiver" %>
+    <%= ck %>: <%= cv.is_a?(Array) ? cv.inspect : cv %>
+<% end %>
+<% end %>
+<% else %>
+  <%= k %>: <%= v.is_a?(Array) ? v.inspect : v %>
+<% end %>
+<% end %>
+
+receivers:
+<% @receivers.each do |receiver| %>
+- name: <%= receiver["name"] %>
+<% if receiver["webhook_configs"] %>
+<% receiver["webhook_configs"].each do |wc| %>
+  webhook_configs:
+  - url: "<%= @webhook_url %>"
+    send_resolved: <%= wc["send_resolved"] %>
+    payload:
+      room: "<%= wc["room"] %>"
+      message: '{{ if eq .Status "resolved" }}✅{{ else if eq .CommonLabels.severity "warning" }}⚠️{{ else }}🚨{{ end }} {{ if eq .Status "resolved" }}RESOLVED{{ else if .CommonLabels.severity }}{{ .CommonLabels.severity | toUpper }}{{ else }}ALERT{{ end }}: {{ .CommonLabels.alertname }} {{ if eq .Status "resolved" }}({{ len .Alerts }} resolved){{ else if eq (len .Alerts.Firing) (len .Alerts) }}({{ len .Alerts }} firing){{ else }}({{ len .Alerts.Firing }} firing, {{ len .Alerts.Resolved }} resolved){{ end }}{{ range $i, $a := .Alerts }}{{ if lt $i 5 }}{{ "\n" }}  - {{ $a.Labels.instance }}{{ if $a.Labels.vm_host }} ({{ $a.Labels.vm_host }}){{ end }}: {{ if eq $a.Status "resolved" }}resolved{{ else }}{{ $a.Annotations.summary }}{{ end }}{{ end }}{{ end }}{{ if gt (len .Alerts) 5 }}{{ "\n" }}  ... and more{{ end }}'
+<% end %>
+<% end %>
+<% if receiver["email_configs"] %>
+<% receiver["email_configs"].each do |ec| %>
+  email_configs:
+  - to: "<%= ec["to"] %>"
+    from: "<%= ec["from"] %>"
+    smarthost: "<%= ec["smarthost"] %>"
+    require_tls: <%= ec["require_tls"] %>
+    send_resolved: <%= ec["send_resolved"] %>
+    text: '{{ if eq .Status "resolved" }}RESOLVED{{ else if .CommonLabels.severity }}{{ .CommonLabels.severity | toUpper }}{{ else }}ALERT{{ end }}: {{ .CommonLabels.alertname }} {{ if eq .Status "resolved" }}({{ len .Alerts }} resolved){{ else if eq (len .Alerts.Firing) (len .Alerts) }}({{ len .Alerts }} firing){{ else }}({{ len .Alerts.Firing }} firing, {{ len .Alerts.Resolved }} resolved){{ end }}{{ range .Alerts }}
+---{{ range .Labels.SortedPairs }}
+  {{ .Name }}: {{ .Value }}{{ end }}{{ range .Annotations.SortedPairs }}
+  {{ .Name }}: {{ .Value }}{{ end }}{{ end }}'
+<% end %>
+<% end %>
+<% end %>

site-cookbooks/kosmos_prometheus/templates/default/prometheus.yml.erb

@@ -0,0 +1,50 @@
+global:
+<% @global_config.each do |k, v| %>
+  <%= k %>: "<%= v %>"
+<% end %>
+
+<% if @alerting %>
+alerting:
+  alertmanagers:
+<% @alerting['alertmanagers'].each do |am| %>
+    - static_configs:
+<% am['static_configs'].each do |sc| %>
+        - targets:
+<% sc['targets'].each do |target| %>
+            - <%= target %>
+<% end %>
+<% end %>
+<% end %>
+<% end %>
+
+scrape_configs:
+<% @jobs.each do |name, job| %>
+- job_name: "<%= name %>"
+<% if job['scrape_interval'] %>
+  scrape_interval: "<%= job['scrape_interval'] %>"
+<% end %>
+<% if job['scrape_timeout'] %>
+  scrape_timeout: "<%= job['scrape_timeout'] %>"
+<% end %>
+  metrics_path: "<%= job.fetch('metrics_path', '/metrics') %>"
+  static_configs:
+<% job['targets'].each do |t| %>
+    - targets:
+        - <%= t['target'] %>
+      labels:
+        instance: <%= t['instance'] %>
+<% if t['env'] %>
+        env: <%= t['env'] %>
+<% end %>
+<% if t['vm_host'] %>
+        vm_host: <%= t['vm_host'] %>
+<% end %>
+<% end %>
+<% end %>
+
+<% if @rule_files && !@rule_files.empty? %>
+rule_files:
+<% @rule_files.each do |filename| %>
+  - <%= filename %>
+<% end %>
+<% end %>