diff --git a/.githooks/pre-commit b/.githooks/pre-commit new file mode 100755 index 0000000..e97f9cb --- /dev/null +++ b/.githooks/pre-commit @@ -0,0 +1,40 @@ +#!/usr/bin/env bash +set -euo pipefail + +SCRIPT="$(git rev-parse --show-toplevel)/scripts/util/sanitize_node.sh" + +FILES=$(git diff --cached --name-only --diff-filter=ACM -- 'nodes/*.json' || true) + +[[ -z "$FILES" ]] && exit 0 + +DIRTY=0 + +while IFS= read -r file; do + staged_tmp=$(mktemp) + sanitized_tmp=$(mktemp) + + # 1. get staged version + git show ":$file" > "$staged_tmp" + + # 2. sanitize IN PLACE (on temp copy) + cp "$staged_tmp" "$sanitized_tmp" + "$SCRIPT" "$sanitized_tmp" + + # 3. if sanitizer changed file, update working tree + if ! diff -q "$staged_tmp" "$sanitized_tmp" >/dev/null; then + cp "$sanitized_tmp" "$file" + echo "Sanitized: $file" + DIRTY=1 + fi + + rm -f "$staged_tmp" "$sanitized_tmp" + +done <<< "$FILES" + +if [[ "$DIRTY" -ne 0 ]]; then + echo "" + echo "Commit paused, sanitized files must be re-staged." + exit 1 +fi + +exit 0 diff --git a/README.md b/README.md index c15ca12..1fcb982 100644 --- a/README.md +++ b/README.md @@ -28,6 +28,10 @@ Clone this repository, `cd` into it, and run: bundle install +### Enable Git hooks + + git config core.hooksPath .githooks + ## Common tasks ### Bootstrap a new host server @@ -54,6 +58,10 @@ Clone this repository, `cd` into it, and run: knife zero converge name:dev.kosmos.org --client-version 15.3.14 +### Upgrade Chef Client on a node: + + knife ssh -a knife_zero.host 'name:foo-1' 'curl -L https://chef.io/chef/install.sh | sudo bash -s -- -v 18.10.17' + ## Managing cookbooks Cookbooks are managed via Berkshelf. Run `berks --help` for command help. diff --git a/clients/garage-10.json b/clients/garage-10.json deleted file mode 100644 index 6ab390b..0000000 --- a/clients/garage-10.json +++ /dev/null @@ -1,4 +0,0 @@ -{ - "name": "garage-10", - "public_key": "-----BEGIN PUBLIC KEY-----\nMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAw2+3Wo+KkXVJCOX1SxT9\nSdwKXgPbCDM3EI9uwoxhMxQfRyN53dxIsBDsQUVOIe1Z8yqm4FenMQlNmeDR+QLE\nvNFf1fisinW+D9VVRm+CjcJy96i/Dyt786Z6YRrDlB860HxCbfTL2Zv5BRtbyIKg\nhz5gO+9PMEpPVR2ij9iue4K6jbM1AAL2ia/P6zDWLJqeIzUocCeHV5N0Z3jXH6qr\nf444v78x35MMJ+3tg5h95SU1/PDCpdSTct4uHEuKIosiN7p4DlYMoM5iSyvVoujr\nflRQPEpGzS9qEt3rDo/F4ltzYMx6bf1tB/0QaBKD+zwPZWTTwf61tSBo5/NkGvJc\nFQIDAQAB\n-----END PUBLIC KEY-----\n" -} \ No newline at end of file diff --git a/clients/garage-12.json b/clients/garage-12.json deleted file mode 100644 index a297b74..0000000 --- a/clients/garage-12.json +++ /dev/null @@ -1,4 +0,0 @@ -{ - "name": "garage-12", - "public_key": "-----BEGIN PUBLIC KEY-----\nMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEA9GtHHi298BjiIqpZ3WkT\nkYAPfWD60hFe/8icYcq/F/6cHLYKZQ4chek9X/hDCMq4tHEN6Oh58T5x/nuNdPrK\nIAMGyVAGk6ekWlmD4jwdEf6TGb/J3ffJTRDvwX/I8xD/DW3wtXsN+X24T59ByGTm\nrnwRmmmwHF3otRx9wnCsIgDQ0AjiUujsfNNv1FcLXD/WJLys9lEeU5aJ4XtHTwDv\ntJM8YyVEFhEnuvgdKmzn5+F5k9VGdUwForlFOBfvzbCnTZMDMmDVeiUtAUv/7xWQ\nQl2mLUGCtgWuYJYXsQacAJ6pa3h+7cQyshC6w3dwUG+1fS9lNO0Yp1GGX1AGYKpp\nPQIDAQAB\n-----END PUBLIC KEY-----\n" -} \ No newline at end of file diff --git a/clients/garage-13.json b/clients/garage-13.json deleted file mode 100644 index 13582ff..0000000 --- a/clients/garage-13.json +++ /dev/null @@ -1,4 +0,0 @@ -{ - "name": "garage-13", - "public_key": "-----BEGIN PUBLIC KEY-----\nMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAvbqWc6OwRxgHfsQuTNL4\naxeVvNen5d9srYpZSHjuBB/k9NHB+9P6vU5qF37XHkw1lVUGeYbPHzhYsx3O0/kZ\nH5f4+4SMy/P9jc6SE7AJF4qtYKgJ88koZdqCww07c6K9g+BnEGFFZui/h3hUBxWj\nTfhBHEWPyQ2bl/lr9sIJwsEz+EN0isGn/eIXkmw9J6LdLJ5Q0LLks33K28FNOU7q\nfeAN4MiBVMUtgCGyT2Voe6WrOXwQLSDXQONOp3sfSfFExsIJ1s24xdd7AMD7/9a7\n4sFDZ4swhqAWgWmW2giR7Kb8wTvGQLO/O/uUbmKz3DZXgkOKXHdHCEB/PZx1mRNM\nEwIDAQAB\n-----END PUBLIC KEY-----\n" -} \ No newline at end of file diff --git a/clients/garage-9.json b/clients/garage-9.json deleted file mode 100644 index 21336dc..0000000 --- a/clients/garage-9.json +++ /dev/null @@ -1,4 +0,0 @@ -{ - "name": "garage-9", - "public_key": "-----BEGIN PUBLIC KEY-----\nMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAnMHzKE8JBrsQkmRDeMjX\n71mBzvRzNM90cwA8xtvIkXesdTyGqohX9k/PJbCY5ySGK9PpMaYDPVAnwnUP8LFQ\n3G98aSbLxUjqU/PBzRsnWpihehr05uz9zYcNFzr4LTNvGQZsq47nN9Tk+LG3zHP7\nAZViv2mJ4ZRnukXf6KHlyoVvhuTu+tiBM8QzjTF97iP/aguNPzYHmrecy9Uf5bSA\nZrbNZT+ayxtgswC2OclhRucx7XLSuHXtpwFqsQzSAhiX1aQ3wwCyH9WJtVwpfUsE\nlxTjcQiSM9aPZ8iSC0shpBaKD1j3iF/2K2Jk+88++zMhJJPLermvaJxzsdePgvyk\nKQIDAQAB\n-----END PUBLIC KEY-----\n" -} \ No newline at end of file diff --git a/clients/prometheus-1.json b/clients/prometheus-1.json new file mode 100644 index 0000000..a72e203 --- /dev/null +++ b/clients/prometheus-1.json @@ -0,0 +1,4 @@ +{ + "name": "prometheus-1", + "public_key": "-----BEGIN PUBLIC KEY-----\nMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAp7T/OBo/TZm3YqnN4+ok\nHwcJ0kW9w2rl9UfrOlWUvoPHBd2LrqpEv3Az3a150IylQ1H/UozmQA7DtjIoTA7d\nV3oLY970vYrYiURcojOo8qAZBy8EH7dfAHxuZryUeELr+3vdcHF5WrrfSt2FdFVX\nPTY95ikafAnOO0Nt8jvnlPoDn7REV8TOE6KOiUzcHKa2xGlfaIe0oRC21LD86uQm\nR09xY1YaJkVgZfeN/opoRjZawkU3FFs3jlUEVBF8k153oOw9W3bgsFFjSOtRtRRg\nDwyQ7oDeMH83kXnaCdpkNZd59wjzPcpxYAL4LRN52ZXA4Btr4DTi+GxHz98Dr0kU\nUQIDAQAB\n-----END PUBLIC KEY-----\n" +} \ No newline at end of file diff --git a/clients/uploads-1.json b/clients/uploads-1.json deleted file mode 100644 index fec1329..0000000 --- a/clients/uploads-1.json +++ /dev/null @@ -1,4 +0,0 @@ -{ - "name": "uploads-1", - "public_key": "-----BEGIN PUBLIC KEY-----\nMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAwJwWaz8TeGv3SFlKzLMx\nqN8GTL/c0N9ppBvv8xNSS/yF9Y40SbL418uxYzm9hIhOXgIygIgLT2EKIXX32t+R\neOJCdYycQFM3At2fhMkjhuUW0gmDRcYBcBJLC5hLh2EZ+A8V7k4qgrBpPLOjEv48\nhQY0vuAw2DGndWr4QLh5NLUmQiOrfuzcZSSNCBOTIgUZgNmRd9QcCHDq4WDH3poa\nosJo4a9JGEGUL1irOivvEdyJPwEd2f++nYAdWwj8pjCYgpRshQlLhxOlylMx7MxB\nQt2bgJC9sahfbfJCOqdlCU3DMJL0bRUiuxK77WeSsxWBJmrsiF3+Ljs2Ix+s7fnS\nywIDAQAB\n-----END PUBLIC KEY-----\n" -} \ No newline at end of file diff --git a/nodes/akkounts-1.json b/nodes/akkounts-1.json index 0627a9a..7af0bda 100644 --- a/nodes/akkounts-1.json +++ b/nodes/akkounts-1.json @@ -4,7 +4,8 @@ "normal": { "knife_zero": { "host": "10.1.1.144" - } + }, + "vm_host": "draco" }, "automatic": { "fqdn": "akkounts-1", @@ -24,6 +25,7 @@ "recipes": [ "kosmos-base", "kosmos-base::default", + "kosmos_prometheus::node_exporter", "kosmos_kvm::guest", "kosmos-dirsrv::hostsfile", "kosmos_sentry::client", diff --git a/nodes/bitcoin-2.json b/nodes/bitcoin-2.json index 707584a..eaec80c 100644 --- a/nodes/bitcoin-2.json +++ b/nodes/bitcoin-2.json @@ -3,7 +3,8 @@ "normal": { "knife_zero": { "host": "10.1.1.163" - } + }, + "vm_host": "draco" }, "automatic": { "fqdn": "bitcoin-2", @@ -24,6 +25,7 @@ "recipes": [ "kosmos-base", "kosmos-base::default", + "kosmos_prometheus::node_exporter", "kosmos_kvm::guest", "kosmos_sentry::client", "tor-full", diff --git a/nodes/discourse-2.json b/nodes/discourse-2.json index b47c77d..4e76389 100644 --- a/nodes/discourse-2.json +++ b/nodes/discourse-2.json @@ -4,15 +4,17 @@ "normal": { "knife_zero": { "host": "10.1.1.35" - } + }, + "vm_host": "fornax" }, "automatic": { "fqdn": "discourse-2", "os": "linux", - "os_version": "5.4.0-1087-kvm", + "os_version": "5.4.0-1133-kvm", "hostname": "discourse-2", "ipaddress": "192.168.122.104", "roles": [ + "base", "kvm_guest", "garage_gateway", "kosmos_discourse" @@ -20,6 +22,7 @@ "recipes": [ "kosmos-base", "kosmos-base::default", + "kosmos_prometheus::node_exporter", "kosmos_kvm::guest", "kosmos_garage", "kosmos_garage::default", @@ -32,6 +35,7 @@ "timezone_iii::debian", "ntp::default", "ntp::apparmor", + "kosmos-base::journald_conf", "kosmos-base::systemd_emails", "apt::unattended-upgrades", "kosmos-base::firewall", @@ -42,7 +46,6 @@ "postfix::sasl_auth", "hostname::default", "firewall::default", - "chef-sugar::default", "discourse::default" ], "platform": "ubuntu", @@ -50,18 +53,18 @@ "cloud": null, "chef_packages": { "chef": { - "version": "17.9.52", - "chef_root": "/opt/chef/embedded/lib/ruby/gems/3.0.0/gems/chef-17.9.52/lib", + "version": "18.10.17", + "chef_root": "/opt/chef/embedded/lib/ruby/gems/3.1.0/gems/chef-18.10.17/lib", "chef_effortless": null }, "ohai": { - "version": "17.9.0", - "ohai_root": "/opt/chef/embedded/lib/ruby/gems/3.0.0/gems/ohai-17.9.0/lib/ohai" + "version": "18.2.13", + "ohai_root": "/opt/chef/embedded/lib/ruby/gems/3.1.0/gems/ohai-18.2.13/lib/ohai" } } }, "run_list": [ - "recipe[kosmos-base]", + "role[base]", "role[kvm_guest]", "role[garage_gateway]", "role[kosmos_discourse]" diff --git a/nodes/draco.kosmos.org.json b/nodes/draco.kosmos.org.json index 8ae21b4..2c17307 100644 --- a/nodes/draco.kosmos.org.json +++ b/nodes/draco.kosmos.org.json @@ -36,6 +36,7 @@ "recipes": [ "kosmos-base", "kosmos-base::default", + "kosmos_prometheus::node_exporter", "kosmos_kvm::host", "kosmos_kvm::backup", "kosmos_email::firewall", diff --git a/nodes/drone-1.json b/nodes/drone-1.json index 40b57fc..7b9e733 100644 --- a/nodes/drone-1.json +++ b/nodes/drone-1.json @@ -3,7 +3,8 @@ "normal": { "knife_zero": { "host": "10.1.1.128" - } + }, + "vm_host": "fornax" }, "automatic": { "fqdn": "drone-1", @@ -12,6 +13,7 @@ "hostname": "drone-1", "ipaddress": "192.168.122.200", "roles": [ + "base", "kvm_guest", "drone", "postgresql_client" @@ -19,6 +21,7 @@ "recipes": [ "kosmos-base", "kosmos-base::default", + "kosmos_prometheus::node_exporter", "kosmos_kvm::guest", "kosmos_postgresql::hostsfile", "kosmos_drone", @@ -37,7 +40,8 @@ "postfix::_common", "postfix::_attributes", "postfix::sasl_auth", - "hostname::default" + "hostname::default", + "firewall::default" ], "platform": "ubuntu", "platform_version": "20.04", @@ -55,7 +59,7 @@ } }, "run_list": [ - "recipe[kosmos-base]", + "role[base]", "role[kvm_guest]", "role[drone]" ] diff --git a/nodes/ejabberd-4.json b/nodes/ejabberd-4.json index 8f88713..2fba750 100644 --- a/nodes/ejabberd-4.json +++ b/nodes/ejabberd-4.json @@ -4,7 +4,8 @@ "normal": { "knife_zero": { "host": "10.1.1.113" - } + }, + "vm_host": "draco" }, "automatic": { "fqdn": "ejabberd-4", @@ -23,6 +24,7 @@ "recipes": [ "kosmos-base", "kosmos-base::default", + "kosmos_prometheus::node_exporter", "kosmos_kvm::guest", "kosmos-dirsrv::hostsfile", "kosmos_postgresql::hostsfile", @@ -72,4 +74,4 @@ "role[ldap_client]", "role[ejabberd]" ] -} +} \ No newline at end of file diff --git a/nodes/ejabberd-8.json b/nodes/ejabberd-8.json index 4aab853..81a0b09 100644 --- a/nodes/ejabberd-8.json +++ b/nodes/ejabberd-8.json @@ -4,7 +4,8 @@ "normal": { "knife_zero": { "host": "10.1.1.123" - } + }, + "vm_host": "fornax" }, "automatic": { "fqdn": "ejabberd-8", @@ -23,6 +24,7 @@ "recipes": [ "kosmos-base", "kosmos-base::default", + "kosmos_prometheus::node_exporter", "kosmos_kvm::guest", "kosmos-dirsrv::hostsfile", "kosmos_postgresql::hostsfile", @@ -72,4 +74,4 @@ "role[ldap_client]", "role[ejabberd]" ] -} +} \ No newline at end of file diff --git a/nodes/fornax.kosmos.org.json b/nodes/fornax.kosmos.org.json index a57af22..41f19ef 100644 --- a/nodes/fornax.kosmos.org.json +++ b/nodes/fornax.kosmos.org.json @@ -30,6 +30,7 @@ "recipes": [ "kosmos-base", "kosmos-base::default", + "kosmos_prometheus::node_exporter", "kosmos_kvm::host", "kosmos_kvm::backup", "kosmos_openresty", diff --git a/nodes/garage-10.json b/nodes/garage-10.json deleted file mode 100644 index 7228ca1..0000000 --- a/nodes/garage-10.json +++ /dev/null @@ -1,64 +0,0 @@ -{ - "name": "garage-10", - "chef_environment": "production", - "normal": { - "knife_zero": { - "host": "10.1.1.27" - } - }, - "automatic": { - "fqdn": "garage-10", - "os": "linux", - "os_version": "5.4.0-1090-kvm", - "hostname": "garage-10", - "ipaddress": "192.168.122.70", - "roles": [ - "base", - "kvm_guest", - "garage_node" - ], - "recipes": [ - "kosmos-base", - "kosmos-base::default", - "kosmos_kvm::guest", - "kosmos_garage", - "kosmos_garage::default", - "kosmos_garage::firewall_rpc", - "kosmos_garage::firewall_apis", - "apt::default", - "timezone_iii::default", - "timezone_iii::debian", - "ntp::default", - "ntp::apparmor", - "kosmos-base::systemd_emails", - "apt::unattended-upgrades", - "kosmos-base::firewall", - "kosmos-postfix::default", - "postfix::default", - "postfix::_common", - "postfix::_attributes", - "postfix::sasl_auth", - "hostname::default", - "firewall::default" - ], - "platform": "ubuntu", - "platform_version": "20.04", - "cloud": null, - "chef_packages": { - "chef": { - "version": "18.5.0", - "chef_root": "/opt/chef/embedded/lib/ruby/gems/3.1.0/gems/chef-18.5.0/lib", - "chef_effortless": null - }, - "ohai": { - "version": "18.1.11", - "ohai_root": "/opt/chef/embedded/lib/ruby/gems/3.1.0/gems/ohai-18.1.11/lib/ohai" - } - } - }, - "run_list": [ - "role[base]", - "role[kvm_guest]", - "role[garage_node]" - ] -} diff --git a/nodes/garage-11.json b/nodes/garage-11.json index 4f6c161..930d94b 100644 --- a/nodes/garage-11.json +++ b/nodes/garage-11.json @@ -4,7 +4,8 @@ "normal": { "knife_zero": { "host": "10.1.1.165" - } + }, + "vm_host": "fornax" }, "automatic": { "fqdn": "garage-11", @@ -20,6 +21,7 @@ "recipes": [ "kosmos-base", "kosmos-base::default", + "kosmos_prometheus::node_exporter", "kosmos_kvm::guest", "kosmos_garage", "kosmos_garage::default", @@ -30,6 +32,7 @@ "timezone_iii::debian", "ntp::default", "ntp::apparmor", + "kosmos-base::journald_conf", "kosmos-base::systemd_emails", "apt::unattended-upgrades", "kosmos-base::firewall", @@ -61,4 +64,4 @@ "role[kvm_guest]", "role[garage_node]" ] -} +} \ No newline at end of file diff --git a/nodes/garage-12.json b/nodes/garage-12.json deleted file mode 100644 index f5be8b5..0000000 --- a/nodes/garage-12.json +++ /dev/null @@ -1,65 +0,0 @@ -{ - "name": "garage-12", - "chef_environment": "production", - "normal": { - "knife_zero": { - "host": "10.1.1.224" - } - }, - "automatic": { - "fqdn": "garage-12", - "os": "linux", - "os_version": "5.15.0-1059-kvm", - "hostname": "garage-12", - "ipaddress": "192.168.122.173", - "roles": [ - "base", - "kvm_guest", - "garage_node" - ], - "recipes": [ - "kosmos-base", - "kosmos-base::default", - "kosmos_kvm::guest", - "kosmos_garage", - "kosmos_garage::default", - "kosmos_garage::firewall_rpc", - "kosmos_garage::firewall_apis", - "apt::default", - "timezone_iii::default", - "timezone_iii::debian", - "ntp::default", - "ntp::apparmor", - "kosmos-base::journald_conf", - "kosmos-base::systemd_emails", - "apt::unattended-upgrades", - "kosmos-base::firewall", - "kosmos-postfix::default", - "postfix::default", - "postfix::_common", - "postfix::_attributes", - "postfix::sasl_auth", - "hostname::default", - "firewall::default" - ], - "platform": "ubuntu", - "platform_version": "22.04", - "cloud": null, - "chef_packages": { - "chef": { - "version": "18.7.10", - "chef_root": "/opt/chef/embedded/lib/ruby/gems/3.1.0/gems/chef-18.7.10/lib", - "chef_effortless": null - }, - "ohai": { - "version": "18.2.5", - "ohai_root": "/opt/chef/embedded/lib/ruby/gems/3.1.0/gems/ohai-18.2.5/lib/ohai" - } - } - }, - "run_list": [ - "role[base]", - "role[kvm_guest]", - "role[garage_node]" - ] -} diff --git a/nodes/garage-14.json b/nodes/garage-14.json index 2affca0..8d9d69c 100644 --- a/nodes/garage-14.json +++ b/nodes/garage-14.json @@ -4,7 +4,8 @@ "normal": { "knife_zero": { "host": "10.1.1.151" - } + }, + "vm_host": "leo" }, "automatic": { "fqdn": "garage-14", @@ -20,6 +21,7 @@ "recipes": [ "kosmos-base", "kosmos-base::default", + "kosmos_prometheus::node_exporter", "kosmos_kvm::guest", "kosmos_garage", "kosmos_garage::default", @@ -62,4 +64,4 @@ "role[kvm_guest]", "role[garage_node]" ] -} +} \ No newline at end of file diff --git a/nodes/garage-15.json b/nodes/garage-15.json index eb00ff8..d7b5260 100644 --- a/nodes/garage-15.json +++ b/nodes/garage-15.json @@ -4,7 +4,8 @@ "normal": { "knife_zero": { "host": "10.1.1.82" - } + }, + "vm_host": "leo" }, "automatic": { "fqdn": "garage-15", @@ -20,6 +21,7 @@ "recipes": [ "kosmos-base", "kosmos-base::default", + "kosmos_prometheus::node_exporter", "kosmos_kvm::guest", "kosmos_garage", "kosmos_garage::default", @@ -62,4 +64,4 @@ "role[kvm_guest]", "role[garage_node]" ] -} +} \ No newline at end of file diff --git a/nodes/garage-16.json b/nodes/garage-16.json index b4da55d..50a0d2b 100644 --- a/nodes/garage-16.json +++ b/nodes/garage-16.json @@ -4,7 +4,8 @@ "normal": { "knife_zero": { "host": "10.1.1.153" - } + }, + "vm_host": "leo" }, "automatic": { "fqdn": "garage-16", @@ -20,6 +21,7 @@ "recipes": [ "kosmos-base", "kosmos-base::default", + "kosmos_prometheus::node_exporter", "kosmos_kvm::guest", "kosmos_garage", "kosmos_garage::default", @@ -60,4 +62,4 @@ "role[kvm_guest]", "role[garage_node]" ] -} +} \ No newline at end of file diff --git a/nodes/garage-2.json b/nodes/garage-2.json index 93641a6..42994c7 100644 --- a/nodes/garage-2.json +++ b/nodes/garage-2.json @@ -4,7 +4,8 @@ "normal": { "knife_zero": { "host": "10.1.1.40" - } + }, + "vm_host": "draco" }, "automatic": { "fqdn": "garage-2", @@ -20,6 +21,7 @@ "recipes": [ "kosmos-base", "kosmos-base::default", + "kosmos_prometheus::node_exporter", "kosmos_kvm::guest", "kosmos_garage", "kosmos_garage::default", @@ -30,6 +32,7 @@ "timezone_iii::debian", "ntp::default", "ntp::apparmor", + "kosmos-base::journald_conf", "kosmos-base::systemd_emails", "apt::unattended-upgrades", "kosmos-base::firewall", @@ -46,13 +49,13 @@ "cloud": null, "chef_packages": { "chef": { - "version": "18.3.0", - "chef_root": "/opt/chef/embedded/lib/ruby/gems/3.1.0/gems/chef-18.3.0/lib", + "version": "18.10.17", + "chef_root": "/opt/chef/embedded/lib/ruby/gems/3.1.0/gems/chef-18.10.17/lib", "chef_effortless": null }, "ohai": { - "version": "18.1.4", - "ohai_root": "/opt/chef/embedded/lib/ruby/gems/3.1.0/gems/ohai-18.1.4/lib/ohai" + "version": "18.2.13", + "ohai_root": "/opt/chef/embedded/lib/ruby/gems/3.1.0/gems/ohai-18.2.13/lib/ohai" } } }, diff --git a/nodes/garage-7.json b/nodes/garage-7.json index 5d87c28..30604e1 100644 --- a/nodes/garage-7.json +++ b/nodes/garage-7.json @@ -4,7 +4,8 @@ "normal": { "knife_zero": { "host": "10.1.1.182" - } + }, + "vm_host": "draco" }, "automatic": { "fqdn": "garage-7", @@ -20,6 +21,7 @@ "recipes": [ "kosmos-base", "kosmos-base::default", + "kosmos_prometheus::node_exporter", "kosmos_kvm::guest", "kosmos_garage", "kosmos_garage::default", @@ -30,6 +32,7 @@ "timezone_iii::debian", "ntp::default", "ntp::apparmor", + "kosmos-base::journald_conf", "kosmos-base::systemd_emails", "apt::unattended-upgrades", "kosmos-base::firewall", @@ -61,4 +64,4 @@ "role[kvm_guest]", "role[garage_node]" ] -} +} \ No newline at end of file diff --git a/nodes/garage-8.json b/nodes/garage-8.json index 3800a43..b2ccff8 100644 --- a/nodes/garage-8.json +++ b/nodes/garage-8.json @@ -4,7 +4,8 @@ "normal": { "knife_zero": { "host": "10.1.1.61" - } + }, + "vm_host": "fornax" }, "automatic": { "fqdn": "garage-8", @@ -20,6 +21,7 @@ "recipes": [ "kosmos-base", "kosmos-base::default", + "kosmos_prometheus::node_exporter", "kosmos_kvm::guest", "kosmos_garage", "kosmos_garage::default", @@ -30,6 +32,7 @@ "timezone_iii::debian", "ntp::default", "ntp::apparmor", + "kosmos-base::journald_conf", "kosmos-base::systemd_emails", "apt::unattended-upgrades", "kosmos-base::firewall", @@ -61,4 +64,4 @@ "role[kvm_guest]", "role[garage_node]" ] -} +} \ No newline at end of file diff --git a/nodes/garage-9.json b/nodes/garage-9.json deleted file mode 100644 index b733284..0000000 --- a/nodes/garage-9.json +++ /dev/null @@ -1,64 +0,0 @@ -{ - "name": "garage-9", - "chef_environment": "production", - "normal": { - "knife_zero": { - "host": "10.1.1.223" - } - }, - "automatic": { - "fqdn": "garage-9", - "os": "linux", - "os_version": "5.4.0-1090-kvm", - "hostname": "garage-9", - "ipaddress": "192.168.122.21", - "roles": [ - "base", - "kvm_guest", - "garage_node" - ], - "recipes": [ - "kosmos-base", - "kosmos-base::default", - "kosmos_kvm::guest", - "kosmos_garage", - "kosmos_garage::default", - "kosmos_garage::firewall_rpc", - "kosmos_garage::firewall_apis", - "apt::default", - "timezone_iii::default", - "timezone_iii::debian", - "ntp::default", - "ntp::apparmor", - "kosmos-base::systemd_emails", - "apt::unattended-upgrades", - "kosmos-base::firewall", - "kosmos-postfix::default", - "postfix::default", - "postfix::_common", - "postfix::_attributes", - "postfix::sasl_auth", - "hostname::default", - "firewall::default" - ], - "platform": "ubuntu", - "platform_version": "20.04", - "cloud": null, - "chef_packages": { - "chef": { - "version": "18.5.0", - "chef_root": "/opt/chef/embedded/lib/ruby/gems/3.1.0/gems/chef-18.5.0/lib", - "chef_effortless": null - }, - "ohai": { - "version": "18.1.11", - "ohai_root": "/opt/chef/embedded/lib/ruby/gems/3.1.0/gems/ohai-18.1.11/lib/ohai" - } - } - }, - "run_list": [ - "role[base]", - "role[kvm_guest]", - "role[garage_node]" - ] -} diff --git a/nodes/gitea-2.json b/nodes/gitea-2.json index cddfdd9..b726be6 100644 --- a/nodes/gitea-2.json +++ b/nodes/gitea-2.json @@ -4,7 +4,8 @@ "normal": { "knife_zero": { "host": "10.1.1.21" - } + }, + "vm_host": "fornax" }, "automatic": { "fqdn": "gitea-2", @@ -24,6 +25,7 @@ "recipes": [ "kosmos-base", "kosmos-base::default", + "kosmos_prometheus::node_exporter", "kosmos_kvm::guest", "kosmos-dirsrv::hostsfile", "kosmos_garage", @@ -58,13 +60,13 @@ "cloud": null, "chef_packages": { "chef": { - "version": "18.2.7", - "chef_root": "/opt/chef/embedded/lib/ruby/gems/3.1.0/gems/chef-18.2.7/lib", + "version": "18.10.17", + "chef_root": "/opt/chef/embedded/lib/ruby/gems/3.1.0/gems/chef-18.10.17/lib", "chef_effortless": null }, "ohai": { - "version": "18.1.4", - "ohai_root": "/opt/chef/embedded/lib/ruby/gems/3.1.0/gems/ohai-18.1.4/lib/ohai" + "version": "18.2.13", + "ohai_root": "/opt/chef/embedded/lib/ruby/gems/3.1.0/gems/ohai-18.2.13/lib/ohai" } } }, @@ -76,4 +78,4 @@ "role[gitea]", "role[gitea_actions_runner]" ] -} +} \ No newline at end of file diff --git a/nodes/her.json b/nodes/her.json index 43402f3..53e6ffc 100644 --- a/nodes/her.json +++ b/nodes/her.json @@ -9,9 +9,9 @@ "automatic": { "fqdn": "her", "os": "linux", - "os_version": "5.15.0-84-generic", + "os_version": "5.15.0-101-generic", "hostname": "her", - "ipaddress": "192.168.30.172", + "ipaddress": "192.168.30.173", "roles": [ "base", "kvm_host" @@ -19,6 +19,7 @@ "recipes": [ "kosmos-base", "kosmos-base::default", + "kosmos_prometheus::node_exporter", "kosmos_kvm::host", "kosmos_kvm::backup", "apt::default", @@ -26,6 +27,7 @@ "timezone_iii::debian", "ntp::default", "ntp::apparmor", + "kosmos-base::journald_conf", "kosmos-base::systemd_emails", "apt::unattended-upgrades", "kosmos-base::firewall", @@ -55,4 +57,4 @@ "role[base]", "role[kvm_host]" ] -} +} \ No newline at end of file diff --git a/nodes/ipfs-1.json b/nodes/ipfs-1.json index 8022309..c4f0118 100644 --- a/nodes/ipfs-1.json +++ b/nodes/ipfs-1.json @@ -3,7 +3,8 @@ "normal": { "knife_zero": { "host": "10.1.1.198" - } + }, + "vm_host": "draco" }, "automatic": { "fqdn": "ipfs-1", @@ -12,12 +13,14 @@ "hostname": "ipfs-1", "ipaddress": "192.168.122.195", "roles": [ + "base", "kvm_guest", "ipfs_gateway" ], "recipes": [ "kosmos-base", "kosmos-base::default", + "kosmos_prometheus::node_exporter", "kosmos_kvm::guest", "kosmos-ipfs", "kosmos-ipfs::default", @@ -27,6 +30,7 @@ "timezone_iii::debian", "ntp::default", "ntp::apparmor", + "kosmos-base::journald_conf", "kosmos-base::systemd_emails", "apt::unattended-upgrades", "kosmos-base::firewall", @@ -60,7 +64,7 @@ } }, "run_list": [ - "recipe[kosmos-base]", + "role[base]", "role[kvm_guest]", "role[ipfs_gateway]" ] diff --git a/nodes/ldap-3.kosmos.org.json b/nodes/ldap-3.kosmos.org.json index 637595e..cd743f3 100644 --- a/nodes/ldap-3.kosmos.org.json +++ b/nodes/ldap-3.kosmos.org.json @@ -3,7 +3,8 @@ "normal": { "knife_zero": { "host": "10.1.1.6" - } + }, + "vm_host": "draco" }, "automatic": { "fqdn": "ldap-3.kosmos.org", @@ -12,12 +13,14 @@ "hostname": "ldap-3", "ipaddress": "192.168.122.34", "roles": [ + "base", "kvm_guest", - "dirsrv_primary" + "dirsrv_supplier" ], "recipes": [ "kosmos-base", "kosmos-base::default", + "kosmos_prometheus::node_exporter", "kosmos_kvm::guest", "kosmos-dirsrv", "kosmos-dirsrv::default", @@ -26,6 +29,7 @@ "timezone_iii::debian", "ntp::default", "ntp::apparmor", + "kosmos-base::journald_conf", "kosmos-base::systemd_emails", "apt::unattended-upgrades", "kosmos-base::firewall", @@ -35,7 +39,7 @@ "postfix::_attributes", "postfix::sasl_auth", "hostname::default", - "kosmos-dirsrv::hostsfile", + "firewall::default", "kosmos-dirsrv::firewall", "backup::default", "logrotate::default", @@ -46,19 +50,19 @@ "cloud": null, "chef_packages": { "chef": { - "version": "17.10.3", - "chef_root": "/opt/chef/embedded/lib/ruby/gems/3.0.0/gems/chef-17.10.3/lib", + "version": "18.10.17", + "chef_root": "/opt/chef/embedded/lib/ruby/gems/3.1.0/gems/chef-18.10.17/lib", "chef_effortless": null }, "ohai": { - "version": "17.9.0", - "ohai_root": "/opt/chef/embedded/lib/ruby/gems/3.0.0/gems/ohai-17.9.0/lib/ohai" + "version": "18.2.13", + "ohai_root": "/opt/chef/embedded/lib/ruby/gems/3.1.0/gems/ohai-18.2.13/lib/ohai" } } }, "run_list": [ - "recipe[kosmos-base]", + "role[base]", "role[kvm_guest]", "role[dirsrv_supplier]" ] -} +} \ No newline at end of file diff --git a/nodes/ldap-4.kosmos.org.json b/nodes/ldap-4.kosmos.org.json index f66c37e..672b7b4 100644 --- a/nodes/ldap-4.kosmos.org.json +++ b/nodes/ldap-4.kosmos.org.json @@ -4,7 +4,8 @@ "normal": { "knife_zero": { "host": "10.1.1.106" - } + }, + "vm_host": "fornax" }, "automatic": { "fqdn": "ldap-4.kosmos.org", @@ -14,17 +15,22 @@ "ipaddress": "192.168.122.73", "roles": [ "base", - "kvm_guest" + "kvm_guest", + "dirsrv_supplier" ], "recipes": [ "kosmos-base", "kosmos-base::default", + "kosmos_prometheus::node_exporter", "kosmos_kvm::guest", + "kosmos-dirsrv", + "kosmos-dirsrv::default", "apt::default", "timezone_iii::default", "timezone_iii::debian", "ntp::default", "ntp::apparmor", + "kosmos-base::journald_conf", "kosmos-base::systemd_emails", "apt::unattended-upgrades", "kosmos-base::firewall", @@ -33,20 +39,25 @@ "postfix::_common", "postfix::_attributes", "postfix::sasl_auth", - "hostname::default" + "hostname::default", + "firewall::default", + "kosmos-dirsrv::firewall", + "backup::default", + "logrotate::default", + "ulimit::default" ], "platform": "ubuntu", "platform_version": "20.04", "cloud": null, "chef_packages": { "chef": { - "version": "17.10.3", - "chef_root": "/opt/chef/embedded/lib/ruby/gems/3.0.0/gems/chef-17.10.3/lib", + "version": "18.10.17", + "chef_root": "/opt/chef/embedded/lib/ruby/gems/3.1.0/gems/chef-18.10.17/lib", "chef_effortless": null }, "ohai": { - "version": "17.9.0", - "ohai_root": "/opt/chef/embedded/lib/ruby/gems/3.0.0/gems/ohai-17.9.0/lib/ohai" + "version": "18.2.13", + "ohai_root": "/opt/chef/embedded/lib/ruby/gems/3.1.0/gems/ohai-18.2.13/lib/ohai" } } }, @@ -55,4 +66,4 @@ "role[kvm_guest]", "role[dirsrv_supplier]" ] -} +} \ No newline at end of file diff --git a/nodes/leo.json b/nodes/leo.kosmos.org.json similarity index 84% rename from nodes/leo.json rename to nodes/leo.kosmos.org.json index ed647d2..144eeb0 100644 --- a/nodes/leo.json +++ b/nodes/leo.kosmos.org.json @@ -1,12 +1,13 @@ { - "name": "leo", + "name": "leo.kosmos.org", + "chef_environment": "production", "normal": { "knife_zero": { - "host": "leo.kosmos.org" + "host": "10.1.1.204" } }, "automatic": { - "fqdn": "leo", + "fqdn": "leo.kosmos.org", "os": "linux", "os_version": "5.15.0-173-generic", "hostname": "leo", @@ -17,6 +18,7 @@ "recipes": [ "kosmos-base", "kosmos-base::default", + "kosmos_prometheus::node_exporter", "kosmos_kvm::host", "apt::default", "timezone_iii::default", @@ -32,7 +34,8 @@ "postfix::_common", "postfix::_attributes", "postfix::sasl_auth", - "hostname::default" + "hostname::default", + "firewall::default" ], "platform": "ubuntu", "platform_version": "22.04", @@ -53,4 +56,4 @@ "role[base]", "recipe[kosmos_kvm::host]" ] -} +} \ No newline at end of file diff --git a/nodes/lq-1.json b/nodes/lq-1.json index 574ec90..a729c80 100644 --- a/nodes/lq-1.json +++ b/nodes/lq-1.json @@ -4,7 +4,8 @@ "normal": { "knife_zero": { "host": "10.1.1.87" - } + }, + "vm_host": "draco" }, "automatic": { "fqdn": "lq-1", @@ -21,6 +22,7 @@ "recipes": [ "kosmos-base", "kosmos-base::default", + "kosmos_prometheus::node_exporter", "kosmos_kvm::guest", "kosmos_garage", "kosmos_garage::default", @@ -32,6 +34,7 @@ "timezone_iii::debian", "ntp::default", "ntp::apparmor", + "kosmos-base::journald_conf", "kosmos-base::systemd_emails", "apt::unattended-upgrades", "kosmos-base::firewall", @@ -49,13 +52,13 @@ "cloud": null, "chef_packages": { "chef": { - "version": "18.2.7", - "chef_root": "/opt/chef/embedded/lib/ruby/gems/3.1.0/gems/chef-18.2.7/lib", + "version": "18.10.17", + "chef_root": "/opt/chef/embedded/lib/ruby/gems/3.1.0/gems/chef-18.10.17/lib", "chef_effortless": null }, "ohai": { - "version": "18.1.4", - "ohai_root": "/opt/chef/embedded/lib/ruby/gems/3.1.0/gems/ohai-18.1.4/lib/ohai" + "version": "18.2.13", + "ohai_root": "/opt/chef/embedded/lib/ruby/gems/3.1.0/gems/ohai-18.2.13/lib/ohai" } } }, diff --git a/nodes/lq-2.json b/nodes/lq-2.json index 5adfc66..4acf21d 100644 --- a/nodes/lq-2.json +++ b/nodes/lq-2.json @@ -4,7 +4,8 @@ "normal": { "knife_zero": { "host": "10.1.1.188" - } + }, + "vm_host": "fornax" }, "automatic": { "fqdn": "lq-2", @@ -21,6 +22,7 @@ "recipes": [ "kosmos-base", "kosmos-base::default", + "kosmos_prometheus::node_exporter", "kosmos_kvm::guest", "kosmos_garage", "kosmos_garage::default", @@ -32,6 +34,7 @@ "timezone_iii::debian", "ntp::default", "ntp::apparmor", + "kosmos-base::journald_conf", "kosmos-base::systemd_emails", "apt::unattended-upgrades", "kosmos-base::firewall", @@ -49,13 +52,13 @@ "cloud": null, "chef_packages": { "chef": { - "version": "18.2.7", - "chef_root": "/opt/chef/embedded/lib/ruby/gems/3.1.0/gems/chef-18.2.7/lib", + "version": "18.10.17", + "chef_root": "/opt/chef/embedded/lib/ruby/gems/3.1.0/gems/chef-18.10.17/lib", "chef_effortless": null }, "ohai": { - "version": "18.1.4", - "ohai_root": "/opt/chef/embedded/lib/ruby/gems/3.1.0/gems/ohai-18.1.4/lib/ohai" + "version": "18.2.13", + "ohai_root": "/opt/chef/embedded/lib/ruby/gems/3.1.0/gems/ohai-18.2.13/lib/ohai" } } }, diff --git a/nodes/mail.kosmos.org.json b/nodes/mail.kosmos.org.json index 4f116ed..7dbb29e 100644 --- a/nodes/mail.kosmos.org.json +++ b/nodes/mail.kosmos.org.json @@ -4,13 +4,14 @@ "normal": { "knife_zero": { "host": "10.1.1.95" - } + }, + "vm_host": "draco" }, "automatic": { "fqdn": "mail.kosmos.org", "os": "linux", "os_version": "5.15.0-1048-kvm", - "hostname": "mail.kosmos.org", + "hostname": "mail", "ipaddress": "192.168.122.131", "roles": [ "base", @@ -21,6 +22,7 @@ "recipes": [ "kosmos-base", "kosmos-base::default", + "kosmos_prometheus::node_exporter", "kosmos_kvm::guest", "kosmos-dirsrv::hostsfile", "kosmos_email", @@ -68,4 +70,4 @@ "role[kvm_guest]", "role[email_server]" ] -} +} \ No newline at end of file diff --git a/nodes/mastodon-3.json b/nodes/mastodon-3.json index c5dba55..29bb553 100644 --- a/nodes/mastodon-3.json +++ b/nodes/mastodon-3.json @@ -4,7 +4,8 @@ "normal": { "knife_zero": { "host": "10.1.1.30" - } + }, + "vm_host": "fornax" }, "automatic": { "fqdn": "mastodon-3", @@ -83,7 +84,7 @@ } }, "run_list": [ - "recipe[kosmos-base]", + "role[base]", "role[kvm_guest]", "role[ldap_client]", "role[garage_gateway]", diff --git a/nodes/nodejs-4.json b/nodes/nodejs-4.json index 659966b..48ab1fa 100644 --- a/nodes/nodejs-4.json +++ b/nodes/nodejs-4.json @@ -3,7 +3,8 @@ "normal": { "knife_zero": { "host": "10.1.1.138" - } + }, + "vm_host": "fornax" }, "automatic": { "fqdn": "nodejs-4", @@ -21,6 +22,7 @@ "recipes": [ "kosmos-base", "kosmos-base::default", + "kosmos_prometheus::node_exporter", "kosmos_kvm::guest", "kosmos-hubot::botka_irc-libera-chat", "kosmos-hubot::hal8000_xmpp", @@ -35,6 +37,7 @@ "timezone_iii::debian", "ntp::default", "ntp::apparmor", + "kosmos-base::journald_conf", "kosmos-base::systemd_emails", "apt::unattended-upgrades", "kosmos-base::firewall", diff --git a/nodes/postgres-11.json b/nodes/postgres-11.json index a01909f..8f434cc 100644 --- a/nodes/postgres-11.json +++ b/nodes/postgres-11.json @@ -4,7 +4,8 @@ "normal": { "knife_zero": { "host": "10.1.1.91" - } + }, + "vm_host": "leo" }, "automatic": { "fqdn": "postgres-11", @@ -20,6 +21,7 @@ "recipes": [ "kosmos-base", "kosmos-base::default", + "kosmos_prometheus::node_exporter", "kosmos_kvm::guest", "kosmos_postgresql::primary", "kosmos-akkounts::pg_db", @@ -66,4 +68,4 @@ "role[kvm_guest]", "role[postgresql_primary]" ] -} +} \ No newline at end of file diff --git a/nodes/postgres-12.json b/nodes/postgres-12.json index d2359b2..2de0881 100644 --- a/nodes/postgres-12.json +++ b/nodes/postgres-12.json @@ -4,7 +4,8 @@ "normal": { "knife_zero": { "host": "10.1.1.134" - } + }, + "vm_host": "her" }, "automatic": { "fqdn": "postgres-12", @@ -20,6 +21,7 @@ "recipes": [ "kosmos-base", "kosmos-base::default", + "kosmos_prometheus::node_exporter", "kosmos_kvm::guest", "kosmos_postgresql::hostsfile", "kosmos_postgresql::replica", @@ -61,4 +63,4 @@ "role[kvm_guest]", "role[postgresql_replica]" ] -} +} \ No newline at end of file diff --git a/nodes/garage-13.json b/nodes/prometheus-1.json similarity index 65% rename from nodes/garage-13.json rename to nodes/prometheus-1.json index 3c0e075..f834101 100644 --- a/nodes/garage-13.json +++ b/nodes/prometheus-1.json @@ -1,35 +1,33 @@ { - "name": "garage-13", + "name": "prometheus-1", "chef_environment": "production", "normal": { "knife_zero": { - "host": "10.1.1.179" - } + "host": "10.1.1.146" + }, + "vm_host": "leo" }, "automatic": { - "fqdn": "garage-13", + "fqdn": "prometheus-1", "os": "linux", - "os_version": "5.15.0-1059-kvm", - "hostname": "garage-13", - "ipaddress": "192.168.122.27", + "os_version": "6.8.0-134-generic", + "hostname": "prometheus-1", + "ipaddress": "192.168.122.166", "roles": [ "base", "kvm_guest", - "garage_node" + "prometheus_server" ], "recipes": [ "kosmos-base", "kosmos-base::default", + "kosmos_prometheus::node_exporter", "kosmos_kvm::guest", - "kosmos_garage", - "kosmos_garage::default", - "kosmos_garage::firewall_rpc", - "kosmos_garage::firewall_apis", + "kosmos_prometheus::server", + "kosmos_prometheus::alertmanager", "apt::default", "timezone_iii::default", "timezone_iii::debian", - "ntp::default", - "ntp::apparmor", "kosmos-base::journald_conf", "kosmos-base::systemd_emails", "apt::unattended-upgrades", @@ -43,23 +41,23 @@ "firewall::default" ], "platform": "ubuntu", - "platform_version": "22.04", + "platform_version": "24.04", "cloud": null, "chef_packages": { "chef": { - "version": "18.7.10", - "chef_root": "/opt/chef/embedded/lib/ruby/gems/3.1.0/gems/chef-18.7.10/lib", + "version": "18.10.17", + "chef_root": "/opt/chef/embedded/lib/ruby/gems/3.1.0/gems/chef-18.10.17/lib", "chef_effortless": null }, "ohai": { - "version": "18.2.5", - "ohai_root": "/opt/chef/embedded/lib/ruby/gems/3.1.0/gems/ohai-18.2.5/lib/ohai" + "version": "18.2.13", + "ohai_root": "/opt/chef/embedded/lib/ruby/gems/3.1.0/gems/ohai-18.2.13/lib/ohai" } } }, "run_list": [ "role[base]", "role[kvm_guest]", - "role[garage_node]" + "role[prometheus_server]" ] -} +} \ No newline at end of file diff --git a/nodes/redis-1.json b/nodes/redis-1.json index cd0fb79..23d6420 100644 --- a/nodes/redis-1.json +++ b/nodes/redis-1.json @@ -4,7 +4,8 @@ "normal": { "knife_zero": { "host": "10.1.1.225" - } + }, + "vm_host": "draco" }, "automatic": { "fqdn": "redis-1", @@ -20,6 +21,7 @@ "recipes": [ "kosmos-base", "kosmos-base::default", + "kosmos_prometheus::node_exporter", "kosmos_kvm::guest", "kosmos_redis", "kosmos_redis::default", @@ -30,6 +32,7 @@ "timezone_iii::debian", "ntp::default", "ntp::apparmor", + "kosmos-base::journald_conf", "kosmos-base::systemd_emails", "apt::unattended-upgrades", "kosmos-base::firewall", @@ -39,6 +42,7 @@ "postfix::_attributes", "postfix::sasl_auth", "hostname::default", + "firewall::default", "redisio::default", "redisio::_install_prereqs", "redisio::install", diff --git a/nodes/redis-2.json b/nodes/redis-2.json index 07be0f5..cbd65d4 100644 --- a/nodes/redis-2.json +++ b/nodes/redis-2.json @@ -4,7 +4,8 @@ "normal": { "knife_zero": { "host": "10.1.1.208" - } + }, + "vm_host": "fornax" }, "automatic": { "fqdn": "redis-2", @@ -20,6 +21,7 @@ "recipes": [ "kosmos-base", "kosmos-base::default", + "kosmos_prometheus::node_exporter", "kosmos_kvm::guest", "kosmos_redis::replica", "kosmos_redis::firewall", @@ -28,6 +30,7 @@ "timezone_iii::debian", "ntp::default", "ntp::apparmor", + "kosmos-base::journald_conf", "kosmos-base::systemd_emails", "apt::unattended-upgrades", "kosmos-base::firewall", @@ -37,6 +40,7 @@ "postfix::_attributes", "postfix::sasl_auth", "hostname::default", + "firewall::default", "redisio::default", "redisio::_install_prereqs", "redisio::install", diff --git a/nodes/rs-discourse-1.json b/nodes/rs-discourse-1.json index ef67953..ddc0b12 100644 --- a/nodes/rs-discourse-1.json +++ b/nodes/rs-discourse-1.json @@ -3,29 +3,33 @@ "normal": { "knife_zero": { "host": "10.1.1.187" - } + }, + "vm_host": "fornax" }, "automatic": { "fqdn": "rs-discourse-1", "os": "linux", - "os_version": "5.4.0-1076-kvm", + "os_version": "5.4.0-1118-kvm", "hostname": "rs-discourse-1", "ipaddress": "192.168.122.30", "roles": [ - "remotestorage_discourse", - "kvm_guest" + "base", + "kvm_guest", + "remotestorage_discourse" ], "recipes": [ "kosmos-base", "kosmos-base::default", + "kosmos_prometheus::node_exporter", + "kosmos_kvm::guest", "remotestorage_discourse", "remotestorage_discourse::default", - "kosmos_kvm::guest", "apt::default", "timezone_iii::default", "timezone_iii::debian", "ntp::default", "ntp::apparmor", + "kosmos-base::journald_conf", "kosmos-base::systemd_emails", "apt::unattended-upgrades", "kosmos-base::firewall", @@ -35,28 +39,27 @@ "postfix::_attributes", "postfix::sasl_auth", "hostname::default", - "discourse::default", "firewall::default", - "chef-sugar::default" + "discourse::default" ], "platform": "ubuntu", "platform_version": "20.04", "cloud": null, "chef_packages": { "chef": { - "version": "17.10.3", - "chef_root": "/opt/chef/embedded/lib/ruby/gems/3.0.0/gems/chef-17.10.3/lib", + "version": "18.10.17", + "chef_root": "/opt/chef/embedded/lib/ruby/gems/3.1.0/gems/chef-18.10.17/lib", "chef_effortless": null }, "ohai": { - "version": "17.9.0", - "ohai_root": "/opt/chef/embedded/lib/ruby/gems/3.0.0/gems/ohai-17.9.0/lib/ohai" + "version": "18.2.13", + "ohai_root": "/opt/chef/embedded/lib/ruby/gems/3.1.0/gems/ohai-18.2.13/lib/ohai" } } }, "run_list": [ - "recipe[kosmos-base]", + "role[base]", "role[kvm_guest]", "role[remotestorage_discourse]" ] -} +} \ No newline at end of file diff --git a/nodes/rsk-mainnet-3.json b/nodes/rsk-mainnet-3.json index e6ad2d6..8593c70 100644 --- a/nodes/rsk-mainnet-3.json +++ b/nodes/rsk-mainnet-3.json @@ -3,7 +3,8 @@ "normal": { "knife_zero": { "host": "10.1.1.150" - } + }, + "vm_host": "her" }, "automatic": { "fqdn": "rsk-mainnet-3", @@ -58,4 +59,4 @@ "role[kvm_guest]", "role[rskj_mainnet]" ] -} +} \ No newline at end of file diff --git a/nodes/rsk-testnet-4.json b/nodes/rsk-testnet-4.json index 21ab8f2..51e5d16 100644 --- a/nodes/rsk-testnet-4.json +++ b/nodes/rsk-testnet-4.json @@ -3,7 +3,8 @@ "normal": { "knife_zero": { "host": "10.1.1.126" - } + }, + "vm_host": "her" }, "automatic": { "fqdn": "rsk-testnet-4", @@ -57,4 +58,4 @@ "role[base]", "role[kvm_guest]" ] -} +} \ No newline at end of file diff --git a/nodes/rsk-testnet-6.json b/nodes/rsk-testnet-6.json index a7c4ee0..ad6795b 100644 --- a/nodes/rsk-testnet-6.json +++ b/nodes/rsk-testnet-6.json @@ -3,7 +3,8 @@ "normal": { "knife_zero": { "host": "10.1.1.20" - } + }, + "vm_host": "leo" }, "automatic": { "fqdn": "rsk-testnet-6", @@ -19,6 +20,7 @@ "recipes": [ "kosmos-base", "kosmos-base::default", + "kosmos_prometheus::node_exporter", "kosmos_kvm::guest", "kosmos_rsk::rskj", "apt::default", @@ -34,8 +36,8 @@ "postfix::_attributes", "postfix::sasl_auth", "hostname::default", - "kosmos_rsk::firewall", - "firewall::default" + "firewall::default", + "kosmos_rsk::firewall" ], "platform": "ubuntu", "platform_version": "24.04", @@ -57,4 +59,4 @@ "role[kvm_guest]", "role[rskj_testnet]" ] -} +} \ No newline at end of file diff --git a/nodes/sentry-1.json b/nodes/sentry-1.json index a037ffb..52e353d 100644 --- a/nodes/sentry-1.json +++ b/nodes/sentry-1.json @@ -4,7 +4,8 @@ "normal": { "knife_zero": { "host": "10.1.1.132" - } + }, + "vm_host": "her" }, "automatic": { "fqdn": "sentry-1", diff --git a/nodes/strfry-1.json b/nodes/strfry-1.json index 1e9de13..3b60167 100644 --- a/nodes/strfry-1.json +++ b/nodes/strfry-1.json @@ -4,7 +4,8 @@ "normal": { "knife_zero": { "host": "10.1.1.164" - } + }, + "vm_host": "draco" }, "automatic": { "fqdn": "strfry-1", @@ -22,6 +23,7 @@ "recipes": [ "kosmos-base", "kosmos-base::default", + "kosmos_prometheus::node_exporter", "kosmos_kvm::guest", "kosmos-dirsrv::hostsfile", "strfry", @@ -70,4 +72,4 @@ "role[strfry]", "role[blossom]" ] -} +} \ No newline at end of file diff --git a/nodes/uploads-1.json b/nodes/uploads-1.json deleted file mode 100644 index f2af120..0000000 --- a/nodes/uploads-1.json +++ /dev/null @@ -1,67 +0,0 @@ -{ - "name": "uploads-1", - "normal": { - "knife_zero": { - "host": "10.1.1.98" - } - }, - "automatic": { - "fqdn": "uploads-1", - "os": "linux", - "os_version": "5.4.0-128-generic", - "hostname": "uploads-1", - "ipaddress": "192.168.122.230", - "roles": [ - "kvm_guest" - ], - "recipes": [ - "kosmos-base", - "kosmos-base::default", - "kosmos_kvm::guest", - "kosmos-ejabberd::upload_service", - "apt::default", - "timezone_iii::default", - "timezone_iii::debian", - "ntp::default", - "ntp::apparmor", - "kosmos-base::systemd_emails", - "apt::unattended-upgrades", - "kosmos-base::firewall", - "kosmos-postfix::default", - "postfix::default", - "postfix::_common", - "postfix::_attributes", - "postfix::sasl_auth", - "hostname::default", - "kosmos-nginx::with_perl", - "nginx::default", - "nginx::package", - "nginx::ohai_plugin", - "nginx::repo", - "nginx::commons", - "nginx::commons_dir", - "nginx::commons_script", - "nginx::commons_conf", - "kosmos-nginx::firewall", - "kosmos-base::letsencrypt" - ], - "platform": "ubuntu", - "platform_version": "20.04", - "cloud": null, - "chef_packages": { - "chef": { - "version": "15.14.0", - "chef_root": "/opt/chef/embedded/lib/ruby/gems/2.6.0/gems/chef-15.14.0/lib" - }, - "ohai": { - "version": "15.12.0", - "ohai_root": "/opt/chef/embedded/lib/ruby/gems/2.6.0/gems/ohai-15.12.0/lib/ohai" - } - } - }, - "run_list": [ - "recipe[kosmos-base]", - "role[kvm_guest]", - "recipe[kosmos-ejabberd::upload_service]" - ] -} \ No newline at end of file diff --git a/nodes/wiki-1.json b/nodes/wiki-1.json index 0dcd3fb..9b43155 100644 --- a/nodes/wiki-1.json +++ b/nodes/wiki-1.json @@ -3,7 +3,8 @@ "normal": { "knife_zero": { "host": "10.1.1.206" - } + }, + "vm_host": "draco" }, "automatic": { "fqdn": "wiki-1", @@ -19,6 +20,7 @@ "recipes": [ "kosmos-base", "kosmos-base::default", + "kosmos_prometheus::node_exporter", "kosmos_kvm::guest", "kosmos-dirsrv::hostsfile", "kosmos-mediawiki", diff --git a/roles/base.rb b/roles/base.rb index 86d3ab5..dc838b3 100644 --- a/roles/base.rb +++ b/roles/base.rb @@ -2,4 +2,5 @@ name "base" run_list %w( kosmos-base::default + kosmos_prometheus::node_exporter ) diff --git a/roles/prometheus_server.rb b/roles/prometheus_server.rb new file mode 100644 index 0000000..dc0bfea --- /dev/null +++ b/roles/prometheus_server.rb @@ -0,0 +1,12 @@ +name "prometheus_server" + +default_run_list = [ + "kosmos_prometheus::server", + "kosmos_prometheus::alertmanager" +] + +env_run_lists( + "_default" => default_run_list, + "development" => default_run_list, + "production" => default_run_list +) diff --git a/scripts/util/sanitize_node.sh b/scripts/util/sanitize_node.sh new file mode 100755 index 0000000..99d4ec3 --- /dev/null +++ b/scripts/util/sanitize_node.sh @@ -0,0 +1,69 @@ +#!/usr/bin/env bash +# Strip the top-level "override" and "default" attributes from Chef node JSON files. +# +# Usage: +# ./scripts/util/strip_node_attrs.sh [ ...] +# ./scripts/util/strip_node_attrs.sh --all +# +# A may be a bare node name (e.g. "prometheus-1"), a path relative to the +# repository root (e.g. "nodes/prometheus-1.json"), or an absolute path. +# +# Exit codes: +# 0 - all target files processed successfully +# 1 - one or more targets could not be processed (missing args, missing jq, +# file not found, invalid JSON, or write failure) + +set -uo pipefail + +command -v jq >/dev/null 2>&1 || exit 1 + +root="$(git rev-parse --show-toplevel 2>/dev/null)" || root="$(pwd)" + +targets=() +if [ "$#" -eq 0 ]; then + exit 1 +fi + +for arg in "$@"; do + if [ "$arg" = "--all" ]; then + shopt -s nullglob + for f in "$root"/nodes/*.json; do + targets+=("$f") + done + shopt -u nullglob + continue + fi + + if [[ "$arg" == */* ]]; then + if [[ "$arg" = /* ]]; then + targets+=("$arg") + else + targets+=("$root/$arg") + fi + else + targets+=("$root/nodes/$arg.json") + fi +done + +if [ "${#targets[@]}" -eq 0 ]; then + exit 1 +fi + +rc=0 +for file in "${targets[@]}"; do + [ -f "$file" ] || { rc=1; continue; } + + jq -e . "$file" >/dev/null 2>&1 || { rc=1; continue; } + + tmp="$(mktemp "${file}.XXXXXX")" || { rc=1; continue; } + if jq --indent 2 'del(.override, .default)' "$file" > "$tmp" 2>/dev/null; then + truncate -s -1 "$tmp" 2>/dev/null || true + mv "$tmp" "$file" || { rm -f "$tmp"; rc=1; continue; } + else + rm -f "$tmp" + rc=1 + continue + fi +done + +exit "$rc" diff --git a/site-cookbooks/kosmos-postfix/recipes/default.rb b/site-cookbooks/kosmos-postfix/recipes/default.rb index 1afd892..c689928 100644 --- a/site-cookbooks/kosmos-postfix/recipes/default.rb +++ b/site-cookbooks/kosmos-postfix/recipes/default.rb @@ -21,5 +21,6 @@ node.default["postfix"]["sasl"] = { node.default["postfix"]["main"]["relayhost"] = smtp_credentials["relayhost"] node.default["postfix"]["main"]["smtp_sasl_auth_enable"] = "yes" node.default["postfix"]["main"]["smtp_sasl_security_options"] = "noanonymous" +node.default["postfix"]["main"]["smtpd_relay_restrictions"] = "permit_mynetworks, reject" include_recipe "postfix::default" diff --git a/site-cookbooks/kosmos_prometheus/.gitignore b/site-cookbooks/kosmos_prometheus/.gitignore new file mode 100644 index 0000000..f1e57b8 --- /dev/null +++ b/site-cookbooks/kosmos_prometheus/.gitignore @@ -0,0 +1,25 @@ +.vagrant +*~ +*# +.#* +\#*# +.*.sw[a-z] +*.un~ + +# Bundler +Gemfile.lock +gems.locked +bin/* +.bundle/* + +# test kitchen +.kitchen/ +kitchen.local.yml + +# Chef Infra +Berksfile.lock +.zero-knife.rb +Policyfile.lock.json + +.idea/ + diff --git a/site-cookbooks/kosmos_prometheus/CHANGELOG.md b/site-cookbooks/kosmos_prometheus/CHANGELOG.md new file mode 100644 index 0000000..2aa0619 --- /dev/null +++ b/site-cookbooks/kosmos_prometheus/CHANGELOG.md @@ -0,0 +1,7 @@ +# kosmos_prometheus CHANGELOG + +This file is used to list changes made in each version of the kosmos_prometheus cookbook. + +## 0.1.0 + +Initial release. diff --git a/site-cookbooks/kosmos_prometheus/LICENSE b/site-cookbooks/kosmos_prometheus/LICENSE new file mode 100644 index 0000000..98808c8 --- /dev/null +++ b/site-cookbooks/kosmos_prometheus/LICENSE @@ -0,0 +1,20 @@ +Copyright (c) 2026 Kosmos Developers + +Permission is hereby granted, free of charge, to any person obtaining +a copy of this software and associated documentation files (the +"Software"), to deal in the Software without restriction, including +without limitation the rights to use, copy, modify, merge, publish, +distribute, sublicense, and/or sell copies of the Software, and to +permit persons to whom the Software is furnished to do so, subject to +the following conditions: + +The above copyright notice and this permission notice shall be +included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE +LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION +WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. diff --git a/site-cookbooks/kosmos_prometheus/README.md b/site-cookbooks/kosmos_prometheus/README.md new file mode 100644 index 0000000..5c56b8e --- /dev/null +++ b/site-cookbooks/kosmos_prometheus/README.md @@ -0,0 +1,4 @@ +# kosmos_prometheus + +TODO: Enter the cookbook description here. + diff --git a/site-cookbooks/kosmos_prometheus/attributes/default.rb b/site-cookbooks/kosmos_prometheus/attributes/default.rb new file mode 100644 index 0000000..5851470 --- /dev/null +++ b/site-cookbooks/kosmos_prometheus/attributes/default.rb @@ -0,0 +1,85 @@ +node.default["kosmos_prometheus"]["version"] = "3.13.0" +node.default["kosmos_prometheus"]["checksum"] = "744d93324cc024d82089921737bd797474d7f1e5dbbfd1c6b387bad258538cb9" + +node.default["kosmos_prometheus"]["data_dir"] = "/var/lib/prometheus" +node.default["kosmos_prometheus"]["config_dir"] = "/etc/prometheus" +node.default["kosmos_prometheus"]["rules_dir"] = "#{node["kosmos_prometheus"]["config_dir"]}/rules" + +node.default["kosmos_prometheus"]["web"]["port"] = 9090 + +node.default["kosmos_prometheus"]["alertmanager"]["version"] = "0.33.0" +node.default["kosmos_prometheus"]["alertmanager"]["checksum"] = "8ce11c42e8a6dfbbf93a59c0b193cb1329210b36d0c7ef3df7b745608675a1d1" + +node.default["kosmos_prometheus"]["node_exporter"]["version"] = "1.11.1" +node.default["kosmos_prometheus"]["node_exporter"]["checksum"] = "9f5ea48e5bc7b656f8a91a32e7d7deb89f70f73dabd0d974418aca15f37d6810" +node.default["kosmos_prometheus"]["node_exporter"]["port"] = 9100 + +node.default["kosmos_prometheus"]["alertmanager"]["route"] = { + "receiver" => "xmpp", + "group_by" => ["alertname", "severity"], + "group_wait" => "20s", + "group_interval" => "30s", + "repeat_interval" => "3h", + "routes" => [ + { + "receiver" => "xmpp", + "continue" => true, + }, + { + "receiver" => "email", + "matchers" => ['severity="critical"'], + "continue" => true, + } + ], +} + +node.default["kosmos_prometheus"]["alertmanager"]["port"] = 9093 + +node.default["kosmos_prometheus"]["alertmanager"]["receivers"] = [ + { + "name" => "xmpp", + "webhook_configs" => [ + { + "send_resolved" => true, + "room" => "info@kosmos.chat", + } + ] + }, + { + "name" => "email", + "email_configs" => [ + { + "send_resolved" => true, + "to" => "ops@kosmos.org", + "from" => "ops@kosmos.org", + "smarthost" => "localhost:25", + "require_tls" => false, + } + ] + } +] + +node.default["kosmos_prometheus"]["global"] = { + "scrape_interval" => "15s", + "evaluation_interval" => "15s", +} + +node.default["kosmos_prometheus"]["alerting"] = { + "alertmanagers" => [ + "static_configs" => [ + { "targets" => ["127.0.0.1:#{node["kosmos_prometheus"]["alertmanager"]["port"]}"] } + ] + ] +} + +node.default["kosmos_prometheus"]["jobs"] = { + "prometheus" => { "targets" => [{ "target" => "localhost:#{node["kosmos_prometheus"]["web"]["port"]}", "instance" => "localhost" }] }, +} + +node.default["kosmos_prometheus"]["rule_files"] = [ + "#{node["kosmos_prometheus"]["rules_dir"]}/*.yml" +] + +node.default["kosmos_prometheus"]["rules"] = [ + "node_exporter" +] diff --git a/site-cookbooks/kosmos_prometheus/chefignore b/site-cookbooks/kosmos_prometheus/chefignore new file mode 100644 index 0000000..cc170ea --- /dev/null +++ b/site-cookbooks/kosmos_prometheus/chefignore @@ -0,0 +1,115 @@ +# Put files/directories that should be ignored in this file when uploading +# to a Chef Infra Server or Supermarket. +# Lines that start with '# ' are comments. + +# OS generated files # +###################### +.DS_Store +ehthumbs.db +Icon? +nohup.out +Thumbs.db +.envrc + +# EDITORS # +########### +.#* +.project +.settings +*_flymake +*_flymake.* +*.bak +*.sw[a-z] +*.tmproj +*~ +\#* +REVISION +TAGS* +tmtags +.vscode +.editorconfig + +## COMPILED ## +############## +*.class +*.com +*.dll +*.exe +*.o +*.pyc +*.so +*/rdoc/ +a.out +mkmf.log + +# Testing # +########### +.circleci/* +.codeclimate.yml +.delivery/* +.foodcritic +.kitchen* +.mdlrc +.overcommit.yml +.rspec +.rubocop.yml +.travis.yml +.watchr +.yamllint +azure-pipelines.yml +Dangerfile +examples/* +features/* +Guardfile +kitchen.yml* +mlc_config.json +Procfile +Rakefile +spec/* +test/* + +# SCM # +####### +.git +.gitattributes +.gitconfig +.github/* +.gitignore +.gitkeep +.gitmodules +.svn +*/.bzr/* +*/.git +*/.hg/* +*/.svn/* + +# Berkshelf # +############# +Berksfile +Berksfile.lock +cookbooks/* +tmp + +# Bundler # +########### +vendor/* +Gemfile +Gemfile.lock + +# Policyfile # +############## +Policyfile.rb +Policyfile.lock.json + +# Documentation # +############# +CODE_OF_CONDUCT* +CONTRIBUTING* +documentation/* +TESTING* +UPGRADING* + +# Vagrant # +########### +.vagrant +Vagrantfile diff --git a/site-cookbooks/kosmos_prometheus/files/default/rules/node_exporter.yml b/site-cookbooks/kosmos_prometheus/files/default/rules/node_exporter.yml new file mode 100644 index 0000000..854e0f5 --- /dev/null +++ b/site-cookbooks/kosmos_prometheus/files/default/rules/node_exporter.yml @@ -0,0 +1,94 @@ +groups: + - name: NodeExporter + rules: + - alert: NodeDown + expr: up{job="node"} == 0 + for: 20s + labels: + severity: warning + annotations: + summary: "Node is down" + + - alert: NodeFilesystemAlmostOutOfSpace + expr: | + (node_filesystem_avail_bytes{fstype!~"tmpfs|fuse.lxcfs|overlay|squashfs|ramfs",mountpoint!~"/run.*|/var/lib/docker/.*"} + / node_filesystem_size_bytes{fstype!~"tmpfs|fuse.lxcfs|overlay|squashfs|ramfs",mountpoint!~"/run.*|/var/lib/docker/.*"}) * 100 < 10 + for: 5m + labels: + severity: warning + annotations: + summary: "Filesystem has < 10% free space" + + - alert: NodeFilesystemAlmostOutOfSpace + expr: | + (node_filesystem_avail_bytes{fstype!~"tmpfs|fuse.lxcfs|overlay|squashfs|ramfs",mountpoint!~"/run.*|/var/lib/docker/.*"} + / node_filesystem_size_bytes{fstype!~"tmpfs|fuse.lxcfs|overlay|squashfs|ramfs",mountpoint!~"/run.*|/var/lib/docker/.*"}) * 100 < 5 + for: 5m + labels: + severity: critical + annotations: + summary: "Filesystem has < 5% free space" + + - alert: NodeFilesystemAlmostOutOfInodes + expr: (node_filesystem_files_free / node_filesystem_files) * 100 < 10 + for: 5m + labels: + severity: warning + annotations: + summary: "Filesystem has < 10% free inodes" + + - alert: NodeFilesystemReadOnly + expr: node_filesystem_readonly == 1 + for: 2m + labels: + severity: critical + annotations: + summary: "Filesystem remounted read-only" + + - alert: NodeMemoryHigh + expr: (1 - node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes) * 100 > 90 + for: 10m + labels: + severity: warning + annotations: + summary: "Memory usage > 90%" + + - alert: NodeMemoryHigh + expr: (1 - node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes) * 100 > 95 + for: 5m + labels: + severity: critical + annotations: + summary: "Memory usage > 95%" + + - alert: NodeSwapHigh + expr: (node_memory_SwapTotal_bytes > 0) and ((1 - node_memory_SwapFree_bytes / node_memory_SwapTotal_bytes) * 100 > 50) + for: 10m + labels: + severity: warning + annotations: + summary: "Swap usage > 50%" + + - alert: NodeCpuHigh + expr: 100 - (avg by (instance) (rate(node_cpu_seconds_total{mode="idle"}[5m])) * 100) > 80 + for: 10m + labels: + severity: warning + annotations: + summary: "CPU usage > 80% for 10m" + + - alert: NodeCpuHigh + expr: 100 - (avg by (instance) (rate(node_cpu_seconds_total{mode="idle"}[5m])) * 100) > 95 + for: 5m + labels: + severity: critical + annotations: + summary: "CPU usage > 95% for 5m" + + - alert: NodeLoadHigh + expr: node_load1 / count by (instance) (node_cpu_seconds_total{mode="idle"}) > 1.5 + for: 10m + labels: + severity: warning + annotations: + summary: "System load1 > 1.5x CPU cores" diff --git a/site-cookbooks/kosmos_prometheus/metadata.rb b/site-cookbooks/kosmos_prometheus/metadata.rb new file mode 100644 index 0000000..1860c0f --- /dev/null +++ b/site-cookbooks/kosmos_prometheus/metadata.rb @@ -0,0 +1,21 @@ +name 'kosmos_prometheus' +maintainer 'Kosmos Developers' +maintainer_email 'mail@kosmos.org' +license 'MIT' +description 'Installs/Configures prometheus' +version '0.1.0' +chef_version '>= 16.0' + +depends "firewall" + +# The `issues_url` points to the location where issues for this cookbook are +# tracked. A `View Issues` link will be displayed on this cookbook's page when +# uploaded to a Supermarket. +# +# issues_url 'https://github.com//kosmos_prometheus/issues' + +# The `source_url` points to the development repository for this cookbook. A +# `View Source` link will be displayed on this cookbook's page when uploaded to +# a Supermarket. +# +# source_url 'https://github.com//kosmos_prometheus' diff --git a/site-cookbooks/kosmos_prometheus/recipes/alertmanager.rb b/site-cookbooks/kosmos_prometheus/recipes/alertmanager.rb new file mode 100644 index 0000000..b8625a9 --- /dev/null +++ b/site-cookbooks/kosmos_prometheus/recipes/alertmanager.rb @@ -0,0 +1,107 @@ +# +# Cookbook:: kosmos_prometheus +# Recipe:: alertmanager +# + +include_recipe "firewall" + +version = node["kosmos_prometheus"]["alertmanager"]["version"] +checksum = node["kosmos_prometheus"]["alertmanager"]["checksum"] +tarball = "#{Chef::Config[:file_cache_path]}/alertmanager-#{version}.linux-amd64.tar.gz" +binary_url = "https://github.com/prometheus/alertmanager/releases/download/v#{version}/alertmanager-#{version}.linux-amd64.tar.gz" + +alertmanager_port = node["kosmos_prometheus"]["alertmanager"]["port"] + +group "alertmanager" + +user "alertmanager" do + gid "alertmanager" + system true + shell "/bin/false" + home "/nonexistent" +end + +directory "/var/lib/alertmanager" do + owner "alertmanager" + group "alertmanager" + mode "0755" + recursive true +end + +directory "/etc/prometheus" do + owner "root" + group "root" + mode "0755" + recursive true +end + +package %w(tar bzip2) + +remote_file tarball do + source binary_url + checksum checksum + action :create + notifies :run, "execute[install_alertmanager]", :immediately +end + +execute "install_alertmanager" do + command "tar -xzf #{tarball} -C /usr/local/bin --strip-components=1 alertmanager-#{version}.linux-amd64/alertmanager" + action :nothing + notifies :restart, "service[alertmanager]", :delayed +end + +file "/usr/local/bin/alertmanager" do + owner "root" + group "root" + mode "0755" + notifies :restart, "service[alertmanager]", :delayed +end + +# TODO use Auth header once supported by hubot-incoming-webhook +credentials = data_bag_item("credentials", "hal8000_xmpp") +webhook_url = "https://hal8000.kosmos.chat/incoming/#{credentials['webhook_token']}" + +template "/etc/prometheus/alertmanager.yml" do + source "alertmanager.yml.erb" + owner "root" + group "alertmanager" + mode "0644" + variables route: node["kosmos_prometheus"]["alertmanager"]["route"], + receivers: node["kosmos_prometheus"]["alertmanager"]["receivers"], + webhook_url: webhook_url + notifies :restart, "service[alertmanager]", :delayed + sensitive true +end + +systemd_unit "alertmanager.service" do + content({ + Unit: { + Description: "Prometheus Alertmanager", + After: "network.target", + }, + Service: { + Type: "simple", + User: "alertmanager", + Group: "alertmanager", + ExecStart: "/usr/local/bin/alertmanager --config.file=/etc/prometheus/alertmanager.yml --storage.path=/var/lib/alertmanager --web.listen-address=:#{alertmanager_port}", + Restart: "on-failure", + RestartSec: "5", + }, + Install: { + WantedBy: "multi-user.target", + }, + }) + triggers_reload true + action :create +end + +service "alertmanager" do + action [:enable, :start] +end + +firewall_rule "prometheus alertmanager" do + port alertmanager_port + source "10.1.1.0/24" + protocol :tcp + command :allow +end diff --git a/site-cookbooks/kosmos_prometheus/recipes/node_exporter.rb b/site-cookbooks/kosmos_prometheus/recipes/node_exporter.rb new file mode 100644 index 0000000..f365b78 --- /dev/null +++ b/site-cookbooks/kosmos_prometheus/recipes/node_exporter.rb @@ -0,0 +1,85 @@ +# +# Cookbook:: kosmos_prometheus +# Recipe:: node_exporter +# + +version = node["kosmos_prometheus"]["node_exporter"]["version"] +checksum = node["kosmos_prometheus"]["node_exporter"]["checksum"] +tarball = "#{Chef::Config[:file_cache_path]}/node_exporter-#{version}.linux-amd64.tar.gz" +binary_url = "https://github.com/prometheus/node_exporter/releases/download/v#{version}/node_exporter-#{version}.linux-amd64.tar.gz" + +node_exporter_port = node["kosmos_prometheus"]["node_exporter"]["port"] + +group "node_exporter" + +user "node_exporter" do + gid "node_exporter" + system true + shell "/bin/false" + home "/nonexistent" +end + +directory "/var/lib/node_exporter/textfile" do + owner "node_exporter" + group "node_exporter" + mode "0755" + recursive true +end + +package %w(tar bzip2) + +remote_file tarball do + source binary_url + checksum checksum + action :create + notifies :run, "execute[install_node_exporter]", :immediately +end + +execute "install_node_exporter" do + command "tar -xzf #{tarball} -C /usr/local/bin --strip-components=1 node_exporter-#{version}.linux-amd64/node_exporter" + action :nothing + notifies :restart, "service[node_exporter]", :delayed +end + +file "/usr/local/bin/node_exporter" do + owner "root" + group "root" + mode "0755" + notifies :restart, "service[node_exporter]", :delayed +end + +systemd_unit "node_exporter.service" do + content({ + Unit: { + Description: "Prometheus node exporter", + Documentation: ["https://github.com/prometheus/node_exporter"], + }, + Service: { + Type: "simple", + User: "node_exporter", + Group: "node_exporter", + ExecStart: "/usr/local/bin/node_exporter --web.listen-address=:#{node_exporter_port} --collector.textfile.directory=/var/lib/node_exporter/textfile", + Restart: "on-failure", + RestartSec: "5", + NoNewPrivileges: "yes", + ProtectSystem: "full", + ProtectHome: "yes", + }, + Install: { + WantedBy: "multi-user.target", + }, + }) + triggers_reload true + action :create +end + +service "node_exporter" do + action [:enable, :start] +end + +firewall_rule "node_exporter" do + port node_exporter_port + source "10.1.1.0/24" + protocol :tcp + command :allow +end diff --git a/site-cookbooks/kosmos_prometheus/recipes/server.rb b/site-cookbooks/kosmos_prometheus/recipes/server.rb new file mode 100644 index 0000000..ef73331 --- /dev/null +++ b/site-cookbooks/kosmos_prometheus/recipes/server.rb @@ -0,0 +1,150 @@ +# +# Cookbook:: kosmos_prometheus +# Recipe:: server +# + +include_recipe "firewall" + +version = node["kosmos_prometheus"]["version"] +checksum = node["kosmos_prometheus"]["checksum"] +tarball = "#{Chef::Config[:file_cache_path]}/prometheus-#{version}.linux-amd64.tar.gz" +binary_url = "https://github.com/prometheus/prometheus/releases/download/v#{version}/prometheus-#{version}.linux-amd64.tar.gz" + +data_dir = node["kosmos_prometheus"]["data_dir"] +config_dir = node["kosmos_prometheus"]["config_dir"] +rules_dir = node["kosmos_prometheus"]["rules_dir"] +web_port = node["kosmos_prometheus"]["web"]["port"] + +group "prometheus" + +user "prometheus" do + gid "prometheus" + system true + shell "/bin/false" + home "/nonexistent" +end + +directory data_dir do + owner "prometheus" + group "prometheus" + mode "0750" + recursive true +end + +directory config_dir do + owner "root" + group "root" + mode "0755" + recursive true +end + +directory rules_dir do + owner "root" + group "prometheus" + mode "0755" + recursive true +end + +node["kosmos_prometheus"]["rules"].each do |rule_name| + cookbook_file "#{rules_dir}/#{rule_name}.yml" do + source "rules/#{rule_name}.yml" + owner "root" + group "prometheus" + mode "0644" + notifies :reload, "service[prometheus]", :delayed + end +end + +package %w(tar bzip2) + +remote_file tarball do + source binary_url + checksum checksum + action :create + notifies :run, "execute[install_prometheus]", :immediately +end + +execute "install_prometheus" do + command "tar -xzf #{tarball} -C /usr/local/bin --strip-components=1 prometheus-#{version}.linux-amd64/prometheus" + action :nothing + notifies :restart, "service[prometheus]", :delayed +end + +file "/usr/local/bin/prometheus" do + owner "root" + group "root" + mode "0755" + notifies :restart, "service[prometheus]", :delayed +end + +jobs = node["kosmos_prometheus"]["jobs"].merge( +{ + # node exporter + "node" => { + "query" => "role:base", + "port" => node["kosmos_prometheus"]["node_exporter"]["port"] + }, + # garage metrics + "garage" => { + "query" => "role:garage_node", + "port" => 3903 + }, +}.transform_values do |config| + { + "targets" => search(:node, config["query"]).map do |n| + target = { "target" => "#{n['knife_zero']['host']}:#{config['port']}", "instance" => n.name } + target["env"] = n.chef_environment if n.chef_environment + target["vm_host"] = n["vm_host"] if n["vm_host"] + target + end.compact.sort_by { |t| t["instance"] }, + } + end +) + +template "#{config_dir}/prometheus.yml" do + source "prometheus.yml.erb" + owner "root" + group "prometheus" + mode "0644" + variables( + global_config: node["kosmos_prometheus"]["global"], + alerting: node["kosmos_prometheus"]["alerting"], + jobs: jobs, + rule_files: node["kosmos_prometheus"]["rule_files"] + ) + notifies :reload, "service[prometheus]", :delayed +end + +systemd_unit "prometheus.service" do + content({ + Unit: { + Description: "Prometheus", + After: "network.target", + }, + Service: { + Type: "simple", + User: "prometheus", + Group: "prometheus", + ExecStart: "/usr/local/bin/prometheus --config.file=/etc/prometheus/prometheus.yml --storage.tsdb.path=/var/lib/prometheus --storage.tsdb.retention.time=15d --web.listen-address=:#{web_port} --web.enable-lifecycle", + ExecReload: "/bin/kill -HUP $MAINPID", + Restart: "on-failure", + RestartSec: "5", + }, + Install: { + WantedBy: "multi-user.target", + }, + }) + triggers_reload true + action :create +end + +service "prometheus" do + action [:enable, :start] +end + +firewall_rule "prometheus web" do + port web_port + source "10.1.1.0/24" + protocol :tcp + command :allow +end diff --git a/site-cookbooks/kosmos_prometheus/templates/default/alertmanager.yml.erb b/site-cookbooks/kosmos_prometheus/templates/default/alertmanager.yml.erb new file mode 100644 index 0000000..381c538 --- /dev/null +++ b/site-cookbooks/kosmos_prometheus/templates/default/alertmanager.yml.erb @@ -0,0 +1,47 @@ +global: + resolve_timeout: 5m + +route: +<% @route.each do |k, v| %> +<% if k == "routes" && v.is_a?(Array) && !v.empty? %> + routes: +<% v.each do |child| %> + - receiver: <%= child["receiver"] %> +<% child.each do |ck, cv| %> +<% next if ck == "receiver" %> + <%= ck %>: <%= cv.is_a?(Array) ? cv.inspect : cv %> +<% end %> +<% end %> +<% else %> + <%= k %>: <%= v.is_a?(Array) ? v.inspect : v %> +<% end %> +<% end %> + +receivers: +<% @receivers.each do |receiver| %> +- name: <%= receiver["name"] %> +<% if receiver["webhook_configs"] %> +<% receiver["webhook_configs"].each do |wc| %> + webhook_configs: + - url: "<%= @webhook_url %>" + send_resolved: <%= wc["send_resolved"] %> + payload: + room: "<%= wc["room"] %>" + message: '{{ if eq .Status "resolved" }}✅{{ else if eq .CommonLabels.severity "warning" }}⚠️{{ else }}🚨{{ end }} {{ if eq .Status "resolved" }}RESOLVED{{ else if .CommonLabels.severity }}{{ .CommonLabels.severity | toUpper }}{{ else }}ALERT{{ end }}: {{ .CommonLabels.alertname }} {{ if eq .Status "resolved" }}({{ len .Alerts }} resolved){{ else if eq (len .Alerts.Firing) (len .Alerts) }}({{ len .Alerts }} firing){{ else }}({{ len .Alerts.Firing }} firing, {{ len .Alerts.Resolved }} resolved){{ end }}{{ range $i, $a := .Alerts }}{{ if lt $i 5 }}{{ "\n" }} - {{ $a.Labels.instance }}{{ if $a.Labels.vm_host }} ({{ $a.Labels.vm_host }}){{ end }}: {{ if eq $a.Status "resolved" }}resolved{{ else }}{{ $a.Annotations.summary }}{{ end }}{{ end }}{{ end }}{{ if gt (len .Alerts) 5 }}{{ "\n" }} ... and more{{ end }}' +<% end %> +<% end %> +<% if receiver["email_configs"] %> +<% receiver["email_configs"].each do |ec| %> + email_configs: + - to: "<%= ec["to"] %>" + from: "<%= ec["from"] %>" + smarthost: "<%= ec["smarthost"] %>" + require_tls: <%= ec["require_tls"] %> + send_resolved: <%= ec["send_resolved"] %> + text: '{{ if eq .Status "resolved" }}RESOLVED{{ else if .CommonLabels.severity }}{{ .CommonLabels.severity | toUpper }}{{ else }}ALERT{{ end }}: {{ .CommonLabels.alertname }} {{ if eq .Status "resolved" }}({{ len .Alerts }} resolved){{ else if eq (len .Alerts.Firing) (len .Alerts) }}({{ len .Alerts }} firing){{ else }}({{ len .Alerts.Firing }} firing, {{ len .Alerts.Resolved }} resolved){{ end }}{{ range .Alerts }} +---{{ range .Labels.SortedPairs }} + {{ .Name }}: {{ .Value }}{{ end }}{{ range .Annotations.SortedPairs }} + {{ .Name }}: {{ .Value }}{{ end }}{{ end }}' +<% end %> +<% end %> +<% end %> diff --git a/site-cookbooks/kosmos_prometheus/templates/default/prometheus.yml.erb b/site-cookbooks/kosmos_prometheus/templates/default/prometheus.yml.erb new file mode 100644 index 0000000..a7bd9f3 --- /dev/null +++ b/site-cookbooks/kosmos_prometheus/templates/default/prometheus.yml.erb @@ -0,0 +1,50 @@ +global: +<% @global_config.each do |k, v| %> + <%= k %>: "<%= v %>" +<% end %> + +<% if @alerting %> +alerting: + alertmanagers: +<% @alerting['alertmanagers'].each do |am| %> + - static_configs: +<% am['static_configs'].each do |sc| %> + - targets: +<% sc['targets'].each do |target| %> + - <%= target %> +<% end %> +<% end %> +<% end %> +<% end %> + +scrape_configs: +<% @jobs.each do |name, job| %> +- job_name: "<%= name %>" +<% if job['scrape_interval'] %> + scrape_interval: "<%= job['scrape_interval'] %>" +<% end %> +<% if job['scrape_timeout'] %> + scrape_timeout: "<%= job['scrape_timeout'] %>" +<% end %> + metrics_path: "<%= job.fetch('metrics_path', '/metrics') %>" + static_configs: +<% job['targets'].each do |t| %> + - targets: + - <%= t['target'] %> + labels: + instance: <%= t['instance'] %> +<% if t['env'] %> + env: <%= t['env'] %> +<% end %> +<% if t['vm_host'] %> + vm_host: <%= t['vm_host'] %> +<% end %> +<% end %> +<% end %> + +<% if @rule_files && !@rule_files.empty? %> +rule_files: +<% @rule_files.each do |filename| %> + - <%= filename %> +<% end %> +<% end %>