From 850db344b7b5b6cab20af96098a7e6e51d73c325 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A2u=20Cao?= Date: Fri, 3 Jul 2026 15:50:15 +0200 Subject: [PATCH 01/47] Add prometheus node --- clients/prometheus-1.json | 4 +++ nodes/prometheus-1.json | 55 +++++++++++++++++++++++++++++++++++++++ 2 files changed, 59 insertions(+) create mode 100644 clients/prometheus-1.json create mode 100644 nodes/prometheus-1.json diff --git a/clients/prometheus-1.json b/clients/prometheus-1.json new file mode 100644 index 0000000..a72e203 --- /dev/null +++ b/clients/prometheus-1.json @@ -0,0 +1,4 @@ +{ + "name": "prometheus-1", + "public_key": "-----BEGIN PUBLIC KEY-----\nMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAp7T/OBo/TZm3YqnN4+ok\nHwcJ0kW9w2rl9UfrOlWUvoPHBd2LrqpEv3Az3a150IylQ1H/UozmQA7DtjIoTA7d\nV3oLY970vYrYiURcojOo8qAZBy8EH7dfAHxuZryUeELr+3vdcHF5WrrfSt2FdFVX\nPTY95ikafAnOO0Nt8jvnlPoDn7REV8TOE6KOiUzcHKa2xGlfaIe0oRC21LD86uQm\nR09xY1YaJkVgZfeN/opoRjZawkU3FFs3jlUEVBF8k153oOw9W3bgsFFjSOtRtRRg\nDwyQ7oDeMH83kXnaCdpkNZd59wjzPcpxYAL4LRN52ZXA4Btr4DTi+GxHz98Dr0kU\nUQIDAQAB\n-----END PUBLIC KEY-----\n" +} \ No newline at end of file diff --git a/nodes/prometheus-1.json b/nodes/prometheus-1.json new file mode 100644 index 0000000..829e405 --- /dev/null +++ b/nodes/prometheus-1.json @@ -0,0 +1,55 @@ +{ + "name": "prometheus-1", + "normal": { + "knife_zero": { + "host": "10.1.1.146" + } + }, + "automatic": { + "fqdn": "prometheus-1", + "os": "linux", + "os_version": "6.8.0-106-generic", + "hostname": "prometheus-1", + "ipaddress": "192.168.122.166", + "roles": [ + "base", + "kvm_guest" + ], + "recipes": [ + "kosmos-base", + "kosmos-base::default", + "kosmos_kvm::guest", + "apt::default", + "timezone_iii::default", + "timezone_iii::debian", + "kosmos-base::journald_conf", + "kosmos-base::systemd_emails", + "apt::unattended-upgrades", + "kosmos-base::firewall", + "kosmos-postfix::default", + "postfix::default", + "postfix::_common", + "postfix::_attributes", + "postfix::sasl_auth", + "hostname::default" + ], + "platform": "ubuntu", + "platform_version": "24.04", + "cloud": null, + "chef_packages": { + "chef": { + "version": "18.10.17", + "chef_root": "/opt/chef/embedded/lib/ruby/gems/3.1.0/gems/chef-18.10.17/lib", + "chef_effortless": null + }, + "ohai": { + "version": "18.2.13", + "ohai_root": "/opt/chef/embedded/lib/ruby/gems/3.1.0/gems/ohai-18.2.13/lib/ohai" + } + } + }, + "run_list": [ + "role[base]", + "role[kvm_guest]" + ] +} From ec73dd5b57c397fe1c62106171968d9e96db0837 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A2u=20Cao?= Date: Fri, 3 Jul 2026 15:52:46 +0200 Subject: [PATCH 02/47] Set Chef environment for node --- nodes/prometheus-1.json | 1 + 1 file changed, 1 insertion(+) diff --git a/nodes/prometheus-1.json b/nodes/prometheus-1.json index 829e405..29e6cfe 100644 --- a/nodes/prometheus-1.json +++ b/nodes/prometheus-1.json @@ -1,5 +1,6 @@ { "name": "prometheus-1", + "chef_environment": "production", "normal": { "knife_zero": { "host": "10.1.1.146" From 4cd6c412545a33864dba5a118b796aa25e4635ef Mon Sep 17 00:00:00 2001 From: Greg Karekinian Date: Fri, 3 Jul 2026 16:35:26 +0200 Subject: [PATCH 03/47] Add community prometheus cookbook --- Berksfile | 1 + Berksfile.lock | 3 + cookbooks/prometheus/.hound.yml | 8 + cookbooks/prometheus/.markdownlint-cli2.yaml | 9 + .../prometheus/.release-please-manifest.json | 3 + cookbooks/prometheus/CHANGELOG.md | 104 +++++++++ cookbooks/prometheus/LICENSE | 201 ++++++++++++++++++ cookbooks/prometheus/LIMITATIONS.md | 36 ++++ cookbooks/prometheus/README.md | 67 ++++++ cookbooks/prometheus/Thorfile | 10 + cookbooks/prometheus/chefignore | 115 ++++++++++ cookbooks/prometheus/libraries/helpers.rb | 78 +++++++ cookbooks/prometheus/metadata.json | 44 ++++ cookbooks/prometheus/metadata.rb | 23 ++ cookbooks/prometheus/migration.md | 36 ++++ .../prometheus/release-please-config.json | 12 ++ cookbooks/prometheus/renovate.json | 18 ++ .../resources/_partial/_alertmanager.rb | 21 ++ .../prometheus/resources/_partial/_common.rb | 8 + .../resources/_partial/_prometheus.rb | 71 +++++++ .../resources/alertmanager_config.rb | 30 +++ .../resources/alertmanager_install.rb | 89 ++++++++ .../resources/alertmanager_service.rb | 56 +++++ cookbooks/prometheus/resources/config.rb | 44 ++++ cookbooks/prometheus/resources/install.rb | 98 +++++++++ cookbooks/prometheus/resources/job.rb | 41 ++++ cookbooks/prometheus/resources/service.rb | 56 +++++ .../templates/default/alertmanager.yml.erb | 115 ++++++++++ .../templates/default/prometheus.yml.erb | 33 +++ 29 files changed, 1430 insertions(+) create mode 100644 cookbooks/prometheus/.hound.yml create mode 100644 cookbooks/prometheus/.markdownlint-cli2.yaml create mode 100644 cookbooks/prometheus/.release-please-manifest.json create mode 100644 cookbooks/prometheus/CHANGELOG.md create mode 100644 cookbooks/prometheus/LICENSE create mode 100644 cookbooks/prometheus/LIMITATIONS.md create mode 100644 cookbooks/prometheus/README.md create mode 100644 cookbooks/prometheus/Thorfile create mode 100644 cookbooks/prometheus/chefignore create mode 100644 cookbooks/prometheus/libraries/helpers.rb create mode 100644 cookbooks/prometheus/metadata.json create mode 100644 cookbooks/prometheus/metadata.rb create mode 100644 cookbooks/prometheus/migration.md create mode 100644 cookbooks/prometheus/release-please-config.json create mode 100644 cookbooks/prometheus/renovate.json create mode 100644 cookbooks/prometheus/resources/_partial/_alertmanager.rb create mode 100644 cookbooks/prometheus/resources/_partial/_common.rb create mode 100644 cookbooks/prometheus/resources/_partial/_prometheus.rb create mode 100644 cookbooks/prometheus/resources/alertmanager_config.rb create mode 100644 cookbooks/prometheus/resources/alertmanager_install.rb create mode 100644 cookbooks/prometheus/resources/alertmanager_service.rb create mode 100644 cookbooks/prometheus/resources/config.rb create mode 100644 cookbooks/prometheus/resources/install.rb create mode 100644 cookbooks/prometheus/resources/job.rb create mode 100644 cookbooks/prometheus/resources/service.rb create mode 100644 cookbooks/prometheus/templates/default/alertmanager.yml.erb create mode 100644 cookbooks/prometheus/templates/default/prometheus.yml.erb diff --git a/Berksfile b/Berksfile index 5019974..c30c798 100644 --- a/Berksfile +++ b/Berksfile @@ -41,6 +41,7 @@ cookbook 'ulimit', '~> 1.0.0' cookbook 'users', '~> 5.3.1' cookbook 'zerotier', '~> 1.0.7' cookbook 'unbound', '~> 3.0.2' +cookbook 'prometheus', '~> 1.0.0' # openresty dependency cookbook 'jemalloc', '~> 0.1.7' diff --git a/Berksfile.lock b/Berksfile.lock index 1651b1b..887e8f2 100644 --- a/Berksfile.lock +++ b/Berksfile.lock @@ -33,6 +33,7 @@ DEPENDENCIES git: https://gitea.kosmos.org/kosmos/postfix-cookbook.git revision: dd6598572a775ae73f17527260ec8097b52d385b ref: bugfix/ + prometheus (~> 1.0.0) redisio (~> 6.4.1) ruby_build (~> 2.5.0) timezone_iii (= 1.0.4) @@ -95,6 +96,8 @@ GRAPH php (8.0.1) yum-epel (>= 0.0.0) postfix (6.4.1) + prometheus (1.0.0) + ark (>= 0.0.0) redisio (6.4.1) selinux (>= 0.0.0) ruby_build (2.5.0) diff --git a/cookbooks/prometheus/.hound.yml b/cookbooks/prometheus/.hound.yml new file mode 100644 index 0000000..eddba4f --- /dev/null +++ b/cookbooks/prometheus/.hound.yml @@ -0,0 +1,8 @@ +ruby: + config_file: .rubocop.yml + +java_script: + enabled: false + +scss: + enabled: false diff --git a/cookbooks/prometheus/.markdownlint-cli2.yaml b/cookbooks/prometheus/.markdownlint-cli2.yaml new file mode 100644 index 0000000..ac5076b --- /dev/null +++ b/cookbooks/prometheus/.markdownlint-cli2.yaml @@ -0,0 +1,9 @@ +config: + ul-indent: false # MD007 + line-length: false # MD013 + no-duplicate-heading: false # MD024 + reference-links-images: false # MD052 + no-multiple-blanks: + maximum: 2 +ignores: + - .github/copilot-instructions.md diff --git a/cookbooks/prometheus/.release-please-manifest.json b/cookbooks/prometheus/.release-please-manifest.json new file mode 100644 index 0000000..37fcefa --- /dev/null +++ b/cookbooks/prometheus/.release-please-manifest.json @@ -0,0 +1,3 @@ +{ + ".": "1.0.0" +} diff --git a/cookbooks/prometheus/CHANGELOG.md b/cookbooks/prometheus/CHANGELOG.md new file mode 100644 index 0000000..a83d864 --- /dev/null +++ b/cookbooks/prometheus/CHANGELOG.md @@ -0,0 +1,104 @@ +# Change Log + +All notable changes to this project will be documented in this file. + +## [1.0.0](https://github.com/sous-chefs/prometheus/compare/v0.7.6...v1.0.0) (2026-04-29) + + +### ⚠ BREAKING CHANGES + +* migrate prometheus resources ([#36](https://github.com/sous-chefs/prometheus/issues/36)) + +### Features + +* migrate prometheus resources ([#36](https://github.com/sous-chefs/prometheus/issues/36)) ([6698649](https://github.com/sous-chefs/prometheus/commit/66986492fe0f069bc69474e19423b16e21563c16)) + + +### Bug Fixes + +* **ci:** Update workflows to use release pipeline ([#29](https://github.com/sous-chefs/prometheus/issues/29)) ([b991214](https://github.com/sous-chefs/prometheus/commit/b991214225164d55694966e535478a38e720a8f0)) + +## [0.7.6](https://github.com/sous-chefs/prometheus/compare/0.7.5...v0.7.6) (2025-10-16) + + +### Bug Fixes + +* **ci:** Update workflows to use release pipeline ([#29](https://github.com/sous-chefs/prometheus/issues/29)) ([b991214](https://github.com/sous-chefs/prometheus/commit/b991214225164d55694966e535478a38e720a8f0)) + +## 0.7.2 - *2022-02-08* + +* Remove delivery folder + +Standardise files with files in sous-chefs/repo-management + +## 0.7.1 - *2021-06-01* + +* resolved cookstyle error: Thorfile:1:1 convention: `Style/Encoding` +* resolved cookstyle error: test/shared/spec_helper.rb:6:10 convention: `Style/ExpandPathArguments` + +## [0.7.1 - *2021-06-01*][0.7.1 - *2021-06-01*] + +### Changed + +* Updated attributes and templates for Prometheus 0.15 release. + +### Added + +* Added upstart init for ubuntu platform. + +## [0.5.1] - 2015-03-25 + +Changed + +* Updated documentation. + +## [0.5.0] - 2015-03-25 + +Added + +* Added systemd init for redhat platform family version 7 or greater. +* Default init style per platform. +* Install Prometheus via pre-compiled binary. +* Added the prometheus_job resource for defining Prometheus scraping jobs. +* Attribute flag to externally manage prometheus.conf file. + +Changed + +* Removed flags that were deprecated in the prometheus 0.12.0 release. + +### Contributors for this release + +* [Eric Richardson](https://github.com/ewr) - External jobs config and prometheus job resource. + +Thank You! + +## [0.4.0] - 2015-03-12 + +### Fixed + +* Fix init template path bug on chef 11.x. + +## [0.3.0] - 2015-03-11 + +Fixed + +* Fixed cookbook badge in README + +## [0.2.0] - 2015-03-11 + +Fixed + +* License defined in metadata. + +## 0.1.0 - 2015-03-11 + +Changed + +* Initial release of prometheus cookbook + +[0.7.1 - *2021-06-01*]: https://github.com/rayrod2030/chef-prometheus/compare/0.5.1...HEAD +[0.5.1]: https://github.com/rayrod2030/chef-prometheus/compare/0.5.0...0.5.1 +[0.5.0]: https://github.com/rayrod2030/chef-prometheus/compare/0.4.0...0.5.0 +[0.4.0]: https://github.com/rayrod2030/chef-prometheus/compare/0.3.0...0.4.0 +[0.3.0]: https://github.com/rayrod2030/chef-prometheus/compare/0.2.0...0.3.0 +[0.2.0]: https://github.com/rayrod2030/chef-prometheus/compare/0.1.0...0.2.0 diff --git a/cookbooks/prometheus/LICENSE b/cookbooks/prometheus/LICENSE new file mode 100644 index 0000000..11069ed --- /dev/null +++ b/cookbooks/prometheus/LICENSE @@ -0,0 +1,201 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + +TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + +1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + +2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + +3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + +4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + +5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + +6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + +7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + +8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + +9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + +END OF TERMS AND CONDITIONS + +APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + +Copyright [yyyy] [name of copyright owner] + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. diff --git a/cookbooks/prometheus/LIMITATIONS.md b/cookbooks/prometheus/LIMITATIONS.md new file mode 100644 index 0000000..6085fa8 --- /dev/null +++ b/cookbooks/prometheus/LIMITATIONS.md @@ -0,0 +1,36 @@ +# Limitations + +This cookbook manages Prometheus and Alertmanager from upstream release archives or source builds. + +## Upstream Support + +Prometheus publishes precompiled archives for Prometheus and Alertmanager from the official download page and GitHub releases. As of April 28, 2026, the cookbook defaults to: + +* Prometheus 3.11.2 `linux-amd64` and `linux-arm64` +* Alertmanager 0.32.0 `linux-amd64` and `linux-arm64` + +The upstream projects also publish Docker images and archives for other operating systems. This cookbook only supports Linux systemd hosts. + +## Platform Support + +Supported platforms are modern systemd Linux distributions declared in `metadata.rb`: + +* AlmaLinux 8+ +* Amazon Linux 2023+ +* CentOS Stream 9+ +* Debian 12+ +* Fedora +* Oracle Linux 8+ +* Red Hat Enterprise Linux 8+ +* Rocky Linux 8+ +* Ubuntu 20.04+ + +Legacy init systems are not supported. The cookbook no longer manages runit, upstart, or SysV init services. + +## Installation Constraints + +The default binary installation path uses official Linux tarballs for `amd64` or `arm64`. Override `architecture`, `binary_url`, `checksum`, and `file_extension` when using another upstream artifact or a private mirror. + +Source installs require build tooling, Git, and the upstream Go build chain expected by the selected Prometheus or Alertmanager version. Source builds are retained for compatibility but binary installs are the primary supported path. + +Prometheus and Alertmanager are not installed from operating system package repositories by these resources. diff --git a/cookbooks/prometheus/README.md b/cookbooks/prometheus/README.md new file mode 100644 index 0000000..855db77 --- /dev/null +++ b/cookbooks/prometheus/README.md @@ -0,0 +1,67 @@ +# prometheus Cookbook + +This cookbook provides custom resources for installing and configuring [Prometheus][] and Alertmanager. + +## Requirements + +* Chef Infra Client 16.0 or later +* A systemd Linux platform listed in [LIMITATIONS.md](LIMITATIONS.md) + +See [LIMITATIONS.md](LIMITATIONS.md) for upstream platform and installation constraints. + +## Resources + +* [prometheus_install](documentation/prometheus_install.md) +* [prometheus_config](documentation/prometheus_config.md) +* [prometheus_service](documentation/prometheus_service.md) +* [prometheus_job](documentation/prometheus_job.md) +* [prometheus_alertmanager_install](documentation/prometheus_alertmanager_install.md) +* [prometheus_alertmanager_config](documentation/prometheus_alertmanager_config.md) +* [prometheus_alertmanager_service](documentation/prometheus_alertmanager_service.md) + +See [migration.md](migration.md) for migration notes from the legacy recipe and attribute interface. + +## Usage + +```ruby +prometheus_install 'prometheus' + +prometheus_config 'prometheus' + +prometheus_job 'prometheus' do + scrape_interval '15s' + target 'localhost:9090' +end + +prometheus_service 'prometheus' +``` + +Alertmanager: + +```ruby +prometheus_alertmanager_install 'alertmanager' +prometheus_alertmanager_config 'alertmanager' +prometheus_alertmanager_service 'alertmanager' +``` + +## License & Authors + +* Author: Ray Rodriguez +* Author: kristian järvenpää +* Maintainer: Sous Chefs + +```text +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +``` + +[Prometheus]: https://github.com/prometheus/prometheus diff --git a/cookbooks/prometheus/Thorfile b/cookbooks/prometheus/Thorfile new file mode 100644 index 0000000..75ec16f --- /dev/null +++ b/cookbooks/prometheus/Thorfile @@ -0,0 +1,10 @@ +require 'bundler' +require 'bundler/setup' +require 'berkshelf/thor' + +begin + require 'kitchen/thor_tasks' + Kitchen::ThorTasks.new +rescue LoadError + puts '>>>>> Kitchen gem not loaded, omitting tasks' unless ENV['CI'] +end diff --git a/cookbooks/prometheus/chefignore b/cookbooks/prometheus/chefignore new file mode 100644 index 0000000..a27b0b2 --- /dev/null +++ b/cookbooks/prometheus/chefignore @@ -0,0 +1,115 @@ +# Put files/directories that should be ignored in this file when uploading +# to a Chef Infra Server or Supermarket. +# Lines that start with '# ' are comments. + +# OS generated files # +###################### +.DS_Store +ehthumbs.db +Icon? +nohup.out +Thumbs.db +.envrc + +# EDITORS # +########### +.#* +.project +.settings +*_flymake +*_flymake.* +*.bak +*.sw[a-z] +*.tmproj +*~ +\#* +REVISION +TAGS* +tmtags +.vscode +.editorconfig + +## COMPILED ## +############## +*.class +*.com +*.dll +*.exe +*.o +*.pyc +*.so +*/rdoc/ +a.out +mkmf.log + +# Testing # +########### +.circleci/* +.codeclimate.yml +.delivery/* +.foodcritic +.kitchen* +.mdlrc +.overcommit.yml +.rspec +.rubocop.yml +.travis.yml +.watchr +.yamllint +azure-pipelines.yml +Dangerfile +examples/* +features/* +Guardfile +kitchen*.yml +mlc_config.json +Procfile +Rakefile +spec/* +test/* + +# SCM # +####### +.git +.gitattributes +.gitconfig +.github/* +.gitignore +.gitkeep +.gitmodules +.svn +*/.bzr/* +*/.git +*/.hg/* +*/.svn/* + +# Berkshelf # +############# +Berksfile +Berksfile.lock +cookbooks/* +tmp + +# Bundler # +########### +vendor/* +Gemfile +Gemfile.lock + +# Policyfile # +############## +Policyfile.rb +Policyfile.lock.json + +# Documentation # +############# +CODE_OF_CONDUCT* +CONTRIBUTING* +documentation/* +TESTING* +UPGRADING* + +# Vagrant # +########### +.vagrant +Vagrantfile diff --git a/cookbooks/prometheus/libraries/helpers.rb b/cookbooks/prometheus/libraries/helpers.rb new file mode 100644 index 0000000..c52f5fe --- /dev/null +++ b/cookbooks/prometheus/libraries/helpers.rb @@ -0,0 +1,78 @@ +# frozen_string_literal: true + +require 'uri' + +module PrometheusCookbook + module Helpers + def archive_name(component, version, url) + basename = ::File.basename(URI.parse(url).path) + basename.sub(/(?:\.tar\.gz|\.tgz|\.tar\.bz2|\.tar\.xz|\.zip)\z/, '') + rescue URI::InvalidURIError + "#{component}-#{version}" + end + + def install_dir_parent(install_dir) + ::File.dirname(install_dir) + end + + def install_dir_name(install_dir) + ::File.basename(install_dir) + end + + def prometheus_flags(resource) + flag_pairs = if Gem::Version.new(resource.version) < Gem::Version.new('2.0.0-alpha.0') + resource.flags.map { |key, value| "-#{key}=#{value}" unless value == '' } + else + resource.cli_options.map { |key, value| "--#{key}=#{value}" unless value == '' } + + resource.cli_flags.map { |flag| "--#{flag}" unless flag == '' } + end + + flag_pairs.compact.join(' ') + end + + def prometheus_unit_content(resource) + { + Unit: { + Description: 'Prometheus', + After: 'network.target auditd.service', + }, + Service: { + Type: 'simple', + Environment: "GOMAXPROCS=#{node['cpu']['total'] || 1}", + User: resource.user, + Group: resource.group, + ExecStart: "#{resource.binary} #{prometheus_flags(resource)}", + ExecReload: '/bin/kill -HUP $MAINPID', + Restart: 'always', + }, + Install: { + WantedBy: 'multi-user.target', + }, + } + end + + def alertmanager_unit_content(resource) + { + Unit: { + Description: 'Prometheus Alertmanager', + After: 'network.target', + }, + Service: { + User: resource.user, + Group: resource.group, + ExecStart: [ + resource.binary, + "--log.level=#{resource.log_level}", + "--storage.path=#{resource.storage_path}", + "--config.file=#{resource.config_file}", + "--web.external-url=#{resource.external_url}", + ].join(' '), + Restart: 'always', + }, + Install: { + WantedBy: 'multi-user.target', + }, + } + end + end +end diff --git a/cookbooks/prometheus/metadata.json b/cookbooks/prometheus/metadata.json new file mode 100644 index 0000000..36c5be3 --- /dev/null +++ b/cookbooks/prometheus/metadata.json @@ -0,0 +1,44 @@ +{ + "name": "prometheus", + "description": "Provides custom resources for installing and configuring Prometheus and Alertmanager", + "long_description": "", + "maintainer": "Sous Chefs", + "maintainer_email": "help@sous-chefs.org", + "license": "Apache-2.0", + "platforms": { + "almalinux": ">= 8.0", + "amazon": ">= 2023.0", + "centos_stream": ">= 9.0", + "debian": ">= 12.0", + "fedora": ">= 0.0.0", + "oracle": ">= 8.0", + "redhat": ">= 8.0", + "rocky": ">= 8.0", + "ubuntu": ">= 20.04" + }, + "dependencies": { + "ark": ">= 0.0.0" + }, + "providing": { + + }, + "recipes": { + + }, + "version": "1.0.0", + "source_url": "https://github.com/sous-chefs/prometheus", + "issues_url": "https://github.com/sous-chefs/prometheus/issues", + "privacy": false, + "chef_versions": [ + [ + ">= 16.0" + ] + ], + "ohai_versions": [ + + ], + "gems": [ + + ], + "eager_load_libraries": true +} diff --git a/cookbooks/prometheus/metadata.rb b/cookbooks/prometheus/metadata.rb new file mode 100644 index 0000000..20c6d4f --- /dev/null +++ b/cookbooks/prometheus/metadata.rb @@ -0,0 +1,23 @@ +# frozen_string_literal: true + +name 'prometheus' +maintainer 'Sous Chefs' +maintainer_email 'help@sous-chefs.org' +license 'Apache-2.0' +description 'Provides custom resources for installing and configuring Prometheus and Alertmanager' +version '1.0.0' +source_url 'https://github.com/sous-chefs/prometheus' +issues_url 'https://github.com/sous-chefs/prometheus/issues' +chef_version '>= 16.0' + +depends 'ark' + +supports 'almalinux', '>= 8.0' +supports 'amazon', '>= 2023.0' +supports 'centos_stream', '>= 9.0' +supports 'debian', '>= 12.0' +supports 'fedora' +supports 'oracle', '>= 8.0' +supports 'redhat', '>= 8.0' +supports 'rocky', '>= 8.0' +supports 'ubuntu', '>= 20.04' diff --git a/cookbooks/prometheus/migration.md b/cookbooks/prometheus/migration.md new file mode 100644 index 0000000..51ebc2c --- /dev/null +++ b/cookbooks/prometheus/migration.md @@ -0,0 +1,36 @@ +# Migration + +This cookbook has been migrated from recipes and attributes to custom resources. + +## Removed Entry Points + +The `recipes/` and `attributes/` directories were removed. Wrapper cookbooks should call the resources directly and pass property values instead of overriding `node['prometheus']` attributes. + +Legacy runit, upstart, and SysV init behavior was removed. Services are managed with Chef's `systemd_unit` resource. + +## Resource Mapping + +Use these resources in place of the old recipes: + +* `prometheus_install` replaces `prometheus::binary`, `prometheus::shell_binary`, and `prometheus::source`. +* `prometheus_config` replaces the Prometheus configuration portion of `prometheus::default`. +* `prometheus_service` replaces `prometheus::service`. +* `prometheus_alertmanager_install` replaces `prometheus::alertmanager_binary` and `prometheus::alertmanager_source`. +* `prometheus_alertmanager_config` replaces the Alertmanager configuration portion of `prometheus::alertmanager`. +* `prometheus_alertmanager_service` replaces the Alertmanager service portion of `prometheus::alertmanager`. +* `prometheus_job` remains available and now uses explicit properties instead of node attributes. + +## Example + +```ruby +prometheus_install 'prometheus' + +prometheus_config 'prometheus' + +prometheus_job 'prometheus' do + scrape_interval '15s' + target 'localhost:9090' +end + +prometheus_service 'prometheus' +``` diff --git a/cookbooks/prometheus/release-please-config.json b/cookbooks/prometheus/release-please-config.json new file mode 100644 index 0000000..148dfd5 --- /dev/null +++ b/cookbooks/prometheus/release-please-config.json @@ -0,0 +1,12 @@ +{ + "packages": { + ".": { + "package-name": "prometheus", + "changelog-path": "CHANGELOG.md", + "release-type": "ruby", + "include-component-in-tag": false, + "version-file": "metadata.rb" + } + }, + "$schema": "https://raw.githubusercontent.com/googleapis/release-please/main/schemas/config.json" +} diff --git a/cookbooks/prometheus/renovate.json b/cookbooks/prometheus/renovate.json new file mode 100644 index 0000000..a0b29c8 --- /dev/null +++ b/cookbooks/prometheus/renovate.json @@ -0,0 +1,18 @@ +{ + "$schema": "https://docs.renovatebot.com/renovate-schema.json", + "extends": ["config:base"], + "packageRules": [ + { + "groupName": "Actions", + "matchUpdateTypes": ["minor", "patch", "pin"], + "automerge": true, + "addLabels": ["Release: Patch", "Skip: Announcements"] + }, + { + "groupName": "Actions", + "matchUpdateTypes": ["major"], + "automerge": false, + "addLabels": ["Release: Patch", "Skip: Announcements"] + } + ] +} diff --git a/cookbooks/prometheus/resources/_partial/_alertmanager.rb b/cookbooks/prometheus/resources/_partial/_alertmanager.rb new file mode 100644 index 0000000..c63bfa1 --- /dev/null +++ b/cookbooks/prometheus/resources/_partial/_alertmanager.rb @@ -0,0 +1,21 @@ +# frozen_string_literal: true + +property :version, String, default: '0.32.0' +property :binary, String, default: '/opt/prometheus/alertmanager' +property :architecture, String, equal_to: %w(amd64 arm64), default: lazy { + node['kernel']['machine'] == 'aarch64' ? 'arm64' : 'amd64' +} +property :binary_url, String, default: lazy { "https://github.com/prometheus/alertmanager/releases/download/v#{version}/alertmanager-#{version}.linux-#{architecture}.tar.gz" } +property :checksum, String, default: lazy { + { + 'amd64' => 'be72f50f6124ec53d944c0f100f8ec8108d969bade02fcc9f06a3068ff6c726f', + 'arm64' => '7812e12699694974f57ecc0b0400913c6c0d90190630d4332a7994a44982b1ed', + }[architecture] +} +property :file_extension, String, default: '' +property :source_repository, String, default: 'https://github.com/prometheus/alertmanager.git' +property :source_revision, String, default: lazy { "v#{version}" } +property :config_file, String, default: '/opt/prometheus/alertmanager.yml' +property :storage_path, String, default: '/opt/prometheus/data' +property :external_url, String, default: 'http://127.0.0.1/alert-manager/' +property :log_level, String, default: 'debug' diff --git a/cookbooks/prometheus/resources/_partial/_common.rb b/cookbooks/prometheus/resources/_partial/_common.rb new file mode 100644 index 0000000..5c6a534 --- /dev/null +++ b/cookbooks/prometheus/resources/_partial/_common.rb @@ -0,0 +1,8 @@ +# frozen_string_literal: true + +property :install_dir, String, default: '/opt/prometheus' +property :log_dir, String, default: '/var/log/prometheus' +property :user, String, default: 'prometheus' +property :group, String, default: 'prometheus' +property :use_existing_user, [true, false], default: false +property :install_method, String, equal_to: %w(binary shell_binary source), default: 'binary' diff --git a/cookbooks/prometheus/resources/_partial/_prometheus.rb b/cookbooks/prometheus/resources/_partial/_prometheus.rb new file mode 100644 index 0000000..37c9f7c --- /dev/null +++ b/cookbooks/prometheus/resources/_partial/_prometheus.rb @@ -0,0 +1,71 @@ +# frozen_string_literal: true + +property :version, String, default: '3.11.2' +property :binary, String, default: '/opt/prometheus/prometheus' +property :architecture, String, equal_to: %w(amd64 arm64), default: lazy { + node['kernel']['machine'] == 'aarch64' ? 'arm64' : 'amd64' +} +property :binary_url, String, default: lazy { "https://github.com/prometheus/prometheus/releases/download/v#{version}/prometheus-#{version}.linux-#{architecture}.tar.gz" } +property :checksum, String, default: lazy { + { + 'amd64' => 'f643ea1ee90d109329302d27bddb1fb2e52655b1fa84e9e26f9a6f340da144a6', + 'arm64' => '4e40f115655a3021744137f49287846bc5a59e02835565748ff66b23e776a73d', + }[architecture] +} +property :file_extension, String, default: '' +property :source_repository, String, default: 'https://github.com/prometheus/prometheus.git' +property :source_revision, String, default: lazy { "v#{version}" } +property :config_file, String, default: '/opt/prometheus/prometheus.yml' +property :storage_path, String, default: '/var/lib/prometheus' +property :flags, Hash, default: lazy { + legacy_flags = { + 'config.file' => config_file, + 'log.level' => 'info', + 'alertmanager.timeout' => '10s', + 'alertmanager.notification-queue-capacity' => 100, + 'alertmanager.url' => 'http://127.0.0.1/alert-manager/', + 'query.max-concurrency' => 20, + 'query.staleness-delta' => '5m', + 'query.timeout' => '2m', + 'storage.local.checkpoint-dirty-series-limit' => 5000, + 'storage.local.checkpoint-interval' => '5m', + 'storage.local.dirty' => false, + 'storage.local.index-cache-size.fingerprint-to-metric' => 10_485_760, + 'storage.local.index-cache-size.fingerprint-to-timerange' => 5_242_880, + 'storage.local.index-cache-size.label-name-to-label-values' => 10_485_760, + 'storage.local.index-cache-size.label-pair-to-fingerprints' => 20_971_520, + 'storage.local.memory-chunks' => 1_048_576, + 'storage.local.path' => storage_path, + 'storage.local.pedantic-checks' => false, + 'storage.local.retention' => '360h0m0s', + 'storage.local.series-sync-strategy' => 'adaptive', + 'storage.remote.influxdb-url' => '', + 'storage.remote.influxdb.database' => 'prometheus', + 'storage.remote.influxdb.retention-policy' => 'default', + 'storage.remote.opentsdb-url' => '', + 'storage.remote.timeout' => '30s', + 'web.console.libraries' => 'console_libraries', + 'web.console.templates' => 'consoles', + 'web.enable-remote-shutdown' => false, + 'web.external-url' => '', + 'web.listen-address' => ':9090', + 'web.telemetry-path' => '/metrics', + 'web.user-assets' => '', + } + legacy_flags['web.use-local-assets'] = false if Gem::Version.new(version) <= Gem::Version.new('0.16.2') + legacy_flags +} +property :cli_options, Hash, default: lazy { + { + 'config.file' => config_file, + 'log.level' => 'info', + 'query.max-concurrency' => 20, + 'query.lookback-delta' => '5m', + 'query.timeout' => '2m', + 'storage.tsdb.path' => storage_path, + 'storage.tsdb.retention.time' => '15d', + 'web.listen-address' => ':9090', + 'web.telemetry-path' => '/metrics', + } +} +property :cli_flags, Array, default: ['web.enable-lifecycle'] diff --git a/cookbooks/prometheus/resources/alertmanager_config.rb b/cookbooks/prometheus/resources/alertmanager_config.rb new file mode 100644 index 0000000..65dbd8a --- /dev/null +++ b/cookbooks/prometheus/resources/alertmanager_config.rb @@ -0,0 +1,30 @@ +# frozen_string_literal: true + +provides :prometheus_alertmanager_config +unified_mode true + +use '_partial/_common' +use '_partial/_alertmanager' + +property :template_cookbook, String, default: 'prometheus' +property :template_source, String, default: 'alertmanager.yml.erb' +property :notification_config, Hash, default: {} + +default_action :create + +action :create do + template new_resource.config_file do + cookbook new_resource.template_cookbook + source new_resource.template_source + mode '0644' + owner new_resource.user + group new_resource.group + variables(notification_config: new_resource.notification_config) + end +end + +action :delete do + file new_resource.config_file do + action :delete + end +end diff --git a/cookbooks/prometheus/resources/alertmanager_install.rb b/cookbooks/prometheus/resources/alertmanager_install.rb new file mode 100644 index 0000000..c918f7f --- /dev/null +++ b/cookbooks/prometheus/resources/alertmanager_install.rb @@ -0,0 +1,89 @@ +# frozen_string_literal: true + +provides :prometheus_alertmanager_install +unified_mode true + +use '_partial/_common' +use '_partial/_alertmanager' + +default_action :install + +action_class do + include PrometheusCookbook::Helpers +end + +action :install do + user new_resource.user do + system true + shell '/bin/false' + home new_resource.install_dir + not_if { new_resource.use_existing_user || new_resource.user == 'root' } + end + + directory new_resource.install_dir do + owner new_resource.user + group new_resource.group + mode '0755' + recursive true + end + + directory new_resource.log_dir do + owner new_resource.user + group new_resource.group + mode '0755' + recursive true + end + + directory new_resource.storage_path do + owner new_resource.user + group new_resource.group + mode '0755' + recursive true + end + + case new_resource.install_method + when 'binary' + package %w(tar bzip2) + + ark install_dir_name(new_resource.install_dir) do + url new_resource.binary_url + checksum new_resource.checksum + version new_resource.version + prefix_root Chef::Config['file_cache_path'] + path install_dir_parent(new_resource.install_dir) + owner new_resource.user + group new_resource.group + extension new_resource.file_extension unless new_resource.file_extension.empty? + action :put + end + when 'shell_binary' + package %w(tar bzip2) + + remote_file "#{Chef::Config[:file_cache_path]}/alertmanager-#{new_resource.version}.tar.gz" do + source new_resource.binary_url + checksum new_resource.checksum + action :create + notifies :run, 'execute[install_alertmanager_archive]', :immediately + end + + execute 'install_alertmanager_archive' do + command "tar -xzf #{Chef::Config[:file_cache_path]}/alertmanager-#{new_resource.version}.tar.gz -C #{new_resource.install_dir} --strip-components=1" + action :nothing + end + when 'source' + build_essential 'install compilation tools' + + package %w(curl git-core mercurial gzip sed) + + git "#{Chef::Config[:file_cache_path]}/alertmanager-#{new_resource.version}" do + repository new_resource.source_repository + revision new_resource.source_revision + action :checkout + end + + bash 'compile_alertmanager_source' do + cwd "#{Chef::Config[:file_cache_path]}/alertmanager-#{new_resource.version}" + code "make && mv alertmanager #{new_resource.install_dir}" + end + end +end diff --git a/cookbooks/prometheus/resources/alertmanager_service.rb b/cookbooks/prometheus/resources/alertmanager_service.rb new file mode 100644 index 0000000..8b4cb9f --- /dev/null +++ b/cookbooks/prometheus/resources/alertmanager_service.rb @@ -0,0 +1,56 @@ +# frozen_string_literal: true + +provides :prometheus_alertmanager_service +unified_mode true + +use '_partial/_common' +use '_partial/_alertmanager' + +default_action :create + +action_class do + include PrometheusCookbook::Helpers +end + +action :create do + systemd_unit 'alertmanager.service' do + content alertmanager_unit_content(new_resource) + action [:create, :enable, :start] + end +end + +action :enable do + systemd_unit 'alertmanager.service' do + action :enable + end +end + +action :start do + systemd_unit 'alertmanager.service' do + action :start + end +end + +action :restart do + systemd_unit 'alertmanager.service' do + action :restart + end +end + +action :reload do + systemd_unit 'alertmanager.service' do + action :reload + end +end + +action :stop do + systemd_unit 'alertmanager.service' do + action [:stop, :disable] + end +end + +action :delete do + systemd_unit 'alertmanager.service' do + action [:stop, :disable, :delete] + end +end diff --git a/cookbooks/prometheus/resources/config.rb b/cookbooks/prometheus/resources/config.rb new file mode 100644 index 0000000..20de8d5 --- /dev/null +++ b/cookbooks/prometheus/resources/config.rb @@ -0,0 +1,44 @@ +# frozen_string_literal: true + +provides :prometheus_config +unified_mode true + +use '_partial/_common' +use '_partial/_prometheus' + +property :template_cookbook, String, default: 'prometheus' +property :template_source, String, default: 'prometheus.yml.erb' +property :rule_filenames, [Array, nil], default: nil +property :global_config, Hash, default: { + 'scrape_interval' => '60s', + 'evaluation_interval' => '60s', +} +property :allow_external_config, [true, false], default: false + +default_action :create + +action :create do + config_resource = new_resource + + with_run_context :root do + template config_resource.config_file do + cookbook config_resource.template_cookbook + source config_resource.template_source + mode '0644' + owner config_resource.user + group config_resource.group + variables( + global_config: config_resource.global_config, + jobs: {}, + rule_filenames: config_resource.rule_filenames + ) + not_if { config_resource.allow_external_config } + end + end +end + +action :delete do + file new_resource.config_file do + action :delete + end +end diff --git a/cookbooks/prometheus/resources/install.rb b/cookbooks/prometheus/resources/install.rb new file mode 100644 index 0000000..856254d --- /dev/null +++ b/cookbooks/prometheus/resources/install.rb @@ -0,0 +1,98 @@ +# frozen_string_literal: true + +provides :prometheus_install +unified_mode true + +use '_partial/_common' +use '_partial/_prometheus' + +default_action :install + +action_class do + include PrometheusCookbook::Helpers +end + +action :install do + user new_resource.user do + system true + shell '/bin/false' + home new_resource.install_dir + not_if { new_resource.use_existing_user || new_resource.user == 'root' } + end + + directory new_resource.install_dir do + owner new_resource.user + group new_resource.group + mode '0755' + recursive true + end + + directory new_resource.log_dir do + owner new_resource.user + group new_resource.group + mode '0755' + recursive true + end + + directory new_resource.storage_path do + owner new_resource.user + group new_resource.group + mode '0755' + recursive true + end + + case new_resource.install_method + when 'binary' + package %w(tar bzip2) + + ark install_dir_name(new_resource.install_dir) do + url new_resource.binary_url + checksum new_resource.checksum + version new_resource.version + prefix_root Chef::Config['file_cache_path'] + path install_dir_parent(new_resource.install_dir) + owner new_resource.user + group new_resource.group + extension new_resource.file_extension unless new_resource.file_extension.empty? + action :put + end + when 'shell_binary' + package %w(tar bzip2) + + remote_file "#{Chef::Config[:file_cache_path]}/prometheus-#{new_resource.version}.tar.gz" do + source new_resource.binary_url + checksum new_resource.checksum + action :create + notifies :run, 'execute[install_prometheus_archive]', :immediately + end + + execute 'install_prometheus_archive' do + command "tar -xzf #{Chef::Config[:file_cache_path]}/prometheus-#{new_resource.version}.tar.gz -C #{new_resource.install_dir} --strip-components=1" + action :nothing + end + when 'source' + build_essential 'install compilation tools' + + package %w(curl git-core mercurial gzip sed) + + git "#{Chef::Config[:file_cache_path]}/prometheus-#{new_resource.version}" do + repository new_resource.source_repository + revision new_resource.source_revision + action :checkout + end + + bash 'compile_prometheus_source' do + cwd "#{Chef::Config[:file_cache_path]}/prometheus-#{new_resource.version}" + environment( + 'PATH' => "/usr/local/go/bin:#{ENV.fetch('PATH', nil)}", + 'GOPATH' => '/opt/go:/opt/go/src/github.com/prometheus/promu/vendor' + ) + code <<~EOH + make build + mv prometheus #{new_resource.install_dir} + cp -R console_libraries #{new_resource.install_dir} + cp -R consoles #{new_resource.install_dir} + EOH + end + end +end diff --git a/cookbooks/prometheus/resources/job.rb b/cookbooks/prometheus/resources/job.rb new file mode 100644 index 0000000..7d104a5 --- /dev/null +++ b/cookbooks/prometheus/resources/job.rb @@ -0,0 +1,41 @@ +# frozen_string_literal: true + +provides :prometheus_job +unified_mode true + +property :scrape_interval, String +property :scrape_timeout, String +property :labels, Hash +property :target, [Array, String], required: true +property :metrics_path, String, default: '/metrics' +property :config_file, String, default: '/opt/prometheus/prometheus.yml' +property :allow_external_config, [true, false], default: false + +default_action :create + +action :create do + job_resource = new_resource + + with_run_context :root do + edit_resource(:template, job_resource.config_file) do + variables[:jobs] ||= {} + variables[:jobs][job_resource.name] ||= {} + variables[:jobs][job_resource.name]['scrape_interval'] = job_resource.scrape_interval + variables[:jobs][job_resource.name]['scrape_timeout'] = job_resource.scrape_timeout + variables[:jobs][job_resource.name]['target'] = job_resource.target + variables[:jobs][job_resource.name]['metrics_path'] = job_resource.metrics_path + variables[:jobs][job_resource.name]['labels'] = job_resource.labels + + action :nothing + delayed_action :create + + not_if { job_resource.allow_external_config } + end + end +end + +action :delete do + file new_resource.config_file do + action :delete + end +end diff --git a/cookbooks/prometheus/resources/service.rb b/cookbooks/prometheus/resources/service.rb new file mode 100644 index 0000000..1a883f2 --- /dev/null +++ b/cookbooks/prometheus/resources/service.rb @@ -0,0 +1,56 @@ +# frozen_string_literal: true + +provides :prometheus_service +unified_mode true + +use '_partial/_common' +use '_partial/_prometheus' + +default_action :create + +action_class do + include PrometheusCookbook::Helpers +end + +action :create do + systemd_unit 'prometheus.service' do + content prometheus_unit_content(new_resource) + action [:create, :enable, :start] + end +end + +action :enable do + systemd_unit 'prometheus.service' do + action :enable + end +end + +action :start do + systemd_unit 'prometheus.service' do + action :start + end +end + +action :restart do + systemd_unit 'prometheus.service' do + action :restart + end +end + +action :reload do + systemd_unit 'prometheus.service' do + action :reload + end +end + +action :stop do + systemd_unit 'prometheus.service' do + action [:stop, :disable] + end +end + +action :delete do + systemd_unit 'prometheus.service' do + action [:stop, :disable, :delete] + end +end diff --git a/cookbooks/prometheus/templates/default/alertmanager.yml.erb b/cookbooks/prometheus/templates/default/alertmanager.yml.erb new file mode 100644 index 0000000..949efac --- /dev/null +++ b/cookbooks/prometheus/templates/default/alertmanager.yml.erb @@ -0,0 +1,115 @@ +global: + # The smarthost and SMTP sender used for mail notifications. + smtp_smarthost: 'localhost:25' + smtp_from: 'alertmanager@example.org' + +# The root route on which each incoming alert enters. +route: + # The root route must not have any matchers as it is the entry point for + # all alerts. It needs to have a receiver configured so alerts that do not + # match any of the sub-routes are sent to someone. + receiver: 'team-X-mails' + + # The labels by which incoming alerts are grouped together. For example, + # multiple alerts coming in for cluster=A and alertname=LatencyHigh would + # be batched into a single group. + group_by: ['alertname', 'cluster'] + + # When a new group of alerts is created by an incoming alert, wait at + # least 'group_wait' to send the initial notification. + # This way ensures that you get multiple alerts for the same group that start + # firing shortly after another are batched together on the first + # notification. + group_wait: 30s + + # When the first notification was sent, wait 'group_interval' to send a batch + # of new alerts that started firing for that group. + group_interval: 5m + + # If an alert has successfully been sent, wait 'repeat_interval' to + # resend them. + repeat_interval: 3h + + # All the above attributes are inherited by all child routes and can + # overwritten on each. + + # The child route trees. + routes: + # This routes performs a regular expression match on alert labels to + # catch alerts that are related to a list of services. + - match_re: + service: ^(foo1|foo2|baz)$ + receiver: team-X-mails + + # The service has a sub-route for critical alerts, any alerts + # that do not match, i.e. severity != critical, fall-back to the + # parent node and are sent to 'team-X-mails' + routes: + - match: + severity: critical + receiver: team-X-pager + + - match: + service: files + receiver: team-Y-mails + + routes: + - match: + severity: critical + receiver: team-Y-pager + + # This route handles all alerts coming from a database service. If there's + # no team to handle it, it defaults to the DB team. + - match: + service: database + + receiver: team-DB-pager + # Also group alerts by affected database. + group_by: [alertname, cluster, database] + + routes: + - match: + owner: team-X + receiver: team-X-pager + + - match: + owner: team-Y + receiver: team-Y-pager + + +# Inhibition rules allow to mute a set of alerts given that another alert is +# firing. +# We use this to mute any warning-level notifications if the same alert is +# already critical. +inhibit_rules: +- source_match: + severity: 'critical' + target_match: + severity: 'warning' + # Apply inhibition if the alertname is the same. + equal: ['alertname'] + + +receivers: +- name: 'team-X-mails' + email_configs: + - to: 'team-X+alerts@example.org' + +- name: 'team-X-pager' + email_configs: + - to: 'team-X+alerts-critical@example.org' + pagerduty_configs: + - service_key: + +- name: 'team-Y-mails' + email_configs: + - to: 'team-Y+alerts@example.org' + +- name: 'team-Y-pager' + pagerduty_configs: + - service_key: + +- name: 'team-DB-pager' + pagerduty_configs: + - service_key: + diff --git a/cookbooks/prometheus/templates/default/prometheus.yml.erb b/cookbooks/prometheus/templates/default/prometheus.yml.erb new file mode 100644 index 0000000..be8ed7a --- /dev/null +++ b/cookbooks/prometheus/templates/default/prometheus.yml.erb @@ -0,0 +1,33 @@ +# Global default settings. +global: + <% @global_config.each do |k,v| %> + <%=k%>: "<%=v%>" + <% end %> + +scrape_configs: +<% @jobs.each do |name, job| %> +- job_name: "<%= name %>" + <% if job['scrape_interval'] %> + scrape_interval: "<%= job['scrape_interval'] %>" + <% end %> + <% if job['scrape_timeout'] %> + scrape_timeout: "<%= job['scrape_timeout'] %>" + <% end %> + metrics_path: "<%= job['metrics_path'] %>" + static_configs: + - targets: <%= Array(job['target']) %> + <%if job['labels'] %> + labels: + <% job['labels'].each do |label,label_config| %> + <%=label%>: <%=label_config%> + <%end%> + <%end%> + + <% end %> + +<% if @rule_filenames %> +rule_files: +<% @rule_filenames.each do |filename| %> + - <%= filename %> +<% end %> +<% end %> From 765d0b080e804052c76239096fb5ff0452fc4429 Mon Sep 17 00:00:00 2001 From: Greg Karekinian Date: Fri, 3 Jul 2026 17:47:13 +0200 Subject: [PATCH 04/47] WIP Initial kosmos_prometheus wrapper cookbook --- nodes/prometheus-1.json | 13 +- roles/prometheus_server.rb | 12 ++ site-cookbooks/kosmos_prometheus/.gitignore | 25 ++++ site-cookbooks/kosmos_prometheus/CHANGELOG.md | 7 ++ site-cookbooks/kosmos_prometheus/LICENSE | 20 +++ site-cookbooks/kosmos_prometheus/README.md | 4 + .../kosmos_prometheus/attributes/default.rb | 5 + site-cookbooks/kosmos_prometheus/chefignore | 115 ++++++++++++++++++ site-cookbooks/kosmos_prometheus/metadata.rb | 21 ++++ .../kosmos_prometheus/recipes/alertmanager.rb | 21 ++++ .../kosmos_prometheus/recipes/server.rb | 41 +++++++ 11 files changed, 280 insertions(+), 4 deletions(-) create mode 100644 roles/prometheus_server.rb create mode 100644 site-cookbooks/kosmos_prometheus/.gitignore create mode 100644 site-cookbooks/kosmos_prometheus/CHANGELOG.md create mode 100644 site-cookbooks/kosmos_prometheus/LICENSE create mode 100644 site-cookbooks/kosmos_prometheus/README.md create mode 100644 site-cookbooks/kosmos_prometheus/attributes/default.rb create mode 100644 site-cookbooks/kosmos_prometheus/chefignore create mode 100644 site-cookbooks/kosmos_prometheus/metadata.rb create mode 100644 site-cookbooks/kosmos_prometheus/recipes/alertmanager.rb create mode 100644 site-cookbooks/kosmos_prometheus/recipes/server.rb diff --git a/nodes/prometheus-1.json b/nodes/prometheus-1.json index 29e6cfe..63084e5 100644 --- a/nodes/prometheus-1.json +++ b/nodes/prometheus-1.json @@ -9,17 +9,20 @@ "automatic": { "fqdn": "prometheus-1", "os": "linux", - "os_version": "6.8.0-106-generic", + "os_version": "6.8.0-134-generic", "hostname": "prometheus-1", "ipaddress": "192.168.122.166", "roles": [ "base", - "kvm_guest" + "kvm_guest", + "prometheus_server" ], "recipes": [ "kosmos-base", "kosmos-base::default", "kosmos_kvm::guest", + "kosmos_prometheus::server", + "kosmos_prometheus::alertmanager", "apt::default", "timezone_iii::default", "timezone_iii::debian", @@ -32,7 +35,8 @@ "postfix::_common", "postfix::_attributes", "postfix::sasl_auth", - "hostname::default" + "hostname::default", + "firewall::default" ], "platform": "ubuntu", "platform_version": "24.04", @@ -51,6 +55,7 @@ }, "run_list": [ "role[base]", - "role[kvm_guest]" + "role[kvm_guest]", + "role[prometheus_server]" ] } diff --git a/roles/prometheus_server.rb b/roles/prometheus_server.rb new file mode 100644 index 0000000..dc0bfea --- /dev/null +++ b/roles/prometheus_server.rb @@ -0,0 +1,12 @@ +name "prometheus_server" + +default_run_list = [ + "kosmos_prometheus::server", + "kosmos_prometheus::alertmanager" +] + +env_run_lists( + "_default" => default_run_list, + "development" => default_run_list, + "production" => default_run_list +) diff --git a/site-cookbooks/kosmos_prometheus/.gitignore b/site-cookbooks/kosmos_prometheus/.gitignore new file mode 100644 index 0000000..f1e57b8 --- /dev/null +++ b/site-cookbooks/kosmos_prometheus/.gitignore @@ -0,0 +1,25 @@ +.vagrant +*~ +*# +.#* +\#*# +.*.sw[a-z] +*.un~ + +# Bundler +Gemfile.lock +gems.locked +bin/* +.bundle/* + +# test kitchen +.kitchen/ +kitchen.local.yml + +# Chef Infra +Berksfile.lock +.zero-knife.rb +Policyfile.lock.json + +.idea/ + diff --git a/site-cookbooks/kosmos_prometheus/CHANGELOG.md b/site-cookbooks/kosmos_prometheus/CHANGELOG.md new file mode 100644 index 0000000..2aa0619 --- /dev/null +++ b/site-cookbooks/kosmos_prometheus/CHANGELOG.md @@ -0,0 +1,7 @@ +# kosmos_prometheus CHANGELOG + +This file is used to list changes made in each version of the kosmos_prometheus cookbook. + +## 0.1.0 + +Initial release. diff --git a/site-cookbooks/kosmos_prometheus/LICENSE b/site-cookbooks/kosmos_prometheus/LICENSE new file mode 100644 index 0000000..f3b5d1c --- /dev/null +++ b/site-cookbooks/kosmos_prometheus/LICENSE @@ -0,0 +1,20 @@ +Copyright (c) 2019 Kosmos Developers + +Permission is hereby granted, free of charge, to any person obtaining +a copy of this software and associated documentation files (the +"Software"), to deal in the Software without restriction, including +without limitation the rights to use, copy, modify, merge, publish, +distribute, sublicense, and/or sell copies of the Software, and to +permit persons to whom the Software is furnished to do so, subject to +the following conditions: + +The above copyright notice and this permission notice shall be +included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE +LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION +WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. diff --git a/site-cookbooks/kosmos_prometheus/README.md b/site-cookbooks/kosmos_prometheus/README.md new file mode 100644 index 0000000..5c56b8e --- /dev/null +++ b/site-cookbooks/kosmos_prometheus/README.md @@ -0,0 +1,4 @@ +# kosmos_prometheus + +TODO: Enter the cookbook description here. + diff --git a/site-cookbooks/kosmos_prometheus/attributes/default.rb b/site-cookbooks/kosmos_prometheus/attributes/default.rb new file mode 100644 index 0000000..7e5005b --- /dev/null +++ b/site-cookbooks/kosmos_prometheus/attributes/default.rb @@ -0,0 +1,5 @@ +node.default["prometheus"]["version"] = "3.13.0" +node.default["prometheus"]["checksum"] = "744d93324cc024d82089921737bd797474d7f1e5dbbfd1c6b387bad258538cb9" + +node.default["prometheus"]["alertmanager"]["version"] = "0.33.0" +node.default["prometheus"]["alertmanager"]["checksum"] = "8ce11c42e8a6dfbbf93a59c0b193cb1329210b36d0c7ef3df7b745608675a1d1" diff --git a/site-cookbooks/kosmos_prometheus/chefignore b/site-cookbooks/kosmos_prometheus/chefignore new file mode 100644 index 0000000..cc170ea --- /dev/null +++ b/site-cookbooks/kosmos_prometheus/chefignore @@ -0,0 +1,115 @@ +# Put files/directories that should be ignored in this file when uploading +# to a Chef Infra Server or Supermarket. +# Lines that start with '# ' are comments. + +# OS generated files # +###################### +.DS_Store +ehthumbs.db +Icon? +nohup.out +Thumbs.db +.envrc + +# EDITORS # +########### +.#* +.project +.settings +*_flymake +*_flymake.* +*.bak +*.sw[a-z] +*.tmproj +*~ +\#* +REVISION +TAGS* +tmtags +.vscode +.editorconfig + +## COMPILED ## +############## +*.class +*.com +*.dll +*.exe +*.o +*.pyc +*.so +*/rdoc/ +a.out +mkmf.log + +# Testing # +########### +.circleci/* +.codeclimate.yml +.delivery/* +.foodcritic +.kitchen* +.mdlrc +.overcommit.yml +.rspec +.rubocop.yml +.travis.yml +.watchr +.yamllint +azure-pipelines.yml +Dangerfile +examples/* +features/* +Guardfile +kitchen.yml* +mlc_config.json +Procfile +Rakefile +spec/* +test/* + +# SCM # +####### +.git +.gitattributes +.gitconfig +.github/* +.gitignore +.gitkeep +.gitmodules +.svn +*/.bzr/* +*/.git +*/.hg/* +*/.svn/* + +# Berkshelf # +############# +Berksfile +Berksfile.lock +cookbooks/* +tmp + +# Bundler # +########### +vendor/* +Gemfile +Gemfile.lock + +# Policyfile # +############## +Policyfile.rb +Policyfile.lock.json + +# Documentation # +############# +CODE_OF_CONDUCT* +CONTRIBUTING* +documentation/* +TESTING* +UPGRADING* + +# Vagrant # +########### +.vagrant +Vagrantfile diff --git a/site-cookbooks/kosmos_prometheus/metadata.rb b/site-cookbooks/kosmos_prometheus/metadata.rb new file mode 100644 index 0000000..2c04d3b --- /dev/null +++ b/site-cookbooks/kosmos_prometheus/metadata.rb @@ -0,0 +1,21 @@ +name 'kosmos_prometheus' +maintainer 'Kosmos Developers' +maintainer_email 'mail@kosmos.org' +license 'MIT' +description 'Installs/Configures prometheus' +version '0.1.0' +chef_version '>= 16.0' + +depends "prometheus" + +# The `issues_url` points to the location where issues for this cookbook are +# tracked. A `View Issues` link will be displayed on this cookbook's page when +# uploaded to a Supermarket. +# +# issues_url 'https://github.com//kosmos_prometheus/issues' + +# The `source_url` points to the development repository for this cookbook. A +# `View Source` link will be displayed on this cookbook's page when uploaded to +# a Supermarket. +# +# source_url 'https://github.com//kosmos_prometheus' diff --git a/site-cookbooks/kosmos_prometheus/recipes/alertmanager.rb b/site-cookbooks/kosmos_prometheus/recipes/alertmanager.rb new file mode 100644 index 0000000..1984732 --- /dev/null +++ b/site-cookbooks/kosmos_prometheus/recipes/alertmanager.rb @@ -0,0 +1,21 @@ +# +# Cookbook:: kosmos_prometheus +# Recipe:: alertmanager +# + +include_recipe "firewall" + +prometheus_alertmanager_install "alertmanager" do + version node["prometheus"]["alertmanager"]["version"] + checksum node["prometheus"]["alertmanager"]["checksum"] +end + +prometheus_alertmanager_config "alertmanager" +prometheus_alertmanager_service "alertmanager" + +firewall_rule "prometheus alertmanager" do + port 9093 + source "10.1.1.0/24" + protocol :tcp + command :allow +end diff --git a/site-cookbooks/kosmos_prometheus/recipes/server.rb b/site-cookbooks/kosmos_prometheus/recipes/server.rb new file mode 100644 index 0000000..6cea020 --- /dev/null +++ b/site-cookbooks/kosmos_prometheus/recipes/server.rb @@ -0,0 +1,41 @@ +# +# Cookbook:: kosmos_prometheus +# Recipe:: server +# + +include_recipe "firewall" + +prometheus_install "prometheus" do + version node["prometheus"]["version"] + checksum node["prometheus"]["checksum"] +end + +prometheus_config "prometheus" do + # scrape_interval "60s" + # evaluation_interval "60s" +end + +prometheus_job "prometheus" do + target "localhost:9090" +end + +prometheus_service "prometheus" do + cli_options({ + "config.file" => "/opt/prometheus/prometheus.yml", + "log.level" => "info", + "query.max-concurrency" => 20, + "query.lookback-delta" => "5m", + "query.timeout" => "2m", + "storage.tsdb.path" => "/var/lib/prometheus", + "storage.tsdb.retention.time" => "15d", + "web.listen-address" => ":9090", + "web.telemetry-path" => "" + }) +end + +firewall_rule "prometheus web" do + port 9090 + source "10.1.1.0/24" + protocol :tcp + command :allow +end From df8c8d174256aa21349503ea4e3d36169edb4ac5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A2u=20Cao?= Date: Sat, 4 Jul 2026 13:29:38 +0200 Subject: [PATCH 05/47] Remove obsolete CLI option Was left over from trying to overwrite the defaults before --- site-cookbooks/kosmos_prometheus/recipes/server.rb | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/site-cookbooks/kosmos_prometheus/recipes/server.rb b/site-cookbooks/kosmos_prometheus/recipes/server.rb index 6cea020..d770ee5 100644 --- a/site-cookbooks/kosmos_prometheus/recipes/server.rb +++ b/site-cookbooks/kosmos_prometheus/recipes/server.rb @@ -28,8 +28,7 @@ prometheus_service "prometheus" do "query.timeout" => "2m", "storage.tsdb.path" => "/var/lib/prometheus", "storage.tsdb.retention.time" => "15d", - "web.listen-address" => ":9090", - "web.telemetry-path" => "" + "web.listen-address" => ":9090" }) end From 7396af5ca42d96732959c61170fd09823927c765 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A2u=20Cao?= Date: Sat, 4 Jul 2026 13:30:52 +0200 Subject: [PATCH 06/47] WIP Add node exporter --- roles/prometheus_server.rb | 3 +- .../kosmos_prometheus/attributes/default.rb | 3 + .../recipes/node_exporter.rb | 85 +++++++++++++++++++ .../kosmos_prometheus/recipes/server.rb | 4 + 4 files changed, 94 insertions(+), 1 deletion(-) create mode 100644 site-cookbooks/kosmos_prometheus/recipes/node_exporter.rb diff --git a/roles/prometheus_server.rb b/roles/prometheus_server.rb index dc0bfea..a91fc15 100644 --- a/roles/prometheus_server.rb +++ b/roles/prometheus_server.rb @@ -2,7 +2,8 @@ name "prometheus_server" default_run_list = [ "kosmos_prometheus::server", - "kosmos_prometheus::alertmanager" + "kosmos_prometheus::alertmanager", + "kosmos_prometheus::node_exporter" ] env_run_lists( diff --git a/site-cookbooks/kosmos_prometheus/attributes/default.rb b/site-cookbooks/kosmos_prometheus/attributes/default.rb index 7e5005b..89714ac 100644 --- a/site-cookbooks/kosmos_prometheus/attributes/default.rb +++ b/site-cookbooks/kosmos_prometheus/attributes/default.rb @@ -3,3 +3,6 @@ node.default["prometheus"]["checksum"] = "744d93324cc024d82089921737bd797474d7f1 node.default["prometheus"]["alertmanager"]["version"] = "0.33.0" node.default["prometheus"]["alertmanager"]["checksum"] = "8ce11c42e8a6dfbbf93a59c0b193cb1329210b36d0c7ef3df7b745608675a1d1" + +node.default["prometheus"]["node_exporter"]["version"] = "1.11.1" +node.default["prometheus"]["node_exporter"]["checksum"] = "9f5ea48e5bc7b656f8a91a32e7d7deb89f70f73dabd0d974418aca15f37d6810" diff --git a/site-cookbooks/kosmos_prometheus/recipes/node_exporter.rb b/site-cookbooks/kosmos_prometheus/recipes/node_exporter.rb new file mode 100644 index 0000000..7e6a9dc --- /dev/null +++ b/site-cookbooks/kosmos_prometheus/recipes/node_exporter.rb @@ -0,0 +1,85 @@ +# +# Cookbook:: kosmos_prometheus +# Recipe:: node_exporter +# + +include_recipe "firewall" + +version = node["prometheus"]["node_exporter"]["version"] +checksum = node["prometheus"]["node_exporter"]["checksum"] +tarball = "#{Chef::Config[:file_cache_path]}/node_exporter-#{version}.linux-amd64.tar.gz" +binary_url = "https://github.com/prometheus/node_exporter/releases/download/v#{version}/node_exporter-#{version}.linux-amd64.tar.gz" + +group "node_exporter" + +user "node_exporter" do + gid "node_exporter" + system true + shell "/bin/false" + home "/nonexistent" +end + +directory "/var/lib/node_exporter/textfile" do + owner "node_exporter" + group "node_exporter" + mode "0755" + recursive true +end + +package %w(tar bzip2) + +remote_file tarball do + source binary_url + checksum checksum + action :create + notifies :run, "execute[install_node_exporter]", :immediately +end + +execute "install_node_exporter" do + command "tar -xzf #{tarball} -C /usr/local/bin --strip-components=1 node_exporter-#{version}.linux-amd64/node_exporter" + action :nothing + notifies :restart, "service[node_exporter]", :delayed +end + +file "/usr/local/bin/node_exporter" do + owner "root" + group "root" + mode "0755" + notifies :restart, "service[node_exporter]", :delayed +end + +systemd_unit "node_exporter.service" do + content({ + Unit: { + Description: "Prometheus node exporter", + Documentation: ["https://github.com/prometheus/node_exporter"], + }, + Service: { + Type: "simple", + User: "node_exporter", + Group: "node_exporter", + ExecStart: "/usr/local/bin/node_exporter --web.listen-address=:9100 --collector.textfile.directory=/var/lib/node_exporter/textfile", + Restart: "on-failure", + RestartSec: "5", + NoNewPrivileges: "yes", + ProtectSystem: "full", + ProtectHome: "yes", + }, + Install: { + WantedBy: "multi-user.target", + }, + }) + triggers_reload true + action :create +end + +service "node_exporter" do + action [:enable, :start] +end + +firewall_rule "node_exporter" do + port 9100 + source "10.1.1.0/24" + protocol :tcp + command :allow +end diff --git a/site-cookbooks/kosmos_prometheus/recipes/server.rb b/site-cookbooks/kosmos_prometheus/recipes/server.rb index d770ee5..16867c2 100644 --- a/site-cookbooks/kosmos_prometheus/recipes/server.rb +++ b/site-cookbooks/kosmos_prometheus/recipes/server.rb @@ -19,6 +19,10 @@ prometheus_job "prometheus" do target "localhost:9090" end +prometheus_job "node" do + target "localhost:9100" +end + prometheus_service "prometheus" do cli_options({ "config.file" => "/opt/prometheus/prometheus.yml", From e21797b40246f53d586e4d20b55ee12a11c03c7e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A2u=20Cao?= Date: Sat, 4 Jul 2026 14:12:26 +0200 Subject: [PATCH 07/47] Apply changed configs to prometheus and alertmanager --- site-cookbooks/kosmos_prometheus/recipes/alertmanager.rb | 7 +++++++ site-cookbooks/kosmos_prometheus/recipes/server.rb | 8 ++++++++ 2 files changed, 15 insertions(+) diff --git a/site-cookbooks/kosmos_prometheus/recipes/alertmanager.rb b/site-cookbooks/kosmos_prometheus/recipes/alertmanager.rb index 1984732..4ed73ca 100644 --- a/site-cookbooks/kosmos_prometheus/recipes/alertmanager.rb +++ b/site-cookbooks/kosmos_prometheus/recipes/alertmanager.rb @@ -11,6 +11,13 @@ prometheus_alertmanager_install "alertmanager" do end prometheus_alertmanager_config "alertmanager" + +execute "restart alertmanager config" do + command "systemctl restart alertmanager.service" + action :nothing + subscribes :run, "template[/opt/prometheus/alertmanager.yml]", :delayed +end + prometheus_alertmanager_service "alertmanager" firewall_rule "prometheus alertmanager" do diff --git a/site-cookbooks/kosmos_prometheus/recipes/server.rb b/site-cookbooks/kosmos_prometheus/recipes/server.rb index 16867c2..3256944 100644 --- a/site-cookbooks/kosmos_prometheus/recipes/server.rb +++ b/site-cookbooks/kosmos_prometheus/recipes/server.rb @@ -23,6 +23,14 @@ prometheus_job "node" do target "localhost:9100" end +with_run_context :root do + execute "reload prometheus config" do + command "systemctl reload prometheus.service" + action :nothing + subscribes :run, "template[/opt/prometheus/prometheus.yml]", :delayed + end +end + prometheus_service "prometheus" do cli_options({ "config.file" => "/opt/prometheus/prometheus.yml", From 2d835335b586ecc8604ba671b69adf97a623c617 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A2u=20Cao?= Date: Sat, 4 Jul 2026 14:15:55 +0200 Subject: [PATCH 08/47] Re-add global config, change values --- site-cookbooks/kosmos_prometheus/recipes/server.rb | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/site-cookbooks/kosmos_prometheus/recipes/server.rb b/site-cookbooks/kosmos_prometheus/recipes/server.rb index 3256944..43b17aa 100644 --- a/site-cookbooks/kosmos_prometheus/recipes/server.rb +++ b/site-cookbooks/kosmos_prometheus/recipes/server.rb @@ -11,8 +11,10 @@ prometheus_install "prometheus" do end prometheus_config "prometheus" do - # scrape_interval "60s" - # evaluation_interval "60s" + global_config( + "scrape_interval" => "30s", + "evaluation_interval" => "30s" + ) end prometheus_job "prometheus" do From 63534e1cf59ec72292381afcd56ed11ed75644d6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A2u=20Cao?= Date: Sat, 4 Jul 2026 15:27:18 +0200 Subject: [PATCH 09/47] Remove upstream prometheus cookbook, migrate to our own --- Berksfile | 1 - Berksfile.lock | 3 - cookbooks/prometheus/.hound.yml | 8 - cookbooks/prometheus/.markdownlint-cli2.yaml | 9 - .../prometheus/.release-please-manifest.json | 3 - cookbooks/prometheus/CHANGELOG.md | 104 --------- cookbooks/prometheus/LICENSE | 201 ------------------ cookbooks/prometheus/LIMITATIONS.md | 36 ---- cookbooks/prometheus/README.md | 67 ------ cookbooks/prometheus/Thorfile | 10 - cookbooks/prometheus/chefignore | 115 ---------- cookbooks/prometheus/libraries/helpers.rb | 78 ------- cookbooks/prometheus/metadata.json | 44 ---- cookbooks/prometheus/metadata.rb | 23 -- cookbooks/prometheus/migration.md | 36 ---- .../prometheus/release-please-config.json | 12 -- cookbooks/prometheus/renovate.json | 18 -- .../resources/_partial/_alertmanager.rb | 21 -- .../prometheus/resources/_partial/_common.rb | 8 - .../resources/_partial/_prometheus.rb | 71 ------- .../resources/alertmanager_config.rb | 30 --- .../resources/alertmanager_install.rb | 89 -------- .../resources/alertmanager_service.rb | 56 ----- cookbooks/prometheus/resources/config.rb | 44 ---- cookbooks/prometheus/resources/install.rb | 98 --------- cookbooks/prometheus/resources/job.rb | 41 ---- cookbooks/prometheus/resources/service.rb | 56 ----- .../templates/default/alertmanager.yml.erb | 115 ---------- .../templates/default/prometheus.yml.erb | 33 --- nodes/prometheus-1.json | 1 + .../kosmos_prometheus/attributes/default.rb | 23 +- site-cookbooks/kosmos_prometheus/metadata.rb | 2 +- .../kosmos_prometheus/recipes/alertmanager.rb | 85 +++++++- .../recipes/node_exporter.rb | 4 +- .../kosmos_prometheus/recipes/server.rb | 122 ++++++++--- .../templates/default/alertmanager.yml.erb | 12 ++ .../templates/default/prometheus.yml.erb | 31 +++ 37 files changed, 230 insertions(+), 1480 deletions(-) delete mode 100644 cookbooks/prometheus/.hound.yml delete mode 100644 cookbooks/prometheus/.markdownlint-cli2.yaml delete mode 100644 cookbooks/prometheus/.release-please-manifest.json delete mode 100644 cookbooks/prometheus/CHANGELOG.md delete mode 100644 cookbooks/prometheus/LICENSE delete mode 100644 cookbooks/prometheus/LIMITATIONS.md delete mode 100644 cookbooks/prometheus/README.md delete mode 100644 cookbooks/prometheus/Thorfile delete mode 100644 cookbooks/prometheus/chefignore delete mode 100644 cookbooks/prometheus/libraries/helpers.rb delete mode 100644 cookbooks/prometheus/metadata.json delete mode 100644 cookbooks/prometheus/metadata.rb delete mode 100644 cookbooks/prometheus/migration.md delete mode 100644 cookbooks/prometheus/release-please-config.json delete mode 100644 cookbooks/prometheus/renovate.json delete mode 100644 cookbooks/prometheus/resources/_partial/_alertmanager.rb delete mode 100644 cookbooks/prometheus/resources/_partial/_common.rb delete mode 100644 cookbooks/prometheus/resources/_partial/_prometheus.rb delete mode 100644 cookbooks/prometheus/resources/alertmanager_config.rb delete mode 100644 cookbooks/prometheus/resources/alertmanager_install.rb delete mode 100644 cookbooks/prometheus/resources/alertmanager_service.rb delete mode 100644 cookbooks/prometheus/resources/config.rb delete mode 100644 cookbooks/prometheus/resources/install.rb delete mode 100644 cookbooks/prometheus/resources/job.rb delete mode 100644 cookbooks/prometheus/resources/service.rb delete mode 100644 cookbooks/prometheus/templates/default/alertmanager.yml.erb delete mode 100644 cookbooks/prometheus/templates/default/prometheus.yml.erb create mode 100644 site-cookbooks/kosmos_prometheus/templates/default/alertmanager.yml.erb create mode 100644 site-cookbooks/kosmos_prometheus/templates/default/prometheus.yml.erb diff --git a/Berksfile b/Berksfile index c30c798..5019974 100644 --- a/Berksfile +++ b/Berksfile @@ -41,7 +41,6 @@ cookbook 'ulimit', '~> 1.0.0' cookbook 'users', '~> 5.3.1' cookbook 'zerotier', '~> 1.0.7' cookbook 'unbound', '~> 3.0.2' -cookbook 'prometheus', '~> 1.0.0' # openresty dependency cookbook 'jemalloc', '~> 0.1.7' diff --git a/Berksfile.lock b/Berksfile.lock index 887e8f2..1651b1b 100644 --- a/Berksfile.lock +++ b/Berksfile.lock @@ -33,7 +33,6 @@ DEPENDENCIES git: https://gitea.kosmos.org/kosmos/postfix-cookbook.git revision: dd6598572a775ae73f17527260ec8097b52d385b ref: bugfix/ - prometheus (~> 1.0.0) redisio (~> 6.4.1) ruby_build (~> 2.5.0) timezone_iii (= 1.0.4) @@ -96,8 +95,6 @@ GRAPH php (8.0.1) yum-epel (>= 0.0.0) postfix (6.4.1) - prometheus (1.0.0) - ark (>= 0.0.0) redisio (6.4.1) selinux (>= 0.0.0) ruby_build (2.5.0) diff --git a/cookbooks/prometheus/.hound.yml b/cookbooks/prometheus/.hound.yml deleted file mode 100644 index eddba4f..0000000 --- a/cookbooks/prometheus/.hound.yml +++ /dev/null @@ -1,8 +0,0 @@ -ruby: - config_file: .rubocop.yml - -java_script: - enabled: false - -scss: - enabled: false diff --git a/cookbooks/prometheus/.markdownlint-cli2.yaml b/cookbooks/prometheus/.markdownlint-cli2.yaml deleted file mode 100644 index ac5076b..0000000 --- a/cookbooks/prometheus/.markdownlint-cli2.yaml +++ /dev/null @@ -1,9 +0,0 @@ -config: - ul-indent: false # MD007 - line-length: false # MD013 - no-duplicate-heading: false # MD024 - reference-links-images: false # MD052 - no-multiple-blanks: - maximum: 2 -ignores: - - .github/copilot-instructions.md diff --git a/cookbooks/prometheus/.release-please-manifest.json b/cookbooks/prometheus/.release-please-manifest.json deleted file mode 100644 index 37fcefa..0000000 --- a/cookbooks/prometheus/.release-please-manifest.json +++ /dev/null @@ -1,3 +0,0 @@ -{ - ".": "1.0.0" -} diff --git a/cookbooks/prometheus/CHANGELOG.md b/cookbooks/prometheus/CHANGELOG.md deleted file mode 100644 index a83d864..0000000 --- a/cookbooks/prometheus/CHANGELOG.md +++ /dev/null @@ -1,104 +0,0 @@ -# Change Log - -All notable changes to this project will be documented in this file. - -## [1.0.0](https://github.com/sous-chefs/prometheus/compare/v0.7.6...v1.0.0) (2026-04-29) - - -### ⚠ BREAKING CHANGES - -* migrate prometheus resources ([#36](https://github.com/sous-chefs/prometheus/issues/36)) - -### Features - -* migrate prometheus resources ([#36](https://github.com/sous-chefs/prometheus/issues/36)) ([6698649](https://github.com/sous-chefs/prometheus/commit/66986492fe0f069bc69474e19423b16e21563c16)) - - -### Bug Fixes - -* **ci:** Update workflows to use release pipeline ([#29](https://github.com/sous-chefs/prometheus/issues/29)) ([b991214](https://github.com/sous-chefs/prometheus/commit/b991214225164d55694966e535478a38e720a8f0)) - -## [0.7.6](https://github.com/sous-chefs/prometheus/compare/0.7.5...v0.7.6) (2025-10-16) - - -### Bug Fixes - -* **ci:** Update workflows to use release pipeline ([#29](https://github.com/sous-chefs/prometheus/issues/29)) ([b991214](https://github.com/sous-chefs/prometheus/commit/b991214225164d55694966e535478a38e720a8f0)) - -## 0.7.2 - *2022-02-08* - -* Remove delivery folder - -Standardise files with files in sous-chefs/repo-management - -## 0.7.1 - *2021-06-01* - -* resolved cookstyle error: Thorfile:1:1 convention: `Style/Encoding` -* resolved cookstyle error: test/shared/spec_helper.rb:6:10 convention: `Style/ExpandPathArguments` - -## [0.7.1 - *2021-06-01*][0.7.1 - *2021-06-01*] - -### Changed - -* Updated attributes and templates for Prometheus 0.15 release. - -### Added - -* Added upstart init for ubuntu platform. - -## [0.5.1] - 2015-03-25 - -Changed - -* Updated documentation. - -## [0.5.0] - 2015-03-25 - -Added - -* Added systemd init for redhat platform family version 7 or greater. -* Default init style per platform. -* Install Prometheus via pre-compiled binary. -* Added the prometheus_job resource for defining Prometheus scraping jobs. -* Attribute flag to externally manage prometheus.conf file. - -Changed - -* Removed flags that were deprecated in the prometheus 0.12.0 release. - -### Contributors for this release - -* [Eric Richardson](https://github.com/ewr) - External jobs config and prometheus job resource. - -Thank You! - -## [0.4.0] - 2015-03-12 - -### Fixed - -* Fix init template path bug on chef 11.x. - -## [0.3.0] - 2015-03-11 - -Fixed - -* Fixed cookbook badge in README - -## [0.2.0] - 2015-03-11 - -Fixed - -* License defined in metadata. - -## 0.1.0 - 2015-03-11 - -Changed - -* Initial release of prometheus cookbook - -[0.7.1 - *2021-06-01*]: https://github.com/rayrod2030/chef-prometheus/compare/0.5.1...HEAD -[0.5.1]: https://github.com/rayrod2030/chef-prometheus/compare/0.5.0...0.5.1 -[0.5.0]: https://github.com/rayrod2030/chef-prometheus/compare/0.4.0...0.5.0 -[0.4.0]: https://github.com/rayrod2030/chef-prometheus/compare/0.3.0...0.4.0 -[0.3.0]: https://github.com/rayrod2030/chef-prometheus/compare/0.2.0...0.3.0 -[0.2.0]: https://github.com/rayrod2030/chef-prometheus/compare/0.1.0...0.2.0 diff --git a/cookbooks/prometheus/LICENSE b/cookbooks/prometheus/LICENSE deleted file mode 100644 index 11069ed..0000000 --- a/cookbooks/prometheus/LICENSE +++ /dev/null @@ -1,201 +0,0 @@ - Apache License - Version 2.0, January 2004 - http://www.apache.org/licenses/ - -TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION - -1. Definitions. - - "License" shall mean the terms and conditions for use, reproduction, - and distribution as defined by Sections 1 through 9 of this document. - - "Licensor" shall mean the copyright owner or entity authorized by - the copyright owner that is granting the License. - - "Legal Entity" shall mean the union of the acting entity and all - other entities that control, are controlled by, or are under common - control with that entity. For the purposes of this definition, - "control" means (i) the power, direct or indirect, to cause the - direction or management of such entity, whether by contract or - otherwise, or (ii) ownership of fifty percent (50%) or more of the - outstanding shares, or (iii) beneficial ownership of such entity. - - "You" (or "Your") shall mean an individual or Legal Entity - exercising permissions granted by this License. - - "Source" form shall mean the preferred form for making modifications, - including but not limited to software source code, documentation - source, and configuration files. - - "Object" form shall mean any form resulting from mechanical - transformation or translation of a Source form, including but - not limited to compiled object code, generated documentation, - and conversions to other media types. - - "Work" shall mean the work of authorship, whether in Source or - Object form, made available under the License, as indicated by a - copyright notice that is included in or attached to the work - (an example is provided in the Appendix below). - - "Derivative Works" shall mean any work, whether in Source or Object - form, that is based on (or derived from) the Work and for which the - editorial revisions, annotations, elaborations, or other modifications - represent, as a whole, an original work of authorship. For the purposes - of this License, Derivative Works shall not include works that remain - separable from, or merely link (or bind by name) to the interfaces of, - the Work and Derivative Works thereof. - - "Contribution" shall mean any work of authorship, including - the original version of the Work and any modifications or additions - to that Work or Derivative Works thereof, that is intentionally - submitted to Licensor for inclusion in the Work by the copyright owner - or by an individual or Legal Entity authorized to submit on behalf of - the copyright owner. For the purposes of this definition, "submitted" - means any form of electronic, verbal, or written communication sent - to the Licensor or its representatives, including but not limited to - communication on electronic mailing lists, source code control systems, - and issue tracking systems that are managed by, or on behalf of, the - Licensor for the purpose of discussing and improving the Work, but - excluding communication that is conspicuously marked or otherwise - designated in writing by the copyright owner as "Not a Contribution." - - "Contributor" shall mean Licensor and any individual or Legal Entity - on behalf of whom a Contribution has been received by Licensor and - subsequently incorporated within the Work. - -2. Grant of Copyright License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - copyright license to reproduce, prepare Derivative Works of, - publicly display, publicly perform, sublicense, and distribute the - Work and such Derivative Works in Source or Object form. - -3. Grant of Patent License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - (except as stated in this section) patent license to make, have made, - use, offer to sell, sell, import, and otherwise transfer the Work, - where such license applies only to those patent claims licensable - by such Contributor that are necessarily infringed by their - Contribution(s) alone or by combination of their Contribution(s) - with the Work to which such Contribution(s) was submitted. If You - institute patent litigation against any entity (including a - cross-claim or counterclaim in a lawsuit) alleging that the Work - or a Contribution incorporated within the Work constitutes direct - or contributory patent infringement, then any patent licenses - granted to You under this License for that Work shall terminate - as of the date such litigation is filed. - -4. Redistribution. You may reproduce and distribute copies of the - Work or Derivative Works thereof in any medium, with or without - modifications, and in Source or Object form, provided that You - meet the following conditions: - - (a) You must give any other recipients of the Work or - Derivative Works a copy of this License; and - - (b) You must cause any modified files to carry prominent notices - stating that You changed the files; and - - (c) You must retain, in the Source form of any Derivative Works - that You distribute, all copyright, patent, trademark, and - attribution notices from the Source form of the Work, - excluding those notices that do not pertain to any part of - the Derivative Works; and - - (d) If the Work includes a "NOTICE" text file as part of its - distribution, then any Derivative Works that You distribute must - include a readable copy of the attribution notices contained - within such NOTICE file, excluding those notices that do not - pertain to any part of the Derivative Works, in at least one - of the following places: within a NOTICE text file distributed - as part of the Derivative Works; within the Source form or - documentation, if provided along with the Derivative Works; or, - within a display generated by the Derivative Works, if and - wherever such third-party notices normally appear. The contents - of the NOTICE file are for informational purposes only and - do not modify the License. You may add Your own attribution - notices within Derivative Works that You distribute, alongside - or as an addendum to the NOTICE text from the Work, provided - that such additional attribution notices cannot be construed - as modifying the License. - - You may add Your own copyright statement to Your modifications and - may provide additional or different license terms and conditions - for use, reproduction, or distribution of Your modifications, or - for any such Derivative Works as a whole, provided Your use, - reproduction, and distribution of the Work otherwise complies with - the conditions stated in this License. - -5. Submission of Contributions. Unless You explicitly state otherwise, - any Contribution intentionally submitted for inclusion in the Work - by You to the Licensor shall be under the terms and conditions of - this License, without any additional terms or conditions. - Notwithstanding the above, nothing herein shall supersede or modify - the terms of any separate license agreement you may have executed - with Licensor regarding such Contributions. - -6. Trademarks. This License does not grant permission to use the trade - names, trademarks, service marks, or product names of the Licensor, - except as required for reasonable and customary use in describing the - origin of the Work and reproducing the content of the NOTICE file. - -7. Disclaimer of Warranty. Unless required by applicable law or - agreed to in writing, Licensor provides the Work (and each - Contributor provides its Contributions) on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or - implied, including, without limitation, any warranties or conditions - of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A - PARTICULAR PURPOSE. You are solely responsible for determining the - appropriateness of using or redistributing the Work and assume any - risks associated with Your exercise of permissions under this License. - -8. Limitation of Liability. In no event and under no legal theory, - whether in tort (including negligence), contract, or otherwise, - unless required by applicable law (such as deliberate and grossly - negligent acts) or agreed to in writing, shall any Contributor be - liable to You for damages, including any direct, indirect, special, - incidental, or consequential damages of any character arising as a - result of this License or out of the use or inability to use the - Work (including but not limited to damages for loss of goodwill, - work stoppage, computer failure or malfunction, or any and all - other commercial damages or losses), even if such Contributor - has been advised of the possibility of such damages. - -9. Accepting Warranty or Additional Liability. While redistributing - the Work or Derivative Works thereof, You may choose to offer, - and charge a fee for, acceptance of support, warranty, indemnity, - or other liability obligations and/or rights consistent with this - License. However, in accepting such obligations, You may act only - on Your own behalf and on Your sole responsibility, not on behalf - of any other Contributor, and only if You agree to indemnify, - defend, and hold each Contributor harmless for any liability - incurred by, or claims asserted against, such Contributor by reason - of your accepting any such warranty or additional liability. - -END OF TERMS AND CONDITIONS - -APPENDIX: How to apply the Apache License to your work. - - To apply the Apache License to your work, attach the following - boilerplate notice, with the fields enclosed by brackets "[]" - replaced with your own identifying information. (Don't include - the brackets!) The text should be enclosed in the appropriate - comment syntax for the file format. We also recommend that a - file or class name and description of purpose be included on the - same "printed page" as the copyright notice for easier - identification within third-party archives. - -Copyright [yyyy] [name of copyright owner] - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. diff --git a/cookbooks/prometheus/LIMITATIONS.md b/cookbooks/prometheus/LIMITATIONS.md deleted file mode 100644 index 6085fa8..0000000 --- a/cookbooks/prometheus/LIMITATIONS.md +++ /dev/null @@ -1,36 +0,0 @@ -# Limitations - -This cookbook manages Prometheus and Alertmanager from upstream release archives or source builds. - -## Upstream Support - -Prometheus publishes precompiled archives for Prometheus and Alertmanager from the official download page and GitHub releases. As of April 28, 2026, the cookbook defaults to: - -* Prometheus 3.11.2 `linux-amd64` and `linux-arm64` -* Alertmanager 0.32.0 `linux-amd64` and `linux-arm64` - -The upstream projects also publish Docker images and archives for other operating systems. This cookbook only supports Linux systemd hosts. - -## Platform Support - -Supported platforms are modern systemd Linux distributions declared in `metadata.rb`: - -* AlmaLinux 8+ -* Amazon Linux 2023+ -* CentOS Stream 9+ -* Debian 12+ -* Fedora -* Oracle Linux 8+ -* Red Hat Enterprise Linux 8+ -* Rocky Linux 8+ -* Ubuntu 20.04+ - -Legacy init systems are not supported. The cookbook no longer manages runit, upstart, or SysV init services. - -## Installation Constraints - -The default binary installation path uses official Linux tarballs for `amd64` or `arm64`. Override `architecture`, `binary_url`, `checksum`, and `file_extension` when using another upstream artifact or a private mirror. - -Source installs require build tooling, Git, and the upstream Go build chain expected by the selected Prometheus or Alertmanager version. Source builds are retained for compatibility but binary installs are the primary supported path. - -Prometheus and Alertmanager are not installed from operating system package repositories by these resources. diff --git a/cookbooks/prometheus/README.md b/cookbooks/prometheus/README.md deleted file mode 100644 index 855db77..0000000 --- a/cookbooks/prometheus/README.md +++ /dev/null @@ -1,67 +0,0 @@ -# prometheus Cookbook - -This cookbook provides custom resources for installing and configuring [Prometheus][] and Alertmanager. - -## Requirements - -* Chef Infra Client 16.0 or later -* A systemd Linux platform listed in [LIMITATIONS.md](LIMITATIONS.md) - -See [LIMITATIONS.md](LIMITATIONS.md) for upstream platform and installation constraints. - -## Resources - -* [prometheus_install](documentation/prometheus_install.md) -* [prometheus_config](documentation/prometheus_config.md) -* [prometheus_service](documentation/prometheus_service.md) -* [prometheus_job](documentation/prometheus_job.md) -* [prometheus_alertmanager_install](documentation/prometheus_alertmanager_install.md) -* [prometheus_alertmanager_config](documentation/prometheus_alertmanager_config.md) -* [prometheus_alertmanager_service](documentation/prometheus_alertmanager_service.md) - -See [migration.md](migration.md) for migration notes from the legacy recipe and attribute interface. - -## Usage - -```ruby -prometheus_install 'prometheus' - -prometheus_config 'prometheus' - -prometheus_job 'prometheus' do - scrape_interval '15s' - target 'localhost:9090' -end - -prometheus_service 'prometheus' -``` - -Alertmanager: - -```ruby -prometheus_alertmanager_install 'alertmanager' -prometheus_alertmanager_config 'alertmanager' -prometheus_alertmanager_service 'alertmanager' -``` - -## License & Authors - -* Author: Ray Rodriguez -* Author: kristian järvenpää -* Maintainer: Sous Chefs - -```text -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -``` - -[Prometheus]: https://github.com/prometheus/prometheus diff --git a/cookbooks/prometheus/Thorfile b/cookbooks/prometheus/Thorfile deleted file mode 100644 index 75ec16f..0000000 --- a/cookbooks/prometheus/Thorfile +++ /dev/null @@ -1,10 +0,0 @@ -require 'bundler' -require 'bundler/setup' -require 'berkshelf/thor' - -begin - require 'kitchen/thor_tasks' - Kitchen::ThorTasks.new -rescue LoadError - puts '>>>>> Kitchen gem not loaded, omitting tasks' unless ENV['CI'] -end diff --git a/cookbooks/prometheus/chefignore b/cookbooks/prometheus/chefignore deleted file mode 100644 index a27b0b2..0000000 --- a/cookbooks/prometheus/chefignore +++ /dev/null @@ -1,115 +0,0 @@ -# Put files/directories that should be ignored in this file when uploading -# to a Chef Infra Server or Supermarket. -# Lines that start with '# ' are comments. - -# OS generated files # -###################### -.DS_Store -ehthumbs.db -Icon? -nohup.out -Thumbs.db -.envrc - -# EDITORS # -########### -.#* -.project -.settings -*_flymake -*_flymake.* -*.bak -*.sw[a-z] -*.tmproj -*~ -\#* -REVISION -TAGS* -tmtags -.vscode -.editorconfig - -## COMPILED ## -############## -*.class -*.com -*.dll -*.exe -*.o -*.pyc -*.so -*/rdoc/ -a.out -mkmf.log - -# Testing # -########### -.circleci/* -.codeclimate.yml -.delivery/* -.foodcritic -.kitchen* -.mdlrc -.overcommit.yml -.rspec -.rubocop.yml -.travis.yml -.watchr -.yamllint -azure-pipelines.yml -Dangerfile -examples/* -features/* -Guardfile -kitchen*.yml -mlc_config.json -Procfile -Rakefile -spec/* -test/* - -# SCM # -####### -.git -.gitattributes -.gitconfig -.github/* -.gitignore -.gitkeep -.gitmodules -.svn -*/.bzr/* -*/.git -*/.hg/* -*/.svn/* - -# Berkshelf # -############# -Berksfile -Berksfile.lock -cookbooks/* -tmp - -# Bundler # -########### -vendor/* -Gemfile -Gemfile.lock - -# Policyfile # -############## -Policyfile.rb -Policyfile.lock.json - -# Documentation # -############# -CODE_OF_CONDUCT* -CONTRIBUTING* -documentation/* -TESTING* -UPGRADING* - -# Vagrant # -########### -.vagrant -Vagrantfile diff --git a/cookbooks/prometheus/libraries/helpers.rb b/cookbooks/prometheus/libraries/helpers.rb deleted file mode 100644 index c52f5fe..0000000 --- a/cookbooks/prometheus/libraries/helpers.rb +++ /dev/null @@ -1,78 +0,0 @@ -# frozen_string_literal: true - -require 'uri' - -module PrometheusCookbook - module Helpers - def archive_name(component, version, url) - basename = ::File.basename(URI.parse(url).path) - basename.sub(/(?:\.tar\.gz|\.tgz|\.tar\.bz2|\.tar\.xz|\.zip)\z/, '') - rescue URI::InvalidURIError - "#{component}-#{version}" - end - - def install_dir_parent(install_dir) - ::File.dirname(install_dir) - end - - def install_dir_name(install_dir) - ::File.basename(install_dir) - end - - def prometheus_flags(resource) - flag_pairs = if Gem::Version.new(resource.version) < Gem::Version.new('2.0.0-alpha.0') - resource.flags.map { |key, value| "-#{key}=#{value}" unless value == '' } - else - resource.cli_options.map { |key, value| "--#{key}=#{value}" unless value == '' } + - resource.cli_flags.map { |flag| "--#{flag}" unless flag == '' } - end - - flag_pairs.compact.join(' ') - end - - def prometheus_unit_content(resource) - { - Unit: { - Description: 'Prometheus', - After: 'network.target auditd.service', - }, - Service: { - Type: 'simple', - Environment: "GOMAXPROCS=#{node['cpu']['total'] || 1}", - User: resource.user, - Group: resource.group, - ExecStart: "#{resource.binary} #{prometheus_flags(resource)}", - ExecReload: '/bin/kill -HUP $MAINPID', - Restart: 'always', - }, - Install: { - WantedBy: 'multi-user.target', - }, - } - end - - def alertmanager_unit_content(resource) - { - Unit: { - Description: 'Prometheus Alertmanager', - After: 'network.target', - }, - Service: { - User: resource.user, - Group: resource.group, - ExecStart: [ - resource.binary, - "--log.level=#{resource.log_level}", - "--storage.path=#{resource.storage_path}", - "--config.file=#{resource.config_file}", - "--web.external-url=#{resource.external_url}", - ].join(' '), - Restart: 'always', - }, - Install: { - WantedBy: 'multi-user.target', - }, - } - end - end -end diff --git a/cookbooks/prometheus/metadata.json b/cookbooks/prometheus/metadata.json deleted file mode 100644 index 36c5be3..0000000 --- a/cookbooks/prometheus/metadata.json +++ /dev/null @@ -1,44 +0,0 @@ -{ - "name": "prometheus", - "description": "Provides custom resources for installing and configuring Prometheus and Alertmanager", - "long_description": "", - "maintainer": "Sous Chefs", - "maintainer_email": "help@sous-chefs.org", - "license": "Apache-2.0", - "platforms": { - "almalinux": ">= 8.0", - "amazon": ">= 2023.0", - "centos_stream": ">= 9.0", - "debian": ">= 12.0", - "fedora": ">= 0.0.0", - "oracle": ">= 8.0", - "redhat": ">= 8.0", - "rocky": ">= 8.0", - "ubuntu": ">= 20.04" - }, - "dependencies": { - "ark": ">= 0.0.0" - }, - "providing": { - - }, - "recipes": { - - }, - "version": "1.0.0", - "source_url": "https://github.com/sous-chefs/prometheus", - "issues_url": "https://github.com/sous-chefs/prometheus/issues", - "privacy": false, - "chef_versions": [ - [ - ">= 16.0" - ] - ], - "ohai_versions": [ - - ], - "gems": [ - - ], - "eager_load_libraries": true -} diff --git a/cookbooks/prometheus/metadata.rb b/cookbooks/prometheus/metadata.rb deleted file mode 100644 index 20c6d4f..0000000 --- a/cookbooks/prometheus/metadata.rb +++ /dev/null @@ -1,23 +0,0 @@ -# frozen_string_literal: true - -name 'prometheus' -maintainer 'Sous Chefs' -maintainer_email 'help@sous-chefs.org' -license 'Apache-2.0' -description 'Provides custom resources for installing and configuring Prometheus and Alertmanager' -version '1.0.0' -source_url 'https://github.com/sous-chefs/prometheus' -issues_url 'https://github.com/sous-chefs/prometheus/issues' -chef_version '>= 16.0' - -depends 'ark' - -supports 'almalinux', '>= 8.0' -supports 'amazon', '>= 2023.0' -supports 'centos_stream', '>= 9.0' -supports 'debian', '>= 12.0' -supports 'fedora' -supports 'oracle', '>= 8.0' -supports 'redhat', '>= 8.0' -supports 'rocky', '>= 8.0' -supports 'ubuntu', '>= 20.04' diff --git a/cookbooks/prometheus/migration.md b/cookbooks/prometheus/migration.md deleted file mode 100644 index 51ebc2c..0000000 --- a/cookbooks/prometheus/migration.md +++ /dev/null @@ -1,36 +0,0 @@ -# Migration - -This cookbook has been migrated from recipes and attributes to custom resources. - -## Removed Entry Points - -The `recipes/` and `attributes/` directories were removed. Wrapper cookbooks should call the resources directly and pass property values instead of overriding `node['prometheus']` attributes. - -Legacy runit, upstart, and SysV init behavior was removed. Services are managed with Chef's `systemd_unit` resource. - -## Resource Mapping - -Use these resources in place of the old recipes: - -* `prometheus_install` replaces `prometheus::binary`, `prometheus::shell_binary`, and `prometheus::source`. -* `prometheus_config` replaces the Prometheus configuration portion of `prometheus::default`. -* `prometheus_service` replaces `prometheus::service`. -* `prometheus_alertmanager_install` replaces `prometheus::alertmanager_binary` and `prometheus::alertmanager_source`. -* `prometheus_alertmanager_config` replaces the Alertmanager configuration portion of `prometheus::alertmanager`. -* `prometheus_alertmanager_service` replaces the Alertmanager service portion of `prometheus::alertmanager`. -* `prometheus_job` remains available and now uses explicit properties instead of node attributes. - -## Example - -```ruby -prometheus_install 'prometheus' - -prometheus_config 'prometheus' - -prometheus_job 'prometheus' do - scrape_interval '15s' - target 'localhost:9090' -end - -prometheus_service 'prometheus' -``` diff --git a/cookbooks/prometheus/release-please-config.json b/cookbooks/prometheus/release-please-config.json deleted file mode 100644 index 148dfd5..0000000 --- a/cookbooks/prometheus/release-please-config.json +++ /dev/null @@ -1,12 +0,0 @@ -{ - "packages": { - ".": { - "package-name": "prometheus", - "changelog-path": "CHANGELOG.md", - "release-type": "ruby", - "include-component-in-tag": false, - "version-file": "metadata.rb" - } - }, - "$schema": "https://raw.githubusercontent.com/googleapis/release-please/main/schemas/config.json" -} diff --git a/cookbooks/prometheus/renovate.json b/cookbooks/prometheus/renovate.json deleted file mode 100644 index a0b29c8..0000000 --- a/cookbooks/prometheus/renovate.json +++ /dev/null @@ -1,18 +0,0 @@ -{ - "$schema": "https://docs.renovatebot.com/renovate-schema.json", - "extends": ["config:base"], - "packageRules": [ - { - "groupName": "Actions", - "matchUpdateTypes": ["minor", "patch", "pin"], - "automerge": true, - "addLabels": ["Release: Patch", "Skip: Announcements"] - }, - { - "groupName": "Actions", - "matchUpdateTypes": ["major"], - "automerge": false, - "addLabels": ["Release: Patch", "Skip: Announcements"] - } - ] -} diff --git a/cookbooks/prometheus/resources/_partial/_alertmanager.rb b/cookbooks/prometheus/resources/_partial/_alertmanager.rb deleted file mode 100644 index c63bfa1..0000000 --- a/cookbooks/prometheus/resources/_partial/_alertmanager.rb +++ /dev/null @@ -1,21 +0,0 @@ -# frozen_string_literal: true - -property :version, String, default: '0.32.0' -property :binary, String, default: '/opt/prometheus/alertmanager' -property :architecture, String, equal_to: %w(amd64 arm64), default: lazy { - node['kernel']['machine'] == 'aarch64' ? 'arm64' : 'amd64' -} -property :binary_url, String, default: lazy { "https://github.com/prometheus/alertmanager/releases/download/v#{version}/alertmanager-#{version}.linux-#{architecture}.tar.gz" } -property :checksum, String, default: lazy { - { - 'amd64' => 'be72f50f6124ec53d944c0f100f8ec8108d969bade02fcc9f06a3068ff6c726f', - 'arm64' => '7812e12699694974f57ecc0b0400913c6c0d90190630d4332a7994a44982b1ed', - }[architecture] -} -property :file_extension, String, default: '' -property :source_repository, String, default: 'https://github.com/prometheus/alertmanager.git' -property :source_revision, String, default: lazy { "v#{version}" } -property :config_file, String, default: '/opt/prometheus/alertmanager.yml' -property :storage_path, String, default: '/opt/prometheus/data' -property :external_url, String, default: 'http://127.0.0.1/alert-manager/' -property :log_level, String, default: 'debug' diff --git a/cookbooks/prometheus/resources/_partial/_common.rb b/cookbooks/prometheus/resources/_partial/_common.rb deleted file mode 100644 index 5c6a534..0000000 --- a/cookbooks/prometheus/resources/_partial/_common.rb +++ /dev/null @@ -1,8 +0,0 @@ -# frozen_string_literal: true - -property :install_dir, String, default: '/opt/prometheus' -property :log_dir, String, default: '/var/log/prometheus' -property :user, String, default: 'prometheus' -property :group, String, default: 'prometheus' -property :use_existing_user, [true, false], default: false -property :install_method, String, equal_to: %w(binary shell_binary source), default: 'binary' diff --git a/cookbooks/prometheus/resources/_partial/_prometheus.rb b/cookbooks/prometheus/resources/_partial/_prometheus.rb deleted file mode 100644 index 37c9f7c..0000000 --- a/cookbooks/prometheus/resources/_partial/_prometheus.rb +++ /dev/null @@ -1,71 +0,0 @@ -# frozen_string_literal: true - -property :version, String, default: '3.11.2' -property :binary, String, default: '/opt/prometheus/prometheus' -property :architecture, String, equal_to: %w(amd64 arm64), default: lazy { - node['kernel']['machine'] == 'aarch64' ? 'arm64' : 'amd64' -} -property :binary_url, String, default: lazy { "https://github.com/prometheus/prometheus/releases/download/v#{version}/prometheus-#{version}.linux-#{architecture}.tar.gz" } -property :checksum, String, default: lazy { - { - 'amd64' => 'f643ea1ee90d109329302d27bddb1fb2e52655b1fa84e9e26f9a6f340da144a6', - 'arm64' => '4e40f115655a3021744137f49287846bc5a59e02835565748ff66b23e776a73d', - }[architecture] -} -property :file_extension, String, default: '' -property :source_repository, String, default: 'https://github.com/prometheus/prometheus.git' -property :source_revision, String, default: lazy { "v#{version}" } -property :config_file, String, default: '/opt/prometheus/prometheus.yml' -property :storage_path, String, default: '/var/lib/prometheus' -property :flags, Hash, default: lazy { - legacy_flags = { - 'config.file' => config_file, - 'log.level' => 'info', - 'alertmanager.timeout' => '10s', - 'alertmanager.notification-queue-capacity' => 100, - 'alertmanager.url' => 'http://127.0.0.1/alert-manager/', - 'query.max-concurrency' => 20, - 'query.staleness-delta' => '5m', - 'query.timeout' => '2m', - 'storage.local.checkpoint-dirty-series-limit' => 5000, - 'storage.local.checkpoint-interval' => '5m', - 'storage.local.dirty' => false, - 'storage.local.index-cache-size.fingerprint-to-metric' => 10_485_760, - 'storage.local.index-cache-size.fingerprint-to-timerange' => 5_242_880, - 'storage.local.index-cache-size.label-name-to-label-values' => 10_485_760, - 'storage.local.index-cache-size.label-pair-to-fingerprints' => 20_971_520, - 'storage.local.memory-chunks' => 1_048_576, - 'storage.local.path' => storage_path, - 'storage.local.pedantic-checks' => false, - 'storage.local.retention' => '360h0m0s', - 'storage.local.series-sync-strategy' => 'adaptive', - 'storage.remote.influxdb-url' => '', - 'storage.remote.influxdb.database' => 'prometheus', - 'storage.remote.influxdb.retention-policy' => 'default', - 'storage.remote.opentsdb-url' => '', - 'storage.remote.timeout' => '30s', - 'web.console.libraries' => 'console_libraries', - 'web.console.templates' => 'consoles', - 'web.enable-remote-shutdown' => false, - 'web.external-url' => '', - 'web.listen-address' => ':9090', - 'web.telemetry-path' => '/metrics', - 'web.user-assets' => '', - } - legacy_flags['web.use-local-assets'] = false if Gem::Version.new(version) <= Gem::Version.new('0.16.2') - legacy_flags -} -property :cli_options, Hash, default: lazy { - { - 'config.file' => config_file, - 'log.level' => 'info', - 'query.max-concurrency' => 20, - 'query.lookback-delta' => '5m', - 'query.timeout' => '2m', - 'storage.tsdb.path' => storage_path, - 'storage.tsdb.retention.time' => '15d', - 'web.listen-address' => ':9090', - 'web.telemetry-path' => '/metrics', - } -} -property :cli_flags, Array, default: ['web.enable-lifecycle'] diff --git a/cookbooks/prometheus/resources/alertmanager_config.rb b/cookbooks/prometheus/resources/alertmanager_config.rb deleted file mode 100644 index 65dbd8a..0000000 --- a/cookbooks/prometheus/resources/alertmanager_config.rb +++ /dev/null @@ -1,30 +0,0 @@ -# frozen_string_literal: true - -provides :prometheus_alertmanager_config -unified_mode true - -use '_partial/_common' -use '_partial/_alertmanager' - -property :template_cookbook, String, default: 'prometheus' -property :template_source, String, default: 'alertmanager.yml.erb' -property :notification_config, Hash, default: {} - -default_action :create - -action :create do - template new_resource.config_file do - cookbook new_resource.template_cookbook - source new_resource.template_source - mode '0644' - owner new_resource.user - group new_resource.group - variables(notification_config: new_resource.notification_config) - end -end - -action :delete do - file new_resource.config_file do - action :delete - end -end diff --git a/cookbooks/prometheus/resources/alertmanager_install.rb b/cookbooks/prometheus/resources/alertmanager_install.rb deleted file mode 100644 index c918f7f..0000000 --- a/cookbooks/prometheus/resources/alertmanager_install.rb +++ /dev/null @@ -1,89 +0,0 @@ -# frozen_string_literal: true - -provides :prometheus_alertmanager_install -unified_mode true - -use '_partial/_common' -use '_partial/_alertmanager' - -default_action :install - -action_class do - include PrometheusCookbook::Helpers -end - -action :install do - user new_resource.user do - system true - shell '/bin/false' - home new_resource.install_dir - not_if { new_resource.use_existing_user || new_resource.user == 'root' } - end - - directory new_resource.install_dir do - owner new_resource.user - group new_resource.group - mode '0755' - recursive true - end - - directory new_resource.log_dir do - owner new_resource.user - group new_resource.group - mode '0755' - recursive true - end - - directory new_resource.storage_path do - owner new_resource.user - group new_resource.group - mode '0755' - recursive true - end - - case new_resource.install_method - when 'binary' - package %w(tar bzip2) - - ark install_dir_name(new_resource.install_dir) do - url new_resource.binary_url - checksum new_resource.checksum - version new_resource.version - prefix_root Chef::Config['file_cache_path'] - path install_dir_parent(new_resource.install_dir) - owner new_resource.user - group new_resource.group - extension new_resource.file_extension unless new_resource.file_extension.empty? - action :put - end - when 'shell_binary' - package %w(tar bzip2) - - remote_file "#{Chef::Config[:file_cache_path]}/alertmanager-#{new_resource.version}.tar.gz" do - source new_resource.binary_url - checksum new_resource.checksum - action :create - notifies :run, 'execute[install_alertmanager_archive]', :immediately - end - - execute 'install_alertmanager_archive' do - command "tar -xzf #{Chef::Config[:file_cache_path]}/alertmanager-#{new_resource.version}.tar.gz -C #{new_resource.install_dir} --strip-components=1" - action :nothing - end - when 'source' - build_essential 'install compilation tools' - - package %w(curl git-core mercurial gzip sed) - - git "#{Chef::Config[:file_cache_path]}/alertmanager-#{new_resource.version}" do - repository new_resource.source_repository - revision new_resource.source_revision - action :checkout - end - - bash 'compile_alertmanager_source' do - cwd "#{Chef::Config[:file_cache_path]}/alertmanager-#{new_resource.version}" - code "make && mv alertmanager #{new_resource.install_dir}" - end - end -end diff --git a/cookbooks/prometheus/resources/alertmanager_service.rb b/cookbooks/prometheus/resources/alertmanager_service.rb deleted file mode 100644 index 8b4cb9f..0000000 --- a/cookbooks/prometheus/resources/alertmanager_service.rb +++ /dev/null @@ -1,56 +0,0 @@ -# frozen_string_literal: true - -provides :prometheus_alertmanager_service -unified_mode true - -use '_partial/_common' -use '_partial/_alertmanager' - -default_action :create - -action_class do - include PrometheusCookbook::Helpers -end - -action :create do - systemd_unit 'alertmanager.service' do - content alertmanager_unit_content(new_resource) - action [:create, :enable, :start] - end -end - -action :enable do - systemd_unit 'alertmanager.service' do - action :enable - end -end - -action :start do - systemd_unit 'alertmanager.service' do - action :start - end -end - -action :restart do - systemd_unit 'alertmanager.service' do - action :restart - end -end - -action :reload do - systemd_unit 'alertmanager.service' do - action :reload - end -end - -action :stop do - systemd_unit 'alertmanager.service' do - action [:stop, :disable] - end -end - -action :delete do - systemd_unit 'alertmanager.service' do - action [:stop, :disable, :delete] - end -end diff --git a/cookbooks/prometheus/resources/config.rb b/cookbooks/prometheus/resources/config.rb deleted file mode 100644 index 20de8d5..0000000 --- a/cookbooks/prometheus/resources/config.rb +++ /dev/null @@ -1,44 +0,0 @@ -# frozen_string_literal: true - -provides :prometheus_config -unified_mode true - -use '_partial/_common' -use '_partial/_prometheus' - -property :template_cookbook, String, default: 'prometheus' -property :template_source, String, default: 'prometheus.yml.erb' -property :rule_filenames, [Array, nil], default: nil -property :global_config, Hash, default: { - 'scrape_interval' => '60s', - 'evaluation_interval' => '60s', -} -property :allow_external_config, [true, false], default: false - -default_action :create - -action :create do - config_resource = new_resource - - with_run_context :root do - template config_resource.config_file do - cookbook config_resource.template_cookbook - source config_resource.template_source - mode '0644' - owner config_resource.user - group config_resource.group - variables( - global_config: config_resource.global_config, - jobs: {}, - rule_filenames: config_resource.rule_filenames - ) - not_if { config_resource.allow_external_config } - end - end -end - -action :delete do - file new_resource.config_file do - action :delete - end -end diff --git a/cookbooks/prometheus/resources/install.rb b/cookbooks/prometheus/resources/install.rb deleted file mode 100644 index 856254d..0000000 --- a/cookbooks/prometheus/resources/install.rb +++ /dev/null @@ -1,98 +0,0 @@ -# frozen_string_literal: true - -provides :prometheus_install -unified_mode true - -use '_partial/_common' -use '_partial/_prometheus' - -default_action :install - -action_class do - include PrometheusCookbook::Helpers -end - -action :install do - user new_resource.user do - system true - shell '/bin/false' - home new_resource.install_dir - not_if { new_resource.use_existing_user || new_resource.user == 'root' } - end - - directory new_resource.install_dir do - owner new_resource.user - group new_resource.group - mode '0755' - recursive true - end - - directory new_resource.log_dir do - owner new_resource.user - group new_resource.group - mode '0755' - recursive true - end - - directory new_resource.storage_path do - owner new_resource.user - group new_resource.group - mode '0755' - recursive true - end - - case new_resource.install_method - when 'binary' - package %w(tar bzip2) - - ark install_dir_name(new_resource.install_dir) do - url new_resource.binary_url - checksum new_resource.checksum - version new_resource.version - prefix_root Chef::Config['file_cache_path'] - path install_dir_parent(new_resource.install_dir) - owner new_resource.user - group new_resource.group - extension new_resource.file_extension unless new_resource.file_extension.empty? - action :put - end - when 'shell_binary' - package %w(tar bzip2) - - remote_file "#{Chef::Config[:file_cache_path]}/prometheus-#{new_resource.version}.tar.gz" do - source new_resource.binary_url - checksum new_resource.checksum - action :create - notifies :run, 'execute[install_prometheus_archive]', :immediately - end - - execute 'install_prometheus_archive' do - command "tar -xzf #{Chef::Config[:file_cache_path]}/prometheus-#{new_resource.version}.tar.gz -C #{new_resource.install_dir} --strip-components=1" - action :nothing - end - when 'source' - build_essential 'install compilation tools' - - package %w(curl git-core mercurial gzip sed) - - git "#{Chef::Config[:file_cache_path]}/prometheus-#{new_resource.version}" do - repository new_resource.source_repository - revision new_resource.source_revision - action :checkout - end - - bash 'compile_prometheus_source' do - cwd "#{Chef::Config[:file_cache_path]}/prometheus-#{new_resource.version}" - environment( - 'PATH' => "/usr/local/go/bin:#{ENV.fetch('PATH', nil)}", - 'GOPATH' => '/opt/go:/opt/go/src/github.com/prometheus/promu/vendor' - ) - code <<~EOH - make build - mv prometheus #{new_resource.install_dir} - cp -R console_libraries #{new_resource.install_dir} - cp -R consoles #{new_resource.install_dir} - EOH - end - end -end diff --git a/cookbooks/prometheus/resources/job.rb b/cookbooks/prometheus/resources/job.rb deleted file mode 100644 index 7d104a5..0000000 --- a/cookbooks/prometheus/resources/job.rb +++ /dev/null @@ -1,41 +0,0 @@ -# frozen_string_literal: true - -provides :prometheus_job -unified_mode true - -property :scrape_interval, String -property :scrape_timeout, String -property :labels, Hash -property :target, [Array, String], required: true -property :metrics_path, String, default: '/metrics' -property :config_file, String, default: '/opt/prometheus/prometheus.yml' -property :allow_external_config, [true, false], default: false - -default_action :create - -action :create do - job_resource = new_resource - - with_run_context :root do - edit_resource(:template, job_resource.config_file) do - variables[:jobs] ||= {} - variables[:jobs][job_resource.name] ||= {} - variables[:jobs][job_resource.name]['scrape_interval'] = job_resource.scrape_interval - variables[:jobs][job_resource.name]['scrape_timeout'] = job_resource.scrape_timeout - variables[:jobs][job_resource.name]['target'] = job_resource.target - variables[:jobs][job_resource.name]['metrics_path'] = job_resource.metrics_path - variables[:jobs][job_resource.name]['labels'] = job_resource.labels - - action :nothing - delayed_action :create - - not_if { job_resource.allow_external_config } - end - end -end - -action :delete do - file new_resource.config_file do - action :delete - end -end diff --git a/cookbooks/prometheus/resources/service.rb b/cookbooks/prometheus/resources/service.rb deleted file mode 100644 index 1a883f2..0000000 --- a/cookbooks/prometheus/resources/service.rb +++ /dev/null @@ -1,56 +0,0 @@ -# frozen_string_literal: true - -provides :prometheus_service -unified_mode true - -use '_partial/_common' -use '_partial/_prometheus' - -default_action :create - -action_class do - include PrometheusCookbook::Helpers -end - -action :create do - systemd_unit 'prometheus.service' do - content prometheus_unit_content(new_resource) - action [:create, :enable, :start] - end -end - -action :enable do - systemd_unit 'prometheus.service' do - action :enable - end -end - -action :start do - systemd_unit 'prometheus.service' do - action :start - end -end - -action :restart do - systemd_unit 'prometheus.service' do - action :restart - end -end - -action :reload do - systemd_unit 'prometheus.service' do - action :reload - end -end - -action :stop do - systemd_unit 'prometheus.service' do - action [:stop, :disable] - end -end - -action :delete do - systemd_unit 'prometheus.service' do - action [:stop, :disable, :delete] - end -end diff --git a/cookbooks/prometheus/templates/default/alertmanager.yml.erb b/cookbooks/prometheus/templates/default/alertmanager.yml.erb deleted file mode 100644 index 949efac..0000000 --- a/cookbooks/prometheus/templates/default/alertmanager.yml.erb +++ /dev/null @@ -1,115 +0,0 @@ -global: - # The smarthost and SMTP sender used for mail notifications. - smtp_smarthost: 'localhost:25' - smtp_from: 'alertmanager@example.org' - -# The root route on which each incoming alert enters. -route: - # The root route must not have any matchers as it is the entry point for - # all alerts. It needs to have a receiver configured so alerts that do not - # match any of the sub-routes are sent to someone. - receiver: 'team-X-mails' - - # The labels by which incoming alerts are grouped together. For example, - # multiple alerts coming in for cluster=A and alertname=LatencyHigh would - # be batched into a single group. - group_by: ['alertname', 'cluster'] - - # When a new group of alerts is created by an incoming alert, wait at - # least 'group_wait' to send the initial notification. - # This way ensures that you get multiple alerts for the same group that start - # firing shortly after another are batched together on the first - # notification. - group_wait: 30s - - # When the first notification was sent, wait 'group_interval' to send a batch - # of new alerts that started firing for that group. - group_interval: 5m - - # If an alert has successfully been sent, wait 'repeat_interval' to - # resend them. - repeat_interval: 3h - - # All the above attributes are inherited by all child routes and can - # overwritten on each. - - # The child route trees. - routes: - # This routes performs a regular expression match on alert labels to - # catch alerts that are related to a list of services. - - match_re: - service: ^(foo1|foo2|baz)$ - receiver: team-X-mails - - # The service has a sub-route for critical alerts, any alerts - # that do not match, i.e. severity != critical, fall-back to the - # parent node and are sent to 'team-X-mails' - routes: - - match: - severity: critical - receiver: team-X-pager - - - match: - service: files - receiver: team-Y-mails - - routes: - - match: - severity: critical - receiver: team-Y-pager - - # This route handles all alerts coming from a database service. If there's - # no team to handle it, it defaults to the DB team. - - match: - service: database - - receiver: team-DB-pager - # Also group alerts by affected database. - group_by: [alertname, cluster, database] - - routes: - - match: - owner: team-X - receiver: team-X-pager - - - match: - owner: team-Y - receiver: team-Y-pager - - -# Inhibition rules allow to mute a set of alerts given that another alert is -# firing. -# We use this to mute any warning-level notifications if the same alert is -# already critical. -inhibit_rules: -- source_match: - severity: 'critical' - target_match: - severity: 'warning' - # Apply inhibition if the alertname is the same. - equal: ['alertname'] - - -receivers: -- name: 'team-X-mails' - email_configs: - - to: 'team-X+alerts@example.org' - -- name: 'team-X-pager' - email_configs: - - to: 'team-X+alerts-critical@example.org' - pagerduty_configs: - - service_key: - -- name: 'team-Y-mails' - email_configs: - - to: 'team-Y+alerts@example.org' - -- name: 'team-Y-pager' - pagerduty_configs: - - service_key: - -- name: 'team-DB-pager' - pagerduty_configs: - - service_key: - diff --git a/cookbooks/prometheus/templates/default/prometheus.yml.erb b/cookbooks/prometheus/templates/default/prometheus.yml.erb deleted file mode 100644 index be8ed7a..0000000 --- a/cookbooks/prometheus/templates/default/prometheus.yml.erb +++ /dev/null @@ -1,33 +0,0 @@ -# Global default settings. -global: - <% @global_config.each do |k,v| %> - <%=k%>: "<%=v%>" - <% end %> - -scrape_configs: -<% @jobs.each do |name, job| %> -- job_name: "<%= name %>" - <% if job['scrape_interval'] %> - scrape_interval: "<%= job['scrape_interval'] %>" - <% end %> - <% if job['scrape_timeout'] %> - scrape_timeout: "<%= job['scrape_timeout'] %>" - <% end %> - metrics_path: "<%= job['metrics_path'] %>" - static_configs: - - targets: <%= Array(job['target']) %> - <%if job['labels'] %> - labels: - <% job['labels'].each do |label,label_config| %> - <%=label%>: <%=label_config%> - <%end%> - <%end%> - - <% end %> - -<% if @rule_filenames %> -rule_files: -<% @rule_filenames.each do |filename| %> - - <%= filename %> -<% end %> -<% end %> diff --git a/nodes/prometheus-1.json b/nodes/prometheus-1.json index 63084e5..cde5cfd 100644 --- a/nodes/prometheus-1.json +++ b/nodes/prometheus-1.json @@ -20,6 +20,7 @@ "recipes": [ "kosmos-base", "kosmos-base::default", + "kosmos_prometheus::node_exporter", "kosmos_kvm::guest", "kosmos_prometheus::server", "kosmos_prometheus::alertmanager", diff --git a/site-cookbooks/kosmos_prometheus/attributes/default.rb b/site-cookbooks/kosmos_prometheus/attributes/default.rb index 89714ac..fe721a8 100644 --- a/site-cookbooks/kosmos_prometheus/attributes/default.rb +++ b/site-cookbooks/kosmos_prometheus/attributes/default.rb @@ -1,8 +1,19 @@ -node.default["prometheus"]["version"] = "3.13.0" -node.default["prometheus"]["checksum"] = "744d93324cc024d82089921737bd797474d7f1e5dbbfd1c6b387bad258538cb9" +node.default["kosmos_prometheus"]["version"] = "3.13.0" +node.default["kosmos_prometheus"]["checksum"] = "744d93324cc024d82089921737bd797474d7f1e5dbbfd1c6b387bad258538cb9" -node.default["prometheus"]["alertmanager"]["version"] = "0.33.0" -node.default["prometheus"]["alertmanager"]["checksum"] = "8ce11c42e8a6dfbbf93a59c0b193cb1329210b36d0c7ef3df7b745608675a1d1" +node.default["kosmos_prometheus"]["alertmanager"]["version"] = "0.33.0" +node.default["kosmos_prometheus"]["alertmanager"]["checksum"] = "8ce11c42e8a6dfbbf93a59c0b193cb1329210b36d0c7ef3df7b745608675a1d1" -node.default["prometheus"]["node_exporter"]["version"] = "1.11.1" -node.default["prometheus"]["node_exporter"]["checksum"] = "9f5ea48e5bc7b656f8a91a32e7d7deb89f70f73dabd0d974418aca15f37d6810" +node.default["kosmos_prometheus"]["node_exporter"]["version"] = "1.11.1" +node.default["kosmos_prometheus"]["node_exporter"]["checksum"] = "9f5ea48e5bc7b656f8a91a32e7d7deb89f70f73dabd0d974418aca15f37d6810" + +node.default["kosmos_prometheus"]["global"] = { + "scrape_interval" => "30s", + "evaluation_interval" => "30s", +} + +node.default["kosmos_prometheus"]["jobs"] = { + "prometheus" => { "targets" => ["localhost:9090"] } +} + +node.default["kosmos_prometheus"]["rule_files"] = [] diff --git a/site-cookbooks/kosmos_prometheus/metadata.rb b/site-cookbooks/kosmos_prometheus/metadata.rb index 2c04d3b..1860c0f 100644 --- a/site-cookbooks/kosmos_prometheus/metadata.rb +++ b/site-cookbooks/kosmos_prometheus/metadata.rb @@ -6,7 +6,7 @@ description 'Installs/Configures prometheus' version '0.1.0' chef_version '>= 16.0' -depends "prometheus" +depends "firewall" # The `issues_url` points to the location where issues for this cookbook are # tracked. A `View Issues` link will be displayed on this cookbook's page when diff --git a/site-cookbooks/kosmos_prometheus/recipes/alertmanager.rb b/site-cookbooks/kosmos_prometheus/recipes/alertmanager.rb index 4ed73ca..4cba895 100644 --- a/site-cookbooks/kosmos_prometheus/recipes/alertmanager.rb +++ b/site-cookbooks/kosmos_prometheus/recipes/alertmanager.rb @@ -5,20 +5,89 @@ include_recipe "firewall" -prometheus_alertmanager_install "alertmanager" do - version node["prometheus"]["alertmanager"]["version"] - checksum node["prometheus"]["alertmanager"]["checksum"] +version = node["kosmos_prometheus"]["alertmanager"]["version"] +checksum = node["kosmos_prometheus"]["alertmanager"]["checksum"] +tarball = "#{Chef::Config[:file_cache_path]}/alertmanager-#{version}.linux-amd64.tar.gz" +binary_url = "https://github.com/prometheus/alertmanager/releases/download/v#{version}/alertmanager-#{version}.linux-amd64.tar.gz" + +group "alertmanager" + +user "alertmanager" do + gid "alertmanager" + system true + shell "/bin/false" + home "/nonexistent" end -prometheus_alertmanager_config "alertmanager" +directory "/var/lib/alertmanager" do + owner "alertmanager" + group "alertmanager" + mode "0755" + recursive true +end -execute "restart alertmanager config" do - command "systemctl restart alertmanager.service" +directory "/etc/prometheus" do + owner "root" + group "root" + mode "0755" + recursive true +end + +package %w(tar bzip2) + +remote_file tarball do + source binary_url + checksum checksum + action :create + notifies :run, "execute[install_alertmanager]", :immediately +end + +execute "install_alertmanager" do + command "tar -xzf #{tarball} -C /usr/local/bin --strip-components=1 alertmanager-#{version}.linux-amd64/alertmanager" action :nothing - subscribes :run, "template[/opt/prometheus/alertmanager.yml]", :delayed + notifies :restart, "service[alertmanager]", :delayed end -prometheus_alertmanager_service "alertmanager" +file "/usr/local/bin/alertmanager" do + owner "root" + group "root" + mode "0755" + notifies :restart, "service[alertmanager]", :delayed +end + +template "/etc/prometheus/alertmanager.yml" do + source "alertmanager.yml.erb" + owner "root" + group "alertmanager" + mode "0644" + notifies :restart, "service[alertmanager]", :delayed +end + +systemd_unit "alertmanager.service" do + content({ + Unit: { + Description: "Prometheus Alertmanager", + After: "network.target", + }, + Service: { + Type: "simple", + User: "alertmanager", + Group: "alertmanager", + ExecStart: "/usr/local/bin/alertmanager --config.file=/etc/prometheus/alertmanager.yml --storage.path=/var/lib/alertmanager --web.listen-address=:9093", + Restart: "on-failure", + RestartSec: "5", + }, + Install: { + WantedBy: "multi-user.target", + }, + }) + triggers_reload true + action :create +end + +service "alertmanager" do + action [:enable, :start] +end firewall_rule "prometheus alertmanager" do port 9093 diff --git a/site-cookbooks/kosmos_prometheus/recipes/node_exporter.rb b/site-cookbooks/kosmos_prometheus/recipes/node_exporter.rb index 7e6a9dc..b4e4a2a 100644 --- a/site-cookbooks/kosmos_prometheus/recipes/node_exporter.rb +++ b/site-cookbooks/kosmos_prometheus/recipes/node_exporter.rb @@ -5,8 +5,8 @@ include_recipe "firewall" -version = node["prometheus"]["node_exporter"]["version"] -checksum = node["prometheus"]["node_exporter"]["checksum"] +version = node["kosmos_prometheus"]["node_exporter"]["version"] +checksum = node["kosmos_prometheus"]["node_exporter"]["checksum"] tarball = "#{Chef::Config[:file_cache_path]}/node_exporter-#{version}.linux-amd64.tar.gz" binary_url = "https://github.com/prometheus/node_exporter/releases/download/v#{version}/node_exporter-#{version}.linux-amd64.tar.gz" diff --git a/site-cookbooks/kosmos_prometheus/recipes/server.rb b/site-cookbooks/kosmos_prometheus/recipes/server.rb index 43b17aa..fea18f0 100644 --- a/site-cookbooks/kosmos_prometheus/recipes/server.rb +++ b/site-cookbooks/kosmos_prometheus/recipes/server.rb @@ -5,45 +5,101 @@ include_recipe "firewall" -prometheus_install "prometheus" do - version node["prometheus"]["version"] - checksum node["prometheus"]["checksum"] +version = node["kosmos_prometheus"]["version"] +checksum = node["kosmos_prometheus"]["checksum"] +tarball = "#{Chef::Config[:file_cache_path]}/prometheus-#{version}.linux-amd64.tar.gz" +binary_url = "https://github.com/prometheus/prometheus/releases/download/v#{version}/prometheus-#{version}.linux-amd64.tar.gz" + +group "prometheus" + +user "prometheus" do + gid "prometheus" + system true + shell "/bin/false" + home "/nonexistent" end -prometheus_config "prometheus" do - global_config( - "scrape_interval" => "30s", - "evaluation_interval" => "30s" +directory "/var/lib/prometheus" do + owner "prometheus" + group "prometheus" + mode "0755" + recursive true +end + +directory "/etc/prometheus" do + owner "root" + group "root" + mode "0755" + recursive true +end + +directory "/etc/prometheus/rules" do + owner "root" + group "root" + mode "0755" + recursive true +end + +package %w(tar bzip2) + +remote_file tarball do + source binary_url + checksum checksum + action :create + notifies :run, "execute[install_prometheus]", :immediately +end + +execute "install_prometheus" do + command "tar -xzf #{tarball} -C /usr/local/bin --strip-components=1 prometheus-#{version}.linux-amd64/prometheus" + action :nothing + notifies :restart, "service[prometheus]", :delayed +end + +file "/usr/local/bin/prometheus" do + owner "root" + group "root" + mode "0755" + notifies :restart, "service[prometheus]", :delayed +end + +template "/etc/prometheus/prometheus.yml" do + source "prometheus.yml.erb" + owner "root" + group "prometheus" + mode "0644" + variables( + global_config: node["kosmos_prometheus"]["global"], + jobs: node["kosmos_prometheus"]["jobs"], + rule_files: node["kosmos_prometheus"]["rule_files"] ) + notifies :reload, "service[prometheus]", :delayed end -prometheus_job "prometheus" do - target "localhost:9090" -end - -prometheus_job "node" do - target "localhost:9100" -end - -with_run_context :root do - execute "reload prometheus config" do - command "systemctl reload prometheus.service" - action :nothing - subscribes :run, "template[/opt/prometheus/prometheus.yml]", :delayed - end -end - -prometheus_service "prometheus" do - cli_options({ - "config.file" => "/opt/prometheus/prometheus.yml", - "log.level" => "info", - "query.max-concurrency" => 20, - "query.lookback-delta" => "5m", - "query.timeout" => "2m", - "storage.tsdb.path" => "/var/lib/prometheus", - "storage.tsdb.retention.time" => "15d", - "web.listen-address" => ":9090" +systemd_unit "prometheus.service" do + content({ + Unit: { + Description: "Prometheus", + After: "network.target", + }, + Service: { + Type: "simple", + User: "prometheus", + Group: "prometheus", + ExecStart: "/usr/local/bin/prometheus --config.file=/etc/prometheus/prometheus.yml --storage.tsdb.path=/var/lib/prometheus --storage.tsdb.retention.time=15d --web.listen-address=:9090 --web.enable-lifecycle", + ExecReload: "/bin/kill -HUP $MAINPID", + Restart: "on-failure", + RestartSec: "5", + }, + Install: { + WantedBy: "multi-user.target", + }, }) + triggers_reload true + action :create +end + +service "prometheus" do + action [:enable, :start] end firewall_rule "prometheus web" do diff --git a/site-cookbooks/kosmos_prometheus/templates/default/alertmanager.yml.erb b/site-cookbooks/kosmos_prometheus/templates/default/alertmanager.yml.erb new file mode 100644 index 0000000..b0fd0a9 --- /dev/null +++ b/site-cookbooks/kosmos_prometheus/templates/default/alertmanager.yml.erb @@ -0,0 +1,12 @@ +global: + resolve_timeout: 5m + +route: + receiver: default + group_by: ['alertname'] + group_wait: 30s + group_interval: 5m + repeat_interval: 3h + +receivers: +- name: default diff --git a/site-cookbooks/kosmos_prometheus/templates/default/prometheus.yml.erb b/site-cookbooks/kosmos_prometheus/templates/default/prometheus.yml.erb new file mode 100644 index 0000000..5a790d7 --- /dev/null +++ b/site-cookbooks/kosmos_prometheus/templates/default/prometheus.yml.erb @@ -0,0 +1,31 @@ +global: +<% @global_config.each do |k, v| %> + <%= k %>: "<%= v %>" +<% end %> + +scrape_configs: +<% @jobs.each do |name, job| %> +- job_name: "<%= name %>" +<% if job['scrape_interval'] %> + scrape_interval: "<%= job['scrape_interval'] %>" +<% end %> +<% if job['scrape_timeout'] %> + scrape_timeout: "<%= job['scrape_timeout'] %>" +<% end %> + metrics_path: "<%= job.fetch('metrics_path', '/metrics') %>" + static_configs: + - targets: <%= Array(job['targets']) %> +<% if job['labels'] %> + labels: +<% job['labels'].each do |label, label_config| %> + <%= label %>: <%= label_config %> +<% end %> +<% end %> +<% end %> + +<% if @rule_files && !@rule_files.empty? %> +rule_files: +<% @rule_files.each do |filename| %> + - <%= filename %> +<% end %> +<% end %> From 5813a459873176ff91ce726df976dc19c1461b61 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A2u=20Cao?= Date: Sat, 4 Jul 2026 15:45:23 +0200 Subject: [PATCH 10/47] Use base role instead of recipe in all runlists --- nodes/discourse-2.json | 2 +- nodes/drone-1.json | 2 +- nodes/ipfs-1.json | 2 +- nodes/ldap-3.kosmos.org.json | 2 +- nodes/mastodon-3.json | 2 +- nodes/rs-discourse-1.json | 2 +- nodes/uploads-1.json | 2 +- 7 files changed, 7 insertions(+), 7 deletions(-) diff --git a/nodes/discourse-2.json b/nodes/discourse-2.json index b47c77d..da4bb5a 100644 --- a/nodes/discourse-2.json +++ b/nodes/discourse-2.json @@ -61,7 +61,7 @@ } }, "run_list": [ - "recipe[kosmos-base]", + "role[base]", "role[kvm_guest]", "role[garage_gateway]", "role[kosmos_discourse]" diff --git a/nodes/drone-1.json b/nodes/drone-1.json index 40b57fc..befd090 100644 --- a/nodes/drone-1.json +++ b/nodes/drone-1.json @@ -55,7 +55,7 @@ } }, "run_list": [ - "recipe[kosmos-base]", + "role[base]", "role[kvm_guest]", "role[drone]" ] diff --git a/nodes/ipfs-1.json b/nodes/ipfs-1.json index 8022309..bb82e47 100644 --- a/nodes/ipfs-1.json +++ b/nodes/ipfs-1.json @@ -60,7 +60,7 @@ } }, "run_list": [ - "recipe[kosmos-base]", + "role[base]", "role[kvm_guest]", "role[ipfs_gateway]" ] diff --git a/nodes/ldap-3.kosmos.org.json b/nodes/ldap-3.kosmos.org.json index 637595e..820c284 100644 --- a/nodes/ldap-3.kosmos.org.json +++ b/nodes/ldap-3.kosmos.org.json @@ -57,7 +57,7 @@ } }, "run_list": [ - "recipe[kosmos-base]", + "role[base]", "role[kvm_guest]", "role[dirsrv_supplier]" ] diff --git a/nodes/mastodon-3.json b/nodes/mastodon-3.json index c5dba55..82aadff 100644 --- a/nodes/mastodon-3.json +++ b/nodes/mastodon-3.json @@ -83,7 +83,7 @@ } }, "run_list": [ - "recipe[kosmos-base]", + "role[base]", "role[kvm_guest]", "role[ldap_client]", "role[garage_gateway]", diff --git a/nodes/rs-discourse-1.json b/nodes/rs-discourse-1.json index ef67953..6d6ade9 100644 --- a/nodes/rs-discourse-1.json +++ b/nodes/rs-discourse-1.json @@ -55,7 +55,7 @@ } }, "run_list": [ - "recipe[kosmos-base]", + "role[base]", "role[kvm_guest]", "role[remotestorage_discourse]" ] diff --git a/nodes/uploads-1.json b/nodes/uploads-1.json index f2af120..348b67d 100644 --- a/nodes/uploads-1.json +++ b/nodes/uploads-1.json @@ -60,7 +60,7 @@ } }, "run_list": [ - "recipe[kosmos-base]", + "role[base]", "role[kvm_guest]", "recipe[kosmos-ejabberd::upload_service]" ] From ea69c7cec697401b4ceb711827ba72a94a1f4e80 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A2u=20Cao?= Date: Sat, 4 Jul 2026 15:45:50 +0200 Subject: [PATCH 11/47] Add prometheus node exporter to base role --- roles/base.rb | 1 + roles/prometheus_server.rb | 3 +-- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/roles/base.rb b/roles/base.rb index 86d3ab5..dc838b3 100644 --- a/roles/base.rb +++ b/roles/base.rb @@ -2,4 +2,5 @@ name "base" run_list %w( kosmos-base::default + kosmos_prometheus::node_exporter ) diff --git a/roles/prometheus_server.rb b/roles/prometheus_server.rb index a91fc15..dc0bfea 100644 --- a/roles/prometheus_server.rb +++ b/roles/prometheus_server.rb @@ -2,8 +2,7 @@ name "prometheus_server" default_run_list = [ "kosmos_prometheus::server", - "kosmos_prometheus::alertmanager", - "kosmos_prometheus::node_exporter" + "kosmos_prometheus::alertmanager" ] env_run_lists( From 153b1e77c5a7b57512831dc6fe3e96da3ebdf5a6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A2u=20Cao?= Date: Sat, 4 Jul 2026 15:46:41 +0200 Subject: [PATCH 12/47] Add all nodes with base role as node exporter targets --- .../kosmos_prometheus/attributes/default.rb | 2 +- site-cookbooks/kosmos_prometheus/recipes/server.rb | 11 ++++++++++- 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/site-cookbooks/kosmos_prometheus/attributes/default.rb b/site-cookbooks/kosmos_prometheus/attributes/default.rb index fe721a8..8c73879 100644 --- a/site-cookbooks/kosmos_prometheus/attributes/default.rb +++ b/site-cookbooks/kosmos_prometheus/attributes/default.rb @@ -13,7 +13,7 @@ node.default["kosmos_prometheus"]["global"] = { } node.default["kosmos_prometheus"]["jobs"] = { - "prometheus" => { "targets" => ["localhost:9090"] } + "prometheus" => { "targets" => ["localhost:9090"] }, } node.default["kosmos_prometheus"]["rule_files"] = [] diff --git a/site-cookbooks/kosmos_prometheus/recipes/server.rb b/site-cookbooks/kosmos_prometheus/recipes/server.rb index fea18f0..38b24d6 100644 --- a/site-cookbooks/kosmos_prometheus/recipes/server.rb +++ b/site-cookbooks/kosmos_prometheus/recipes/server.rb @@ -62,6 +62,15 @@ file "/usr/local/bin/prometheus" do notifies :restart, "service[prometheus]", :delayed end +node_targets = search(:node, "role:base").map { |n| n["knife_zero"]["host"] } + .compact + .sort_by { |ip| ip.split(".").map(&:to_i) } + .map { |ip| "#{ip}:9100" } + +jobs = node["kosmos_prometheus"]["jobs"].merge( + "node" => { "targets" => node_targets } +) + template "/etc/prometheus/prometheus.yml" do source "prometheus.yml.erb" owner "root" @@ -69,7 +78,7 @@ template "/etc/prometheus/prometheus.yml" do mode "0644" variables( global_config: node["kosmos_prometheus"]["global"], - jobs: node["kosmos_prometheus"]["jobs"], + jobs: jobs, rule_files: node["kosmos_prometheus"]["rule_files"] ) notifies :reload, "service[prometheus]", :delayed From 7dc4895da32e52407ee4a3f0ce2f1560dcd752f5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A2u=20Cao?= Date: Sat, 4 Jul 2026 15:51:39 +0200 Subject: [PATCH 13/47] Remove old garage nodes --- clients/garage-10.json | 4 --- clients/garage-12.json | 4 --- clients/garage-13.json | 4 --- clients/garage-9.json | 4 --- nodes/garage-10.json | 64 ----------------------------------------- nodes/garage-12.json | 65 ------------------------------------------ nodes/garage-13.json | 65 ------------------------------------------ nodes/garage-9.json | 64 ----------------------------------------- 8 files changed, 274 deletions(-) delete mode 100644 clients/garage-10.json delete mode 100644 clients/garage-12.json delete mode 100644 clients/garage-13.json delete mode 100644 clients/garage-9.json delete mode 100644 nodes/garage-10.json delete mode 100644 nodes/garage-12.json delete mode 100644 nodes/garage-13.json delete mode 100644 nodes/garage-9.json diff --git a/clients/garage-10.json b/clients/garage-10.json deleted file mode 100644 index 6ab390b..0000000 --- a/clients/garage-10.json +++ /dev/null @@ -1,4 +0,0 @@ -{ - "name": "garage-10", - "public_key": "-----BEGIN PUBLIC KEY-----\nMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAw2+3Wo+KkXVJCOX1SxT9\nSdwKXgPbCDM3EI9uwoxhMxQfRyN53dxIsBDsQUVOIe1Z8yqm4FenMQlNmeDR+QLE\nvNFf1fisinW+D9VVRm+CjcJy96i/Dyt786Z6YRrDlB860HxCbfTL2Zv5BRtbyIKg\nhz5gO+9PMEpPVR2ij9iue4K6jbM1AAL2ia/P6zDWLJqeIzUocCeHV5N0Z3jXH6qr\nf444v78x35MMJ+3tg5h95SU1/PDCpdSTct4uHEuKIosiN7p4DlYMoM5iSyvVoujr\nflRQPEpGzS9qEt3rDo/F4ltzYMx6bf1tB/0QaBKD+zwPZWTTwf61tSBo5/NkGvJc\nFQIDAQAB\n-----END PUBLIC KEY-----\n" -} \ No newline at end of file diff --git a/clients/garage-12.json b/clients/garage-12.json deleted file mode 100644 index a297b74..0000000 --- a/clients/garage-12.json +++ /dev/null @@ -1,4 +0,0 @@ -{ - "name": "garage-12", - "public_key": "-----BEGIN PUBLIC KEY-----\nMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEA9GtHHi298BjiIqpZ3WkT\nkYAPfWD60hFe/8icYcq/F/6cHLYKZQ4chek9X/hDCMq4tHEN6Oh58T5x/nuNdPrK\nIAMGyVAGk6ekWlmD4jwdEf6TGb/J3ffJTRDvwX/I8xD/DW3wtXsN+X24T59ByGTm\nrnwRmmmwHF3otRx9wnCsIgDQ0AjiUujsfNNv1FcLXD/WJLys9lEeU5aJ4XtHTwDv\ntJM8YyVEFhEnuvgdKmzn5+F5k9VGdUwForlFOBfvzbCnTZMDMmDVeiUtAUv/7xWQ\nQl2mLUGCtgWuYJYXsQacAJ6pa3h+7cQyshC6w3dwUG+1fS9lNO0Yp1GGX1AGYKpp\nPQIDAQAB\n-----END PUBLIC KEY-----\n" -} \ No newline at end of file diff --git a/clients/garage-13.json b/clients/garage-13.json deleted file mode 100644 index 13582ff..0000000 --- a/clients/garage-13.json +++ /dev/null @@ -1,4 +0,0 @@ -{ - "name": "garage-13", - "public_key": "-----BEGIN PUBLIC KEY-----\nMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAvbqWc6OwRxgHfsQuTNL4\naxeVvNen5d9srYpZSHjuBB/k9NHB+9P6vU5qF37XHkw1lVUGeYbPHzhYsx3O0/kZ\nH5f4+4SMy/P9jc6SE7AJF4qtYKgJ88koZdqCww07c6K9g+BnEGFFZui/h3hUBxWj\nTfhBHEWPyQ2bl/lr9sIJwsEz+EN0isGn/eIXkmw9J6LdLJ5Q0LLks33K28FNOU7q\nfeAN4MiBVMUtgCGyT2Voe6WrOXwQLSDXQONOp3sfSfFExsIJ1s24xdd7AMD7/9a7\n4sFDZ4swhqAWgWmW2giR7Kb8wTvGQLO/O/uUbmKz3DZXgkOKXHdHCEB/PZx1mRNM\nEwIDAQAB\n-----END PUBLIC KEY-----\n" -} \ No newline at end of file diff --git a/clients/garage-9.json b/clients/garage-9.json deleted file mode 100644 index 21336dc..0000000 --- a/clients/garage-9.json +++ /dev/null @@ -1,4 +0,0 @@ -{ - "name": "garage-9", - "public_key": "-----BEGIN PUBLIC KEY-----\nMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAnMHzKE8JBrsQkmRDeMjX\n71mBzvRzNM90cwA8xtvIkXesdTyGqohX9k/PJbCY5ySGK9PpMaYDPVAnwnUP8LFQ\n3G98aSbLxUjqU/PBzRsnWpihehr05uz9zYcNFzr4LTNvGQZsq47nN9Tk+LG3zHP7\nAZViv2mJ4ZRnukXf6KHlyoVvhuTu+tiBM8QzjTF97iP/aguNPzYHmrecy9Uf5bSA\nZrbNZT+ayxtgswC2OclhRucx7XLSuHXtpwFqsQzSAhiX1aQ3wwCyH9WJtVwpfUsE\nlxTjcQiSM9aPZ8iSC0shpBaKD1j3iF/2K2Jk+88++zMhJJPLermvaJxzsdePgvyk\nKQIDAQAB\n-----END PUBLIC KEY-----\n" -} \ No newline at end of file diff --git a/nodes/garage-10.json b/nodes/garage-10.json deleted file mode 100644 index 7228ca1..0000000 --- a/nodes/garage-10.json +++ /dev/null @@ -1,64 +0,0 @@ -{ - "name": "garage-10", - "chef_environment": "production", - "normal": { - "knife_zero": { - "host": "10.1.1.27" - } - }, - "automatic": { - "fqdn": "garage-10", - "os": "linux", - "os_version": "5.4.0-1090-kvm", - "hostname": "garage-10", - "ipaddress": "192.168.122.70", - "roles": [ - "base", - "kvm_guest", - "garage_node" - ], - "recipes": [ - "kosmos-base", - "kosmos-base::default", - "kosmos_kvm::guest", - "kosmos_garage", - "kosmos_garage::default", - "kosmos_garage::firewall_rpc", - "kosmos_garage::firewall_apis", - "apt::default", - "timezone_iii::default", - "timezone_iii::debian", - "ntp::default", - "ntp::apparmor", - "kosmos-base::systemd_emails", - "apt::unattended-upgrades", - "kosmos-base::firewall", - "kosmos-postfix::default", - "postfix::default", - "postfix::_common", - "postfix::_attributes", - "postfix::sasl_auth", - "hostname::default", - "firewall::default" - ], - "platform": "ubuntu", - "platform_version": "20.04", - "cloud": null, - "chef_packages": { - "chef": { - "version": "18.5.0", - "chef_root": "/opt/chef/embedded/lib/ruby/gems/3.1.0/gems/chef-18.5.0/lib", - "chef_effortless": null - }, - "ohai": { - "version": "18.1.11", - "ohai_root": "/opt/chef/embedded/lib/ruby/gems/3.1.0/gems/ohai-18.1.11/lib/ohai" - } - } - }, - "run_list": [ - "role[base]", - "role[kvm_guest]", - "role[garage_node]" - ] -} diff --git a/nodes/garage-12.json b/nodes/garage-12.json deleted file mode 100644 index f5be8b5..0000000 --- a/nodes/garage-12.json +++ /dev/null @@ -1,65 +0,0 @@ -{ - "name": "garage-12", - "chef_environment": "production", - "normal": { - "knife_zero": { - "host": "10.1.1.224" - } - }, - "automatic": { - "fqdn": "garage-12", - "os": "linux", - "os_version": "5.15.0-1059-kvm", - "hostname": "garage-12", - "ipaddress": "192.168.122.173", - "roles": [ - "base", - "kvm_guest", - "garage_node" - ], - "recipes": [ - "kosmos-base", - "kosmos-base::default", - "kosmos_kvm::guest", - "kosmos_garage", - "kosmos_garage::default", - "kosmos_garage::firewall_rpc", - "kosmos_garage::firewall_apis", - "apt::default", - "timezone_iii::default", - "timezone_iii::debian", - "ntp::default", - "ntp::apparmor", - "kosmos-base::journald_conf", - "kosmos-base::systemd_emails", - "apt::unattended-upgrades", - "kosmos-base::firewall", - "kosmos-postfix::default", - "postfix::default", - "postfix::_common", - "postfix::_attributes", - "postfix::sasl_auth", - "hostname::default", - "firewall::default" - ], - "platform": "ubuntu", - "platform_version": "22.04", - "cloud": null, - "chef_packages": { - "chef": { - "version": "18.7.10", - "chef_root": "/opt/chef/embedded/lib/ruby/gems/3.1.0/gems/chef-18.7.10/lib", - "chef_effortless": null - }, - "ohai": { - "version": "18.2.5", - "ohai_root": "/opt/chef/embedded/lib/ruby/gems/3.1.0/gems/ohai-18.2.5/lib/ohai" - } - } - }, - "run_list": [ - "role[base]", - "role[kvm_guest]", - "role[garage_node]" - ] -} diff --git a/nodes/garage-13.json b/nodes/garage-13.json deleted file mode 100644 index 3c0e075..0000000 --- a/nodes/garage-13.json +++ /dev/null @@ -1,65 +0,0 @@ -{ - "name": "garage-13", - "chef_environment": "production", - "normal": { - "knife_zero": { - "host": "10.1.1.179" - } - }, - "automatic": { - "fqdn": "garage-13", - "os": "linux", - "os_version": "5.15.0-1059-kvm", - "hostname": "garage-13", - "ipaddress": "192.168.122.27", - "roles": [ - "base", - "kvm_guest", - "garage_node" - ], - "recipes": [ - "kosmos-base", - "kosmos-base::default", - "kosmos_kvm::guest", - "kosmos_garage", - "kosmos_garage::default", - "kosmos_garage::firewall_rpc", - "kosmos_garage::firewall_apis", - "apt::default", - "timezone_iii::default", - "timezone_iii::debian", - "ntp::default", - "ntp::apparmor", - "kosmos-base::journald_conf", - "kosmos-base::systemd_emails", - "apt::unattended-upgrades", - "kosmos-base::firewall", - "kosmos-postfix::default", - "postfix::default", - "postfix::_common", - "postfix::_attributes", - "postfix::sasl_auth", - "hostname::default", - "firewall::default" - ], - "platform": "ubuntu", - "platform_version": "22.04", - "cloud": null, - "chef_packages": { - "chef": { - "version": "18.7.10", - "chef_root": "/opt/chef/embedded/lib/ruby/gems/3.1.0/gems/chef-18.7.10/lib", - "chef_effortless": null - }, - "ohai": { - "version": "18.2.5", - "ohai_root": "/opt/chef/embedded/lib/ruby/gems/3.1.0/gems/ohai-18.2.5/lib/ohai" - } - } - }, - "run_list": [ - "role[base]", - "role[kvm_guest]", - "role[garage_node]" - ] -} diff --git a/nodes/garage-9.json b/nodes/garage-9.json deleted file mode 100644 index b733284..0000000 --- a/nodes/garage-9.json +++ /dev/null @@ -1,64 +0,0 @@ -{ - "name": "garage-9", - "chef_environment": "production", - "normal": { - "knife_zero": { - "host": "10.1.1.223" - } - }, - "automatic": { - "fqdn": "garage-9", - "os": "linux", - "os_version": "5.4.0-1090-kvm", - "hostname": "garage-9", - "ipaddress": "192.168.122.21", - "roles": [ - "base", - "kvm_guest", - "garage_node" - ], - "recipes": [ - "kosmos-base", - "kosmos-base::default", - "kosmos_kvm::guest", - "kosmos_garage", - "kosmos_garage::default", - "kosmos_garage::firewall_rpc", - "kosmos_garage::firewall_apis", - "apt::default", - "timezone_iii::default", - "timezone_iii::debian", - "ntp::default", - "ntp::apparmor", - "kosmos-base::systemd_emails", - "apt::unattended-upgrades", - "kosmos-base::firewall", - "kosmos-postfix::default", - "postfix::default", - "postfix::_common", - "postfix::_attributes", - "postfix::sasl_auth", - "hostname::default", - "firewall::default" - ], - "platform": "ubuntu", - "platform_version": "20.04", - "cloud": null, - "chef_packages": { - "chef": { - "version": "18.5.0", - "chef_root": "/opt/chef/embedded/lib/ruby/gems/3.1.0/gems/chef-18.5.0/lib", - "chef_effortless": null - }, - "ohai": { - "version": "18.1.11", - "ohai_root": "/opt/chef/embedded/lib/ruby/gems/3.1.0/gems/ohai-18.1.11/lib/ohai" - } - } - }, - "run_list": [ - "role[base]", - "role[kvm_guest]", - "role[garage_node]" - ] -} From 67f62ebd6c210db8acb70be655e121c0d32d102b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A2u=20Cao?= Date: Sat, 4 Jul 2026 16:14:36 +0200 Subject: [PATCH 14/47] Scrape garage metrics --- site-cookbooks/kosmos_prometheus/recipes/server.rb | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/site-cookbooks/kosmos_prometheus/recipes/server.rb b/site-cookbooks/kosmos_prometheus/recipes/server.rb index 38b24d6..a2049fc 100644 --- a/site-cookbooks/kosmos_prometheus/recipes/server.rb +++ b/site-cookbooks/kosmos_prometheus/recipes/server.rb @@ -67,8 +67,14 @@ node_targets = search(:node, "role:base").map { |n| n["knife_zero"]["host"] } .sort_by { |ip| ip.split(".").map(&:to_i) } .map { |ip| "#{ip}:9100" } +garage_targets = search(:node, "role:garage_node").map { |n| n["knife_zero"]["host"] } + .compact + .sort_by { |ip| ip.split(".").map(&:to_i) } + .map { |ip| "#{ip}:3903" } + jobs = node["kosmos_prometheus"]["jobs"].merge( - "node" => { "targets" => node_targets } + "node" => { "targets" => node_targets }, + "garage" => { "targets" => garage_targets } ) template "/etc/prometheus/prometheus.yml" do From f2ebda4a1a1c4a6dee7c3048a93c3ea3cf8ba9fc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A2u=20Cao?= Date: Sat, 4 Jul 2026 16:30:15 +0200 Subject: [PATCH 15/47] Update node configs --- nodes/garage-11.json | 2 ++ nodes/garage-14.json | 1 + nodes/garage-15.json | 1 + nodes/garage-16.json | 1 + nodes/garage-7.json | 2 ++ nodes/garage-8.json | 2 ++ 6 files changed, 9 insertions(+) diff --git a/nodes/garage-11.json b/nodes/garage-11.json index 4f6c161..52109d3 100644 --- a/nodes/garage-11.json +++ b/nodes/garage-11.json @@ -20,6 +20,7 @@ "recipes": [ "kosmos-base", "kosmos-base::default", + "kosmos_prometheus::node_exporter", "kosmos_kvm::guest", "kosmos_garage", "kosmos_garage::default", @@ -30,6 +31,7 @@ "timezone_iii::debian", "ntp::default", "ntp::apparmor", + "kosmos-base::journald_conf", "kosmos-base::systemd_emails", "apt::unattended-upgrades", "kosmos-base::firewall", diff --git a/nodes/garage-14.json b/nodes/garage-14.json index 2affca0..ad5e5e7 100644 --- a/nodes/garage-14.json +++ b/nodes/garage-14.json @@ -20,6 +20,7 @@ "recipes": [ "kosmos-base", "kosmos-base::default", + "kosmos_prometheus::node_exporter", "kosmos_kvm::guest", "kosmos_garage", "kosmos_garage::default", diff --git a/nodes/garage-15.json b/nodes/garage-15.json index eb00ff8..27b62c1 100644 --- a/nodes/garage-15.json +++ b/nodes/garage-15.json @@ -20,6 +20,7 @@ "recipes": [ "kosmos-base", "kosmos-base::default", + "kosmos_prometheus::node_exporter", "kosmos_kvm::guest", "kosmos_garage", "kosmos_garage::default", diff --git a/nodes/garage-16.json b/nodes/garage-16.json index b4da55d..150584b 100644 --- a/nodes/garage-16.json +++ b/nodes/garage-16.json @@ -20,6 +20,7 @@ "recipes": [ "kosmos-base", "kosmos-base::default", + "kosmos_prometheus::node_exporter", "kosmos_kvm::guest", "kosmos_garage", "kosmos_garage::default", diff --git a/nodes/garage-7.json b/nodes/garage-7.json index 5d87c28..7c80eaf 100644 --- a/nodes/garage-7.json +++ b/nodes/garage-7.json @@ -20,6 +20,7 @@ "recipes": [ "kosmos-base", "kosmos-base::default", + "kosmos_prometheus::node_exporter", "kosmos_kvm::guest", "kosmos_garage", "kosmos_garage::default", @@ -30,6 +31,7 @@ "timezone_iii::debian", "ntp::default", "ntp::apparmor", + "kosmos-base::journald_conf", "kosmos-base::systemd_emails", "apt::unattended-upgrades", "kosmos-base::firewall", diff --git a/nodes/garage-8.json b/nodes/garage-8.json index 3800a43..7291134 100644 --- a/nodes/garage-8.json +++ b/nodes/garage-8.json @@ -20,6 +20,7 @@ "recipes": [ "kosmos-base", "kosmos-base::default", + "kosmos_prometheus::node_exporter", "kosmos_kvm::guest", "kosmos_garage", "kosmos_garage::default", @@ -30,6 +31,7 @@ "timezone_iii::debian", "ntp::default", "ntp::apparmor", + "kosmos-base::journald_conf", "kosmos-base::systemd_emails", "apt::unattended-upgrades", "kosmos-base::firewall", From e29d90d3d083617c856ee147216308ceffb61dca Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A2u=20Cao?= Date: Sun, 5 Jul 2026 12:40:50 +0200 Subject: [PATCH 16/47] Use full domain as fqdn for leo --- nodes/{leo.json => leo.kosmos.org.json} | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) rename nodes/{leo.json => leo.kosmos.org.json} (84%) diff --git a/nodes/leo.json b/nodes/leo.kosmos.org.json similarity index 84% rename from nodes/leo.json rename to nodes/leo.kosmos.org.json index ed647d2..dfef648 100644 --- a/nodes/leo.json +++ b/nodes/leo.kosmos.org.json @@ -1,12 +1,13 @@ { - "name": "leo", + "name": "leo.kosmos.org", + "chef_environment": "production", "normal": { "knife_zero": { - "host": "leo.kosmos.org" + "host": "10.1.1.204" } }, "automatic": { - "fqdn": "leo", + "fqdn": "leo.kosmos.org", "os": "linux", "os_version": "5.15.0-173-generic", "hostname": "leo", @@ -17,6 +18,7 @@ "recipes": [ "kosmos-base", "kosmos-base::default", + "kosmos_prometheus::node_exporter", "kosmos_kvm::host", "apt::default", "timezone_iii::default", @@ -32,7 +34,8 @@ "postfix::_common", "postfix::_attributes", "postfix::sasl_auth", - "hostname::default" + "hostname::default", + "firewall::default" ], "platform": "ubuntu", "platform_version": "22.04", From 821b56e6ab2f17a39ac7b47a7d7effcd8ab00886 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A2u=20Cao?= Date: Sun, 5 Jul 2026 12:41:31 +0200 Subject: [PATCH 17/47] Set node name as prometheus instance name And DRY up the recipe so it's easy and expressive to add more metrics/targets --- .../kosmos_prometheus/attributes/default.rb | 2 +- .../kosmos_prometheus/recipes/server.rb | 32 ++++++++++++------- .../templates/default/prometheus.yml.erb | 10 +++--- 3 files changed, 27 insertions(+), 17 deletions(-) diff --git a/site-cookbooks/kosmos_prometheus/attributes/default.rb b/site-cookbooks/kosmos_prometheus/attributes/default.rb index 8c73879..a90a7c9 100644 --- a/site-cookbooks/kosmos_prometheus/attributes/default.rb +++ b/site-cookbooks/kosmos_prometheus/attributes/default.rb @@ -13,7 +13,7 @@ node.default["kosmos_prometheus"]["global"] = { } node.default["kosmos_prometheus"]["jobs"] = { - "prometheus" => { "targets" => ["localhost:9090"] }, + "prometheus" => { "targets" => [{ "target" => "localhost:9090", "instance" => "localhost" }] }, } node.default["kosmos_prometheus"]["rule_files"] = [] diff --git a/site-cookbooks/kosmos_prometheus/recipes/server.rb b/site-cookbooks/kosmos_prometheus/recipes/server.rb index a2049fc..3e214d3 100644 --- a/site-cookbooks/kosmos_prometheus/recipes/server.rb +++ b/site-cookbooks/kosmos_prometheus/recipes/server.rb @@ -62,19 +62,27 @@ file "/usr/local/bin/prometheus" do notifies :restart, "service[prometheus]", :delayed end -node_targets = search(:node, "role:base").map { |n| n["knife_zero"]["host"] } - .compact - .sort_by { |ip| ip.split(".").map(&:to_i) } - .map { |ip| "#{ip}:9100" } - -garage_targets = search(:node, "role:garage_node").map { |n| n["knife_zero"]["host"] } - .compact - .sort_by { |ip| ip.split(".").map(&:to_i) } - .map { |ip| "#{ip}:3903" } - jobs = node["kosmos_prometheus"]["jobs"].merge( - "node" => { "targets" => node_targets }, - "garage" => { "targets" => garage_targets } +{ + # node exporter + "node" => { + "query" => "role:base", + "port" => 9100 + }, + # garage metrics + "garage" => { + "query" => "role:garage_node", + "port" => 3903 + }, +}.transform_values do |config| + { + "targets" => search(:node, config["query"]).map do |n| + target = { "target" => "#{n['knife_zero']['host']}:#{config['port']}", "instance" => n.name } + target["env"] = n.chef_environment if n.chef_environment + target + end.compact.sort_by { |t| t["instance"] }, + } + end ) template "/etc/prometheus/prometheus.yml" do diff --git a/site-cookbooks/kosmos_prometheus/templates/default/prometheus.yml.erb b/site-cookbooks/kosmos_prometheus/templates/default/prometheus.yml.erb index 5a790d7..3a79fe9 100644 --- a/site-cookbooks/kosmos_prometheus/templates/default/prometheus.yml.erb +++ b/site-cookbooks/kosmos_prometheus/templates/default/prometheus.yml.erb @@ -14,11 +14,13 @@ scrape_configs: <% end %> metrics_path: "<%= job.fetch('metrics_path', '/metrics') %>" static_configs: - - targets: <%= Array(job['targets']) %> -<% if job['labels'] %> +<% job['targets'].each do |t| %> + - targets: + - <%= t['target'] %> labels: -<% job['labels'].each do |label, label_config| %> - <%= label %>: <%= label_config %> + instance: <%= t['instance'] %> +<% if t['env'] %> + env: <%= t['env'] %> <% end %> <% end %> <% end %> From 171db0fb7c223eae2ffc92b4616d6d4766758ce7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A2u=20Cao?= Date: Sun, 5 Jul 2026 13:18:37 +0200 Subject: [PATCH 18/47] Add altermanager section to prometheus config --- .../kosmos_prometheus/attributes/default.rb | 8 ++++++++ site-cookbooks/kosmos_prometheus/recipes/server.rb | 1 + .../templates/default/prometheus.yml.erb | 14 ++++++++++++++ 3 files changed, 23 insertions(+) diff --git a/site-cookbooks/kosmos_prometheus/attributes/default.rb b/site-cookbooks/kosmos_prometheus/attributes/default.rb index a90a7c9..3189a86 100644 --- a/site-cookbooks/kosmos_prometheus/attributes/default.rb +++ b/site-cookbooks/kosmos_prometheus/attributes/default.rb @@ -12,6 +12,14 @@ node.default["kosmos_prometheus"]["global"] = { "evaluation_interval" => "30s", } +node.default["kosmos_prometheus"]["alerting"] = { + "alertmanagers" => [ + "static_configs" => [ + { "targets" => ["127.0.0.1:9093"] } + ] + ] +} + node.default["kosmos_prometheus"]["jobs"] = { "prometheus" => { "targets" => [{ "target" => "localhost:9090", "instance" => "localhost" }] }, } diff --git a/site-cookbooks/kosmos_prometheus/recipes/server.rb b/site-cookbooks/kosmos_prometheus/recipes/server.rb index 3e214d3..a7b2bb1 100644 --- a/site-cookbooks/kosmos_prometheus/recipes/server.rb +++ b/site-cookbooks/kosmos_prometheus/recipes/server.rb @@ -92,6 +92,7 @@ template "/etc/prometheus/prometheus.yml" do mode "0644" variables( global_config: node["kosmos_prometheus"]["global"], + alerting: node["kosmos_prometheus"]["alerting"], jobs: jobs, rule_files: node["kosmos_prometheus"]["rule_files"] ) diff --git a/site-cookbooks/kosmos_prometheus/templates/default/prometheus.yml.erb b/site-cookbooks/kosmos_prometheus/templates/default/prometheus.yml.erb index 3a79fe9..9cc3ea4 100644 --- a/site-cookbooks/kosmos_prometheus/templates/default/prometheus.yml.erb +++ b/site-cookbooks/kosmos_prometheus/templates/default/prometheus.yml.erb @@ -3,6 +3,20 @@ global: <%= k %>: "<%= v %>" <% end %> +<% if @alerting %> +alerting: + alertmanagers: +<% @alerting['alertmanagers'].each do |am| %> + - static_configs: +<% am['static_configs'].each do |sc| %> + - targets: +<% sc['targets'].each do |target| %> + - <%= target %> +<% end %> +<% end %> +<% end %> +<% end %> + scrape_configs: <% @jobs.each do |name, job| %> - job_name: "<%= name %>" From 639800a8384b9fb4dfa67221b9f8a7ff378d750b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A2u=20Cao?= Date: Sun, 5 Jul 2026 17:19:16 +0200 Subject: [PATCH 19/47] Remove decommissioned node --- clients/rsk-testnet-5.json | 4 --- nodes/rsk-testnet-5.json | 61 -------------------------------------- 2 files changed, 65 deletions(-) delete mode 100644 clients/rsk-testnet-5.json delete mode 100644 nodes/rsk-testnet-5.json diff --git a/clients/rsk-testnet-5.json b/clients/rsk-testnet-5.json deleted file mode 100644 index 827f2f7..0000000 --- a/clients/rsk-testnet-5.json +++ /dev/null @@ -1,4 +0,0 @@ -{ - "name": "rsk-testnet-5", - "public_key": "-----BEGIN PUBLIC KEY-----\nMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAx/UHlgcSeh9Do7CTCKXC\n/4/aO2OvT+ijDVmrMYCNtE4sMeuFqKPnV1zxJZmRm4VNhkSQDkdWYD+6XvuFYW60\nyjB/N6D5lLlyjG4HD6fTkfh0K6f7t5mOYV7o4T59OoA3cBZuSROjtWmJ8jEFJ+k9\nII2kcyhPQcFN01ckzvZKRSPbVRccMoc+AKTjB3ZUfs/ERtlVoDrK4jEHluXOxUJO\nBKCcLonjJuLlpRLh7QfKrKFcR4idn5Ir43R6aSUesI/ipKwKsXnR3Bu7vXp74VF3\nMJ3EkdSBG+qJzy51fbRfQiUPAr/vSoVQZwW7FkIhIqqLkMaYCymn7qKfTGujoNU7\nlwIDAQAB\n-----END PUBLIC KEY-----\n" -} \ No newline at end of file diff --git a/nodes/rsk-testnet-5.json b/nodes/rsk-testnet-5.json deleted file mode 100644 index 9df65b3..0000000 --- a/nodes/rsk-testnet-5.json +++ /dev/null @@ -1,61 +0,0 @@ -{ - "name": "rsk-testnet-5", - "normal": { - "knife_zero": { - "host": "10.1.1.194" - } - }, - "automatic": { - "fqdn": "rsk-testnet-5", - "os": "linux", - "os_version": "5.4.0-1103-kvm", - "hostname": "rsk-testnet-5", - "ipaddress": "192.168.122.171", - "roles": [ - "base", - "kvm_guest", - "rskj_testnet" - ], - "recipes": [ - "kosmos-base", - "kosmos-base::default", - "kosmos_kvm::guest", - "kosmos_rsk::rskj", - "apt::default", - "timezone_iii::default", - "timezone_iii::debian", - "ntp::default", - "ntp::apparmor", - "kosmos-base::systemd_emails", - "apt::unattended-upgrades", - "kosmos-base::firewall", - "kosmos-postfix::default", - "postfix::default", - "postfix::_common", - "postfix::_attributes", - "postfix::sasl_auth", - "hostname::default", - "kosmos_rsk::firewall", - "firewall::default" - ], - "platform": "ubuntu", - "platform_version": "20.04", - "cloud": null, - "chef_packages": { - "chef": { - "version": "18.3.0", - "chef_root": "/opt/chef/embedded/lib/ruby/gems/3.1.0/gems/chef-18.3.0/lib", - "chef_effortless": null - }, - "ohai": { - "version": "18.1.4", - "ohai_root": "/opt/chef/embedded/lib/ruby/gems/3.1.0/gems/ohai-18.1.4/lib/ohai" - } - } - }, - "run_list": [ - "role[base]", - "role[kvm_guest]", - "role[rskj_testnet]" - ] -} From 6c3da8a711bd67b7e138dc4b5c3769d64a1e6927 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A2u=20Cao?= Date: Sun, 5 Jul 2026 13:41:28 +0200 Subject: [PATCH 20/47] Add util script for sanitizing nodes Deletes the "override" and "default" properties, which may contain sensitive data --- scripts/util/sanitize_node.sh | 69 +++++++++++++++++++++++++++++++++++ 1 file changed, 69 insertions(+) create mode 100755 scripts/util/sanitize_node.sh diff --git a/scripts/util/sanitize_node.sh b/scripts/util/sanitize_node.sh new file mode 100755 index 0000000..99d4ec3 --- /dev/null +++ b/scripts/util/sanitize_node.sh @@ -0,0 +1,69 @@ +#!/usr/bin/env bash +# Strip the top-level "override" and "default" attributes from Chef node JSON files. +# +# Usage: +# ./scripts/util/strip_node_attrs.sh [ ...] +# ./scripts/util/strip_node_attrs.sh --all +# +# A may be a bare node name (e.g. "prometheus-1"), a path relative to the +# repository root (e.g. "nodes/prometheus-1.json"), or an absolute path. +# +# Exit codes: +# 0 - all target files processed successfully +# 1 - one or more targets could not be processed (missing args, missing jq, +# file not found, invalid JSON, or write failure) + +set -uo pipefail + +command -v jq >/dev/null 2>&1 || exit 1 + +root="$(git rev-parse --show-toplevel 2>/dev/null)" || root="$(pwd)" + +targets=() +if [ "$#" -eq 0 ]; then + exit 1 +fi + +for arg in "$@"; do + if [ "$arg" = "--all" ]; then + shopt -s nullglob + for f in "$root"/nodes/*.json; do + targets+=("$f") + done + shopt -u nullglob + continue + fi + + if [[ "$arg" == */* ]]; then + if [[ "$arg" = /* ]]; then + targets+=("$arg") + else + targets+=("$root/$arg") + fi + else + targets+=("$root/nodes/$arg.json") + fi +done + +if [ "${#targets[@]}" -eq 0 ]; then + exit 1 +fi + +rc=0 +for file in "${targets[@]}"; do + [ -f "$file" ] || { rc=1; continue; } + + jq -e . "$file" >/dev/null 2>&1 || { rc=1; continue; } + + tmp="$(mktemp "${file}.XXXXXX")" || { rc=1; continue; } + if jq --indent 2 'del(.override, .default)' "$file" > "$tmp" 2>/dev/null; then + truncate -s -1 "$tmp" 2>/dev/null || true + mv "$tmp" "$file" || { rm -f "$tmp"; rc=1; continue; } + else + rm -f "$tmp" + rc=1 + continue + fi +done + +exit "$rc" From 8c71261fc9f823874959ae9b8bae067a9e811600 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A2u=20Cao?= Date: Sun, 5 Jul 2026 14:00:10 +0200 Subject: [PATCH 21/47] Add pre-commit hook for sanitizing node configs --- .githooks/pre-commit | 40 ++++++++++++++++++++++++++++++++++++++++ README.md | 4 ++++ 2 files changed, 44 insertions(+) create mode 100755 .githooks/pre-commit diff --git a/.githooks/pre-commit b/.githooks/pre-commit new file mode 100755 index 0000000..e97f9cb --- /dev/null +++ b/.githooks/pre-commit @@ -0,0 +1,40 @@ +#!/usr/bin/env bash +set -euo pipefail + +SCRIPT="$(git rev-parse --show-toplevel)/scripts/util/sanitize_node.sh" + +FILES=$(git diff --cached --name-only --diff-filter=ACM -- 'nodes/*.json' || true) + +[[ -z "$FILES" ]] && exit 0 + +DIRTY=0 + +while IFS= read -r file; do + staged_tmp=$(mktemp) + sanitized_tmp=$(mktemp) + + # 1. get staged version + git show ":$file" > "$staged_tmp" + + # 2. sanitize IN PLACE (on temp copy) + cp "$staged_tmp" "$sanitized_tmp" + "$SCRIPT" "$sanitized_tmp" + + # 3. if sanitizer changed file, update working tree + if ! diff -q "$staged_tmp" "$sanitized_tmp" >/dev/null; then + cp "$sanitized_tmp" "$file" + echo "Sanitized: $file" + DIRTY=1 + fi + + rm -f "$staged_tmp" "$sanitized_tmp" + +done <<< "$FILES" + +if [[ "$DIRTY" -ne 0 ]]; then + echo "" + echo "Commit paused, sanitized files must be re-staged." + exit 1 +fi + +exit 0 diff --git a/README.md b/README.md index c15ca12..ef6c609 100644 --- a/README.md +++ b/README.md @@ -28,6 +28,10 @@ Clone this repository, `cd` into it, and run: bundle install +### Enable Git hooks + + git config core.hooksPath .githooks + ## Common tasks ### Bootstrap a new host server From 959d506cfbd0b496287ffe698142b1e58d7ecc55 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A2u=20Cao?= Date: Sun, 5 Jul 2026 14:28:04 +0200 Subject: [PATCH 22/47] Remove newline from all node files Was actually added by manually editing the files, but Chef writes them without newline, so our sanitization script also does it that way --- nodes/ejabberd-4.json | 2 +- nodes/ejabberd-8.json | 2 +- nodes/garage-11.json | 2 +- nodes/garage-14.json | 2 +- nodes/garage-15.json | 2 +- nodes/garage-16.json | 2 +- nodes/garage-7.json | 2 +- nodes/garage-8.json | 2 +- nodes/gitea-2.json | 2 +- nodes/her.json | 2 +- nodes/ldap-3.kosmos.org.json | 2 +- nodes/ldap-4.kosmos.org.json | 2 +- nodes/leo.kosmos.org.json | 2 +- nodes/mail.kosmos.org.json | 2 +- nodes/postgres-11.json | 2 +- nodes/postgres-12.json | 2 +- nodes/prometheus-1.json | 2 +- nodes/rs-discourse-1.json | 2 +- nodes/rsk-mainnet-3.json | 2 +- nodes/rsk-testnet-4.json | 2 +- nodes/rsk-testnet-6.json | 2 +- nodes/strfry-1.json | 2 +- 22 files changed, 22 insertions(+), 22 deletions(-) diff --git a/nodes/ejabberd-4.json b/nodes/ejabberd-4.json index 8f88713..3a148c8 100644 --- a/nodes/ejabberd-4.json +++ b/nodes/ejabberd-4.json @@ -72,4 +72,4 @@ "role[ldap_client]", "role[ejabberd]" ] -} +} \ No newline at end of file diff --git a/nodes/ejabberd-8.json b/nodes/ejabberd-8.json index 4aab853..8a41ef2 100644 --- a/nodes/ejabberd-8.json +++ b/nodes/ejabberd-8.json @@ -72,4 +72,4 @@ "role[ldap_client]", "role[ejabberd]" ] -} +} \ No newline at end of file diff --git a/nodes/garage-11.json b/nodes/garage-11.json index 52109d3..1177218 100644 --- a/nodes/garage-11.json +++ b/nodes/garage-11.json @@ -63,4 +63,4 @@ "role[kvm_guest]", "role[garage_node]" ] -} +} \ No newline at end of file diff --git a/nodes/garage-14.json b/nodes/garage-14.json index ad5e5e7..2ca3820 100644 --- a/nodes/garage-14.json +++ b/nodes/garage-14.json @@ -63,4 +63,4 @@ "role[kvm_guest]", "role[garage_node]" ] -} +} \ No newline at end of file diff --git a/nodes/garage-15.json b/nodes/garage-15.json index 27b62c1..166d7d2 100644 --- a/nodes/garage-15.json +++ b/nodes/garage-15.json @@ -63,4 +63,4 @@ "role[kvm_guest]", "role[garage_node]" ] -} +} \ No newline at end of file diff --git a/nodes/garage-16.json b/nodes/garage-16.json index 150584b..5c26e0b 100644 --- a/nodes/garage-16.json +++ b/nodes/garage-16.json @@ -61,4 +61,4 @@ "role[kvm_guest]", "role[garage_node]" ] -} +} \ No newline at end of file diff --git a/nodes/garage-7.json b/nodes/garage-7.json index 7c80eaf..ab60013 100644 --- a/nodes/garage-7.json +++ b/nodes/garage-7.json @@ -63,4 +63,4 @@ "role[kvm_guest]", "role[garage_node]" ] -} +} \ No newline at end of file diff --git a/nodes/garage-8.json b/nodes/garage-8.json index 7291134..3d1527e 100644 --- a/nodes/garage-8.json +++ b/nodes/garage-8.json @@ -63,4 +63,4 @@ "role[kvm_guest]", "role[garage_node]" ] -} +} \ No newline at end of file diff --git a/nodes/gitea-2.json b/nodes/gitea-2.json index cddfdd9..931372a 100644 --- a/nodes/gitea-2.json +++ b/nodes/gitea-2.json @@ -76,4 +76,4 @@ "role[gitea]", "role[gitea_actions_runner]" ] -} +} \ No newline at end of file diff --git a/nodes/her.json b/nodes/her.json index 43402f3..ba758d0 100644 --- a/nodes/her.json +++ b/nodes/her.json @@ -55,4 +55,4 @@ "role[base]", "role[kvm_host]" ] -} +} \ No newline at end of file diff --git a/nodes/ldap-3.kosmos.org.json b/nodes/ldap-3.kosmos.org.json index 820c284..9062959 100644 --- a/nodes/ldap-3.kosmos.org.json +++ b/nodes/ldap-3.kosmos.org.json @@ -61,4 +61,4 @@ "role[kvm_guest]", "role[dirsrv_supplier]" ] -} +} \ No newline at end of file diff --git a/nodes/ldap-4.kosmos.org.json b/nodes/ldap-4.kosmos.org.json index f66c37e..d5ff769 100644 --- a/nodes/ldap-4.kosmos.org.json +++ b/nodes/ldap-4.kosmos.org.json @@ -55,4 +55,4 @@ "role[kvm_guest]", "role[dirsrv_supplier]" ] -} +} \ No newline at end of file diff --git a/nodes/leo.kosmos.org.json b/nodes/leo.kosmos.org.json index dfef648..144eeb0 100644 --- a/nodes/leo.kosmos.org.json +++ b/nodes/leo.kosmos.org.json @@ -56,4 +56,4 @@ "role[base]", "recipe[kosmos_kvm::host]" ] -} +} \ No newline at end of file diff --git a/nodes/mail.kosmos.org.json b/nodes/mail.kosmos.org.json index 4f116ed..f3b57b0 100644 --- a/nodes/mail.kosmos.org.json +++ b/nodes/mail.kosmos.org.json @@ -68,4 +68,4 @@ "role[kvm_guest]", "role[email_server]" ] -} +} \ No newline at end of file diff --git a/nodes/postgres-11.json b/nodes/postgres-11.json index a01909f..24cec64 100644 --- a/nodes/postgres-11.json +++ b/nodes/postgres-11.json @@ -66,4 +66,4 @@ "role[kvm_guest]", "role[postgresql_primary]" ] -} +} \ No newline at end of file diff --git a/nodes/postgres-12.json b/nodes/postgres-12.json index d2359b2..7fac349 100644 --- a/nodes/postgres-12.json +++ b/nodes/postgres-12.json @@ -61,4 +61,4 @@ "role[kvm_guest]", "role[postgresql_replica]" ] -} +} \ No newline at end of file diff --git a/nodes/prometheus-1.json b/nodes/prometheus-1.json index cde5cfd..f76c369 100644 --- a/nodes/prometheus-1.json +++ b/nodes/prometheus-1.json @@ -59,4 +59,4 @@ "role[kvm_guest]", "role[prometheus_server]" ] -} +} \ No newline at end of file diff --git a/nodes/rs-discourse-1.json b/nodes/rs-discourse-1.json index 6d6ade9..6c1a826 100644 --- a/nodes/rs-discourse-1.json +++ b/nodes/rs-discourse-1.json @@ -59,4 +59,4 @@ "role[kvm_guest]", "role[remotestorage_discourse]" ] -} +} \ No newline at end of file diff --git a/nodes/rsk-mainnet-3.json b/nodes/rsk-mainnet-3.json index e6ad2d6..985f06f 100644 --- a/nodes/rsk-mainnet-3.json +++ b/nodes/rsk-mainnet-3.json @@ -58,4 +58,4 @@ "role[kvm_guest]", "role[rskj_mainnet]" ] -} +} \ No newline at end of file diff --git a/nodes/rsk-testnet-4.json b/nodes/rsk-testnet-4.json index 21ab8f2..a420cf2 100644 --- a/nodes/rsk-testnet-4.json +++ b/nodes/rsk-testnet-4.json @@ -57,4 +57,4 @@ "role[base]", "role[kvm_guest]" ] -} +} \ No newline at end of file diff --git a/nodes/rsk-testnet-6.json b/nodes/rsk-testnet-6.json index a7c4ee0..6246c25 100644 --- a/nodes/rsk-testnet-6.json +++ b/nodes/rsk-testnet-6.json @@ -57,4 +57,4 @@ "role[kvm_guest]", "role[rskj_testnet]" ] -} +} \ No newline at end of file diff --git a/nodes/strfry-1.json b/nodes/strfry-1.json index 1e9de13..b151730 100644 --- a/nodes/strfry-1.json +++ b/nodes/strfry-1.json @@ -70,4 +70,4 @@ "role[strfry]", "role[blossom]" ] -} +} \ No newline at end of file From 53cf95c4e7df71838a18821ee206349da5b84e12 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A2u=20Cao?= Date: Sun, 5 Jul 2026 15:45:27 +0200 Subject: [PATCH 23/47] Remove decommissioned nodes --- clients/uploads-1.json | 4 --- nodes/uploads-1.json | 67 ------------------------------------------ 2 files changed, 71 deletions(-) delete mode 100644 clients/uploads-1.json delete mode 100644 nodes/uploads-1.json diff --git a/clients/uploads-1.json b/clients/uploads-1.json deleted file mode 100644 index fec1329..0000000 --- a/clients/uploads-1.json +++ /dev/null @@ -1,4 +0,0 @@ -{ - "name": "uploads-1", - "public_key": "-----BEGIN PUBLIC KEY-----\nMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAwJwWaz8TeGv3SFlKzLMx\nqN8GTL/c0N9ppBvv8xNSS/yF9Y40SbL418uxYzm9hIhOXgIygIgLT2EKIXX32t+R\neOJCdYycQFM3At2fhMkjhuUW0gmDRcYBcBJLC5hLh2EZ+A8V7k4qgrBpPLOjEv48\nhQY0vuAw2DGndWr4QLh5NLUmQiOrfuzcZSSNCBOTIgUZgNmRd9QcCHDq4WDH3poa\nosJo4a9JGEGUL1irOivvEdyJPwEd2f++nYAdWwj8pjCYgpRshQlLhxOlylMx7MxB\nQt2bgJC9sahfbfJCOqdlCU3DMJL0bRUiuxK77WeSsxWBJmrsiF3+Ljs2Ix+s7fnS\nywIDAQAB\n-----END PUBLIC KEY-----\n" -} \ No newline at end of file diff --git a/nodes/uploads-1.json b/nodes/uploads-1.json deleted file mode 100644 index 348b67d..0000000 --- a/nodes/uploads-1.json +++ /dev/null @@ -1,67 +0,0 @@ -{ - "name": "uploads-1", - "normal": { - "knife_zero": { - "host": "10.1.1.98" - } - }, - "automatic": { - "fqdn": "uploads-1", - "os": "linux", - "os_version": "5.4.0-128-generic", - "hostname": "uploads-1", - "ipaddress": "192.168.122.230", - "roles": [ - "kvm_guest" - ], - "recipes": [ - "kosmos-base", - "kosmos-base::default", - "kosmos_kvm::guest", - "kosmos-ejabberd::upload_service", - "apt::default", - "timezone_iii::default", - "timezone_iii::debian", - "ntp::default", - "ntp::apparmor", - "kosmos-base::systemd_emails", - "apt::unattended-upgrades", - "kosmos-base::firewall", - "kosmos-postfix::default", - "postfix::default", - "postfix::_common", - "postfix::_attributes", - "postfix::sasl_auth", - "hostname::default", - "kosmos-nginx::with_perl", - "nginx::default", - "nginx::package", - "nginx::ohai_plugin", - "nginx::repo", - "nginx::commons", - "nginx::commons_dir", - "nginx::commons_script", - "nginx::commons_conf", - "kosmos-nginx::firewall", - "kosmos-base::letsencrypt" - ], - "platform": "ubuntu", - "platform_version": "20.04", - "cloud": null, - "chef_packages": { - "chef": { - "version": "15.14.0", - "chef_root": "/opt/chef/embedded/lib/ruby/gems/2.6.0/gems/chef-15.14.0/lib" - }, - "ohai": { - "version": "15.12.0", - "ohai_root": "/opt/chef/embedded/lib/ruby/gems/2.6.0/gems/ohai-15.12.0/lib/ohai" - } - } - }, - "run_list": [ - "role[base]", - "role[kvm_guest]", - "recipe[kosmos-ejabberd::upload_service]" - ] -} \ No newline at end of file From 0e9bcca72da1d1100c6f7df9d3588593f78b86b5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A2u=20Cao?= Date: Sun, 5 Jul 2026 16:19:36 +0200 Subject: [PATCH 24/47] Add vm_host label to prometheus targets --- site-cookbooks/kosmos_prometheus/recipes/server.rb | 1 + .../kosmos_prometheus/templates/default/prometheus.yml.erb | 3 +++ 2 files changed, 4 insertions(+) diff --git a/site-cookbooks/kosmos_prometheus/recipes/server.rb b/site-cookbooks/kosmos_prometheus/recipes/server.rb index a7b2bb1..345bd94 100644 --- a/site-cookbooks/kosmos_prometheus/recipes/server.rb +++ b/site-cookbooks/kosmos_prometheus/recipes/server.rb @@ -79,6 +79,7 @@ jobs = node["kosmos_prometheus"]["jobs"].merge( "targets" => search(:node, config["query"]).map do |n| target = { "target" => "#{n['knife_zero']['host']}:#{config['port']}", "instance" => n.name } target["env"] = n.chef_environment if n.chef_environment + target["vm_host"] = n["vm_host"] if n["vm_host"] target end.compact.sort_by { |t| t["instance"] }, } diff --git a/site-cookbooks/kosmos_prometheus/templates/default/prometheus.yml.erb b/site-cookbooks/kosmos_prometheus/templates/default/prometheus.yml.erb index 9cc3ea4..a7bd9f3 100644 --- a/site-cookbooks/kosmos_prometheus/templates/default/prometheus.yml.erb +++ b/site-cookbooks/kosmos_prometheus/templates/default/prometheus.yml.erb @@ -36,6 +36,9 @@ scrape_configs: <% if t['env'] %> env: <%= t['env'] %> <% end %> +<% if t['vm_host'] %> + vm_host: <%= t['vm_host'] %> +<% end %> <% end %> <% end %> From 6370069e9d73923c34f5e0cbd58e94aa34653267 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A2u=20Cao?= Date: Sun, 5 Jul 2026 16:18:07 +0200 Subject: [PATCH 25/47] Add new vm_host attribute/value to all VM nodes --- .chef/config.rb | 4 ++-- nodes/akkounts-1.json | 3 ++- nodes/bitcoin-2.json | 3 ++- nodes/discourse-2.json | 3 ++- nodes/drone-1.json | 3 ++- nodes/ejabberd-4.json | 3 ++- nodes/ejabberd-8.json | 3 ++- nodes/garage-11.json | 3 ++- nodes/garage-14.json | 3 ++- nodes/garage-15.json | 3 ++- nodes/garage-16.json | 3 ++- nodes/garage-2.json | 3 ++- nodes/garage-7.json | 3 ++- nodes/garage-8.json | 3 ++- nodes/gitea-2.json | 3 ++- nodes/ipfs-1.json | 3 ++- nodes/ldap-3.kosmos.org.json | 3 ++- nodes/ldap-4.kosmos.org.json | 3 ++- nodes/lq-1.json | 3 ++- nodes/lq-2.json | 3 ++- nodes/mail.kosmos.org.json | 3 ++- nodes/mastodon-3.json | 3 ++- nodes/nodejs-4.json | 3 ++- nodes/postgres-11.json | 3 ++- nodes/postgres-12.json | 3 ++- nodes/prometheus-1.json | 3 ++- nodes/redis-1.json | 3 ++- nodes/redis-2.json | 3 ++- nodes/rs-discourse-1.json | 3 ++- nodes/rsk-mainnet-3.json | 3 ++- nodes/rsk-testnet-4.json | 3 ++- nodes/rsk-testnet-6.json | 3 ++- nodes/sentry-1.json | 3 ++- nodes/strfry-1.json | 3 ++- nodes/wiki-1.json | 3 ++- 35 files changed, 70 insertions(+), 36 deletions(-) diff --git a/.chef/config.rb b/.chef/config.rb index a0f2ce6..6751fcd 100644 --- a/.chef/config.rb +++ b/.chef/config.rb @@ -28,5 +28,5 @@ knife[:automatic_attribute_whitelist] = %w[ # Added to /etc/chef/client.rb on node bootstrap # https://docs.chef.io/attribute_persistence/ -knife[:normal_attribute_allowlist] = ['knife_zero', 'kosmos_kvm', 'kosmos-ejabberd', 'openresty'] -knife[:allowed_normal_attributes] = ['knife_zero', 'kosmos_kvm', 'kosmos-ejabberd', 'openresty'] +knife[:normal_attribute_allowlist] = ['knife_zero', 'kosmos_kvm', 'kosmos-ejabberd', 'openresty', 'vm_host'] +knife[:allowed_normal_attributes] = ['knife_zero', 'kosmos_kvm', 'kosmos-ejabberd', 'openresty', 'vm_host'] diff --git a/nodes/akkounts-1.json b/nodes/akkounts-1.json index 0627a9a..c496c52 100644 --- a/nodes/akkounts-1.json +++ b/nodes/akkounts-1.json @@ -4,7 +4,8 @@ "normal": { "knife_zero": { "host": "10.1.1.144" - } + }, + "vm_host": "draco" }, "automatic": { "fqdn": "akkounts-1", diff --git a/nodes/bitcoin-2.json b/nodes/bitcoin-2.json index 707584a..4ccd759 100644 --- a/nodes/bitcoin-2.json +++ b/nodes/bitcoin-2.json @@ -3,7 +3,8 @@ "normal": { "knife_zero": { "host": "10.1.1.163" - } + }, + "vm_host": "draco" }, "automatic": { "fqdn": "bitcoin-2", diff --git a/nodes/discourse-2.json b/nodes/discourse-2.json index da4bb5a..bc40c6a 100644 --- a/nodes/discourse-2.json +++ b/nodes/discourse-2.json @@ -4,7 +4,8 @@ "normal": { "knife_zero": { "host": "10.1.1.35" - } + }, + "vm_host": "fornax" }, "automatic": { "fqdn": "discourse-2", diff --git a/nodes/drone-1.json b/nodes/drone-1.json index befd090..b658d3e 100644 --- a/nodes/drone-1.json +++ b/nodes/drone-1.json @@ -3,7 +3,8 @@ "normal": { "knife_zero": { "host": "10.1.1.128" - } + }, + "vm_host": "fornax" }, "automatic": { "fqdn": "drone-1", diff --git a/nodes/ejabberd-4.json b/nodes/ejabberd-4.json index 3a148c8..72f50f0 100644 --- a/nodes/ejabberd-4.json +++ b/nodes/ejabberd-4.json @@ -4,7 +4,8 @@ "normal": { "knife_zero": { "host": "10.1.1.113" - } + }, + "vm_host": "draco" }, "automatic": { "fqdn": "ejabberd-4", diff --git a/nodes/ejabberd-8.json b/nodes/ejabberd-8.json index 8a41ef2..0e0ef8e 100644 --- a/nodes/ejabberd-8.json +++ b/nodes/ejabberd-8.json @@ -4,7 +4,8 @@ "normal": { "knife_zero": { "host": "10.1.1.123" - } + }, + "vm_host": "fornax" }, "automatic": { "fqdn": "ejabberd-8", diff --git a/nodes/garage-11.json b/nodes/garage-11.json index 1177218..930d94b 100644 --- a/nodes/garage-11.json +++ b/nodes/garage-11.json @@ -4,7 +4,8 @@ "normal": { "knife_zero": { "host": "10.1.1.165" - } + }, + "vm_host": "fornax" }, "automatic": { "fqdn": "garage-11", diff --git a/nodes/garage-14.json b/nodes/garage-14.json index 2ca3820..8d9d69c 100644 --- a/nodes/garage-14.json +++ b/nodes/garage-14.json @@ -4,7 +4,8 @@ "normal": { "knife_zero": { "host": "10.1.1.151" - } + }, + "vm_host": "leo" }, "automatic": { "fqdn": "garage-14", diff --git a/nodes/garage-15.json b/nodes/garage-15.json index 166d7d2..d7b5260 100644 --- a/nodes/garage-15.json +++ b/nodes/garage-15.json @@ -4,7 +4,8 @@ "normal": { "knife_zero": { "host": "10.1.1.82" - } + }, + "vm_host": "leo" }, "automatic": { "fqdn": "garage-15", diff --git a/nodes/garage-16.json b/nodes/garage-16.json index 5c26e0b..50a0d2b 100644 --- a/nodes/garage-16.json +++ b/nodes/garage-16.json @@ -4,7 +4,8 @@ "normal": { "knife_zero": { "host": "10.1.1.153" - } + }, + "vm_host": "leo" }, "automatic": { "fqdn": "garage-16", diff --git a/nodes/garage-2.json b/nodes/garage-2.json index 93641a6..9faadf1 100644 --- a/nodes/garage-2.json +++ b/nodes/garage-2.json @@ -4,7 +4,8 @@ "normal": { "knife_zero": { "host": "10.1.1.40" - } + }, + "vm_host": "draco" }, "automatic": { "fqdn": "garage-2", diff --git a/nodes/garage-7.json b/nodes/garage-7.json index ab60013..30604e1 100644 --- a/nodes/garage-7.json +++ b/nodes/garage-7.json @@ -4,7 +4,8 @@ "normal": { "knife_zero": { "host": "10.1.1.182" - } + }, + "vm_host": "draco" }, "automatic": { "fqdn": "garage-7", diff --git a/nodes/garage-8.json b/nodes/garage-8.json index 3d1527e..b2ccff8 100644 --- a/nodes/garage-8.json +++ b/nodes/garage-8.json @@ -4,7 +4,8 @@ "normal": { "knife_zero": { "host": "10.1.1.61" - } + }, + "vm_host": "fornax" }, "automatic": { "fqdn": "garage-8", diff --git a/nodes/gitea-2.json b/nodes/gitea-2.json index 931372a..9c122a2 100644 --- a/nodes/gitea-2.json +++ b/nodes/gitea-2.json @@ -4,7 +4,8 @@ "normal": { "knife_zero": { "host": "10.1.1.21" - } + }, + "vm_host": "fornax" }, "automatic": { "fqdn": "gitea-2", diff --git a/nodes/ipfs-1.json b/nodes/ipfs-1.json index bb82e47..4b70fb0 100644 --- a/nodes/ipfs-1.json +++ b/nodes/ipfs-1.json @@ -3,7 +3,8 @@ "normal": { "knife_zero": { "host": "10.1.1.198" - } + }, + "vm_host": "draco" }, "automatic": { "fqdn": "ipfs-1", diff --git a/nodes/ldap-3.kosmos.org.json b/nodes/ldap-3.kosmos.org.json index 9062959..f18fd8a 100644 --- a/nodes/ldap-3.kosmos.org.json +++ b/nodes/ldap-3.kosmos.org.json @@ -3,7 +3,8 @@ "normal": { "knife_zero": { "host": "10.1.1.6" - } + }, + "vm_host": "draco" }, "automatic": { "fqdn": "ldap-3.kosmos.org", diff --git a/nodes/ldap-4.kosmos.org.json b/nodes/ldap-4.kosmos.org.json index d5ff769..9f727b6 100644 --- a/nodes/ldap-4.kosmos.org.json +++ b/nodes/ldap-4.kosmos.org.json @@ -4,7 +4,8 @@ "normal": { "knife_zero": { "host": "10.1.1.106" - } + }, + "vm_host": "fornax" }, "automatic": { "fqdn": "ldap-4.kosmos.org", diff --git a/nodes/lq-1.json b/nodes/lq-1.json index 574ec90..9102a55 100644 --- a/nodes/lq-1.json +++ b/nodes/lq-1.json @@ -4,7 +4,8 @@ "normal": { "knife_zero": { "host": "10.1.1.87" - } + }, + "vm_host": "draco" }, "automatic": { "fqdn": "lq-1", diff --git a/nodes/lq-2.json b/nodes/lq-2.json index 5adfc66..09524d4 100644 --- a/nodes/lq-2.json +++ b/nodes/lq-2.json @@ -4,7 +4,8 @@ "normal": { "knife_zero": { "host": "10.1.1.188" - } + }, + "vm_host": "fornax" }, "automatic": { "fqdn": "lq-2", diff --git a/nodes/mail.kosmos.org.json b/nodes/mail.kosmos.org.json index f3b57b0..f0ee81b 100644 --- a/nodes/mail.kosmos.org.json +++ b/nodes/mail.kosmos.org.json @@ -4,7 +4,8 @@ "normal": { "knife_zero": { "host": "10.1.1.95" - } + }, + "vm_host": "draco" }, "automatic": { "fqdn": "mail.kosmos.org", diff --git a/nodes/mastodon-3.json b/nodes/mastodon-3.json index 82aadff..29bb553 100644 --- a/nodes/mastodon-3.json +++ b/nodes/mastodon-3.json @@ -4,7 +4,8 @@ "normal": { "knife_zero": { "host": "10.1.1.30" - } + }, + "vm_host": "fornax" }, "automatic": { "fqdn": "mastodon-3", diff --git a/nodes/nodejs-4.json b/nodes/nodejs-4.json index 659966b..2181fc0 100644 --- a/nodes/nodejs-4.json +++ b/nodes/nodejs-4.json @@ -3,7 +3,8 @@ "normal": { "knife_zero": { "host": "10.1.1.138" - } + }, + "vm_host": "fornax" }, "automatic": { "fqdn": "nodejs-4", diff --git a/nodes/postgres-11.json b/nodes/postgres-11.json index 24cec64..8cb74f9 100644 --- a/nodes/postgres-11.json +++ b/nodes/postgres-11.json @@ -4,7 +4,8 @@ "normal": { "knife_zero": { "host": "10.1.1.91" - } + }, + "vm_host": "leo" }, "automatic": { "fqdn": "postgres-11", diff --git a/nodes/postgres-12.json b/nodes/postgres-12.json index 7fac349..5828eb0 100644 --- a/nodes/postgres-12.json +++ b/nodes/postgres-12.json @@ -4,7 +4,8 @@ "normal": { "knife_zero": { "host": "10.1.1.134" - } + }, + "vm_host": "her" }, "automatic": { "fqdn": "postgres-12", diff --git a/nodes/prometheus-1.json b/nodes/prometheus-1.json index f76c369..f834101 100644 --- a/nodes/prometheus-1.json +++ b/nodes/prometheus-1.json @@ -4,7 +4,8 @@ "normal": { "knife_zero": { "host": "10.1.1.146" - } + }, + "vm_host": "leo" }, "automatic": { "fqdn": "prometheus-1", diff --git a/nodes/redis-1.json b/nodes/redis-1.json index cd0fb79..6698a68 100644 --- a/nodes/redis-1.json +++ b/nodes/redis-1.json @@ -4,7 +4,8 @@ "normal": { "knife_zero": { "host": "10.1.1.225" - } + }, + "vm_host": "draco" }, "automatic": { "fqdn": "redis-1", diff --git a/nodes/redis-2.json b/nodes/redis-2.json index 07be0f5..2f6d10e 100644 --- a/nodes/redis-2.json +++ b/nodes/redis-2.json @@ -4,7 +4,8 @@ "normal": { "knife_zero": { "host": "10.1.1.208" - } + }, + "vm_host": "fornax" }, "automatic": { "fqdn": "redis-2", diff --git a/nodes/rs-discourse-1.json b/nodes/rs-discourse-1.json index 6c1a826..b322827 100644 --- a/nodes/rs-discourse-1.json +++ b/nodes/rs-discourse-1.json @@ -3,7 +3,8 @@ "normal": { "knife_zero": { "host": "10.1.1.187" - } + }, + "vm_host": "fornax" }, "automatic": { "fqdn": "rs-discourse-1", diff --git a/nodes/rsk-mainnet-3.json b/nodes/rsk-mainnet-3.json index 985f06f..8593c70 100644 --- a/nodes/rsk-mainnet-3.json +++ b/nodes/rsk-mainnet-3.json @@ -3,7 +3,8 @@ "normal": { "knife_zero": { "host": "10.1.1.150" - } + }, + "vm_host": "her" }, "automatic": { "fqdn": "rsk-mainnet-3", diff --git a/nodes/rsk-testnet-4.json b/nodes/rsk-testnet-4.json index a420cf2..51e5d16 100644 --- a/nodes/rsk-testnet-4.json +++ b/nodes/rsk-testnet-4.json @@ -3,7 +3,8 @@ "normal": { "knife_zero": { "host": "10.1.1.126" - } + }, + "vm_host": "her" }, "automatic": { "fqdn": "rsk-testnet-4", diff --git a/nodes/rsk-testnet-6.json b/nodes/rsk-testnet-6.json index 6246c25..73ae932 100644 --- a/nodes/rsk-testnet-6.json +++ b/nodes/rsk-testnet-6.json @@ -3,7 +3,8 @@ "normal": { "knife_zero": { "host": "10.1.1.20" - } + }, + "vm_host": "leo" }, "automatic": { "fqdn": "rsk-testnet-6", diff --git a/nodes/sentry-1.json b/nodes/sentry-1.json index a037ffb..52e353d 100644 --- a/nodes/sentry-1.json +++ b/nodes/sentry-1.json @@ -4,7 +4,8 @@ "normal": { "knife_zero": { "host": "10.1.1.132" - } + }, + "vm_host": "her" }, "automatic": { "fqdn": "sentry-1", diff --git a/nodes/strfry-1.json b/nodes/strfry-1.json index b151730..41f868b 100644 --- a/nodes/strfry-1.json +++ b/nodes/strfry-1.json @@ -4,7 +4,8 @@ "normal": { "knife_zero": { "host": "10.1.1.164" - } + }, + "vm_host": "draco" }, "automatic": { "fqdn": "strfry-1", diff --git a/nodes/wiki-1.json b/nodes/wiki-1.json index 0dcd3fb..9216e81 100644 --- a/nodes/wiki-1.json +++ b/nodes/wiki-1.json @@ -3,7 +3,8 @@ "normal": { "knife_zero": { "host": "10.1.1.206" - } + }, + "vm_host": "draco" }, "automatic": { "fqdn": "wiki-1", From 8601e33764570d171c54eb0dfa152d9b128f648d Mon Sep 17 00:00:00 2001 From: Greg Karekinian Date: Sun, 5 Jul 2026 18:03:12 +0200 Subject: [PATCH 26/47] Fix year in license file --- site-cookbooks/kosmos_prometheus/LICENSE | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/site-cookbooks/kosmos_prometheus/LICENSE b/site-cookbooks/kosmos_prometheus/LICENSE index f3b5d1c..98808c8 100644 --- a/site-cookbooks/kosmos_prometheus/LICENSE +++ b/site-cookbooks/kosmos_prometheus/LICENSE @@ -1,4 +1,4 @@ -Copyright (c) 2019 Kosmos Developers +Copyright (c) 2026 Kosmos Developers Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the From 5e7a0c07b0b33abab23fc23e7e3789436d18f0e0 Mon Sep 17 00:00:00 2001 From: Greg Karekinian Date: Sun, 5 Jul 2026 18:16:18 +0200 Subject: [PATCH 27/47] Move web port to an attribute --- site-cookbooks/kosmos_prometheus/attributes/default.rb | 4 +++- site-cookbooks/kosmos_prometheus/recipes/server.rb | 6 ++++-- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/site-cookbooks/kosmos_prometheus/attributes/default.rb b/site-cookbooks/kosmos_prometheus/attributes/default.rb index 3189a86..8bd2964 100644 --- a/site-cookbooks/kosmos_prometheus/attributes/default.rb +++ b/site-cookbooks/kosmos_prometheus/attributes/default.rb @@ -1,6 +1,8 @@ node.default["kosmos_prometheus"]["version"] = "3.13.0" node.default["kosmos_prometheus"]["checksum"] = "744d93324cc024d82089921737bd797474d7f1e5dbbfd1c6b387bad258538cb9" +node.default["kosmos_prometheus"]["web"]["port"] = "9090" + node.default["kosmos_prometheus"]["alertmanager"]["version"] = "0.33.0" node.default["kosmos_prometheus"]["alertmanager"]["checksum"] = "8ce11c42e8a6dfbbf93a59c0b193cb1329210b36d0c7ef3df7b745608675a1d1" @@ -21,7 +23,7 @@ node.default["kosmos_prometheus"]["alerting"] = { } node.default["kosmos_prometheus"]["jobs"] = { - "prometheus" => { "targets" => [{ "target" => "localhost:9090", "instance" => "localhost" }] }, + "prometheus" => { "targets" => [{ "target" => "localhost:#{node["kosmos_prometheus"]["web"]["port"]}", "instance" => "localhost" }] }, } node.default["kosmos_prometheus"]["rule_files"] = [] diff --git a/site-cookbooks/kosmos_prometheus/recipes/server.rb b/site-cookbooks/kosmos_prometheus/recipes/server.rb index 345bd94..22f6a73 100644 --- a/site-cookbooks/kosmos_prometheus/recipes/server.rb +++ b/site-cookbooks/kosmos_prometheus/recipes/server.rb @@ -10,6 +10,8 @@ checksum = node["kosmos_prometheus"]["checksum"] tarball = "#{Chef::Config[:file_cache_path]}/prometheus-#{version}.linux-amd64.tar.gz" binary_url = "https://github.com/prometheus/prometheus/releases/download/v#{version}/prometheus-#{version}.linux-amd64.tar.gz" +web_port = node["kosmos_prometheus"]["web"]["port"] + group "prometheus" user "prometheus" do @@ -110,7 +112,7 @@ systemd_unit "prometheus.service" do Type: "simple", User: "prometheus", Group: "prometheus", - ExecStart: "/usr/local/bin/prometheus --config.file=/etc/prometheus/prometheus.yml --storage.tsdb.path=/var/lib/prometheus --storage.tsdb.retention.time=15d --web.listen-address=:9090 --web.enable-lifecycle", + ExecStart: "/usr/local/bin/prometheus --config.file=/etc/prometheus/prometheus.yml --storage.tsdb.path=/var/lib/prometheus --storage.tsdb.retention.time=15d --web.listen-address=:#{web_port} --web.enable-lifecycle", ExecReload: "/bin/kill -HUP $MAINPID", Restart: "on-failure", RestartSec: "5", @@ -128,7 +130,7 @@ service "prometheus" do end firewall_rule "prometheus web" do - port 9090 + port web_port source "10.1.1.0/24" protocol :tcp command :allow From 65c1933a99078ae57b9a9717db05bcc30aabec83 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A2u=20Cao?= Date: Mon, 6 Jul 2026 11:35:32 +0200 Subject: [PATCH 28/47] WIP Add default alertmanager route and receiver --- .../kosmos_prometheus/attributes/default.rb | 25 ++++++++++++++- .../kosmos_prometheus/recipes/alertmanager.rb | 8 +++++ .../templates/default/alertmanager.yml.erb | 31 +++++++++++++++---- 3 files changed, 57 insertions(+), 7 deletions(-) diff --git a/site-cookbooks/kosmos_prometheus/attributes/default.rb b/site-cookbooks/kosmos_prometheus/attributes/default.rb index 8bd2964..d16e59b 100644 --- a/site-cookbooks/kosmos_prometheus/attributes/default.rb +++ b/site-cookbooks/kosmos_prometheus/attributes/default.rb @@ -9,6 +9,27 @@ node.default["kosmos_prometheus"]["alertmanager"]["checksum"] = "8ce11c42e8a6dfb node.default["kosmos_prometheus"]["node_exporter"]["version"] = "1.11.1" node.default["kosmos_prometheus"]["node_exporter"]["checksum"] = "9f5ea48e5bc7b656f8a91a32e7d7deb89f70f73dabd0d974418aca15f37d6810" +node.default["kosmos_prometheus"]["alertmanager"]["route"] = { + "receiver" => "xmpp", + "group_by" => ["alertname"], + "group_wait" => "30s", + "group_interval" => "5m", + "repeat_interval" => "3h", + "routes" => [], +} + +node.default["kosmos_prometheus"]["alertmanager"]["receivers"] = [ + { + "name" => "xmpp", + "webhook_configs" => [ + { + "send_resolved" => true, + "room" => "info@kosmos.chat", + } + ] + } +] + node.default["kosmos_prometheus"]["global"] = { "scrape_interval" => "30s", "evaluation_interval" => "30s", @@ -26,4 +47,6 @@ node.default["kosmos_prometheus"]["jobs"] = { "prometheus" => { "targets" => [{ "target" => "localhost:#{node["kosmos_prometheus"]["web"]["port"]}", "instance" => "localhost" }] }, } -node.default["kosmos_prometheus"]["rule_files"] = [] +node.default["kosmos_prometheus"]["rule_files"] = [ + "/etc/prometheus/rules/*.yml" +] diff --git a/site-cookbooks/kosmos_prometheus/recipes/alertmanager.rb b/site-cookbooks/kosmos_prometheus/recipes/alertmanager.rb index 4cba895..9ad9799 100644 --- a/site-cookbooks/kosmos_prometheus/recipes/alertmanager.rb +++ b/site-cookbooks/kosmos_prometheus/recipes/alertmanager.rb @@ -55,12 +55,20 @@ file "/usr/local/bin/alertmanager" do notifies :restart, "service[alertmanager]", :delayed end +# TODO use Auth header once supported by hubot-incoming-webhook +credentials = data_bag_item("credentials", "hal8000_xmpp") +webhook_url = "https://hal8000.kosmos.chat/incoming/#{credentials['webhook_token']}" + template "/etc/prometheus/alertmanager.yml" do source "alertmanager.yml.erb" owner "root" group "alertmanager" mode "0644" + variables route: node["kosmos_prometheus"]["alertmanager"]["route"], + receivers: node["kosmos_prometheus"]["alertmanager"]["receivers"], + webhook_url: webhook_url notifies :restart, "service[alertmanager]", :delayed + sensitive true end systemd_unit "alertmanager.service" do diff --git a/site-cookbooks/kosmos_prometheus/templates/default/alertmanager.yml.erb b/site-cookbooks/kosmos_prometheus/templates/default/alertmanager.yml.erb index b0fd0a9..bced457 100644 --- a/site-cookbooks/kosmos_prometheus/templates/default/alertmanager.yml.erb +++ b/site-cookbooks/kosmos_prometheus/templates/default/alertmanager.yml.erb @@ -2,11 +2,30 @@ global: resolve_timeout: 5m route: - receiver: default - group_by: ['alertname'] - group_wait: 30s - group_interval: 5m - repeat_interval: 3h +<% @route.each do |k, v| %> +<% if k == "routes" && v.is_a?(Array) && !v.empty? %> + routes: +<% v.each do |child| %> + - receiver: <%= child["receiver"] %> +<% child.each do |ck, cv| %> +<% next if ck == "receiver" %> + <%= ck %>: <%= cv.is_a?(Array) ? cv.inspect : cv %> +<% end %> +<% end %> +<% else %> + <%= k %>: <%= v.is_a?(Array) ? v.inspect : v %> +<% end %> +<% end %> receivers: -- name: default +<% @receivers.each do |receiver| %> +- name: <%= receiver["name"] %> +<% receiver["webhook_configs"].each do |wc| %> + webhook_configs: + - url: "<%= @webhook_url %>" + send_resolved: <%= wc["send_resolved"] %> + payload: + room: "<%= wc["room"] %>" + message: '{{ .Status | toUpper }}: {{ .CommonLabels.alertname }}{{ if .CommonLabels.instance }} on {{ .CommonLabels.instance }}{{ end }}{{ if .CommonLabels.vm_host }} (host: {{ .CommonLabels.vm_host }}){{ end }} — {{ .CommonAnnotations.summary }}' +<% end %> +<% end %> From 81aae3d1bbcd809a8af4659bf3f6cc5baee10dc3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A2u=20Cao?= Date: Mon, 6 Jul 2026 12:37:26 +0200 Subject: [PATCH 29/47] Add icons to visualize alert severity --- .../kosmos_prometheus/templates/default/alertmanager.yml.erb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/site-cookbooks/kosmos_prometheus/templates/default/alertmanager.yml.erb b/site-cookbooks/kosmos_prometheus/templates/default/alertmanager.yml.erb index bced457..202d98a 100644 --- a/site-cookbooks/kosmos_prometheus/templates/default/alertmanager.yml.erb +++ b/site-cookbooks/kosmos_prometheus/templates/default/alertmanager.yml.erb @@ -26,6 +26,6 @@ receivers: send_resolved: <%= wc["send_resolved"] %> payload: room: "<%= wc["room"] %>" - message: '{{ .Status | toUpper }}: {{ .CommonLabels.alertname }}{{ if .CommonLabels.instance }} on {{ .CommonLabels.instance }}{{ end }}{{ if .CommonLabels.vm_host }} (host: {{ .CommonLabels.vm_host }}){{ end }} — {{ .CommonAnnotations.summary }}' + message: '{{ if eq .Status "resolved" }}✅{{ else if eq .CommonLabels.severity "warning" }}⚠️{{ else }}🚨{{ end }} {{ .Status | toUpper }}: {{ .CommonLabels.alertname }}{{ if .CommonLabels.instance }} on {{ .CommonLabels.instance }}{{ end }}{{ if .CommonLabels.vm_host }} (host: {{ .CommonLabels.vm_host }}){{ end }} — {{ .CommonAnnotations.summary }}' <% end %> <% end %> From 33987402e353e27a62bc0aac2103edfc9d482f15 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A2u=20Cao?= Date: Mon, 6 Jul 2026 13:16:10 +0200 Subject: [PATCH 30/47] Fix port attribute --- site-cookbooks/kosmos_prometheus/attributes/default.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/site-cookbooks/kosmos_prometheus/attributes/default.rb b/site-cookbooks/kosmos_prometheus/attributes/default.rb index d16e59b..53bca16 100644 --- a/site-cookbooks/kosmos_prometheus/attributes/default.rb +++ b/site-cookbooks/kosmos_prometheus/attributes/default.rb @@ -1,7 +1,7 @@ node.default["kosmos_prometheus"]["version"] = "3.13.0" node.default["kosmos_prometheus"]["checksum"] = "744d93324cc024d82089921737bd797474d7f1e5dbbfd1c6b387bad258538cb9" -node.default["kosmos_prometheus"]["web"]["port"] = "9090" +node.default["kosmos_prometheus"]["web"]["port"] = 9090 node.default["kosmos_prometheus"]["alertmanager"]["version"] = "0.33.0" node.default["kosmos_prometheus"]["alertmanager"]["checksum"] = "8ce11c42e8a6dfbbf93a59c0b193cb1329210b36d0c7ef3df7b745608675a1d1" From 21513e6d10dd892d7b9b92f49a72954b9f1981a2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A2u=20Cao?= Date: Mon, 6 Jul 2026 13:53:52 +0200 Subject: [PATCH 31/47] Critical icon only for critical severity --- .../kosmos_prometheus/templates/default/alertmanager.yml.erb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/site-cookbooks/kosmos_prometheus/templates/default/alertmanager.yml.erb b/site-cookbooks/kosmos_prometheus/templates/default/alertmanager.yml.erb index 202d98a..d9dd8d8 100644 --- a/site-cookbooks/kosmos_prometheus/templates/default/alertmanager.yml.erb +++ b/site-cookbooks/kosmos_prometheus/templates/default/alertmanager.yml.erb @@ -26,6 +26,6 @@ receivers: send_resolved: <%= wc["send_resolved"] %> payload: room: "<%= wc["room"] %>" - message: '{{ if eq .Status "resolved" }}✅{{ else if eq .CommonLabels.severity "warning" }}⚠️{{ else }}🚨{{ end }} {{ .Status | toUpper }}: {{ .CommonLabels.alertname }}{{ if .CommonLabels.instance }} on {{ .CommonLabels.instance }}{{ end }}{{ if .CommonLabels.vm_host }} (host: {{ .CommonLabels.vm_host }}){{ end }} — {{ .CommonAnnotations.summary }}' + message: '{{ if eq .Status "resolved" }}✅{{ else if eq .CommonLabels.severity "warning" }}⚠️{{ else if eq .CommonLabels.severity "critical" }}🚨{{ end }} {{ .Status | toUpper }}: {{ .CommonLabels.alertname }}{{ if .CommonLabels.instance }} on {{ .CommonLabels.instance }}{{ end }}{{ if .CommonLabels.vm_host }} (host: {{ .CommonLabels.vm_host }}){{ end }} — {{ .CommonAnnotations.summary }}' <% end %> <% end %> From 4ce77e1a6ef7286021a5e0610fa45016ccb6a913 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A2u=20Cao?= Date: Mon, 6 Jul 2026 14:50:49 +0200 Subject: [PATCH 32/47] Fix postfix relay sending (on newer Ubuntu versions) Requires relay restrictions to be configured explicitly --- site-cookbooks/kosmos-postfix/recipes/default.rb | 1 + 1 file changed, 1 insertion(+) diff --git a/site-cookbooks/kosmos-postfix/recipes/default.rb b/site-cookbooks/kosmos-postfix/recipes/default.rb index 1afd892..c689928 100644 --- a/site-cookbooks/kosmos-postfix/recipes/default.rb +++ b/site-cookbooks/kosmos-postfix/recipes/default.rb @@ -21,5 +21,6 @@ node.default["postfix"]["sasl"] = { node.default["postfix"]["main"]["relayhost"] = smtp_credentials["relayhost"] node.default["postfix"]["main"]["smtp_sasl_auth_enable"] = "yes" node.default["postfix"]["main"]["smtp_sasl_security_options"] = "noanonymous" +node.default["postfix"]["main"]["smtpd_relay_restrictions"] = "permit_mynetworks, reject" include_recipe "postfix::default" From 20f67d7261c0e78599ae502c028518999050d473 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A2u=20Cao?= Date: Mon, 6 Jul 2026 14:51:54 +0200 Subject: [PATCH 33/47] Change prometheus scrape/eval intervals --- site-cookbooks/kosmos_prometheus/attributes/default.rb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/site-cookbooks/kosmos_prometheus/attributes/default.rb b/site-cookbooks/kosmos_prometheus/attributes/default.rb index 53bca16..8d1fcab 100644 --- a/site-cookbooks/kosmos_prometheus/attributes/default.rb +++ b/site-cookbooks/kosmos_prometheus/attributes/default.rb @@ -31,8 +31,8 @@ node.default["kosmos_prometheus"]["alertmanager"]["receivers"] = [ ] node.default["kosmos_prometheus"]["global"] = { - "scrape_interval" => "30s", - "evaluation_interval" => "30s", + "scrape_interval" => "15s", + "evaluation_interval" => "15s", } node.default["kosmos_prometheus"]["alerting"] = { From 1ae52108130a67c30f7bbb71c3a4833bf19195ce Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A2u=20Cao?= Date: Mon, 6 Jul 2026 14:52:17 +0200 Subject: [PATCH 34/47] Send critical alerts via email (in addition to XMPP) --- .../kosmos_prometheus/attributes/default.rb | 24 ++++++++++++++++--- .../templates/default/alertmanager.yml.erb | 13 ++++++++++ 2 files changed, 34 insertions(+), 3 deletions(-) diff --git a/site-cookbooks/kosmos_prometheus/attributes/default.rb b/site-cookbooks/kosmos_prometheus/attributes/default.rb index 8d1fcab..16629ad 100644 --- a/site-cookbooks/kosmos_prometheus/attributes/default.rb +++ b/site-cookbooks/kosmos_prometheus/attributes/default.rb @@ -12,10 +12,16 @@ node.default["kosmos_prometheus"]["node_exporter"]["checksum"] = "9f5ea48e5bc7b6 node.default["kosmos_prometheus"]["alertmanager"]["route"] = { "receiver" => "xmpp", "group_by" => ["alertname"], - "group_wait" => "30s", - "group_interval" => "5m", + "group_wait" => "10s", + "group_interval" => "30s", "repeat_interval" => "3h", - "routes" => [], + "routes" => [ + { + "receiver" => "email", + "matchers" => ['severity="critical"'], + "continue" => true, + } + ], } node.default["kosmos_prometheus"]["alertmanager"]["receivers"] = [ @@ -27,6 +33,18 @@ node.default["kosmos_prometheus"]["alertmanager"]["receivers"] = [ "room" => "info@kosmos.chat", } ] + }, + { + "name" => "email", + "email_configs" => [ + { + "send_resolved" => true, + "to" => "ops@kosmos.org", + "from" => "ops@kosmos.org", + "smarthost" => "localhost:25", + "require_tls" => false, + } + ] } ] diff --git a/site-cookbooks/kosmos_prometheus/templates/default/alertmanager.yml.erb b/site-cookbooks/kosmos_prometheus/templates/default/alertmanager.yml.erb index d9dd8d8..0fa3ad9 100644 --- a/site-cookbooks/kosmos_prometheus/templates/default/alertmanager.yml.erb +++ b/site-cookbooks/kosmos_prometheus/templates/default/alertmanager.yml.erb @@ -20,6 +20,7 @@ route: receivers: <% @receivers.each do |receiver| %> - name: <%= receiver["name"] %> +<% if receiver["webhook_configs"] %> <% receiver["webhook_configs"].each do |wc| %> webhook_configs: - url: "<%= @webhook_url %>" @@ -29,3 +30,15 @@ receivers: message: '{{ if eq .Status "resolved" }}✅{{ else if eq .CommonLabels.severity "warning" }}⚠️{{ else if eq .CommonLabels.severity "critical" }}🚨{{ end }} {{ .Status | toUpper }}: {{ .CommonLabels.alertname }}{{ if .CommonLabels.instance }} on {{ .CommonLabels.instance }}{{ end }}{{ if .CommonLabels.vm_host }} (host: {{ .CommonLabels.vm_host }}){{ end }} — {{ .CommonAnnotations.summary }}' <% end %> <% end %> +<% if receiver["email_configs"] %> +<% receiver["email_configs"].each do |ec| %> + email_configs: + - to: "<%= ec["to"] %>" + from: "<%= ec["from"] %>" + smarthost: "<%= ec["smarthost"] %>" + require_tls: <%= ec["require_tls"] %> + send_resolved: <%= ec["send_resolved"] %> + text: '{{ .Status | toUpper }}: {{ .CommonLabels.alertname }}{{ if .CommonLabels.instance }} on {{ .CommonLabels.instance }}{{ end }}{{ if .CommonLabels.vm_host }} (host: {{ .CommonLabels.vm_host }}){{ end }} — {{ .CommonAnnotations.summary }}' +<% end %> +<% end %> +<% end %> From 2a14837f6425b1f400f7507a52f3827e10736652 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A2u=20Cao?= Date: Mon, 6 Jul 2026 18:01:22 +0200 Subject: [PATCH 35/47] WIP Add prometheus rules, improve alerts --- .../kosmos_prometheus/attributes/default.rb | 16 ++++++- .../files/default/rules/node_exporter.yml | 10 +++++ .../kosmos_prometheus/recipes/server.rb | 43 ++++++++++++------- .../templates/default/alertmanager.yml.erb | 7 ++- 4 files changed, 57 insertions(+), 19 deletions(-) create mode 100644 site-cookbooks/kosmos_prometheus/files/default/rules/node_exporter.yml diff --git a/site-cookbooks/kosmos_prometheus/attributes/default.rb b/site-cookbooks/kosmos_prometheus/attributes/default.rb index 16629ad..4be544c 100644 --- a/site-cookbooks/kosmos_prometheus/attributes/default.rb +++ b/site-cookbooks/kosmos_prometheus/attributes/default.rb @@ -1,6 +1,10 @@ node.default["kosmos_prometheus"]["version"] = "3.13.0" node.default["kosmos_prometheus"]["checksum"] = "744d93324cc024d82089921737bd797474d7f1e5dbbfd1c6b387bad258538cb9" +node.default["kosmos_prometheus"]["data_dir"] = "/var/lib/prometheus" +node.default["kosmos_prometheus"]["config_dir"] = "/etc/prometheus" +node.default["kosmos_prometheus"]["rules_dir"] = "#{node["kosmos_prometheus"]["config_dir"]}/rules" + node.default["kosmos_prometheus"]["web"]["port"] = 9090 node.default["kosmos_prometheus"]["alertmanager"]["version"] = "0.33.0" @@ -12,10 +16,14 @@ node.default["kosmos_prometheus"]["node_exporter"]["checksum"] = "9f5ea48e5bc7b6 node.default["kosmos_prometheus"]["alertmanager"]["route"] = { "receiver" => "xmpp", "group_by" => ["alertname"], - "group_wait" => "10s", + "group_wait" => "20s", "group_interval" => "30s", "repeat_interval" => "3h", "routes" => [ + { + "receiver" => "xmpp", + "continue" => true, + }, { "receiver" => "email", "matchers" => ['severity="critical"'], @@ -66,5 +74,9 @@ node.default["kosmos_prometheus"]["jobs"] = { } node.default["kosmos_prometheus"]["rule_files"] = [ - "/etc/prometheus/rules/*.yml" + "#{node["kosmos_prometheus"]["rules_dir"]}/*.yml" +] + +node.default["kosmos_prometheus"]["rules"] = [ + "node_exporter" ] diff --git a/site-cookbooks/kosmos_prometheus/files/default/rules/node_exporter.yml b/site-cookbooks/kosmos_prometheus/files/default/rules/node_exporter.yml new file mode 100644 index 0000000..b3ffbb1 --- /dev/null +++ b/site-cookbooks/kosmos_prometheus/files/default/rules/node_exporter.yml @@ -0,0 +1,10 @@ +groups: + - name: NodeExporter + rules: + - alert: NodeDown + expr: up{job="node"} == 0 + for: 10s + labels: + severity: warning + annotations: + summary: "Node is down" diff --git a/site-cookbooks/kosmos_prometheus/recipes/server.rb b/site-cookbooks/kosmos_prometheus/recipes/server.rb index 22f6a73..093265b 100644 --- a/site-cookbooks/kosmos_prometheus/recipes/server.rb +++ b/site-cookbooks/kosmos_prometheus/recipes/server.rb @@ -10,7 +10,10 @@ checksum = node["kosmos_prometheus"]["checksum"] tarball = "#{Chef::Config[:file_cache_path]}/prometheus-#{version}.linux-amd64.tar.gz" binary_url = "https://github.com/prometheus/prometheus/releases/download/v#{version}/prometheus-#{version}.linux-amd64.tar.gz" -web_port = node["kosmos_prometheus"]["web"]["port"] +data_dir = node["kosmos_prometheus"]["data_dir"] +config_dir = node["kosmos_prometheus"]["config_dir"] +rules_dir = node["kosmos_prometheus"]["rules_dir"] +web_port = node["kosmos_prometheus"]["web"]["port"] group "prometheus" @@ -21,25 +24,35 @@ user "prometheus" do home "/nonexistent" end -directory "/var/lib/prometheus" do +directory data_dir do owner "prometheus" group "prometheus" + mode "0750" + recursive true +end + +directory config_dir do + owner "root" + group "root" + mode "0755" + recursive true +end + +directory rules_dir do + owner "root" + group "prometheus" mode "0755" recursive true end -directory "/etc/prometheus" do - owner "root" - group "root" - mode "0755" - recursive true -end - -directory "/etc/prometheus/rules" do - owner "root" - group "root" - mode "0755" - recursive true +node["kosmos_prometheus"]["rules"].each do |rule_name| + cookbook_file "#{rules_dir}/#{rule_name}.yml" do + source "rules/#{rule_name}.yml" + owner "root" + group "prometheus" + mode "0644" + notifies :reload, "service[prometheus]", :delayed + end end package %w(tar bzip2) @@ -88,7 +101,7 @@ jobs = node["kosmos_prometheus"]["jobs"].merge( end ) -template "/etc/prometheus/prometheus.yml" do +template "#{config_dir}/prometheus.yml" do source "prometheus.yml.erb" owner "root" group "prometheus" diff --git a/site-cookbooks/kosmos_prometheus/templates/default/alertmanager.yml.erb b/site-cookbooks/kosmos_prometheus/templates/default/alertmanager.yml.erb index 0fa3ad9..451a3fd 100644 --- a/site-cookbooks/kosmos_prometheus/templates/default/alertmanager.yml.erb +++ b/site-cookbooks/kosmos_prometheus/templates/default/alertmanager.yml.erb @@ -27,7 +27,7 @@ receivers: send_resolved: <%= wc["send_resolved"] %> payload: room: "<%= wc["room"] %>" - message: '{{ if eq .Status "resolved" }}✅{{ else if eq .CommonLabels.severity "warning" }}⚠️{{ else if eq .CommonLabels.severity "critical" }}🚨{{ end }} {{ .Status | toUpper }}: {{ .CommonLabels.alertname }}{{ if .CommonLabels.instance }} on {{ .CommonLabels.instance }}{{ end }}{{ if .CommonLabels.vm_host }} (host: {{ .CommonLabels.vm_host }}){{ end }} — {{ .CommonAnnotations.summary }}' + message: '{{ if eq .Status "resolved" }}✅{{ else if eq .CommonLabels.severity "warning" }}⚠️{{ else }}🚨{{ end }} {{ .Status | toUpper }}: {{ .CommonLabels.alertname }} ({{ len .Alerts }} alerts){{ range $i, $a := .Alerts }}{{ if lt $i 5 }}{{ "\n" }} - {{ $a.Labels.instance }}{{ if $a.Labels.vm_host }} ({{ $a.Labels.vm_host }}){{ end }}: {{ $a.Annotations.summary }}{{ end }}{{ end }}{{ if gt (len .Alerts) 5 }}{{ "\n" }} ... and more{{ end }}' <% end %> <% end %> <% if receiver["email_configs"] %> @@ -38,7 +38,10 @@ receivers: smarthost: "<%= ec["smarthost"] %>" require_tls: <%= ec["require_tls"] %> send_resolved: <%= ec["send_resolved"] %> - text: '{{ .Status | toUpper }}: {{ .CommonLabels.alertname }}{{ if .CommonLabels.instance }} on {{ .CommonLabels.instance }}{{ end }}{{ if .CommonLabels.vm_host }} (host: {{ .CommonLabels.vm_host }}){{ end }} — {{ .CommonAnnotations.summary }}' + text: '{{ .Status | toUpper }}: {{ .CommonLabels.alertname }} ({{ len .Alerts }} alerts){{ range .Alerts }} +---{{ range .Labels.SortedPairs }} + {{ .Name }}: {{ .Value }}{{ end }}{{ range .Annotations.SortedPairs }} + {{ .Name }}: {{ .Value }}{{ end }}{{ end }}' <% end %> <% end %> <% end %> From 02dc95b7de97433d33bdd9767ef2f73401f9956c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A2u=20Cao?= Date: Mon, 6 Jul 2026 18:47:51 +0200 Subject: [PATCH 36/47] Update node configs --- nodes/akkounts-1.json | 1 + nodes/drone-1.json | 5 ++++- nodes/ejabberd-4.json | 1 + nodes/ejabberd-8.json | 1 + nodes/garage-2.json | 10 ++++++---- nodes/gitea-2.json | 9 +++++---- nodes/lq-1.json | 10 ++++++---- nodes/lq-2.json | 10 ++++++---- nodes/redis-1.json | 3 +++ nodes/redis-2.json | 3 +++ nodes/rs-discourse-1.json | 22 ++++++++++++---------- nodes/rsk-testnet-6.json | 5 +++-- 12 files changed, 51 insertions(+), 29 deletions(-) diff --git a/nodes/akkounts-1.json b/nodes/akkounts-1.json index c496c52..7af0bda 100644 --- a/nodes/akkounts-1.json +++ b/nodes/akkounts-1.json @@ -25,6 +25,7 @@ "recipes": [ "kosmos-base", "kosmos-base::default", + "kosmos_prometheus::node_exporter", "kosmos_kvm::guest", "kosmos-dirsrv::hostsfile", "kosmos_sentry::client", diff --git a/nodes/drone-1.json b/nodes/drone-1.json index b658d3e..7b9e733 100644 --- a/nodes/drone-1.json +++ b/nodes/drone-1.json @@ -13,6 +13,7 @@ "hostname": "drone-1", "ipaddress": "192.168.122.200", "roles": [ + "base", "kvm_guest", "drone", "postgresql_client" @@ -20,6 +21,7 @@ "recipes": [ "kosmos-base", "kosmos-base::default", + "kosmos_prometheus::node_exporter", "kosmos_kvm::guest", "kosmos_postgresql::hostsfile", "kosmos_drone", @@ -38,7 +40,8 @@ "postfix::_common", "postfix::_attributes", "postfix::sasl_auth", - "hostname::default" + "hostname::default", + "firewall::default" ], "platform": "ubuntu", "platform_version": "20.04", diff --git a/nodes/ejabberd-4.json b/nodes/ejabberd-4.json index 72f50f0..2fba750 100644 --- a/nodes/ejabberd-4.json +++ b/nodes/ejabberd-4.json @@ -24,6 +24,7 @@ "recipes": [ "kosmos-base", "kosmos-base::default", + "kosmos_prometheus::node_exporter", "kosmos_kvm::guest", "kosmos-dirsrv::hostsfile", "kosmos_postgresql::hostsfile", diff --git a/nodes/ejabberd-8.json b/nodes/ejabberd-8.json index 0e0ef8e..81a0b09 100644 --- a/nodes/ejabberd-8.json +++ b/nodes/ejabberd-8.json @@ -24,6 +24,7 @@ "recipes": [ "kosmos-base", "kosmos-base::default", + "kosmos_prometheus::node_exporter", "kosmos_kvm::guest", "kosmos-dirsrv::hostsfile", "kosmos_postgresql::hostsfile", diff --git a/nodes/garage-2.json b/nodes/garage-2.json index 9faadf1..42994c7 100644 --- a/nodes/garage-2.json +++ b/nodes/garage-2.json @@ -21,6 +21,7 @@ "recipes": [ "kosmos-base", "kosmos-base::default", + "kosmos_prometheus::node_exporter", "kosmos_kvm::guest", "kosmos_garage", "kosmos_garage::default", @@ -31,6 +32,7 @@ "timezone_iii::debian", "ntp::default", "ntp::apparmor", + "kosmos-base::journald_conf", "kosmos-base::systemd_emails", "apt::unattended-upgrades", "kosmos-base::firewall", @@ -47,13 +49,13 @@ "cloud": null, "chef_packages": { "chef": { - "version": "18.3.0", - "chef_root": "/opt/chef/embedded/lib/ruby/gems/3.1.0/gems/chef-18.3.0/lib", + "version": "18.10.17", + "chef_root": "/opt/chef/embedded/lib/ruby/gems/3.1.0/gems/chef-18.10.17/lib", "chef_effortless": null }, "ohai": { - "version": "18.1.4", - "ohai_root": "/opt/chef/embedded/lib/ruby/gems/3.1.0/gems/ohai-18.1.4/lib/ohai" + "version": "18.2.13", + "ohai_root": "/opt/chef/embedded/lib/ruby/gems/3.1.0/gems/ohai-18.2.13/lib/ohai" } } }, diff --git a/nodes/gitea-2.json b/nodes/gitea-2.json index 9c122a2..b726be6 100644 --- a/nodes/gitea-2.json +++ b/nodes/gitea-2.json @@ -25,6 +25,7 @@ "recipes": [ "kosmos-base", "kosmos-base::default", + "kosmos_prometheus::node_exporter", "kosmos_kvm::guest", "kosmos-dirsrv::hostsfile", "kosmos_garage", @@ -59,13 +60,13 @@ "cloud": null, "chef_packages": { "chef": { - "version": "18.2.7", - "chef_root": "/opt/chef/embedded/lib/ruby/gems/3.1.0/gems/chef-18.2.7/lib", + "version": "18.10.17", + "chef_root": "/opt/chef/embedded/lib/ruby/gems/3.1.0/gems/chef-18.10.17/lib", "chef_effortless": null }, "ohai": { - "version": "18.1.4", - "ohai_root": "/opt/chef/embedded/lib/ruby/gems/3.1.0/gems/ohai-18.1.4/lib/ohai" + "version": "18.2.13", + "ohai_root": "/opt/chef/embedded/lib/ruby/gems/3.1.0/gems/ohai-18.2.13/lib/ohai" } } }, diff --git a/nodes/lq-1.json b/nodes/lq-1.json index 9102a55..a729c80 100644 --- a/nodes/lq-1.json +++ b/nodes/lq-1.json @@ -22,6 +22,7 @@ "recipes": [ "kosmos-base", "kosmos-base::default", + "kosmos_prometheus::node_exporter", "kosmos_kvm::guest", "kosmos_garage", "kosmos_garage::default", @@ -33,6 +34,7 @@ "timezone_iii::debian", "ntp::default", "ntp::apparmor", + "kosmos-base::journald_conf", "kosmos-base::systemd_emails", "apt::unattended-upgrades", "kosmos-base::firewall", @@ -50,13 +52,13 @@ "cloud": null, "chef_packages": { "chef": { - "version": "18.2.7", - "chef_root": "/opt/chef/embedded/lib/ruby/gems/3.1.0/gems/chef-18.2.7/lib", + "version": "18.10.17", + "chef_root": "/opt/chef/embedded/lib/ruby/gems/3.1.0/gems/chef-18.10.17/lib", "chef_effortless": null }, "ohai": { - "version": "18.1.4", - "ohai_root": "/opt/chef/embedded/lib/ruby/gems/3.1.0/gems/ohai-18.1.4/lib/ohai" + "version": "18.2.13", + "ohai_root": "/opt/chef/embedded/lib/ruby/gems/3.1.0/gems/ohai-18.2.13/lib/ohai" } } }, diff --git a/nodes/lq-2.json b/nodes/lq-2.json index 09524d4..4acf21d 100644 --- a/nodes/lq-2.json +++ b/nodes/lq-2.json @@ -22,6 +22,7 @@ "recipes": [ "kosmos-base", "kosmos-base::default", + "kosmos_prometheus::node_exporter", "kosmos_kvm::guest", "kosmos_garage", "kosmos_garage::default", @@ -33,6 +34,7 @@ "timezone_iii::debian", "ntp::default", "ntp::apparmor", + "kosmos-base::journald_conf", "kosmos-base::systemd_emails", "apt::unattended-upgrades", "kosmos-base::firewall", @@ -50,13 +52,13 @@ "cloud": null, "chef_packages": { "chef": { - "version": "18.2.7", - "chef_root": "/opt/chef/embedded/lib/ruby/gems/3.1.0/gems/chef-18.2.7/lib", + "version": "18.10.17", + "chef_root": "/opt/chef/embedded/lib/ruby/gems/3.1.0/gems/chef-18.10.17/lib", "chef_effortless": null }, "ohai": { - "version": "18.1.4", - "ohai_root": "/opt/chef/embedded/lib/ruby/gems/3.1.0/gems/ohai-18.1.4/lib/ohai" + "version": "18.2.13", + "ohai_root": "/opt/chef/embedded/lib/ruby/gems/3.1.0/gems/ohai-18.2.13/lib/ohai" } } }, diff --git a/nodes/redis-1.json b/nodes/redis-1.json index 6698a68..23d6420 100644 --- a/nodes/redis-1.json +++ b/nodes/redis-1.json @@ -21,6 +21,7 @@ "recipes": [ "kosmos-base", "kosmos-base::default", + "kosmos_prometheus::node_exporter", "kosmos_kvm::guest", "kosmos_redis", "kosmos_redis::default", @@ -31,6 +32,7 @@ "timezone_iii::debian", "ntp::default", "ntp::apparmor", + "kosmos-base::journald_conf", "kosmos-base::systemd_emails", "apt::unattended-upgrades", "kosmos-base::firewall", @@ -40,6 +42,7 @@ "postfix::_attributes", "postfix::sasl_auth", "hostname::default", + "firewall::default", "redisio::default", "redisio::_install_prereqs", "redisio::install", diff --git a/nodes/redis-2.json b/nodes/redis-2.json index 2f6d10e..cbd65d4 100644 --- a/nodes/redis-2.json +++ b/nodes/redis-2.json @@ -21,6 +21,7 @@ "recipes": [ "kosmos-base", "kosmos-base::default", + "kosmos_prometheus::node_exporter", "kosmos_kvm::guest", "kosmos_redis::replica", "kosmos_redis::firewall", @@ -29,6 +30,7 @@ "timezone_iii::debian", "ntp::default", "ntp::apparmor", + "kosmos-base::journald_conf", "kosmos-base::systemd_emails", "apt::unattended-upgrades", "kosmos-base::firewall", @@ -38,6 +40,7 @@ "postfix::_attributes", "postfix::sasl_auth", "hostname::default", + "firewall::default", "redisio::default", "redisio::_install_prereqs", "redisio::install", diff --git a/nodes/rs-discourse-1.json b/nodes/rs-discourse-1.json index b322827..ddc0b12 100644 --- a/nodes/rs-discourse-1.json +++ b/nodes/rs-discourse-1.json @@ -9,24 +9,27 @@ "automatic": { "fqdn": "rs-discourse-1", "os": "linux", - "os_version": "5.4.0-1076-kvm", + "os_version": "5.4.0-1118-kvm", "hostname": "rs-discourse-1", "ipaddress": "192.168.122.30", "roles": [ - "remotestorage_discourse", - "kvm_guest" + "base", + "kvm_guest", + "remotestorage_discourse" ], "recipes": [ "kosmos-base", "kosmos-base::default", + "kosmos_prometheus::node_exporter", + "kosmos_kvm::guest", "remotestorage_discourse", "remotestorage_discourse::default", - "kosmos_kvm::guest", "apt::default", "timezone_iii::default", "timezone_iii::debian", "ntp::default", "ntp::apparmor", + "kosmos-base::journald_conf", "kosmos-base::systemd_emails", "apt::unattended-upgrades", "kosmos-base::firewall", @@ -36,22 +39,21 @@ "postfix::_attributes", "postfix::sasl_auth", "hostname::default", - "discourse::default", "firewall::default", - "chef-sugar::default" + "discourse::default" ], "platform": "ubuntu", "platform_version": "20.04", "cloud": null, "chef_packages": { "chef": { - "version": "17.10.3", - "chef_root": "/opt/chef/embedded/lib/ruby/gems/3.0.0/gems/chef-17.10.3/lib", + "version": "18.10.17", + "chef_root": "/opt/chef/embedded/lib/ruby/gems/3.1.0/gems/chef-18.10.17/lib", "chef_effortless": null }, "ohai": { - "version": "17.9.0", - "ohai_root": "/opt/chef/embedded/lib/ruby/gems/3.0.0/gems/ohai-17.9.0/lib/ohai" + "version": "18.2.13", + "ohai_root": "/opt/chef/embedded/lib/ruby/gems/3.1.0/gems/ohai-18.2.13/lib/ohai" } } }, diff --git a/nodes/rsk-testnet-6.json b/nodes/rsk-testnet-6.json index 73ae932..ad6795b 100644 --- a/nodes/rsk-testnet-6.json +++ b/nodes/rsk-testnet-6.json @@ -20,6 +20,7 @@ "recipes": [ "kosmos-base", "kosmos-base::default", + "kosmos_prometheus::node_exporter", "kosmos_kvm::guest", "kosmos_rsk::rskj", "apt::default", @@ -35,8 +36,8 @@ "postfix::_attributes", "postfix::sasl_auth", "hostname::default", - "kosmos_rsk::firewall", - "firewall::default" + "firewall::default", + "kosmos_rsk::firewall" ], "platform": "ubuntu", "platform_version": "24.04", From 704d0bca0e52554f37358e49532b54b476443eda Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A2u=20Cao?= Date: Mon, 6 Jul 2026 18:50:07 +0200 Subject: [PATCH 37/47] Document upgrading Chef client --- README.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/README.md b/README.md index ef6c609..1fcb982 100644 --- a/README.md +++ b/README.md @@ -58,6 +58,10 @@ Clone this repository, `cd` into it, and run: knife zero converge name:dev.kosmos.org --client-version 15.3.14 +### Upgrade Chef Client on a node: + + knife ssh -a knife_zero.host 'name:foo-1' 'curl -L https://chef.io/chef/install.sh | sudo bash -s -- -v 18.10.17' + ## Managing cookbooks Cookbooks are managed via Berkshelf. Run `berks --help` for command help. From 94a3c419649bd232e82d1894678cb54deb341fdf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A2u=20Cao?= Date: Tue, 7 Jul 2026 14:26:21 +0200 Subject: [PATCH 38/47] Show severity for new alerts, number of firing vs resolved --- .../kosmos_prometheus/templates/default/alertmanager.yml.erb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/site-cookbooks/kosmos_prometheus/templates/default/alertmanager.yml.erb b/site-cookbooks/kosmos_prometheus/templates/default/alertmanager.yml.erb index 451a3fd..381c538 100644 --- a/site-cookbooks/kosmos_prometheus/templates/default/alertmanager.yml.erb +++ b/site-cookbooks/kosmos_prometheus/templates/default/alertmanager.yml.erb @@ -27,7 +27,7 @@ receivers: send_resolved: <%= wc["send_resolved"] %> payload: room: "<%= wc["room"] %>" - message: '{{ if eq .Status "resolved" }}✅{{ else if eq .CommonLabels.severity "warning" }}⚠️{{ else }}🚨{{ end }} {{ .Status | toUpper }}: {{ .CommonLabels.alertname }} ({{ len .Alerts }} alerts){{ range $i, $a := .Alerts }}{{ if lt $i 5 }}{{ "\n" }} - {{ $a.Labels.instance }}{{ if $a.Labels.vm_host }} ({{ $a.Labels.vm_host }}){{ end }}: {{ $a.Annotations.summary }}{{ end }}{{ end }}{{ if gt (len .Alerts) 5 }}{{ "\n" }} ... and more{{ end }}' + message: '{{ if eq .Status "resolved" }}✅{{ else if eq .CommonLabels.severity "warning" }}⚠️{{ else }}🚨{{ end }} {{ if eq .Status "resolved" }}RESOLVED{{ else if .CommonLabels.severity }}{{ .CommonLabels.severity | toUpper }}{{ else }}ALERT{{ end }}: {{ .CommonLabels.alertname }} {{ if eq .Status "resolved" }}({{ len .Alerts }} resolved){{ else if eq (len .Alerts.Firing) (len .Alerts) }}({{ len .Alerts }} firing){{ else }}({{ len .Alerts.Firing }} firing, {{ len .Alerts.Resolved }} resolved){{ end }}{{ range $i, $a := .Alerts }}{{ if lt $i 5 }}{{ "\n" }} - {{ $a.Labels.instance }}{{ if $a.Labels.vm_host }} ({{ $a.Labels.vm_host }}){{ end }}: {{ if eq $a.Status "resolved" }}resolved{{ else }}{{ $a.Annotations.summary }}{{ end }}{{ end }}{{ end }}{{ if gt (len .Alerts) 5 }}{{ "\n" }} ... and more{{ end }}' <% end %> <% end %> <% if receiver["email_configs"] %> @@ -38,7 +38,7 @@ receivers: smarthost: "<%= ec["smarthost"] %>" require_tls: <%= ec["require_tls"] %> send_resolved: <%= ec["send_resolved"] %> - text: '{{ .Status | toUpper }}: {{ .CommonLabels.alertname }} ({{ len .Alerts }} alerts){{ range .Alerts }} + text: '{{ if eq .Status "resolved" }}RESOLVED{{ else if .CommonLabels.severity }}{{ .CommonLabels.severity | toUpper }}{{ else }}ALERT{{ end }}: {{ .CommonLabels.alertname }} {{ if eq .Status "resolved" }}({{ len .Alerts }} resolved){{ else if eq (len .Alerts.Firing) (len .Alerts) }}({{ len .Alerts }} firing){{ else }}({{ len .Alerts.Firing }} firing, {{ len .Alerts.Resolved }} resolved){{ end }}{{ range .Alerts }} ---{{ range .Labels.SortedPairs }} {{ .Name }}: {{ .Value }}{{ end }}{{ range .Annotations.SortedPairs }} {{ .Name }}: {{ .Value }}{{ end }}{{ end }}' From e4ad52d4137eb36a91d09574b149f4c92c787947 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A2u=20Cao?= Date: Tue, 7 Jul 2026 15:05:25 +0200 Subject: [PATCH 39/47] Update dirsrv node configs --- nodes/ldap-3.kosmos.org.json | 15 +++++++++------ nodes/ldap-4.kosmos.org.json | 22 ++++++++++++++++------ 2 files changed, 25 insertions(+), 12 deletions(-) diff --git a/nodes/ldap-3.kosmos.org.json b/nodes/ldap-3.kosmos.org.json index f18fd8a..cd743f3 100644 --- a/nodes/ldap-3.kosmos.org.json +++ b/nodes/ldap-3.kosmos.org.json @@ -13,12 +13,14 @@ "hostname": "ldap-3", "ipaddress": "192.168.122.34", "roles": [ + "base", "kvm_guest", - "dirsrv_primary" + "dirsrv_supplier" ], "recipes": [ "kosmos-base", "kosmos-base::default", + "kosmos_prometheus::node_exporter", "kosmos_kvm::guest", "kosmos-dirsrv", "kosmos-dirsrv::default", @@ -27,6 +29,7 @@ "timezone_iii::debian", "ntp::default", "ntp::apparmor", + "kosmos-base::journald_conf", "kosmos-base::systemd_emails", "apt::unattended-upgrades", "kosmos-base::firewall", @@ -36,7 +39,7 @@ "postfix::_attributes", "postfix::sasl_auth", "hostname::default", - "kosmos-dirsrv::hostsfile", + "firewall::default", "kosmos-dirsrv::firewall", "backup::default", "logrotate::default", @@ -47,13 +50,13 @@ "cloud": null, "chef_packages": { "chef": { - "version": "17.10.3", - "chef_root": "/opt/chef/embedded/lib/ruby/gems/3.0.0/gems/chef-17.10.3/lib", + "version": "18.10.17", + "chef_root": "/opt/chef/embedded/lib/ruby/gems/3.1.0/gems/chef-18.10.17/lib", "chef_effortless": null }, "ohai": { - "version": "17.9.0", - "ohai_root": "/opt/chef/embedded/lib/ruby/gems/3.0.0/gems/ohai-17.9.0/lib/ohai" + "version": "18.2.13", + "ohai_root": "/opt/chef/embedded/lib/ruby/gems/3.1.0/gems/ohai-18.2.13/lib/ohai" } } }, diff --git a/nodes/ldap-4.kosmos.org.json b/nodes/ldap-4.kosmos.org.json index 9f727b6..672b7b4 100644 --- a/nodes/ldap-4.kosmos.org.json +++ b/nodes/ldap-4.kosmos.org.json @@ -15,17 +15,22 @@ "ipaddress": "192.168.122.73", "roles": [ "base", - "kvm_guest" + "kvm_guest", + "dirsrv_supplier" ], "recipes": [ "kosmos-base", "kosmos-base::default", + "kosmos_prometheus::node_exporter", "kosmos_kvm::guest", + "kosmos-dirsrv", + "kosmos-dirsrv::default", "apt::default", "timezone_iii::default", "timezone_iii::debian", "ntp::default", "ntp::apparmor", + "kosmos-base::journald_conf", "kosmos-base::systemd_emails", "apt::unattended-upgrades", "kosmos-base::firewall", @@ -34,20 +39,25 @@ "postfix::_common", "postfix::_attributes", "postfix::sasl_auth", - "hostname::default" + "hostname::default", + "firewall::default", + "kosmos-dirsrv::firewall", + "backup::default", + "logrotate::default", + "ulimit::default" ], "platform": "ubuntu", "platform_version": "20.04", "cloud": null, "chef_packages": { "chef": { - "version": "17.10.3", - "chef_root": "/opt/chef/embedded/lib/ruby/gems/3.0.0/gems/chef-17.10.3/lib", + "version": "18.10.17", + "chef_root": "/opt/chef/embedded/lib/ruby/gems/3.1.0/gems/chef-18.10.17/lib", "chef_effortless": null }, "ohai": { - "version": "17.9.0", - "ohai_root": "/opt/chef/embedded/lib/ruby/gems/3.0.0/gems/ohai-17.9.0/lib/ohai" + "version": "18.2.13", + "ohai_root": "/opt/chef/embedded/lib/ruby/gems/3.1.0/gems/ohai-18.2.13/lib/ohai" } } }, From 4976888358b9835c09d3c3e3e66a2b8c662b653a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A2u=20Cao?= Date: Tue, 7 Jul 2026 15:20:33 +0200 Subject: [PATCH 40/47] Remove unnecessary include --- site-cookbooks/kosmos_prometheus/recipes/node_exporter.rb | 2 -- 1 file changed, 2 deletions(-) diff --git a/site-cookbooks/kosmos_prometheus/recipes/node_exporter.rb b/site-cookbooks/kosmos_prometheus/recipes/node_exporter.rb index b4e4a2a..fc43a0c 100644 --- a/site-cookbooks/kosmos_prometheus/recipes/node_exporter.rb +++ b/site-cookbooks/kosmos_prometheus/recipes/node_exporter.rb @@ -3,8 +3,6 @@ # Recipe:: node_exporter # -include_recipe "firewall" - version = node["kosmos_prometheus"]["node_exporter"]["version"] checksum = node["kosmos_prometheus"]["node_exporter"]["checksum"] tarball = "#{Chef::Config[:file_cache_path]}/node_exporter-#{version}.linux-amd64.tar.gz" From b6ba5ba679aa4bca222148204919d0abb062634c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A2u=20Cao?= Date: Tue, 7 Jul 2026 15:20:48 +0200 Subject: [PATCH 41/47] Update node configs --- nodes/postgres-11.json | 1 + nodes/postgres-12.json | 1 + 2 files changed, 2 insertions(+) diff --git a/nodes/postgres-11.json b/nodes/postgres-11.json index 8cb74f9..8f434cc 100644 --- a/nodes/postgres-11.json +++ b/nodes/postgres-11.json @@ -21,6 +21,7 @@ "recipes": [ "kosmos-base", "kosmos-base::default", + "kosmos_prometheus::node_exporter", "kosmos_kvm::guest", "kosmos_postgresql::primary", "kosmos-akkounts::pg_db", diff --git a/nodes/postgres-12.json b/nodes/postgres-12.json index 5828eb0..2de0881 100644 --- a/nodes/postgres-12.json +++ b/nodes/postgres-12.json @@ -21,6 +21,7 @@ "recipes": [ "kosmos-base", "kosmos-base::default", + "kosmos_prometheus::node_exporter", "kosmos_kvm::guest", "kosmos_postgresql::hostsfile", "kosmos_postgresql::replica", From 9fca5386f34e14e512f1ee5893c4257d301abccc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A2u=20Cao?= Date: Tue, 7 Jul 2026 16:57:22 +0200 Subject: [PATCH 42/47] Increase padding for NodeDown rule --- .../kosmos_prometheus/files/default/rules/node_exporter.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/site-cookbooks/kosmos_prometheus/files/default/rules/node_exporter.yml b/site-cookbooks/kosmos_prometheus/files/default/rules/node_exporter.yml index b3ffbb1..b80cef4 100644 --- a/site-cookbooks/kosmos_prometheus/files/default/rules/node_exporter.yml +++ b/site-cookbooks/kosmos_prometheus/files/default/rules/node_exporter.yml @@ -3,7 +3,7 @@ groups: rules: - alert: NodeDown expr: up{job="node"} == 0 - for: 10s + for: 20s labels: severity: warning annotations: From 6f93820936f563b93f253872e9070d91bc40c9c1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A2u=20Cao?= Date: Tue, 7 Jul 2026 16:58:36 +0200 Subject: [PATCH 43/47] Update node configs --- nodes/bitcoin-2.json | 1 + nodes/discourse-2.json | 14 ++++++++------ nodes/draco.kosmos.org.json | 1 + nodes/fornax.kosmos.org.json | 1 + nodes/her.json | 6 ++++-- nodes/ipfs-1.json | 3 +++ nodes/mail.kosmos.org.json | 3 ++- nodes/nodejs-4.json | 2 ++ nodes/strfry-1.json | 1 + nodes/wiki-1.json | 1 + 10 files changed, 24 insertions(+), 9 deletions(-) diff --git a/nodes/bitcoin-2.json b/nodes/bitcoin-2.json index 4ccd759..eaec80c 100644 --- a/nodes/bitcoin-2.json +++ b/nodes/bitcoin-2.json @@ -25,6 +25,7 @@ "recipes": [ "kosmos-base", "kosmos-base::default", + "kosmos_prometheus::node_exporter", "kosmos_kvm::guest", "kosmos_sentry::client", "tor-full", diff --git a/nodes/discourse-2.json b/nodes/discourse-2.json index bc40c6a..4e76389 100644 --- a/nodes/discourse-2.json +++ b/nodes/discourse-2.json @@ -10,10 +10,11 @@ "automatic": { "fqdn": "discourse-2", "os": "linux", - "os_version": "5.4.0-1087-kvm", + "os_version": "5.4.0-1133-kvm", "hostname": "discourse-2", "ipaddress": "192.168.122.104", "roles": [ + "base", "kvm_guest", "garage_gateway", "kosmos_discourse" @@ -21,6 +22,7 @@ "recipes": [ "kosmos-base", "kosmos-base::default", + "kosmos_prometheus::node_exporter", "kosmos_kvm::guest", "kosmos_garage", "kosmos_garage::default", @@ -33,6 +35,7 @@ "timezone_iii::debian", "ntp::default", "ntp::apparmor", + "kosmos-base::journald_conf", "kosmos-base::systemd_emails", "apt::unattended-upgrades", "kosmos-base::firewall", @@ -43,7 +46,6 @@ "postfix::sasl_auth", "hostname::default", "firewall::default", - "chef-sugar::default", "discourse::default" ], "platform": "ubuntu", @@ -51,13 +53,13 @@ "cloud": null, "chef_packages": { "chef": { - "version": "17.9.52", - "chef_root": "/opt/chef/embedded/lib/ruby/gems/3.0.0/gems/chef-17.9.52/lib", + "version": "18.10.17", + "chef_root": "/opt/chef/embedded/lib/ruby/gems/3.1.0/gems/chef-18.10.17/lib", "chef_effortless": null }, "ohai": { - "version": "17.9.0", - "ohai_root": "/opt/chef/embedded/lib/ruby/gems/3.0.0/gems/ohai-17.9.0/lib/ohai" + "version": "18.2.13", + "ohai_root": "/opt/chef/embedded/lib/ruby/gems/3.1.0/gems/ohai-18.2.13/lib/ohai" } } }, diff --git a/nodes/draco.kosmos.org.json b/nodes/draco.kosmos.org.json index 8ae21b4..2c17307 100644 --- a/nodes/draco.kosmos.org.json +++ b/nodes/draco.kosmos.org.json @@ -36,6 +36,7 @@ "recipes": [ "kosmos-base", "kosmos-base::default", + "kosmos_prometheus::node_exporter", "kosmos_kvm::host", "kosmos_kvm::backup", "kosmos_email::firewall", diff --git a/nodes/fornax.kosmos.org.json b/nodes/fornax.kosmos.org.json index a57af22..41f19ef 100644 --- a/nodes/fornax.kosmos.org.json +++ b/nodes/fornax.kosmos.org.json @@ -30,6 +30,7 @@ "recipes": [ "kosmos-base", "kosmos-base::default", + "kosmos_prometheus::node_exporter", "kosmos_kvm::host", "kosmos_kvm::backup", "kosmos_openresty", diff --git a/nodes/her.json b/nodes/her.json index ba758d0..53e6ffc 100644 --- a/nodes/her.json +++ b/nodes/her.json @@ -9,9 +9,9 @@ "automatic": { "fqdn": "her", "os": "linux", - "os_version": "5.15.0-84-generic", + "os_version": "5.15.0-101-generic", "hostname": "her", - "ipaddress": "192.168.30.172", + "ipaddress": "192.168.30.173", "roles": [ "base", "kvm_host" @@ -19,6 +19,7 @@ "recipes": [ "kosmos-base", "kosmos-base::default", + "kosmos_prometheus::node_exporter", "kosmos_kvm::host", "kosmos_kvm::backup", "apt::default", @@ -26,6 +27,7 @@ "timezone_iii::debian", "ntp::default", "ntp::apparmor", + "kosmos-base::journald_conf", "kosmos-base::systemd_emails", "apt::unattended-upgrades", "kosmos-base::firewall", diff --git a/nodes/ipfs-1.json b/nodes/ipfs-1.json index 4b70fb0..c4f0118 100644 --- a/nodes/ipfs-1.json +++ b/nodes/ipfs-1.json @@ -13,12 +13,14 @@ "hostname": "ipfs-1", "ipaddress": "192.168.122.195", "roles": [ + "base", "kvm_guest", "ipfs_gateway" ], "recipes": [ "kosmos-base", "kosmos-base::default", + "kosmos_prometheus::node_exporter", "kosmos_kvm::guest", "kosmos-ipfs", "kosmos-ipfs::default", @@ -28,6 +30,7 @@ "timezone_iii::debian", "ntp::default", "ntp::apparmor", + "kosmos-base::journald_conf", "kosmos-base::systemd_emails", "apt::unattended-upgrades", "kosmos-base::firewall", diff --git a/nodes/mail.kosmos.org.json b/nodes/mail.kosmos.org.json index f0ee81b..7dbb29e 100644 --- a/nodes/mail.kosmos.org.json +++ b/nodes/mail.kosmos.org.json @@ -11,7 +11,7 @@ "fqdn": "mail.kosmos.org", "os": "linux", "os_version": "5.15.0-1048-kvm", - "hostname": "mail.kosmos.org", + "hostname": "mail", "ipaddress": "192.168.122.131", "roles": [ "base", @@ -22,6 +22,7 @@ "recipes": [ "kosmos-base", "kosmos-base::default", + "kosmos_prometheus::node_exporter", "kosmos_kvm::guest", "kosmos-dirsrv::hostsfile", "kosmos_email", diff --git a/nodes/nodejs-4.json b/nodes/nodejs-4.json index 2181fc0..48ab1fa 100644 --- a/nodes/nodejs-4.json +++ b/nodes/nodejs-4.json @@ -22,6 +22,7 @@ "recipes": [ "kosmos-base", "kosmos-base::default", + "kosmos_prometheus::node_exporter", "kosmos_kvm::guest", "kosmos-hubot::botka_irc-libera-chat", "kosmos-hubot::hal8000_xmpp", @@ -36,6 +37,7 @@ "timezone_iii::debian", "ntp::default", "ntp::apparmor", + "kosmos-base::journald_conf", "kosmos-base::systemd_emails", "apt::unattended-upgrades", "kosmos-base::firewall", diff --git a/nodes/strfry-1.json b/nodes/strfry-1.json index 41f868b..3b60167 100644 --- a/nodes/strfry-1.json +++ b/nodes/strfry-1.json @@ -23,6 +23,7 @@ "recipes": [ "kosmos-base", "kosmos-base::default", + "kosmos_prometheus::node_exporter", "kosmos_kvm::guest", "kosmos-dirsrv::hostsfile", "strfry", diff --git a/nodes/wiki-1.json b/nodes/wiki-1.json index 9216e81..9b43155 100644 --- a/nodes/wiki-1.json +++ b/nodes/wiki-1.json @@ -20,6 +20,7 @@ "recipes": [ "kosmos-base", "kosmos-base::default", + "kosmos_prometheus::node_exporter", "kosmos_kvm::guest", "kosmos-dirsrv::hostsfile", "kosmos-mediawiki", From b700baac370820475fc0eec2c3127d8f55acb3fe Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A2u=20Cao?= Date: Thu, 9 Jul 2026 14:04:27 +0200 Subject: [PATCH 44/47] Configure alerts for disk/filesystem usage, memory, CPU --- .../kosmos_prometheus/attributes/default.rb | 2 +- .../files/default/rules/node_exporter.yml | 84 +++++++++++++++++++ 2 files changed, 85 insertions(+), 1 deletion(-) diff --git a/site-cookbooks/kosmos_prometheus/attributes/default.rb b/site-cookbooks/kosmos_prometheus/attributes/default.rb index 4be544c..83275db 100644 --- a/site-cookbooks/kosmos_prometheus/attributes/default.rb +++ b/site-cookbooks/kosmos_prometheus/attributes/default.rb @@ -15,7 +15,7 @@ node.default["kosmos_prometheus"]["node_exporter"]["checksum"] = "9f5ea48e5bc7b6 node.default["kosmos_prometheus"]["alertmanager"]["route"] = { "receiver" => "xmpp", - "group_by" => ["alertname"], + "group_by" => ["alertname", "severity"], "group_wait" => "20s", "group_interval" => "30s", "repeat_interval" => "3h", diff --git a/site-cookbooks/kosmos_prometheus/files/default/rules/node_exporter.yml b/site-cookbooks/kosmos_prometheus/files/default/rules/node_exporter.yml index b80cef4..854e0f5 100644 --- a/site-cookbooks/kosmos_prometheus/files/default/rules/node_exporter.yml +++ b/site-cookbooks/kosmos_prometheus/files/default/rules/node_exporter.yml @@ -8,3 +8,87 @@ groups: severity: warning annotations: summary: "Node is down" + + - alert: NodeFilesystemAlmostOutOfSpace + expr: | + (node_filesystem_avail_bytes{fstype!~"tmpfs|fuse.lxcfs|overlay|squashfs|ramfs",mountpoint!~"/run.*|/var/lib/docker/.*"} + / node_filesystem_size_bytes{fstype!~"tmpfs|fuse.lxcfs|overlay|squashfs|ramfs",mountpoint!~"/run.*|/var/lib/docker/.*"}) * 100 < 10 + for: 5m + labels: + severity: warning + annotations: + summary: "Filesystem has < 10% free space" + + - alert: NodeFilesystemAlmostOutOfSpace + expr: | + (node_filesystem_avail_bytes{fstype!~"tmpfs|fuse.lxcfs|overlay|squashfs|ramfs",mountpoint!~"/run.*|/var/lib/docker/.*"} + / node_filesystem_size_bytes{fstype!~"tmpfs|fuse.lxcfs|overlay|squashfs|ramfs",mountpoint!~"/run.*|/var/lib/docker/.*"}) * 100 < 5 + for: 5m + labels: + severity: critical + annotations: + summary: "Filesystem has < 5% free space" + + - alert: NodeFilesystemAlmostOutOfInodes + expr: (node_filesystem_files_free / node_filesystem_files) * 100 < 10 + for: 5m + labels: + severity: warning + annotations: + summary: "Filesystem has < 10% free inodes" + + - alert: NodeFilesystemReadOnly + expr: node_filesystem_readonly == 1 + for: 2m + labels: + severity: critical + annotations: + summary: "Filesystem remounted read-only" + + - alert: NodeMemoryHigh + expr: (1 - node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes) * 100 > 90 + for: 10m + labels: + severity: warning + annotations: + summary: "Memory usage > 90%" + + - alert: NodeMemoryHigh + expr: (1 - node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes) * 100 > 95 + for: 5m + labels: + severity: critical + annotations: + summary: "Memory usage > 95%" + + - alert: NodeSwapHigh + expr: (node_memory_SwapTotal_bytes > 0) and ((1 - node_memory_SwapFree_bytes / node_memory_SwapTotal_bytes) * 100 > 50) + for: 10m + labels: + severity: warning + annotations: + summary: "Swap usage > 50%" + + - alert: NodeCpuHigh + expr: 100 - (avg by (instance) (rate(node_cpu_seconds_total{mode="idle"}[5m])) * 100) > 80 + for: 10m + labels: + severity: warning + annotations: + summary: "CPU usage > 80% for 10m" + + - alert: NodeCpuHigh + expr: 100 - (avg by (instance) (rate(node_cpu_seconds_total{mode="idle"}[5m])) * 100) > 95 + for: 5m + labels: + severity: critical + annotations: + summary: "CPU usage > 95% for 5m" + + - alert: NodeLoadHigh + expr: node_load1 / count by (instance) (node_cpu_seconds_total{mode="idle"}) > 1.5 + for: 10m + labels: + severity: warning + annotations: + summary: "System load1 > 1.5x CPU cores" From ae14a1f35372b8ac7696fc082eb4dcc27b043556 Mon Sep 17 00:00:00 2001 From: Greg Karekinian Date: Thu, 9 Jul 2026 15:18:06 +0200 Subject: [PATCH 45/47] Move the node_exporter port to an attribute --- site-cookbooks/kosmos_prometheus/attributes/default.rb | 1 + site-cookbooks/kosmos_prometheus/recipes/node_exporter.rb | 6 ++++-- site-cookbooks/kosmos_prometheus/recipes/server.rb | 2 +- 3 files changed, 6 insertions(+), 3 deletions(-) diff --git a/site-cookbooks/kosmos_prometheus/attributes/default.rb b/site-cookbooks/kosmos_prometheus/attributes/default.rb index 83275db..added9b 100644 --- a/site-cookbooks/kosmos_prometheus/attributes/default.rb +++ b/site-cookbooks/kosmos_prometheus/attributes/default.rb @@ -12,6 +12,7 @@ node.default["kosmos_prometheus"]["alertmanager"]["checksum"] = "8ce11c42e8a6dfb node.default["kosmos_prometheus"]["node_exporter"]["version"] = "1.11.1" node.default["kosmos_prometheus"]["node_exporter"]["checksum"] = "9f5ea48e5bc7b656f8a91a32e7d7deb89f70f73dabd0d974418aca15f37d6810" +node.default["kosmos_prometheus"]["node_exporter"]["port"] = 9100 node.default["kosmos_prometheus"]["alertmanager"]["route"] = { "receiver" => "xmpp", diff --git a/site-cookbooks/kosmos_prometheus/recipes/node_exporter.rb b/site-cookbooks/kosmos_prometheus/recipes/node_exporter.rb index fc43a0c..ed6f178 100644 --- a/site-cookbooks/kosmos_prometheus/recipes/node_exporter.rb +++ b/site-cookbooks/kosmos_prometheus/recipes/node_exporter.rb @@ -8,6 +8,8 @@ checksum = node["kosmos_prometheus"]["node_exporter"]["checksum"] tarball = "#{Chef::Config[:file_cache_path]}/node_exporter-#{version}.linux-amd64.tar.gz" binary_url = "https://github.com/prometheus/node_exporter/releases/download/v#{version}/node_exporter-#{version}.linux-amd64.tar.gz" +node_exporter_port = node["node_exporter"]["port"] + group "node_exporter" user "node_exporter" do @@ -56,7 +58,7 @@ systemd_unit "node_exporter.service" do Type: "simple", User: "node_exporter", Group: "node_exporter", - ExecStart: "/usr/local/bin/node_exporter --web.listen-address=:9100 --collector.textfile.directory=/var/lib/node_exporter/textfile", + ExecStart: "/usr/local/bin/node_exporter --web.listen-address=:#{node_exporter_port} --collector.textfile.directory=/var/lib/node_exporter/textfile", Restart: "on-failure", RestartSec: "5", NoNewPrivileges: "yes", @@ -76,7 +78,7 @@ service "node_exporter" do end firewall_rule "node_exporter" do - port 9100 + port node_exporter_port source "10.1.1.0/24" protocol :tcp command :allow diff --git a/site-cookbooks/kosmos_prometheus/recipes/server.rb b/site-cookbooks/kosmos_prometheus/recipes/server.rb index 093265b..ef73331 100644 --- a/site-cookbooks/kosmos_prometheus/recipes/server.rb +++ b/site-cookbooks/kosmos_prometheus/recipes/server.rb @@ -82,7 +82,7 @@ jobs = node["kosmos_prometheus"]["jobs"].merge( # node exporter "node" => { "query" => "role:base", - "port" => 9100 + "port" => node["kosmos_prometheus"]["node_exporter"]["port"] }, # garage metrics "garage" => { From f15d5cb958eedfaecaf14367c4091069e6a69f1c Mon Sep 17 00:00:00 2001 From: Greg Karekinian Date: Thu, 9 Jul 2026 15:23:46 +0200 Subject: [PATCH 46/47] Move the alertmanager port to an attribute --- site-cookbooks/kosmos_prometheus/attributes/default.rb | 4 +++- site-cookbooks/kosmos_prometheus/recipes/alertmanager.rb | 6 ++++-- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/site-cookbooks/kosmos_prometheus/attributes/default.rb b/site-cookbooks/kosmos_prometheus/attributes/default.rb index added9b..5851470 100644 --- a/site-cookbooks/kosmos_prometheus/attributes/default.rb +++ b/site-cookbooks/kosmos_prometheus/attributes/default.rb @@ -33,6 +33,8 @@ node.default["kosmos_prometheus"]["alertmanager"]["route"] = { ], } +node.default["kosmos_prometheus"]["alertmanager"]["port"] = 9093 + node.default["kosmos_prometheus"]["alertmanager"]["receivers"] = [ { "name" => "xmpp", @@ -65,7 +67,7 @@ node.default["kosmos_prometheus"]["global"] = { node.default["kosmos_prometheus"]["alerting"] = { "alertmanagers" => [ "static_configs" => [ - { "targets" => ["127.0.0.1:9093"] } + { "targets" => ["127.0.0.1:#{node["kosmos_prometheus"]["alertmanager"]["port"]}"] } ] ] } diff --git a/site-cookbooks/kosmos_prometheus/recipes/alertmanager.rb b/site-cookbooks/kosmos_prometheus/recipes/alertmanager.rb index 9ad9799..b8625a9 100644 --- a/site-cookbooks/kosmos_prometheus/recipes/alertmanager.rb +++ b/site-cookbooks/kosmos_prometheus/recipes/alertmanager.rb @@ -10,6 +10,8 @@ checksum = node["kosmos_prometheus"]["alertmanager"]["checksum"] tarball = "#{Chef::Config[:file_cache_path]}/alertmanager-#{version}.linux-amd64.tar.gz" binary_url = "https://github.com/prometheus/alertmanager/releases/download/v#{version}/alertmanager-#{version}.linux-amd64.tar.gz" +alertmanager_port = node["kosmos_prometheus"]["alertmanager"]["port"] + group "alertmanager" user "alertmanager" do @@ -81,7 +83,7 @@ systemd_unit "alertmanager.service" do Type: "simple", User: "alertmanager", Group: "alertmanager", - ExecStart: "/usr/local/bin/alertmanager --config.file=/etc/prometheus/alertmanager.yml --storage.path=/var/lib/alertmanager --web.listen-address=:9093", + ExecStart: "/usr/local/bin/alertmanager --config.file=/etc/prometheus/alertmanager.yml --storage.path=/var/lib/alertmanager --web.listen-address=:#{alertmanager_port}", Restart: "on-failure", RestartSec: "5", }, @@ -98,7 +100,7 @@ service "alertmanager" do end firewall_rule "prometheus alertmanager" do - port 9093 + port alertmanager_port source "10.1.1.0/24" protocol :tcp command :allow From 51af9c76e0e8c97a919ef890c0c9a7ece147fd11 Mon Sep 17 00:00:00 2001 From: Greg Karekinian Date: Thu, 9 Jul 2026 15:30:38 +0200 Subject: [PATCH 47/47] Fix node attribute --- site-cookbooks/kosmos_prometheus/recipes/node_exporter.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/site-cookbooks/kosmos_prometheus/recipes/node_exporter.rb b/site-cookbooks/kosmos_prometheus/recipes/node_exporter.rb index ed6f178..f365b78 100644 --- a/site-cookbooks/kosmos_prometheus/recipes/node_exporter.rb +++ b/site-cookbooks/kosmos_prometheus/recipes/node_exporter.rb @@ -8,7 +8,7 @@ checksum = node["kosmos_prometheus"]["node_exporter"]["checksum"] tarball = "#{Chef::Config[:file_cache_path]}/node_exporter-#{version}.linux-amd64.tar.gz" binary_url = "https://github.com/prometheus/node_exporter/releases/download/v#{version}/node_exporter-#{version}.linux-amd64.tar.gz" -node_exporter_port = node["node_exporter"]["port"] +node_exporter_port = node["kosmos_prometheus"]["node_exporter"]["port"] group "node_exporter"