WIP Add default alertmanager route and receiver

site-cookbooks/kosmos_prometheus/attributes/default.rb

@@ -9,6 +9,27 @@ node.default["kosmos_prometheus"]["alertmanager"]["checksum"] = "8ce11c42e8a6dfb
 node.default["kosmos_prometheus"]["node_exporter"]["version"] = "1.11.1"
 node.default["kosmos_prometheus"]["node_exporter"]["checksum"] = "9f5ea48e5bc7b656f8a91a32e7d7deb89f70f73dabd0d974418aca15f37d6810"
 
+node.default["kosmos_prometheus"]["alertmanager"]["route"] = {
+  "receiver" => "xmpp",
+  "group_by" => ["alertname"],
+  "group_wait" => "30s",
+  "group_interval" => "5m",
+  "repeat_interval" => "3h",
+  "routes" => [],
+}
+
+node.default["kosmos_prometheus"]["alertmanager"]["receivers"] = [
+  {
+    "name" => "xmpp",
+    "webhook_configs" => [
+      {
+        "send_resolved" => true,
+        "room" => "info@kosmos.chat",
+      }
+    ]
+  }
+]
+
 node.default["kosmos_prometheus"]["global"] = {
   "scrape_interval" => "30s",
   "evaluation_interval" => "30s",
@@ -26,4 +47,6 @@ node.default["kosmos_prometheus"]["jobs"] = {
   "prometheus" => { "targets" => [{ "target" => "localhost:#{node["kosmos_prometheus"]["web"]["port"]}", "instance" => "localhost" }] },
 }
 
-node.default["kosmos_prometheus"]["rule_files"] = []
+node.default["kosmos_prometheus"]["rule_files"] = [
+  "/etc/prometheus/rules/*.yml"
+]

site-cookbooks/kosmos_prometheus/recipes/alertmanager.rb

@@ -55,12 +55,20 @@ file "/usr/local/bin/alertmanager" do
   notifies :restart, "service[alertmanager]", :delayed
 end
 
+# TODO use Auth header once supported by hubot-incoming-webhook
+credentials = data_bag_item("credentials", "hal8000_xmpp")
+webhook_url = "https://hal8000.kosmos.chat/incoming/#{credentials['webhook_token']}"
+
 template "/etc/prometheus/alertmanager.yml" do
   source "alertmanager.yml.erb"
   owner "root"
   group "alertmanager"
   mode "0644"
+  variables route: node["kosmos_prometheus"]["alertmanager"]["route"],
+            receivers: node["kosmos_prometheus"]["alertmanager"]["receivers"],
+            webhook_url: webhook_url
   notifies :restart, "service[alertmanager]", :delayed
+  sensitive true
 end
 
 systemd_unit "alertmanager.service" do

site-cookbooks/kosmos_prometheus/templates/default/alertmanager.yml.erb

@@ -2,11 +2,30 @@ global:
   resolve_timeout: 5m
 
 route:
-  receiver: default
-  group_by: ['alertname']
-  group_wait: 30s
-  group_interval: 5m
-  repeat_interval: 3h
+<% @route.each do |k, v| %>
+<% if k == "routes" && v.is_a?(Array) && !v.empty? %>
+  routes:
+<% v.each do |child| %>
+  - receiver: <%= child["receiver"] %>
+<% child.each do |ck, cv| %>
+<% next if ck == "receiver" %>
+    <%= ck %>: <%= cv.is_a?(Array) ? cv.inspect : cv %>
+<% end %>
+<% end %>
+<% else %>
+  <%= k %>: <%= v.is_a?(Array) ? v.inspect : v %>
+<% end %>
+<% end %>
 
 receivers:
-- name: default
+<% @receivers.each do |receiver| %>
+- name: <%= receiver["name"] %>
+<% receiver["webhook_configs"].each do |wc| %>
+  webhook_configs:
+  - url: "<%= @webhook_url %>"
+    send_resolved: <%= wc["send_resolved"] %>
+    payload:
+      room: "<%= wc["room"] %>"
+      message: '{{ .Status | toUpper }}: {{ .CommonLabels.alertname }}{{ if .CommonLabels.instance }} on {{ .CommonLabels.instance }}{{ end }}{{ if .CommonLabels.vm_host }} (host: {{ .CommonLabels.vm_host }}){{ end }} — {{ .CommonAnnotations.summary }}'
+<% end %>
+<% end %>