Send critical alerts via email (in addition to XMPP)

site-cookbooks/kosmos_prometheus/attributes/default.rb

@@ -12,10 +12,16 @@ node.default["kosmos_prometheus"]["node_exporter"]["checksum"] = "9f5ea48e5bc7b6
 node.default["kosmos_prometheus"]["alertmanager"]["route"] = {
   "receiver" => "xmpp",
   "group_by" => ["alertname"],
-  "group_wait" => "30s",
+  "group_wait" => "10s",
-  "group_interval" => "5m",
+  "group_interval" => "30s",
   "repeat_interval" => "3h",
-  "routes" => [],
+  "routes" => [
+    {
+      "receiver" => "email",
+      "matchers" => ['severity="critical"'],
+      "continue" => true,
+    }
+  ],
 }
 
 node.default["kosmos_prometheus"]["alertmanager"]["receivers"] = [
@@ -27,6 +33,18 @@ node.default["kosmos_prometheus"]["alertmanager"]["receivers"] = [
         "room" => "info@kosmos.chat",
       }
     ]
+  },
+  {
+    "name" => "email",
+    "email_configs" => [
+      {
+        "send_resolved" => true,
+        "to" => "ops@kosmos.org",
+        "from" => "ops@kosmos.org",
+        "smarthost" => "localhost:25",
+        "require_tls" => false,
+      }
+    ]
   }
 ]
 

site-cookbooks/kosmos_prometheus/templates/default/alertmanager.yml.erb

@@ -20,6 +20,7 @@ route:
 receivers:
 <% @receivers.each do |receiver| %>
 - name: <%= receiver["name"] %>
+<% if receiver["webhook_configs"] %>
 <% receiver["webhook_configs"].each do |wc| %>
   webhook_configs:
   - url: "<%= @webhook_url %>"
@@ -29,3 +30,15 @@ receivers:
       message: '{{ if eq .Status "resolved" }}✅{{ else if eq .CommonLabels.severity "warning" }}⚠️{{ else if eq .CommonLabels.severity "critical" }}🚨{{ end }} {{ .Status | toUpper }}: {{ .CommonLabels.alertname }}{{ if .CommonLabels.instance }} on {{ .CommonLabels.instance }}{{ end }}{{ if .CommonLabels.vm_host }} (host: {{ .CommonLabels.vm_host }}){{ end }} — {{ .CommonAnnotations.summary }}'
 <% end %>
 <% end %>
+<% if receiver["email_configs"] %>
+<% receiver["email_configs"].each do |ec| %>
+  email_configs:
+  - to: "<%= ec["to"] %>"
+    from: "<%= ec["from"] %>"
+    smarthost: "<%= ec["smarthost"] %>"
+    require_tls: <%= ec["require_tls"] %>
+    send_resolved: <%= ec["send_resolved"] %>
+    text: '{{ .Status | toUpper }}: {{ .CommonLabels.alertname }}{{ if .CommonLabels.instance }} on {{ .CommonLabels.instance }}{{ end }}{{ if .CommonLabels.vm_host }} (host: {{ .CommonLabels.vm_host }}){{ end }} — {{ .CommonAnnotations.summary }}'
+<% end %>
+<% end %>
+<% end %>