Send critical alerts via email (in addition to XMPP)

Requires relay restrictions to be configured explicitly

site-cookbooks/kosmos-postfix/recipes/default.rb

@@ -21,5 +21,6 @@ node.default["postfix"]["sasl"] = {
 node.default["postfix"]["main"]["relayhost"]                  = smtp_credentials["relayhost"]
 node.default["postfix"]["main"]["smtp_sasl_auth_enable"]      = "yes"
 node.default["postfix"]["main"]["smtp_sasl_security_options"] = "noanonymous"
+node.default["postfix"]["main"]["smtpd_relay_restrictions"] = "permit_mynetworks, reject"
 
 include_recipe "postfix::default"

site-cookbooks/kosmos_prometheus/attributes/default.rb

@@ -1,7 +1,7 @@
 node.default["kosmos_prometheus"]["version"] = "3.13.0"
 node.default["kosmos_prometheus"]["checksum"] = "744d93324cc024d82089921737bd797474d7f1e5dbbfd1c6b387bad258538cb9"
 
-node.default["kosmos_prometheus"]["web"]["port"] = "9090"
+node.default["kosmos_prometheus"]["web"]["port"] = 9090
 
 node.default["kosmos_prometheus"]["alertmanager"]["version"] = "0.33.0"
 node.default["kosmos_prometheus"]["alertmanager"]["checksum"] = "8ce11c42e8a6dfbbf93a59c0b193cb1329210b36d0c7ef3df7b745608675a1d1"
@@ -12,10 +12,16 @@ node.default["kosmos_prometheus"]["node_exporter"]["checksum"] = "9f5ea48e5bc7b6
 node.default["kosmos_prometheus"]["alertmanager"]["route"] = {
   "receiver" => "xmpp",
   "group_by" => ["alertname"],
-  "group_wait" => "30s",
+  "group_wait" => "10s",
-  "group_interval" => "5m",
+  "group_interval" => "30s",
   "repeat_interval" => "3h",
-  "routes" => [],
+  "routes" => [
+    {
+      "receiver" => "email",
+      "matchers" => ['severity="critical"'],
+      "continue" => true,
+    }
+  ],
 }
 
 node.default["kosmos_prometheus"]["alertmanager"]["receivers"] = [
@@ -27,12 +33,24 @@ node.default["kosmos_prometheus"]["alertmanager"]["receivers"] = [
         "room" => "info@kosmos.chat",
       }
     ]
+  },
+  {
+    "name" => "email",
+    "email_configs" => [
+      {
+        "send_resolved" => true,
+        "to" => "ops@kosmos.org",
+        "from" => "ops@kosmos.org",
+        "smarthost" => "localhost:25",
+        "require_tls" => false,
+      }
+    ]
   }
 ]
 
 node.default["kosmos_prometheus"]["global"] = {
-  "scrape_interval" => "30s",
+  "scrape_interval" => "15s",
-  "evaluation_interval" => "30s",
+  "evaluation_interval" => "15s",
 }
 
 node.default["kosmos_prometheus"]["alerting"] = {

site-cookbooks/kosmos_prometheus/templates/default/alertmanager.yml.erb

@@ -20,12 +20,25 @@ route:
 receivers:
 <% @receivers.each do |receiver| %>
 - name: <%= receiver["name"] %>
+<% if receiver["webhook_configs"] %>
 <% receiver["webhook_configs"].each do |wc| %>
   webhook_configs:
   - url: "<%= @webhook_url %>"
     send_resolved: <%= wc["send_resolved"] %>
     payload:
       room: "<%= wc["room"] %>"
-      message: '{{ if eq .Status "resolved" }}✅{{ else if eq .CommonLabels.severity "warning" }}⚠️{{ else }}🚨{{ end }} {{ .Status | toUpper }}: {{ .CommonLabels.alertname }}{{ if .CommonLabels.instance }} on {{ .CommonLabels.instance }}{{ end }}{{ if .CommonLabels.vm_host }} (host: {{ .CommonLabels.vm_host }}){{ end }} — {{ .CommonAnnotations.summary }}'
+      message: '{{ if eq .Status "resolved" }}✅{{ else if eq .CommonLabels.severity "warning" }}⚠️{{ else if eq .CommonLabels.severity "critical" }}🚨{{ end }} {{ .Status | toUpper }}: {{ .CommonLabels.alertname }}{{ if .CommonLabels.instance }} on {{ .CommonLabels.instance }}{{ end }}{{ if .CommonLabels.vm_host }} (host: {{ .CommonLabels.vm_host }}){{ end }} — {{ .CommonAnnotations.summary }}'
+<% end %>
+<% end %>
+<% if receiver["email_configs"] %>
+<% receiver["email_configs"].each do |ec| %>
+  email_configs:
+  - to: "<%= ec["to"] %>"
+    from: "<%= ec["from"] %>"
+    smarthost: "<%= ec["smarthost"] %>"
+    require_tls: <%= ec["require_tls"] %>
+    send_resolved: <%= ec["send_resolved"] %>
+    text: '{{ .Status | toUpper }}: {{ .CommonLabels.alertname }}{{ if .CommonLabels.instance }} on {{ .CommonLabels.instance }}{{ end }}{{ if .CommonLabels.vm_host }} (host: {{ .CommonLabels.vm_host }}){{ end }} — {{ .CommonAnnotations.summary }}'
+<% end %>
 <% end %>
 <% end %>