diff --git a/site-cookbooks/kosmos_prometheus/attributes/default.rb b/site-cookbooks/kosmos_prometheus/attributes/default.rb index 8d1fcab..16629ad 100644 --- a/site-cookbooks/kosmos_prometheus/attributes/default.rb +++ b/site-cookbooks/kosmos_prometheus/attributes/default.rb @@ -12,10 +12,16 @@ node.default["kosmos_prometheus"]["node_exporter"]["checksum"] = "9f5ea48e5bc7b6 node.default["kosmos_prometheus"]["alertmanager"]["route"] = { "receiver" => "xmpp", "group_by" => ["alertname"], - "group_wait" => "30s", - "group_interval" => "5m", + "group_wait" => "10s", + "group_interval" => "30s", "repeat_interval" => "3h", - "routes" => [], + "routes" => [ + { + "receiver" => "email", + "matchers" => ['severity="critical"'], + "continue" => true, + } + ], } node.default["kosmos_prometheus"]["alertmanager"]["receivers"] = [ @@ -27,6 +33,18 @@ node.default["kosmos_prometheus"]["alertmanager"]["receivers"] = [ "room" => "info@kosmos.chat", } ] + }, + { + "name" => "email", + "email_configs" => [ + { + "send_resolved" => true, + "to" => "ops@kosmos.org", + "from" => "ops@kosmos.org", + "smarthost" => "localhost:25", + "require_tls" => false, + } + ] } ] diff --git a/site-cookbooks/kosmos_prometheus/templates/default/alertmanager.yml.erb b/site-cookbooks/kosmos_prometheus/templates/default/alertmanager.yml.erb index d9dd8d8..0fa3ad9 100644 --- a/site-cookbooks/kosmos_prometheus/templates/default/alertmanager.yml.erb +++ b/site-cookbooks/kosmos_prometheus/templates/default/alertmanager.yml.erb @@ -20,6 +20,7 @@ route: receivers: <% @receivers.each do |receiver| %> - name: <%= receiver["name"] %> +<% if receiver["webhook_configs"] %> <% receiver["webhook_configs"].each do |wc| %> webhook_configs: - url: "<%= @webhook_url %>" @@ -29,3 +30,15 @@ receivers: message: '{{ if eq .Status "resolved" }}✅{{ else if eq .CommonLabels.severity "warning" }}⚠️{{ else if eq .CommonLabels.severity "critical" }}🚨{{ end }} {{ .Status | toUpper }}: {{ .CommonLabels.alertname }}{{ if .CommonLabels.instance }} on {{ .CommonLabels.instance }}{{ end }}{{ if .CommonLabels.vm_host }} (host: {{ .CommonLabels.vm_host }}){{ end }} — {{ .CommonAnnotations.summary }}' <% end %> <% end %> +<% if receiver["email_configs"] %> +<% receiver["email_configs"].each do |ec| %> + email_configs: + - to: "<%= ec["to"] %>" + from: "<%= ec["from"] %>" + smarthost: "<%= ec["smarthost"] %>" + require_tls: <%= ec["require_tls"] %> + send_resolved: <%= ec["send_resolved"] %> + text: '{{ .Status | toUpper }}: {{ .CommonLabels.alertname }}{{ if .CommonLabels.instance }} on {{ .CommonLabels.instance }}{{ end }}{{ if .CommonLabels.vm_host }} (host: {{ .CommonLabels.vm_host }}){{ end }} — {{ .CommonAnnotations.summary }}' +<% end %> +<% end %> +<% end %>