WIP Add prometheus rules, improve alerts

site-cookbooks/kosmos_prometheus/attributes/default.rb

@@ -1,6 +1,10 @@
 node.default["kosmos_prometheus"]["version"] = "3.13.0"
 node.default["kosmos_prometheus"]["checksum"] = "744d93324cc024d82089921737bd797474d7f1e5dbbfd1c6b387bad258538cb9"
 
+node.default["kosmos_prometheus"]["data_dir"] = "/var/lib/prometheus"
+node.default["kosmos_prometheus"]["config_dir"] = "/etc/prometheus"
+node.default["kosmos_prometheus"]["rules_dir"] = "#{node["kosmos_prometheus"]["config_dir"]}/rules"
+
 node.default["kosmos_prometheus"]["web"]["port"] = 9090
 
 node.default["kosmos_prometheus"]["alertmanager"]["version"] = "0.33.0"
@@ -12,10 +16,14 @@ node.default["kosmos_prometheus"]["node_exporter"]["checksum"] = "9f5ea48e5bc7b6
 node.default["kosmos_prometheus"]["alertmanager"]["route"] = {
   "receiver" => "xmpp",
   "group_by" => ["alertname"],
-  "group_wait" => "10s",
+  "group_wait" => "20s",
   "group_interval" => "30s",
   "repeat_interval" => "3h",
   "routes" => [
+    {
+      "receiver" => "xmpp",
+      "continue" => true,
+    },
     {
       "receiver" => "email",
       "matchers" => ['severity="critical"'],
@@ -66,5 +74,9 @@ node.default["kosmos_prometheus"]["jobs"] = {
 }
 
 node.default["kosmos_prometheus"]["rule_files"] = [
-  "/etc/prometheus/rules/*.yml"
+  "#{node["kosmos_prometheus"]["rules_dir"]}/*.yml"
+]
+
+node.default["kosmos_prometheus"]["rules"] = [
+  "node_exporter"
 ]

site-cookbooks/kosmos_prometheus/files/default/rules/node_exporter.yml

@@ -0,0 +1,10 @@
+groups:
+  - name: NodeExporter
+    rules:
+      - alert: NodeDown
+        expr: up{job="node"} == 0
+        for: 10s
+        labels:
+          severity: warning
+        annotations:
+          summary: "Node is down"

site-cookbooks/kosmos_prometheus/recipes/server.rb

@@ -10,6 +10,9 @@ checksum = node["kosmos_prometheus"]["checksum"]
 tarball = "#{Chef::Config[:file_cache_path]}/prometheus-#{version}.linux-amd64.tar.gz"
 binary_url = "https://github.com/prometheus/prometheus/releases/download/v#{version}/prometheus-#{version}.linux-amd64.tar.gz"
 
+data_dir   = node["kosmos_prometheus"]["data_dir"]
+config_dir = node["kosmos_prometheus"]["config_dir"]
+rules_dir  = node["kosmos_prometheus"]["rules_dir"]
 web_port   = node["kosmos_prometheus"]["web"]["port"]
 
 group "prometheus"
@@ -21,25 +24,35 @@ user "prometheus" do
   home "/nonexistent"
 end
 
-directory "/var/lib/prometheus" do
+directory data_dir do
   owner "prometheus"
   group "prometheus"
+  mode "0750"
+  recursive true
+end
+
+directory config_dir do
+  owner "root"
+  group "root"
+  mode "0755"
+  recursive true
+end
+
+directory rules_dir do
+  owner "root"
+  group "prometheus"
   mode "0755"
   recursive true
 end
 
-directory "/etc/prometheus" do
+node["kosmos_prometheus"]["rules"].each do |rule_name|
+  cookbook_file "#{rules_dir}/#{rule_name}.yml" do
+    source "rules/#{rule_name}.yml"
     owner "root"
-  group "root"
+    group "prometheus"
-  mode "0755"
+    mode "0644"
-  recursive true
+    notifies :reload, "service[prometheus]", :delayed
   end
-
-directory "/etc/prometheus/rules" do
-  owner "root"
-  group "root"
-  mode "0755"
-  recursive true
 end
 
 package %w(tar bzip2)
@@ -88,7 +101,7 @@ jobs = node["kosmos_prometheus"]["jobs"].merge(
   end
 )
 
-template "/etc/prometheus/prometheus.yml" do
+template "#{config_dir}/prometheus.yml" do
   source "prometheus.yml.erb"
   owner "root"
   group "prometheus"

site-cookbooks/kosmos_prometheus/templates/default/alertmanager.yml.erb

@@ -27,7 +27,7 @@ receivers:
     send_resolved: <%= wc["send_resolved"] %>
     payload:
       room: "<%= wc["room"] %>"
-      message: '{{ if eq .Status "resolved" }}✅{{ else if eq .CommonLabels.severity "warning" }}⚠️{{ else if eq .CommonLabels.severity "critical" }}🚨{{ end }} {{ .Status | toUpper }}: {{ .CommonLabels.alertname }}{{ if .CommonLabels.instance }} on {{ .CommonLabels.instance }}{{ end }}{{ if .CommonLabels.vm_host }} (host: {{ .CommonLabels.vm_host }}){{ end }} — {{ .CommonAnnotations.summary }}'
+      message: '{{ if eq .Status "resolved" }}✅{{ else if eq .CommonLabels.severity "warning" }}⚠️{{ else }}🚨{{ end }} {{ .Status | toUpper }}: {{ .CommonLabels.alertname }} ({{ len .Alerts }} alerts){{ range $i, $a := .Alerts }}{{ if lt $i 5 }}{{ "\n" }}    - {{ $a.Labels.instance }}{{ if $a.Labels.vm_host }} ({{ $a.Labels.vm_host }}){{ end }}: {{ $a.Annotations.summary }}{{ end }}{{ end }}{{ if gt (len .Alerts) 5 }}{{ "\n" }}  ... and more{{ end }}'
 <% end %>
 <% end %>
 <% if receiver["email_configs"] %>
@@ -38,7 +38,10 @@ receivers:
     smarthost: "<%= ec["smarthost"] %>"
     require_tls: <%= ec["require_tls"] %>
     send_resolved: <%= ec["send_resolved"] %>
-    text: '{{ .Status | toUpper }}: {{ .CommonLabels.alertname }}{{ if .CommonLabels.instance }} on {{ .CommonLabels.instance }}{{ end }}{{ if .CommonLabels.vm_host }} (host: {{ .CommonLabels.vm_host }}){{ end }} — {{ .CommonAnnotations.summary }}'
+    text: '{{ .Status | toUpper }}: {{ .CommonLabels.alertname }} ({{ len .Alerts }} alerts){{ range .Alerts }}
+---{{ range .Labels.SortedPairs }}
+  {{ .Name }}: {{ .Value }}{{ end }}{{ range .Annotations.SortedPairs }}
+  {{ .Name }}: {{ .Value }}{{ end }}{{ end }}'
 <% end %>
 <% end %>
 <% end %>