WIP Add prometheus rules, improve alerts

site-cookbooks/kosmos_prometheus/attributes/default.rb

@@ -1,6 +1,10 @@
 node.default["kosmos_prometheus"]["version"] = "3.13.0"
 node.default["kosmos_prometheus"]["checksum"] = "744d93324cc024d82089921737bd797474d7f1e5dbbfd1c6b387bad258538cb9"
 
+node.default["kosmos_prometheus"]["data_dir"] = "/var/lib/prometheus"
+node.default["kosmos_prometheus"]["config_dir"] = "/etc/prometheus"
+node.default["kosmos_prometheus"]["rules_dir"] = "#{node["kosmos_prometheus"]["config_dir"]}/rules"
+
 node.default["kosmos_prometheus"]["web"]["port"] = 9090
 
 node.default["kosmos_prometheus"]["alertmanager"]["version"] = "0.33.0"
@@ -12,10 +16,14 @@ node.default["kosmos_prometheus"]["node_exporter"]["checksum"] = "9f5ea48e5bc7b6
 node.default["kosmos_prometheus"]["alertmanager"]["route"] = {
   "receiver" => "xmpp",
   "group_by" => ["alertname"],
-  "group_wait" => "10s",
+  "group_wait" => "20s",
   "group_interval" => "30s",
   "repeat_interval" => "3h",
   "routes" => [
+    {
+      "receiver" => "xmpp",
+      "continue" => true,
+    },
     {
       "receiver" => "email",
       "matchers" => ['severity="critical"'],
@@ -66,5 +74,9 @@ node.default["kosmos_prometheus"]["jobs"] = {
 }
 
 node.default["kosmos_prometheus"]["rule_files"] = [
-  "/etc/prometheus/rules/*.yml"
+  "#{node["kosmos_prometheus"]["rules_dir"]}/*.yml"
+]
+
+node.default["kosmos_prometheus"]["rules"] = [
+  "node_exporter"
 ]