WIP Add prometheus rules, improve alerts

site-cookbooks/kosmos_prometheus/recipes/server.rb

@@ -10,7 +10,10 @@ checksum = node["kosmos_prometheus"]["checksum"]
 tarball = "#{Chef::Config[:file_cache_path]}/prometheus-#{version}.linux-amd64.tar.gz"
 binary_url = "https://github.com/prometheus/prometheus/releases/download/v#{version}/prometheus-#{version}.linux-amd64.tar.gz"
 
-web_port = node["kosmos_prometheus"]["web"]["port"]
+data_dir   = node["kosmos_prometheus"]["data_dir"]
+config_dir = node["kosmos_prometheus"]["config_dir"]
+rules_dir  = node["kosmos_prometheus"]["rules_dir"]
+web_port   = node["kosmos_prometheus"]["web"]["port"]
 
 group "prometheus"
 
@@ -21,25 +24,35 @@ user "prometheus" do
   home "/nonexistent"
 end
 
-directory "/var/lib/prometheus" do
+directory data_dir do
   owner "prometheus"
   group "prometheus"
+  mode "0750"
+  recursive true
+end
+
+directory config_dir do
+  owner "root"
+  group "root"
+  mode "0755"
+  recursive true
+end
+
+directory rules_dir do
+  owner "root"
+  group "prometheus"
   mode "0755"
   recursive true
 end
 
-directory "/etc/prometheus" do
-  owner "root"
-  group "root"
-  mode "0755"
-  recursive true
-end
-
-directory "/etc/prometheus/rules" do
-  owner "root"
-  group "root"
-  mode "0755"
-  recursive true
+node["kosmos_prometheus"]["rules"].each do |rule_name|
+  cookbook_file "#{rules_dir}/#{rule_name}.yml" do
+    source "rules/#{rule_name}.yml"
+    owner "root"
+    group "prometheus"
+    mode "0644"
+    notifies :reload, "service[prometheus]", :delayed
+  end
 end
 
 package %w(tar bzip2)
@@ -88,7 +101,7 @@ jobs = node["kosmos_prometheus"]["jobs"].merge(
   end
 )
 
-template "/etc/prometheus/prometheus.yml" do
+template "#{config_dir}/prometheus.yml" do
   source "prometheus.yml.erb"
   owner "root"
   group "prometheus"