diff --git a/site-cookbooks/kosmos_prometheus/attributes/default.rb b/site-cookbooks/kosmos_prometheus/attributes/default.rb index 16629ad..4be544c 100644 --- a/site-cookbooks/kosmos_prometheus/attributes/default.rb +++ b/site-cookbooks/kosmos_prometheus/attributes/default.rb @@ -1,6 +1,10 @@ node.default["kosmos_prometheus"]["version"] = "3.13.0" node.default["kosmos_prometheus"]["checksum"] = "744d93324cc024d82089921737bd797474d7f1e5dbbfd1c6b387bad258538cb9" +node.default["kosmos_prometheus"]["data_dir"] = "/var/lib/prometheus" +node.default["kosmos_prometheus"]["config_dir"] = "/etc/prometheus" +node.default["kosmos_prometheus"]["rules_dir"] = "#{node["kosmos_prometheus"]["config_dir"]}/rules" + node.default["kosmos_prometheus"]["web"]["port"] = 9090 node.default["kosmos_prometheus"]["alertmanager"]["version"] = "0.33.0" @@ -12,10 +16,14 @@ node.default["kosmos_prometheus"]["node_exporter"]["checksum"] = "9f5ea48e5bc7b6 node.default["kosmos_prometheus"]["alertmanager"]["route"] = { "receiver" => "xmpp", "group_by" => ["alertname"], - "group_wait" => "10s", + "group_wait" => "20s", "group_interval" => "30s", "repeat_interval" => "3h", "routes" => [ + { + "receiver" => "xmpp", + "continue" => true, + }, { "receiver" => "email", "matchers" => ['severity="critical"'], @@ -66,5 +74,9 @@ node.default["kosmos_prometheus"]["jobs"] = { } node.default["kosmos_prometheus"]["rule_files"] = [ - "/etc/prometheus/rules/*.yml" + "#{node["kosmos_prometheus"]["rules_dir"]}/*.yml" +] + +node.default["kosmos_prometheus"]["rules"] = [ + "node_exporter" ] diff --git a/site-cookbooks/kosmos_prometheus/files/default/rules/node_exporter.yml b/site-cookbooks/kosmos_prometheus/files/default/rules/node_exporter.yml new file mode 100644 index 0000000..b3ffbb1 --- /dev/null +++ b/site-cookbooks/kosmos_prometheus/files/default/rules/node_exporter.yml @@ -0,0 +1,10 @@ +groups: + - name: NodeExporter + rules: + - alert: NodeDown + expr: up{job="node"} == 0 + for: 10s + labels: + severity: warning + annotations: + summary: "Node is down" diff --git a/site-cookbooks/kosmos_prometheus/recipes/server.rb b/site-cookbooks/kosmos_prometheus/recipes/server.rb index 22f6a73..093265b 100644 --- a/site-cookbooks/kosmos_prometheus/recipes/server.rb +++ b/site-cookbooks/kosmos_prometheus/recipes/server.rb @@ -10,7 +10,10 @@ checksum = node["kosmos_prometheus"]["checksum"] tarball = "#{Chef::Config[:file_cache_path]}/prometheus-#{version}.linux-amd64.tar.gz" binary_url = "https://github.com/prometheus/prometheus/releases/download/v#{version}/prometheus-#{version}.linux-amd64.tar.gz" -web_port = node["kosmos_prometheus"]["web"]["port"] +data_dir = node["kosmos_prometheus"]["data_dir"] +config_dir = node["kosmos_prometheus"]["config_dir"] +rules_dir = node["kosmos_prometheus"]["rules_dir"] +web_port = node["kosmos_prometheus"]["web"]["port"] group "prometheus" @@ -21,25 +24,35 @@ user "prometheus" do home "/nonexistent" end -directory "/var/lib/prometheus" do +directory data_dir do owner "prometheus" group "prometheus" + mode "0750" + recursive true +end + +directory config_dir do + owner "root" + group "root" + mode "0755" + recursive true +end + +directory rules_dir do + owner "root" + group "prometheus" mode "0755" recursive true end -directory "/etc/prometheus" do - owner "root" - group "root" - mode "0755" - recursive true -end - -directory "/etc/prometheus/rules" do - owner "root" - group "root" - mode "0755" - recursive true +node["kosmos_prometheus"]["rules"].each do |rule_name| + cookbook_file "#{rules_dir}/#{rule_name}.yml" do + source "rules/#{rule_name}.yml" + owner "root" + group "prometheus" + mode "0644" + notifies :reload, "service[prometheus]", :delayed + end end package %w(tar bzip2) @@ -88,7 +101,7 @@ jobs = node["kosmos_prometheus"]["jobs"].merge( end ) -template "/etc/prometheus/prometheus.yml" do +template "#{config_dir}/prometheus.yml" do source "prometheus.yml.erb" owner "root" group "prometheus" diff --git a/site-cookbooks/kosmos_prometheus/templates/default/alertmanager.yml.erb b/site-cookbooks/kosmos_prometheus/templates/default/alertmanager.yml.erb index 0fa3ad9..451a3fd 100644 --- a/site-cookbooks/kosmos_prometheus/templates/default/alertmanager.yml.erb +++ b/site-cookbooks/kosmos_prometheus/templates/default/alertmanager.yml.erb @@ -27,7 +27,7 @@ receivers: send_resolved: <%= wc["send_resolved"] %> payload: room: "<%= wc["room"] %>" - message: '{{ if eq .Status "resolved" }}✅{{ else if eq .CommonLabels.severity "warning" }}⚠️{{ else if eq .CommonLabels.severity "critical" }}🚨{{ end }} {{ .Status | toUpper }}: {{ .CommonLabels.alertname }}{{ if .CommonLabels.instance }} on {{ .CommonLabels.instance }}{{ end }}{{ if .CommonLabels.vm_host }} (host: {{ .CommonLabels.vm_host }}){{ end }} — {{ .CommonAnnotations.summary }}' + message: '{{ if eq .Status "resolved" }}✅{{ else if eq .CommonLabels.severity "warning" }}⚠️{{ else }}🚨{{ end }} {{ .Status | toUpper }}: {{ .CommonLabels.alertname }} ({{ len .Alerts }} alerts){{ range $i, $a := .Alerts }}{{ if lt $i 5 }}{{ "\n" }} - {{ $a.Labels.instance }}{{ if $a.Labels.vm_host }} ({{ $a.Labels.vm_host }}){{ end }}: {{ $a.Annotations.summary }}{{ end }}{{ end }}{{ if gt (len .Alerts) 5 }}{{ "\n" }} ... and more{{ end }}' <% end %> <% end %> <% if receiver["email_configs"] %> @@ -38,7 +38,10 @@ receivers: smarthost: "<%= ec["smarthost"] %>" require_tls: <%= ec["require_tls"] %> send_resolved: <%= ec["send_resolved"] %> - text: '{{ .Status | toUpper }}: {{ .CommonLabels.alertname }}{{ if .CommonLabels.instance }} on {{ .CommonLabels.instance }}{{ end }}{{ if .CommonLabels.vm_host }} (host: {{ .CommonLabels.vm_host }}){{ end }} — {{ .CommonAnnotations.summary }}' + text: '{{ .Status | toUpper }}: {{ .CommonLabels.alertname }} ({{ len .Alerts }} alerts){{ range .Alerts }} +---{{ range .Labels.SortedPairs }} + {{ .Name }}: {{ .Value }}{{ end }}{{ range .Annotations.SortedPairs }} + {{ .Name }}: {{ .Value }}{{ end }}{{ end }}' <% end %> <% end %> <% end %>