WIP Add prometheus rules, improve alerts

This commit is contained in:
2026-07-06 18:01:22 +02:00
parent 1ae5210813
commit 2a14837f64
4 changed files with 57 additions and 19 deletions
@@ -27,7 +27,7 @@ receivers:
send_resolved: <%= wc["send_resolved"] %>
payload:
room: "<%= wc["room"] %>"
message: '{{ if eq .Status "resolved" }}✅{{ else if eq .CommonLabels.severity "warning" }}⚠️{{ else if eq .CommonLabels.severity "critical" }}🚨{{ end }} {{ .Status | toUpper }}: {{ .CommonLabels.alertname }}{{ if .CommonLabels.instance }} on {{ .CommonLabels.instance }}{{ end }}{{ if .CommonLabels.vm_host }} (host: {{ .CommonLabels.vm_host }}){{ end }} {{ .CommonAnnotations.summary }}'
message: '{{ if eq .Status "resolved" }}✅{{ else if eq .CommonLabels.severity "warning" }}⚠️{{ else }}🚨{{ end }} {{ .Status | toUpper }}: {{ .CommonLabels.alertname }} ({{ len .Alerts }} alerts){{ range $i, $a := .Alerts }}{{ if lt $i 5 }}{{ "\n" }} - {{ $a.Labels.instance }}{{ if $a.Labels.vm_host }} ({{ $a.Labels.vm_host }}){{ end }}: {{ $a.Annotations.summary }}{{ end }}{{ end }}{{ if gt (len .Alerts) 5 }}{{ "\n" }} ... and more{{ end }}'
<% end %>
<% end %>
<% if receiver["email_configs"] %>
@@ -38,7 +38,10 @@ receivers:
smarthost: "<%= ec["smarthost"] %>"
require_tls: <%= ec["require_tls"] %>
send_resolved: <%= ec["send_resolved"] %>
text: '{{ .Status | toUpper }}: {{ .CommonLabels.alertname }}{{ if .CommonLabels.instance }} on {{ .CommonLabels.instance }}{{ end }}{{ if .CommonLabels.vm_host }} (host: {{ .CommonLabels.vm_host }}){{ end }} — {{ .CommonAnnotations.summary }}'
text: '{{ .Status | toUpper }}: {{ .CommonLabels.alertname }} ({{ len .Alerts }} alerts){{ range .Alerts }}
---{{ range .Labels.SortedPairs }}
{{ .Name }}: {{ .Value }}{{ end }}{{ range .Annotations.SortedPairs }}
{{ .Name }}: {{ .Value }}{{ end }}{{ end }}'
<% end %>
<% end %>
<% end %>