Set node name as prometheus instance name

And DRY up the recipe so it's easy and expressive to add more
metrics/targets

nodes/leo.kosmos.org.json

@@ -1,12 +1,13 @@
 {
-  "name": "leo",
+  "name": "leo.kosmos.org",
+  "chef_environment": "production",
   "normal": {
     "knife_zero": {
-      "host": "leo.kosmos.org"
+      "host": "10.1.1.204"
     }
   },
   "automatic": {
-    "fqdn": "leo",
+    "fqdn": "leo.kosmos.org",
     "os": "linux",
     "os_version": "5.15.0-173-generic",
     "hostname": "leo",
@@ -17,6 +18,7 @@
     "recipes": [
       "kosmos-base",
       "kosmos-base::default",
+      "kosmos_prometheus::node_exporter",
       "kosmos_kvm::host",
       "apt::default",
       "timezone_iii::default",
@@ -32,7 +34,8 @@
       "postfix::_common",
       "postfix::_attributes",
       "postfix::sasl_auth",
-      "hostname::default"
+      "hostname::default",
+      "firewall::default"
     ],
     "platform": "ubuntu",
     "platform_version": "22.04",

site-cookbooks/kosmos_prometheus/attributes/default.rb

@@ -13,7 +13,7 @@ node.default["kosmos_prometheus"]["global"] = {
 }
 
 node.default["kosmos_prometheus"]["jobs"] = {
-  "prometheus" => { "targets" => ["localhost:9090"] },
+  "prometheus" => { "targets" => [{ "target" => "localhost:9090", "instance" => "localhost" }] },
 }
 
 node.default["kosmos_prometheus"]["rule_files"] = []

site-cookbooks/kosmos_prometheus/recipes/server.rb

@@ -62,19 +62,27 @@ file "/usr/local/bin/prometheus" do
   notifies :restart, "service[prometheus]", :delayed
 end
 
-node_targets = search(:node, "role:base").map { |n| n["knife_zero"]["host"] }
-                .compact
-                .sort_by { |ip| ip.split(".").map(&:to_i) }
-                .map { |ip| "#{ip}:9100" }
-
-garage_targets = search(:node, "role:garage_node").map { |n| n["knife_zero"]["host"] }
-                 .compact
-                 .sort_by { |ip| ip.split(".").map(&:to_i) }
-                 .map { |ip| "#{ip}:3903" }
-
 jobs = node["kosmos_prometheus"]["jobs"].merge(
-  "node" => { "targets" => node_targets },
-  "garage" => { "targets" => garage_targets }
+{
+  # node exporter
+  "node" => {
+    "query" => "role:base",
+    "port" => 9100
+  },
+  # garage metrics
+  "garage" => {
+    "query" => "role:garage_node",
+    "port" => 3903
+  },
+}.transform_values do |config|
+    {
+      "targets" => search(:node, config["query"]).map do |n|
+        target = { "target" => "#{n['knife_zero']['host']}:#{config['port']}", "instance" => n.name }
+        target["env"] = n.chef_environment if n.chef_environment
+        target
+      end.compact.sort_by { |t| t["instance"] },
+    }
+  end
 )
 
 template "/etc/prometheus/prometheus.yml" do

site-cookbooks/kosmos_prometheus/templates/default/prometheus.yml.erb

@@ -14,11 +14,13 @@ scrape_configs:
 <% end %>
   metrics_path: "<%= job.fetch('metrics_path', '/metrics') %>"
   static_configs:
-    - targets: <%= Array(job['targets']) %>
-<% if job['labels'] %>
+<% job['targets'].each do |t| %>
+    - targets:
+        - <%= t['target'] %>
       labels:
-<% job['labels'].each do |label, label_config| %>
-        <%= label %>: <%= label_config %>
+        instance: <%= t['instance'] %>
+<% if t['env'] %>
+        env: <%= t['env'] %>
 <% end %>
 <% end %>
 <% end %>