Set node name as prometheus instance name

And DRY up the recipe so it's easy and expressive to add more
metrics/targets

site-cookbooks/kosmos_prometheus/attributes/default.rb

@@ -13,7 +13,7 @@ node.default["kosmos_prometheus"]["global"] = {
 }
 
 node.default["kosmos_prometheus"]["jobs"] = {
-  "prometheus" => { "targets" => ["localhost:9090"] },
+  "prometheus" => { "targets" => [{ "target" => "localhost:9090", "instance" => "localhost" }] },
 }
 
 node.default["kosmos_prometheus"]["rule_files"] = []

site-cookbooks/kosmos_prometheus/recipes/server.rb

@@ -62,19 +62,27 @@ file "/usr/local/bin/prometheus" do
   notifies :restart, "service[prometheus]", :delayed
 end
 
-node_targets = search(:node, "role:base").map { |n| n["knife_zero"]["host"] }
-                .compact
-                .sort_by { |ip| ip.split(".").map(&:to_i) }
-                .map { |ip| "#{ip}:9100" }
-
-garage_targets = search(:node, "role:garage_node").map { |n| n["knife_zero"]["host"] }
-                 .compact
-                 .sort_by { |ip| ip.split(".").map(&:to_i) }
-                 .map { |ip| "#{ip}:3903" }
-
 jobs = node["kosmos_prometheus"]["jobs"].merge(
-  "node" => { "targets" => node_targets },
+{
-  "garage" => { "targets" => garage_targets }
+  # node exporter
+  "node" => {
+    "query" => "role:base",
+    "port" => 9100
+  },
+  # garage metrics
+  "garage" => {
+    "query" => "role:garage_node",
+    "port" => 3903
+  },
+}.transform_values do |config|
+    {
+      "targets" => search(:node, config["query"]).map do |n|
+        target = { "target" => "#{n['knife_zero']['host']}:#{config['port']}", "instance" => n.name }
+        target["env"] = n.chef_environment if n.chef_environment
+        target
+      end.compact.sort_by { |t| t["instance"] },
+    }
+  end
 )
 
 template "/etc/prometheus/prometheus.yml" do

site-cookbooks/kosmos_prometheus/templates/default/prometheus.yml.erb

@@ -14,11 +14,13 @@ scrape_configs:
 <% end %>
   metrics_path: "<%= job.fetch('metrics_path', '/metrics') %>"
   static_configs:
-    - targets: <%= Array(job['targets']) %>
+<% job['targets'].each do |t| %>
-<% if job['labels'] %>
+    - targets:
+        - <%= t['target'] %>
       labels:
-<% job['labels'].each do |label, label_config| %>
+        instance: <%= t['instance'] %>
-        <%= label %>: <%= label_config %>
+<% if t['env'] %>
+        env: <%= t['env'] %>
 <% end %>
 <% end %>
 <% end %>