Merge pull request 'Sanitize node configs via script, and do it automatically in a Git hook' (#635) from dev/sanitize_node_configs into feature/prometheus_cookbook

Reviewed-on: #635

.githooks/pre-commit

@@ -0,0 +1,40 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+SCRIPT="$(git rev-parse --show-toplevel)/scripts/util/sanitize_node.sh"
+
+FILES=$(git diff --cached --name-only --diff-filter=ACM -- 'nodes/*.json' || true)
+
+[[ -z "$FILES" ]] && exit 0
+
+DIRTY=0
+
+while IFS= read -r file; do
+  staged_tmp=$(mktemp)
+  sanitized_tmp=$(mktemp)
+
+  # 1. get staged version
+  git show ":$file" > "$staged_tmp"
+
+  # 2. sanitize IN PLACE (on temp copy)
+  cp "$staged_tmp" "$sanitized_tmp"
+  "$SCRIPT" "$sanitized_tmp"
+
+  # 3. if sanitizer changed file, update working tree
+  if ! diff -q "$staged_tmp" "$sanitized_tmp" >/dev/null; then
+    cp "$sanitized_tmp" "$file"
+    echo "Sanitized: $file"
+    DIRTY=1
+  fi
+
+  rm -f "$staged_tmp" "$sanitized_tmp"
+
+done <<< "$FILES"
+
+if [[ "$DIRTY" -ne 0 ]]; then
+  echo ""
+  echo "Commit paused, sanitized files must be re-staged."
+  exit 1
+fi
+
+exit 0

README.md

@@ -28,6 +28,10 @@ Clone this repository, `cd` into it, and run:
 
     bundle install
 
+### Enable Git hooks
+
+    git config core.hooksPath .githooks
+
 ## Common tasks
 
 ### Bootstrap a new host server

clients/uploads-1.json

@@ -1,4 +0,0 @@
-{
-  "name": "uploads-1",
-  "public_key": "-----BEGIN PUBLIC KEY-----\nMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAwJwWaz8TeGv3SFlKzLMx\nqN8GTL/c0N9ppBvv8xNSS/yF9Y40SbL418uxYzm9hIhOXgIygIgLT2EKIXX32t+R\neOJCdYycQFM3At2fhMkjhuUW0gmDRcYBcBJLC5hLh2EZ+A8V7k4qgrBpPLOjEv48\nhQY0vuAw2DGndWr4QLh5NLUmQiOrfuzcZSSNCBOTIgUZgNmRd9QcCHDq4WDH3poa\nosJo4a9JGEGUL1irOivvEdyJPwEd2f++nYAdWwj8pjCYgpRshQlLhxOlylMx7MxB\nQt2bgJC9sahfbfJCOqdlCU3DMJL0bRUiuxK77WeSsxWBJmrsiF3+Ljs2Ix+s7fnS\nywIDAQAB\n-----END PUBLIC KEY-----\n"
-}

nodes/ejabberd-4.json

@@ -72,4 +72,4 @@
     "role[ldap_client]",
     "role[ejabberd]"
   ]
-}
+}

nodes/ejabberd-8.json

@@ -72,4 +72,4 @@
     "role[ldap_client]",
     "role[ejabberd]"
   ]
-}
+}

nodes/garage-11.json

@@ -63,4 +63,4 @@
     "role[kvm_guest]",
     "role[garage_node]"
   ]
-}
+}

nodes/garage-14.json

@@ -63,4 +63,4 @@
     "role[kvm_guest]",
     "role[garage_node]"
   ]
-}
+}

nodes/garage-15.json

@@ -63,4 +63,4 @@
     "role[kvm_guest]",
     "role[garage_node]"
   ]
-}
+}

nodes/garage-16.json

@@ -61,4 +61,4 @@
     "role[kvm_guest]",
     "role[garage_node]"
   ]
-}
+}

nodes/garage-7.json

@@ -63,4 +63,4 @@
     "role[kvm_guest]",
     "role[garage_node]"
   ]
-}
+}

nodes/garage-8.json

@@ -63,4 +63,4 @@
     "role[kvm_guest]",
     "role[garage_node]"
   ]
-}
+}

nodes/gitea-2.json

@@ -76,4 +76,4 @@
     "role[gitea]",
     "role[gitea_actions_runner]"
   ]
-}
+}

nodes/her.json

@@ -55,4 +55,4 @@
     "role[base]",
     "role[kvm_host]"
   ]
-}
+}

nodes/ldap-3.kosmos.org.json

@@ -61,4 +61,4 @@
     "role[kvm_guest]",
     "role[dirsrv_supplier]"
   ]
-}
+}

nodes/ldap-4.kosmos.org.json

@@ -55,4 +55,4 @@
     "role[kvm_guest]",
     "role[dirsrv_supplier]"
   ]
-}
+}

nodes/leo.kosmos.org.json

@@ -56,4 +56,4 @@
     "role[base]",
     "recipe[kosmos_kvm::host]"
   ]
-}
+}

nodes/mail.kosmos.org.json

@@ -68,4 +68,4 @@
     "role[kvm_guest]",
     "role[email_server]"
   ]
-}
+}

nodes/postgres-11.json

@@ -66,4 +66,4 @@
     "role[kvm_guest]",
     "role[postgresql_primary]"
   ]
-}
+}

nodes/postgres-12.json

@@ -61,4 +61,4 @@
     "role[kvm_guest]",
     "role[postgresql_replica]"
   ]
-}
+}

nodes/prometheus-1.json

@@ -59,4 +59,4 @@
     "role[kvm_guest]",
     "role[prometheus_server]"
   ]
-}
+}

nodes/rs-discourse-1.json

@@ -59,4 +59,4 @@
     "role[kvm_guest]",
     "role[remotestorage_discourse]"
   ]
-}
+}

nodes/rsk-mainnet-3.json

@@ -58,4 +58,4 @@
     "role[kvm_guest]",
     "role[rskj_mainnet]"
   ]
-}
+}

nodes/rsk-testnet-4.json

@@ -57,4 +57,4 @@
     "role[base]",
     "role[kvm_guest]"
   ]
-}
+}

nodes/rsk-testnet-6.json

@@ -57,4 +57,4 @@
     "role[kvm_guest]",
     "role[rskj_testnet]"
   ]
-}
+}

nodes/strfry-1.json

@@ -70,4 +70,4 @@
     "role[strfry]",
     "role[blossom]"
   ]
-}
+}

nodes/uploads-1.json

@@ -1,67 +0,0 @@
-{
-  "name": "uploads-1",
-  "normal": {
-    "knife_zero": {
-      "host": "10.1.1.98"
-    }
-  },
-  "automatic": {
-    "fqdn": "uploads-1",
-    "os": "linux",
-    "os_version": "5.4.0-128-generic",
-    "hostname": "uploads-1",
-    "ipaddress": "192.168.122.230",
-    "roles": [
-      "kvm_guest"
-    ],
-    "recipes": [
-      "kosmos-base",
-      "kosmos-base::default",
-      "kosmos_kvm::guest",
-      "kosmos-ejabberd::upload_service",
-      "apt::default",
-      "timezone_iii::default",
-      "timezone_iii::debian",
-      "ntp::default",
-      "ntp::apparmor",
-      "kosmos-base::systemd_emails",
-      "apt::unattended-upgrades",
-      "kosmos-base::firewall",
-      "kosmos-postfix::default",
-      "postfix::default",
-      "postfix::_common",
-      "postfix::_attributes",
-      "postfix::sasl_auth",
-      "hostname::default",
-      "kosmos-nginx::with_perl",
-      "nginx::default",
-      "nginx::package",
-      "nginx::ohai_plugin",
-      "nginx::repo",
-      "nginx::commons",
-      "nginx::commons_dir",
-      "nginx::commons_script",
-      "nginx::commons_conf",
-      "kosmos-nginx::firewall",
-      "kosmos-base::letsencrypt"
-    ],
-    "platform": "ubuntu",
-    "platform_version": "20.04",
-    "cloud": null,
-    "chef_packages": {
-      "chef": {
-        "version": "15.14.0",
-        "chef_root": "/opt/chef/embedded/lib/ruby/gems/2.6.0/gems/chef-15.14.0/lib"
-      },
-      "ohai": {
-        "version": "15.12.0",
-        "ohai_root": "/opt/chef/embedded/lib/ruby/gems/2.6.0/gems/ohai-15.12.0/lib/ohai"
-      }
-    }
-  },
-  "run_list": [
-    "role[base]",
-    "role[kvm_guest]",
-    "recipe[kosmos-ejabberd::upload_service]"
-  ]
-}

scripts/util/sanitize_node.sh

@@ -0,0 +1,69 @@
+#!/usr/bin/env bash
+# Strip the top-level "override" and "default" attributes from Chef node JSON files.
+#
+# Usage:
+#   ./scripts/util/strip_node_attrs.sh <node> [<node> ...]
+#   ./scripts/util/strip_node_attrs.sh --all
+#
+# A <node> may be a bare node name (e.g. "prometheus-1"), a path relative to the
+# repository root (e.g. "nodes/prometheus-1.json"), or an absolute path.
+#
+# Exit codes:
+#   0 - all target files processed successfully
+#   1 - one or more targets could not be processed (missing args, missing jq,
+#       file not found, invalid JSON, or write failure)
+
+set -uo pipefail
+
+command -v jq >/dev/null 2>&1 || exit 1
+
+root="$(git rev-parse --show-toplevel 2>/dev/null)" || root="$(pwd)"
+
+targets=()
+if [ "$#" -eq 0 ]; then
+  exit 1
+fi
+
+for arg in "$@"; do
+  if [ "$arg" = "--all" ]; then
+    shopt -s nullglob
+    for f in "$root"/nodes/*.json; do
+      targets+=("$f")
+    done
+    shopt -u nullglob
+    continue
+  fi
+
+  if [[ "$arg" == */* ]]; then
+    if [[ "$arg" = /* ]]; then
+      targets+=("$arg")
+    else
+      targets+=("$root/$arg")
+    fi
+  else
+    targets+=("$root/nodes/$arg.json")
+  fi
+done
+
+if [ "${#targets[@]}" -eq 0 ]; then
+  exit 1
+fi
+
+rc=0
+for file in "${targets[@]}"; do
+  [ -f "$file" ] || { rc=1; continue; }
+
+  jq -e . "$file" >/dev/null 2>&1 || { rc=1; continue; }
+
+  tmp="$(mktemp "${file}.XXXXXX")" || { rc=1; continue; }
+  if jq --indent 2 'del(.override, .default)' "$file" > "$tmp" 2>/dev/null; then
+    truncate -s -1 "$tmp" 2>/dev/null || true
+    mv "$tmp" "$file" || { rm -f "$tmp"; rc=1; continue; }
+  else
+    rm -f "$tmp"
+    rc=1
+    continue
+  fi
+done
+
+exit "$rc"