From 290af8177a377cc6c132850bd4d9272e0dd773ca Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?R=C3=A2u=20Cao?= <raucao@kosmos.org>
Date: Wed, 8 Apr 2026 13:21:42 +0400
Subject: [PATCH 01/18] Refactor postgres server recipes/resource

---
 .../kosmos_postgresql/attributes/default.rb   |  5 ++
 .../kosmos_postgresql/libraries/helpers.rb    | 10 +++-
 .../kosmos_postgresql/recipes/primary.rb      | 24 ----------
 .../kosmos_postgresql/recipes/replica.rb      | 48 +++++--------------
 .../kosmos_postgresql/resources/server.rb     | 18 +++++++
 5 files changed, 44 insertions(+), 61 deletions(-)

diff --git a/site-cookbooks/kosmos_postgresql/attributes/default.rb b/site-cookbooks/kosmos_postgresql/attributes/default.rb
index f3daf9b..29aae1f 100644
--- a/site-cookbooks/kosmos_postgresql/attributes/default.rb
+++ b/site-cookbooks/kosmos_postgresql/attributes/default.rb
@@ -1,3 +1,8 @@
+node.default['kosmos_postgresql']['postgresql_version'] = "14"
+
 # This is set to false by default, and set to true in the server resource
 # for replicas.
 node.default['kosmos_postgresql']['ready_to_set_up_replica'] = false
+
+# Address space from which clients are allowed to connect
+node.default['kosmos_postgresql']['access_addr'] = "10.1.1.0/24"
diff --git a/site-cookbooks/kosmos_postgresql/libraries/helpers.rb b/site-cookbooks/kosmos_postgresql/libraries/helpers.rb
index 18e245d..5de6e76 100644
--- a/site-cookbooks/kosmos_postgresql/libraries/helpers.rb
+++ b/site-cookbooks/kosmos_postgresql/libraries/helpers.rb
@@ -36,10 +36,16 @@ class Chef
       end
     end
 
-    def postgresql_service_name
-      postgresql_version = "12"
+    def postgresql_version
+      node['kosmos_postgresql']['postgresql_version']
+    end
 
+    def postgresql_service
       "postgresql@#{postgresql_version}-main"
     end
+
+    def postgresql_data_dir
+      "/var/lib/postgresql/#{postgresql_version}/main"
+    end
   end
 end
diff --git a/site-cookbooks/kosmos_postgresql/recipes/primary.rb b/site-cookbooks/kosmos_postgresql/recipes/primary.rb
index de7466f..406e2ae 100644
--- a/site-cookbooks/kosmos_postgresql/recipes/primary.rb
+++ b/site-cookbooks/kosmos_postgresql/recipes/primary.rb
@@ -3,31 +3,7 @@
 # Recipe:: primary
 #
 
-postgresql_version = "12"
-postgresql_service = "postgresql@#{postgresql_version}-main"
-
-service postgresql_service do
-  supports restart: true, status: true, reload: true
-end
-
 postgresql_custom_server postgresql_version do
   role "primary"
 end
 
-postgresql_access "zerotier members" do
-  access_type "host"
-  access_db "all"
-  access_user "all"
-  access_addr "10.1.1.0/24"
-  access_method "md5"
-  notifies :reload, "service[#{postgresql_service}]", :immediately
-end
-
-postgresql_access "zerotier members replication" do
-  access_type "host"
-  access_db "replication"
-  access_user "replication"
-  access_addr "10.1.1.0/24"
-  access_method "md5"
-  notifies :reload, "service[#{postgresql_service}]", :immediately
-end
diff --git a/site-cookbooks/kosmos_postgresql/recipes/replica.rb b/site-cookbooks/kosmos_postgresql/recipes/replica.rb
index b1dd345..69d5cd0 100644
--- a/site-cookbooks/kosmos_postgresql/recipes/replica.rb
+++ b/site-cookbooks/kosmos_postgresql/recipes/replica.rb
@@ -3,54 +3,32 @@
 # Recipe:: replica
 #
 
-postgresql_version = "12"
-postgresql_service = "postgresql@#{postgresql_version}-main"
+service postgresql_service do
+  supports restart: true, status: true, reload: true
+end
 
 postgresql_custom_server postgresql_version do
   role "replica"
 end
 
-service postgresql_service do
-  supports restart: true, status: true, reload: true
-end
-
 postgresql_data_bag_item = data_bag_item('credentials', 'postgresql')
 
 primary = postgresql_primary
 
-unless primary.nil?
-  # TODO
-  postgresql_data_dir = "/var/lib/postgresql/#{postgresql_version}/main"
+if primary.nil?
+  Chef::Log.warn("No PostgreSQL primary node found. Skipping replication setup.")
+  return
+end
 
-  # FIXME get zerotier IP
-  execute "set up replication" do
-    command <<-EOF
+execute "set up replication" do
+  command <<-EOF
 systemctl stop #{postgresql_service}
 mv #{postgresql_data_dir} #{postgresql_data_dir}.old
 pg_basebackup -h pg.kosmos.local -U replication -D #{postgresql_data_dir} -R
 chown -R postgres:postgres #{postgresql_data_dir}
 systemctl start #{postgresql_service}
-    EOF
-    environment 'PGPASSWORD' => postgresql_data_bag_item['replication_password']
-    sensitive true
-    not_if { ::File.exist? "#{postgresql_data_dir}/standby.signal" }
-  end
-
-  postgresql_access "zerotier members" do
-    access_type "host"
-    access_db "all"
-    access_user "all"
-    access_addr "10.1.1.0/24"
-    access_method "md5"
-    notifies :reload, "service[#{postgresql_service}]", :immediately
-  end
-
-  postgresql_access "zerotier members replication" do
-    access_type "host"
-    access_db "replication"
-    access_user "replication"
-    access_addr "10.1.1.0/24"
-    access_method "md5"
-    notifies :reload, "service[#{postgresql_service}]", :immediately
-  end
+  EOF
+  environment 'PGPASSWORD' => postgresql_data_bag_item['replication_password']
+  sensitive true
+  not_if { ::File.exist? "#{postgresql_data_dir}/standby.signal" }
 end
diff --git a/site-cookbooks/kosmos_postgresql/resources/server.rb b/site-cookbooks/kosmos_postgresql/resources/server.rb
index d5b38da..e024f78 100644
--- a/site-cookbooks/kosmos_postgresql/resources/server.rb
+++ b/site-cookbooks/kosmos_postgresql/resources/server.rb
@@ -70,6 +70,24 @@ action :create do
     replication true
     password postgresql_credentials['replication_password']
   end
+
+  postgresql_access "all members" do
+    access_type "host"
+    access_db "all"
+    access_user "all"
+    access_addr node['kosmos_postgresql']['access_addr']
+    access_method "md5"
+    notifies :reload, "service[#{postgresql_service}]", :immediately
+  end
+
+  postgresql_access "replication members" do
+    access_type "host"
+    access_db "replication"
+    access_user "replication"
+    access_addr node['kosmos_postgresql']['access_addr']
+    access_method "md5"
+    notifies :reload, "service[#{postgresql_service}]", :immediately
+  end
 end
 
 action_class do

From 6583cd7010983b4ae8a88e83db2e678b8bbd10b5 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?R=C3=A2u=20Cao?= <raucao@kosmos.org>
Date: Wed, 8 Apr 2026 13:22:17 +0400
Subject: [PATCH 02/18] Upgrade WAL config for PG14

---
 site-cookbooks/kosmos_postgresql/resources/server.rb | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/site-cookbooks/kosmos_postgresql/resources/server.rb b/site-cookbooks/kosmos_postgresql/resources/server.rb
index e024f78..f08a70d 100644
--- a/site-cookbooks/kosmos_postgresql/resources/server.rb
+++ b/site-cookbooks/kosmos_postgresql/resources/server.rb
@@ -56,7 +56,7 @@ action :create do
     timezone: "UTC", # default is GMT
     listen_addresses: "0.0.0.0",
     promote_trigger_file: "#{postgresql_data_dir}/failover.trigger",
-    wal_keep_segments: 256
+    wal_keep_size: 4096 # 256 segments, 16MB each
   }
 
   postgresql_server_conf "main" do

From bc3f291bd229b200112758caedf3e4707e39b23f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?R=C3=A2u=20Cao?= <raucao@kosmos.org>
Date: Wed, 8 Apr 2026 15:41:56 +0400
Subject: [PATCH 03/18] WIP Prepare postgres for migration by replication

---
 doc/postgres-migration.md                     | 271 ++++++++++++++++++
 roles/postgresql_replica_logical.rb           |   8 +
 .../files/create_publication.sh               |  31 ++
 .../files/create_publications.sh              |  34 +++
 .../files/drop_publications.sh                |  34 +++
 .../files/drop_subscriptions.sh               |  29 ++
 .../files/dump_all_databases.sh               |   9 +
 .../kosmos_postgresql/files/dump_database.sh  |  10 +
 .../kosmos_postgresql/files/fix_sequences.sh  |  35 +++
 .../files/fix_sequences_in_all_databases.sh   |  38 +++
 .../files/list_publications.sh                |   5 +
 .../files/list_replication_slots.sh           |   5 +
 .../files/list_subscriptions.sh               |   5 +
 .../files/restore_all_databases.sh            |  12 +
 .../files/restore_database.sh                 |  14 +
 .../recipes/management_scripts.rb             | 121 ++++++++
 .../kosmos_postgresql/recipes/primary.rb      |   1 -
 .../kosmos_postgresql/recipes/replica.rb      |   7 +-
 .../recipes/replica_logical.rb                |  15 +
 .../kosmos_postgresql/resources/server.rb     |   6 +-
 .../templates/create_subscription.sh.erb      |  31 ++
 .../templates/create_subscriptions.sh.erb     |  34 +++
 22 files changed, 748 insertions(+), 7 deletions(-)
 create mode 100644 doc/postgres-migration.md
 create mode 100644 roles/postgresql_replica_logical.rb
 create mode 100644 site-cookbooks/kosmos_postgresql/files/create_publication.sh
 create mode 100644 site-cookbooks/kosmos_postgresql/files/create_publications.sh
 create mode 100644 site-cookbooks/kosmos_postgresql/files/drop_publications.sh
 create mode 100644 site-cookbooks/kosmos_postgresql/files/drop_subscriptions.sh
 create mode 100644 site-cookbooks/kosmos_postgresql/files/dump_all_databases.sh
 create mode 100644 site-cookbooks/kosmos_postgresql/files/dump_database.sh
 create mode 100644 site-cookbooks/kosmos_postgresql/files/fix_sequences.sh
 create mode 100644 site-cookbooks/kosmos_postgresql/files/fix_sequences_in_all_databases.sh
 create mode 100644 site-cookbooks/kosmos_postgresql/files/list_publications.sh
 create mode 100644 site-cookbooks/kosmos_postgresql/files/list_replication_slots.sh
 create mode 100644 site-cookbooks/kosmos_postgresql/files/list_subscriptions.sh
 create mode 100644 site-cookbooks/kosmos_postgresql/files/restore_all_databases.sh
 create mode 100644 site-cookbooks/kosmos_postgresql/files/restore_database.sh
 create mode 100644 site-cookbooks/kosmos_postgresql/recipes/management_scripts.rb
 create mode 100644 site-cookbooks/kosmos_postgresql/recipes/replica_logical.rb
 create mode 100644 site-cookbooks/kosmos_postgresql/templates/create_subscription.sh.erb
 create mode 100644 site-cookbooks/kosmos_postgresql/templates/create_subscriptions.sh.erb

diff --git a/doc/postgres-migration.md b/doc/postgres-migration.md
new file mode 100644
index 0000000..e9d24d2
--- /dev/null
+++ b/doc/postgres-migration.md
@@ -0,0 +1,271 @@
+# Migrating PostgreSQL cluster to a new major version
+
+## Summary
+
+1. Dump from a replica
+2. Restore to fresh VM running new major version
+3. Add logical replication for delta sync from current/old primary
+4. Switch primary to new server
+5. Remove logical replication on new server
+
+## Runbook
+
+* Primary host: `PRIMARY_HOST`
+* Replica host: `REPLICA_HOST`
+* New PG14 host: `NEW_HOST`
+* PostgreSQL superuser: `postgres`
+* Running locally on each machine via `sudo -u postgres`
+
+Adjust hostnames/IPs/etc. where needed.
+
+---
+
+### 🟢 0. PRIMARY — Pre-checks
+
+```bash
+sudo -u postgres psql -c "SHOW wal_level;"
+sudo -u postgres psql -c "SHOW max_replication_slots;"
+```
+
+If needed, edit config:
+
+```bash
+sudo -u postgres vi $PGDATA/postgresql.conf
+```
+
+Ensure:
+
+```conf
+wal_level = logical
+max_replication_slots = 10
+```
+
+Restart if changed:
+
+```bash
+sudo systemctl restart postgresql
+```
+
+---
+
+### 🔵🟡 3. Create keypair for syncing dump later
+
+🔵 On NEW_HOST:
+
+```bash
+sudo mkdir -p /home/postgres/.ssh && \
+sudo chown -R postgres:postgres /home/postgres && \
+sudo chmod 700 /home/postgres/.ssh && \
+sudo -u postgres bash -c 'ssh-keygen -t ecdsa -b 256 -f /home/postgres/.ssh/id_ecdsa -N "" -C "postgres@$(hostname)"' && \
+sudo cat /home/postgres/.ssh/id_ecdsa.pub
+```
+
+Copy the public key from the above output
+
+🟡 On replica:
+
+```bash
+sudo mkdir -p /home/postgres/.ssh && \
+sudo chown -R postgres:postgres /home/postgres && \
+sudo chmod 700 /home/postgres/.ssh && \
+echo [public_key] | sudo tee /home/postgres/.ssh/authorized_keys > /dev/null && \
+sudo chmod 700 /home/postgres/.ssh
+```
+
+---
+
+### 🟢 1. PRIMARY — Create publication and replication slots
+
+```bash
+sudo -u postgres pg_create_replication_publications
+```
+
+or
+
+```bash
+sudo -u postgres pg_create_replication_publication [db_name]
+```
+
+Listing publications and slots:
+
+```bash
+sudo -u postgres pg_list_replication_publications
+sudo -u postgres pg_list_replication_slots
+```
+
+---
+
+### 🟡 3. REPLICA — Pause replication
+
+```bash
+sudo -u postgres psql -c "SELECT pg_wal_replay_pause();"
+```
+
+Verify:
+
+```bash
+sudo -u postgres psql -c "SELECT pg_is_wal_replay_paused();"
+```
+
+---
+
+### 🟡 4. REPLICA — Run dump
+
+```bash
+sudo -u postgres pg_dump_all_databases
+```
+
+or
+
+```bash
+sudo -u postgres bash -c "pg_dumpall --globals-only > /tmp/globals.sql"
+sudo -u postgres pg_dump_database [db_name]
+```
+
+---
+
+### 🟡 5. REPLICA — Resume replication
+
+```bash
+sudo -u postgres psql -c "SELECT pg_wal_replay_resume();"
+```
+
+---
+
+### 🔵 6. COPY dumps to NEW HOST
+
+From NEW_HOST:
+
+```bash
+export REPLICA_HOST=[private_ip] && \
+cd /tmp && \
+sudo -u postgres scp "postgres@$REPLICA_HOST:/tmp/globals.sql" . && \
+sudo -u postgres scp "postgres@$REPLICA_HOST:/tmp/dump_*.tar.zst" .
+```
+
+---
+
+### 🔵 7. NEW HOST (PostgreSQL 14) — Restore
+
+#### 7.1 Restore globals
+
+```bash
+sudo -u postgres psql -f /tmp/globals.sql
+```
+
+---
+
+#### 7.2 Create databases
+
+```bash
+sudo -u postgres psql -Atqc "SELECT datname FROM pg_database WHERE datallowconn AND datname NOT IN ('template1')" | \
+xargs -I{} sudo -u postgres createdb {}
+```
+
+or
+
+```bash
+sudo -u postgres createdb [db_name]
+```
+
+---
+
+#### 7.3 Restore each database
+
+```bash
+sudo -u postgres pg_restore_all_databases
+```
+
+or
+
+```bash
+sudo -u postgres pg_restore_database [db_name]
+```
+
+---
+
+### 🔵 8. NEW HOST — Create subscriptions
+
+```bash
+sudo -u postgres pg_create_replication_subscriptions
+```
+
+or
+
+```bash
+sudo -u postgres pg_create_replication_subscription [db_name]
+```
+
+---
+
+### 🔵 9. NEW HOST — Monitor replication
+
+```bash
+sudo -u postgres pg_list_replication_subscriptions
+```
+
+---
+
+### 🔴 11. CUTOVER
+
+#### 11.1 Stop writes on old primary
+
+(app maintenance mode, stop app/daemons)
+
+---
+
+#### 11.2 Wait for replication to catch up
+
+TODO: not the best way to check, since WAL LSNs keep increasing
+
+```bash
+sudo -u postgres psql -d [db_name] -c "SELECT * FROM pg_stat_subscription;"
+```
+
+---
+
+#### 11.3 Fix sequences
+
+Run per DB:
+
+```bash
+sudo -u postgres pg_fix_sequences_in_all_databases
+```
+
+or
+
+```bash
+sudo -u postgres pg_fix_sequences [db_name]
+```
+
+---
+
+#### 11.4 Point app to NEW_HOST
+
+* Update pg.kosmos.local in /etc/hosts on app server(s) (maybe override
+  attribute for role and converge)
+* Start app/daemons, deactivate maintenance mode
+
+---
+
+### 🧹 12. CLEANUP (NEW_HOST)
+
+```bash
+sudo -u postgres pg_drop_replication_subscriptions
+```
+
+---
+
+### 🧹 13. CLEANUP (PRIMARY)
+
+TODO: Looks like slots are dropped automatically, when subscriptions are dropped
+
+```bash
+sudo -u postgres pg_drop_replication_publications
+```
+
+---
+
+### ✅ DONE
+
+---
diff --git a/roles/postgresql_replica_logical.rb b/roles/postgresql_replica_logical.rb
new file mode 100644
index 0000000..1e1a7c9
--- /dev/null
+++ b/roles/postgresql_replica_logical.rb
@@ -0,0 +1,8 @@
+name "postgresql_replica_logical"
+
+run_list [
+  "kosmos_postgresql::hostsfile",
+  "kosmos_postgresql::replica_logical",
+  "kosmos_postgresql::firewall",
+  "kosmos_postgresql::management_scripts"
+]
diff --git a/site-cookbooks/kosmos_postgresql/files/create_publication.sh b/site-cookbooks/kosmos_postgresql/files/create_publication.sh
new file mode 100644
index 0000000..2aa7e8f
--- /dev/null
+++ b/site-cookbooks/kosmos_postgresql/files/create_publication.sh
@@ -0,0 +1,31 @@
+#!/bin/bash
+set -euo pipefail
+
+DB_NAME="${1:?Usage: $0 <database_name>}"
+
+echo "== Processing DB: $DB_NAME =="
+
+# Create publication (idempotent)
+psql -d "$DB_NAME" -v ON_ERROR_STOP=1 <<'SQL'
+DO $$
+BEGIN
+  IF NOT EXISTS (
+    SELECT 1 FROM pg_publication WHERE pubname = 'migrate_pub'
+  ) THEN
+    CREATE PUBLICATION migrate_pub FOR ALL TABLES;
+  END IF;
+END
+$$;
+SQL
+
+# Create logical replication slot (idempotent-ish)
+SLOT="migrate_slot_${DB_NAME}"
+
+if ! psql -d "$DB_NAME" -Atqc "SELECT 1 FROM pg_replication_slots WHERE slot_name = '$SLOT'" | grep -q 1; then
+  echo "  Creating slot: $SLOT"
+  psql -d "$DB_NAME" -c "SELECT pg_create_logical_replication_slot('$SLOT', 'pgoutput');"
+else
+  echo "  Slot already exists: $SLOT"
+fi
+
+echo "== Done =="
diff --git a/site-cookbooks/kosmos_postgresql/files/create_publications.sh b/site-cookbooks/kosmos_postgresql/files/create_publications.sh
new file mode 100644
index 0000000..eda54ee
--- /dev/null
+++ b/site-cookbooks/kosmos_postgresql/files/create_publications.sh
@@ -0,0 +1,34 @@
+#!/bin/bash
+set -e
+
+echo "== Creating publication in each database =="
+
+for db in $(psql -Atqc "SELECT datname FROM pg_database WHERE datallowconn AND datname NOT IN ('template1')"); do
+  echo "Processing DB: $db"
+
+  # Create publication (idempotent)
+  psql -d "$db" -v ON_ERROR_STOP=1 <<SQL
+DO \$\$
+BEGIN
+  IF NOT EXISTS (
+    SELECT 1 FROM pg_publication WHERE pubname = 'migrate_pub'
+  ) THEN
+    CREATE PUBLICATION migrate_pub FOR ALL TABLES;
+  END IF;
+END
+\$\$;
+SQL
+
+  # Create logical replication slot (idempotent-ish)
+  SLOT="migrate_slot_${db}"
+
+  if ! psql -d "$db" -Atqc "SELECT 1 FROM pg_replication_slots WHERE slot_name = '$SLOT'" | grep -q 1; then
+    echo "  Creating slot: $SLOT"
+    psql -d "$db" -c "SELECT pg_create_logical_replication_slot('$SLOT', 'pgoutput');"
+  else
+    echo "  Slot already exists: $SLOT"
+  fi
+
+done
+
+echo "== Done =="
diff --git a/site-cookbooks/kosmos_postgresql/files/drop_publications.sh b/site-cookbooks/kosmos_postgresql/files/drop_publications.sh
new file mode 100644
index 0000000..78c8b1b
--- /dev/null
+++ b/site-cookbooks/kosmos_postgresql/files/drop_publications.sh
@@ -0,0 +1,34 @@
+#!/bin/bash
+set -e
+
+echo "== Dropping subscriptions slots and publications =="
+
+for db in $(psql -Atqc "SELECT datname FROM pg_database WHERE datallowconn AND datname NOT IN ('template1')"); do
+  echo "Processing DB: $db"
+
+  SLOT="migrate_slot_${db}"
+
+  # Drop slot if exists
+  if psql -d "$db" -Atqc "SELECT 1 FROM pg_replication_slots WHERE slot_name = '$SLOT'" | grep -q 1; then
+    echo "  Dropping slot: $SLOT"
+    psql -d "$db" -c "SELECT pg_drop_replication_slot('$SLOT');"
+  else
+    echo "  Slot not found: $SLOT"
+  fi
+
+  # Drop publication if exists
+  psql -d "$db" -v ON_ERROR_STOP=1 <<SQL
+DO \$\$
+BEGIN
+  IF EXISTS (
+    SELECT 1 FROM pg_publication WHERE pubname = 'migrate_pub'
+  ) THEN
+    DROP PUBLICATION migrate_pub;
+  END IF;
+END
+\$\$;
+SQL
+
+done
+
+echo "== Done =="
diff --git a/site-cookbooks/kosmos_postgresql/files/drop_subscriptions.sh b/site-cookbooks/kosmos_postgresql/files/drop_subscriptions.sh
new file mode 100644
index 0000000..a718eb5
--- /dev/null
+++ b/site-cookbooks/kosmos_postgresql/files/drop_subscriptions.sh
@@ -0,0 +1,29 @@
+#!/usr/bin/env bash
+set -e
+
+echo "== Dropping subscriptions =="
+
+for db in $(psql -Atqc "SELECT datname FROM pg_database WHERE datallowconn AND datname NOT IN ('template1')"); do
+  echo "Processing DB: $db"
+
+  SUB="migrate_sub_${db}"
+
+  # Check if subscription exists
+  EXISTS=$(psql -d "$db" -Atqc "SELECT 1 FROM pg_subscription WHERE subname = '$SUB'")
+
+  if [ "$EXISTS" = "1" ]; then
+    echo "  Found subscription: $SUB"
+
+    # Disable first (good practice)
+    psql -d "$db" -c "ALTER SUBSCRIPTION $SUB DISABLE;"
+
+    # Drop it (must be top-level)
+    psql -d "$db" -c "DROP SUBSCRIPTION $SUB;"
+
+  else
+    echo "  No subscription: $SUB"
+  fi
+
+done
+
+echo "== Done =="
diff --git a/site-cookbooks/kosmos_postgresql/files/dump_all_databases.sh b/site-cookbooks/kosmos_postgresql/files/dump_all_databases.sh
new file mode 100644
index 0000000..7339ab5
--- /dev/null
+++ b/site-cookbooks/kosmos_postgresql/files/dump_all_databases.sh
@@ -0,0 +1,9 @@
+#!/bin/bash
+cd /tmp && \
+(pg_dumpall --globals-only > globals.sql) && \
+psql -Atqc "SELECT datname FROM pg_database WHERE datallowconn AND datname NOT IN (''template1'')" | \
+xargs -I{} -P4 sh -c "
+  pg_dump -Fd -j 4 -d \"{}\" -f dump_{} &&
+  tar -cf - dump_{} | zstd -19 -T0 > dump_{}.tar.zst &&
+  rm -rf dump_{}
+"
diff --git a/site-cookbooks/kosmos_postgresql/files/dump_database.sh b/site-cookbooks/kosmos_postgresql/files/dump_database.sh
new file mode 100644
index 0000000..028ad77
--- /dev/null
+++ b/site-cookbooks/kosmos_postgresql/files/dump_database.sh
@@ -0,0 +1,10 @@
+#!/bin/bash
+set -euo pipefail
+
+DB_NAME="${1:?Usage: $0 <database_name>}"
+
+cd /tmp
+
+pg_dump -Fd -j 4 -d "$DB_NAME" -f "dump_${DB_NAME}"
+tar -cf - "dump_${DB_NAME}" | zstd -19 -T0 > "dump_${DB_NAME}.tar.zst"
+rm -rf "dump_${DB_NAME}"
diff --git a/site-cookbooks/kosmos_postgresql/files/fix_sequences.sh b/site-cookbooks/kosmos_postgresql/files/fix_sequences.sh
new file mode 100644
index 0000000..a158cdd
--- /dev/null
+++ b/site-cookbooks/kosmos_postgresql/files/fix_sequences.sh
@@ -0,0 +1,35 @@
+#!/bin/bash
+set -e
+
+DB="$1"
+
+if [ -z "$DB" ]; then
+  echo "Usage: $0 <database>"
+  exit 1
+fi
+
+echo "== Fixing sequences in database: $DB =="
+
+SQL=$(psql -d "$DB" -Atqc "
+  SELECT
+    'SELECT setval(' ||
+    quote_literal(pg_get_serial_sequence(quote_ident(n.nspname)||'.'||quote_ident(c.relname), a.attname)) ||
+    ', COALESCE(MAX(' || quote_ident(a.attname) || '), 0) + 1, false) FROM ' ||
+    quote_ident(n.nspname)||'.'||quote_ident(c.relname) || ';'
+  FROM pg_class c
+  JOIN pg_namespace n ON n.oid = c.relnamespace
+  JOIN pg_attribute a ON a.attrelid = c.oid
+  WHERE c.relkind = 'r'
+    AND a.attnum > 0
+    AND NOT a.attisdropped
+    AND pg_get_serial_sequence(quote_ident(n.nspname)||'.'||quote_ident(c.relname), a.attname) IS NOT NULL;
+")
+
+if [ -z "$SQL" ]; then
+  echo "No sequences found in $DB"
+  exit 0
+fi
+
+echo "$SQL" | psql -d "$DB"
+
+echo "== Done =="
diff --git a/site-cookbooks/kosmos_postgresql/files/fix_sequences_in_all_databases.sh b/site-cookbooks/kosmos_postgresql/files/fix_sequences_in_all_databases.sh
new file mode 100644
index 0000000..73b9a7d
--- /dev/null
+++ b/site-cookbooks/kosmos_postgresql/files/fix_sequences_in_all_databases.sh
@@ -0,0 +1,38 @@
+#!/bin/bash
+set -e
+
+echo "== Fixing sequences across all databases =="
+
+for db in $(psql -Atqc "SELECT datname FROM pg_database WHERE datallowconn AND datname NOT IN ('template1')"); do
+  echo "---- DB: $db ----"
+
+  # Generate fix statements
+  SQL=$(psql -d "$db" -Atqc "
+    SELECT
+      'SELECT setval(' ||
+      quote_literal(pg_get_serial_sequence(quote_ident(n.nspname)||'.'||quote_ident(c.relname), a.attname)) ||
+      ', COALESCE(MAX(' || quote_ident(a.attname) || '), 0) + 1, false) FROM ' ||
+      quote_ident(n.nspname)||'.'||quote_ident(c.relname) || ';'
+    FROM pg_class c
+    JOIN pg_namespace n ON n.oid = c.relnamespace
+    JOIN pg_attribute a ON a.attrelid = c.oid
+    WHERE c.relkind = 'r'
+      AND a.attnum > 0
+      AND NOT a.attisdropped
+      AND pg_get_serial_sequence(quote_ident(n.nspname)||'.'||quote_ident(c.relname), a.attname) IS NOT NULL;
+  ")
+
+  if [ -z "$SQL" ]; then
+    echo "No sequences found in $db"
+    continue
+  fi
+
+  echo "Fixing sequences in $db..."
+
+  # Execute generated statements
+  echo "$SQL" | psql -d "$db"
+
+done
+
+echo "== Done fixing sequences =="
+
diff --git a/site-cookbooks/kosmos_postgresql/files/list_publications.sh b/site-cookbooks/kosmos_postgresql/files/list_publications.sh
new file mode 100644
index 0000000..7459005
--- /dev/null
+++ b/site-cookbooks/kosmos_postgresql/files/list_publications.sh
@@ -0,0 +1,5 @@
+#!/bin/bash
+for db in $(psql -Atqc "SELECT datname FROM pg_database WHERE datallowconn AND datname NOT IN ('template1')"); do
+  echo "DB: $db"
+  psql -d "$db" -Atqc "SELECT pubname FROM pg_publication;"
+done
diff --git a/site-cookbooks/kosmos_postgresql/files/list_replication_slots.sh b/site-cookbooks/kosmos_postgresql/files/list_replication_slots.sh
new file mode 100644
index 0000000..591479b
--- /dev/null
+++ b/site-cookbooks/kosmos_postgresql/files/list_replication_slots.sh
@@ -0,0 +1,5 @@
+#!/bin/bash
+psql -c "
+SELECT slot_name,
+pg_size_pretty(pg_wal_lsn_diff(pg_current_wal_lsn(), restart_lsn))
+FROM pg_replication_slots;"
diff --git a/site-cookbooks/kosmos_postgresql/files/list_subscriptions.sh b/site-cookbooks/kosmos_postgresql/files/list_subscriptions.sh
new file mode 100644
index 0000000..30796d5
--- /dev/null
+++ b/site-cookbooks/kosmos_postgresql/files/list_subscriptions.sh
@@ -0,0 +1,5 @@
+#!/bin/bash
+for db in $(psql -Atqc "SELECT datname FROM pg_database WHERE datallowconn AND datname NOT IN ('template1')"); do
+  echo "==== DB: $db ===="
+  psql -d "$db" -c "SELECT * FROM pg_stat_subscription;"
+done
diff --git a/site-cookbooks/kosmos_postgresql/files/restore_all_databases.sh b/site-cookbooks/kosmos_postgresql/files/restore_all_databases.sh
new file mode 100644
index 0000000..18c4a7b
--- /dev/null
+++ b/site-cookbooks/kosmos_postgresql/files/restore_all_databases.sh
@@ -0,0 +1,12 @@
+#!/bin/bash
+set -euo pipefail
+
+cd /tmp
+
+for f in dump_*.tar.zst; do
+  db=$(echo $f | sed "s/dump_\(.*\)\.tar\.zst/\1/")
+  echo "Restoring $db"
+  zstd -d "$f" -c | tar -xf -
+  pg_restore -j 4 -d "$db" dump_$db
+  rm -rf "dump_$db"
+done
diff --git a/site-cookbooks/kosmos_postgresql/files/restore_database.sh b/site-cookbooks/kosmos_postgresql/files/restore_database.sh
new file mode 100644
index 0000000..49e7ce2
--- /dev/null
+++ b/site-cookbooks/kosmos_postgresql/files/restore_database.sh
@@ -0,0 +1,14 @@
+#!/bin/bash
+set -euo pipefail
+
+DB_NAME="${1:?Usage: $0 <database_name>}"
+
+cd /tmp
+
+FILE="dump_${DB_NAME}.tar.zst"
+DIR="dump_${DB_NAME}"
+
+echo "Restoring $DB_NAME"
+zstd -d "$FILE" -c | tar -xf -
+pg_restore -j 4 -d "$DB_NAME" "$DIR"
+rm -rf "$DIR"
diff --git a/site-cookbooks/kosmos_postgresql/recipes/management_scripts.rb b/site-cookbooks/kosmos_postgresql/recipes/management_scripts.rb
new file mode 100644
index 0000000..7a90bc2
--- /dev/null
+++ b/site-cookbooks/kosmos_postgresql/recipes/management_scripts.rb
@@ -0,0 +1,121 @@
+#
+# Cookbook:: kosmos_postgresql
+# Recipe:: management_scripts
+#
+
+credentials = data_bag_item('credentials', 'postgresql')
+
+cookbook_file "/usr/local/bin/pg_dump_all_databases" do
+  source "dump_all_databases.sh"
+  user "postgres"
+  group "postgres"
+  mode "0744"
+end
+
+cookbook_file "/usr/local/bin/pg_dump_database" do
+  source "dump_database.sh"
+  user "postgres"
+  group "postgres"
+  mode "0744"
+end
+
+cookbook_file "/usr/local/bin/pg_restore_all_databases" do
+  source "restore_all_databases.sh"
+  user "postgres"
+  group "postgres"
+  mode "0744"
+end
+
+cookbook_file "/usr/local/bin/pg_restore_database" do
+  source "restore_database.sh"
+  user "postgres"
+  group "postgres"
+  mode "0744"
+end
+
+cookbook_file "/usr/local/bin/pg_create_replication_publications" do
+  source "create_publications.sh"
+  user "postgres"
+  group "postgres"
+  mode "0744"
+end
+
+cookbook_file "/usr/local/bin/pg_create_replication_publication" do
+  source "create_publication.sh"
+  user "postgres"
+  group "postgres"
+  mode "0744"
+end
+
+cookbook_file "/usr/local/bin/pg_drop_replication_publications" do
+  source "drop_publications.sh"
+  user "postgres"
+  group "postgres"
+  mode "0744"
+end
+
+cookbook_file "/usr/local/bin/pg_list_replication_publications" do
+  source "list_publications.sh"
+  user "postgres"
+  group "postgres"
+  mode "0744"
+end
+
+cookbook_file "/usr/local/bin/pg_list_replication_slots" do
+  source "list_replication_slots.sh"
+  user "postgres"
+  group "postgres"
+  mode "0744"
+end
+
+template "/usr/local/bin/pg_create_replication_subscriptions" do
+  source "create_subscriptions.sh.erb"
+  user "postgres"
+  group "postgres"
+  mode "0740"
+  variables pg_host: "pg.kosmos.local",
+            pg_port: 5432,
+            pg_user: "replication",
+            pg_pass: credentials["replication_password"]
+  sensitive true
+end
+
+template "/usr/local/bin/pg_create_replication_subscription" do
+  source "create_subscription.sh.erb"
+  user "postgres"
+  group "postgres"
+  mode "0740"
+  variables pg_host: "pg.kosmos.local",
+            pg_port: 5432,
+            pg_user: "replication",
+            pg_pass: credentials["replication_password"]
+  sensitive true
+end
+
+cookbook_file "/usr/local/bin/pg_drop_replication_subscriptions" do
+  source "drop_subscriptions.sh"
+  user "postgres"
+  group "postgres"
+  mode "0744"
+end
+
+cookbook_file "/usr/local/bin/pg_list_replication_subscriptions" do
+  source "list_subscriptions.sh"
+  user "postgres"
+  group "postgres"
+  mode "0744"
+end
+
+cookbook_file "/usr/local/bin/pg_fix_sequences_in_all_databases" do
+  source "fix_sequences.sh"
+  user "postgres"
+  group "postgres"
+  mode "0744"
+end
+
+cookbook_file "/usr/local/bin/pg_fix_sequences" do
+  source "fix_sequences.sh"
+  user "postgres"
+  group "postgres"
+  mode "0744"
+end
diff --git a/site-cookbooks/kosmos_postgresql/recipes/primary.rb b/site-cookbooks/kosmos_postgresql/recipes/primary.rb
index 406e2ae..73c0224 100644
--- a/site-cookbooks/kosmos_postgresql/recipes/primary.rb
+++ b/site-cookbooks/kosmos_postgresql/recipes/primary.rb
@@ -6,4 +6,3 @@
 postgresql_custom_server postgresql_version do
   role "primary"
 end
-
diff --git a/site-cookbooks/kosmos_postgresql/recipes/replica.rb b/site-cookbooks/kosmos_postgresql/recipes/replica.rb
index 69d5cd0..95a037a 100644
--- a/site-cookbooks/kosmos_postgresql/recipes/replica.rb
+++ b/site-cookbooks/kosmos_postgresql/recipes/replica.rb
@@ -3,10 +3,6 @@
 # Recipe:: replica
 #
 
-service postgresql_service do
-  supports restart: true, status: true, reload: true
-end
-
 postgresql_custom_server postgresql_version do
   role "replica"
 end
@@ -20,6 +16,9 @@ if primary.nil?
   return
 end
 
+# TODO Replace pg.kosmos.local with private IP once available
+# via proper node attribute
+# https://gitea.kosmos.org/kosmos/chef/issues/263
 execute "set up replication" do
   command <<-EOF
 systemctl stop #{postgresql_service}
diff --git a/site-cookbooks/kosmos_postgresql/recipes/replica_logical.rb b/site-cookbooks/kosmos_postgresql/recipes/replica_logical.rb
new file mode 100644
index 0000000..1f3dab9
--- /dev/null
+++ b/site-cookbooks/kosmos_postgresql/recipes/replica_logical.rb
@@ -0,0 +1,15 @@
+#
+# Cookbook:: kosmos_postgresql
+# Recipe:: replica_logical
+#
+
+postgresql_custom_server postgresql_version do
+  role "replica_logical"
+end
+
+# primary = postgresql_primary
+#
+# if primary.nil?
+#   Chef::Log.warn("No PostgreSQL primary node found. Skipping replication setup.")
+#   return
+# end
diff --git a/site-cookbooks/kosmos_postgresql/resources/server.rb b/site-cookbooks/kosmos_postgresql/resources/server.rb
index f08a70d..e6c72f7 100644
--- a/site-cookbooks/kosmos_postgresql/resources/server.rb
+++ b/site-cookbooks/kosmos_postgresql/resources/server.rb
@@ -56,13 +56,15 @@ action :create do
     timezone: "UTC", # default is GMT
     listen_addresses: "0.0.0.0",
     promote_trigger_file: "#{postgresql_data_dir}/failover.trigger",
-    wal_keep_size: 4096 # 256 segments, 16MB each
+    wal_level: "logical",
+    wal_keep_size: 4096, # 256 segments, 16MB each
+    max_replication_slots: 16
   }
 
   postgresql_server_conf "main" do
     version postgresql_version
     additional_config additional_config
-    notifies :reload, "service[#{postgresql_service}]", :delayed
+    notifies :restart, "service[#{postgresql_service}]", :delayed
   end
 
   postgresql_user "replication" do
diff --git a/site-cookbooks/kosmos_postgresql/templates/create_subscription.sh.erb b/site-cookbooks/kosmos_postgresql/templates/create_subscription.sh.erb
new file mode 100644
index 0000000..0839775
--- /dev/null
+++ b/site-cookbooks/kosmos_postgresql/templates/create_subscription.sh.erb
@@ -0,0 +1,31 @@
+#!/bin/bash
+set -euo pipefail
+
+DB_NAME="${1:?Usage: $0 <database_name>}"
+
+echo "== Processing DB: $DB_NAME =="
+
+SLOT="migrate_slot_${DB_NAME}"
+SUB="migrate_sub_${DB_NAME}"
+
+psql -d "$DB_NAME" -v ON_ERROR_STOP=1 <<SQL
+DO \$\$
+BEGIN
+  IF NOT EXISTS (
+    SELECT 1 FROM pg_subscription WHERE subname = '$SUB'
+  ) THEN
+    CREATE SUBSCRIPTION $SUB
+    CONNECTION 'host=<%= @pg_host %> port=<%= @pg_port %> dbname=$DB_NAME user=<%= @pg_user %> password=<%= @pg_pass %>'
+    PUBLICATION migrate_pub
+    WITH (
+      slot_name = '$SLOT',
+      create_slot = false,
+      copy_data = false,
+      enabled = true
+    );
+  END IF;
+END
+\$\$;
+SQL
+
+echo "== Done =="
diff --git a/site-cookbooks/kosmos_postgresql/templates/create_subscriptions.sh.erb b/site-cookbooks/kosmos_postgresql/templates/create_subscriptions.sh.erb
new file mode 100644
index 0000000..8d79c73
--- /dev/null
+++ b/site-cookbooks/kosmos_postgresql/templates/create_subscriptions.sh.erb
@@ -0,0 +1,34 @@
+#!/bin/bash
+set -e
+
+echo "== Creating subscriptions for all databases =="
+
+for db in $(psql -Atqc "SELECT datname FROM pg_database WHERE datallowconn AND datname NOT IN ('template1')"); do
+  echo "Processing DB: $db"
+
+  SLOT="migrate_slot_${db}"
+  SUB="migrate_sub_${db}"
+
+  psql -d "$db" -v ON_ERROR_STOP=1 <<SQL
+DO \$\$
+BEGIN
+  IF NOT EXISTS (
+    SELECT 1 FROM pg_subscription WHERE subname = '$SUB'
+  ) THEN
+    CREATE SUBSCRIPTION $SUB
+    CONNECTION 'host=<%= @pg_host %> port=<%= @pg_port %> dbname=$db user=<%= @pg_user %> password=<%= @pg_pass %>'
+    PUBLICATION migrate_pub
+    WITH (
+      slot_name = '$SLOT',
+      create_slot = false,
+      copy_data = false,
+      enabled = true
+    );
+  END IF;
+END
+\$\$;
+SQL
+
+done
+
+echo "== Done =="

From 09412f69e89e4b68d5996febb8b8fc45216031e8 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?R=C3=A2u=20Cao?= <raucao@kosmos.org>
Date: Fri, 10 Apr 2026 08:36:54 +0400
Subject: [PATCH 04/18] Move doc

---
 doc/{postgres-migration.md => postgres/migration.md} | 0
 1 file changed, 0 insertions(+), 0 deletions(-)
 rename doc/{postgres-migration.md => postgres/migration.md} (100%)

diff --git a/doc/postgres-migration.md b/doc/postgres/migration.md
similarity index 100%
rename from doc/postgres-migration.md
rename to doc/postgres/migration.md

From 0020677ab298fce1f4c333e0365180d84189f3e3 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?R=C3=A2u=20Cao?= <raucao@kosmos.org>
Date: Fri, 10 Apr 2026 08:37:06 +0400
Subject: [PATCH 05/18] Drone: Make database details configurable

---
 site-cookbooks/kosmos_drone/attributes/default.rb |  4 ++++
 site-cookbooks/kosmos_drone/recipes/default.rb    | 10 +++++-----
 2 files changed, 9 insertions(+), 5 deletions(-)

diff --git a/site-cookbooks/kosmos_drone/attributes/default.rb b/site-cookbooks/kosmos_drone/attributes/default.rb
index ceb7564..498ed5e 100644
--- a/site-cookbooks/kosmos_drone/attributes/default.rb
+++ b/site-cookbooks/kosmos_drone/attributes/default.rb
@@ -1,2 +1,6 @@
 node.default["kosmos_drone"]["domain"] = "drone.kosmos.org"
 node.default["kosmos_drone"]["upstream_port"] = 80
+node.default["kosmos_drone"]["pg_host"] = "pg.kosmos.local"
+node.default["kosmos_drone"]["pg_port"] = 5432
+node.default["kosmos_drone"]["pg_db"] = "drone"
+node.default["kosmos_drone"]["pg_user"] = "drone"
diff --git a/site-cookbooks/kosmos_drone/recipes/default.rb b/site-cookbooks/kosmos_drone/recipes/default.rb
index f816377..f480d37 100644
--- a/site-cookbooks/kosmos_drone/recipes/default.rb
+++ b/site-cookbooks/kosmos_drone/recipes/default.rb
@@ -9,11 +9,11 @@ credentials = data_bag_item("credentials", "drone")
 drone_credentials = data_bag_item('credentials', 'drone')
 
 postgres_config = {
-  username: "drone",
-  password: drone_credentials["postgresql_password"],
-  host: "pg.kosmos.local",
-  port: 5432,
-  database: "drone"
+  host: node["kosmos_drone"]["pg_host"],
+  port: node["kosmos_drone"]["pg_port"],
+  database: node["kosmos_drone"]["pg_db"],
+  username: node["kosmos_drone"]["pg_user"],
+  password: drone_credentials["postgresql_password"]
 }
 
 directory deploy_path do

From 8e11df454485832673ff7ffe5dd0566244291d4f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?R=C3=A2u=20Cao?= <raucao@kosmos.org>
Date: Fri, 10 Apr 2026 12:49:28 +0400
Subject: [PATCH 06/18] Update PG migration runbook

---
 doc/postgres/migration.md | 28 ++++++++++++++++++++++------
 1 file changed, 22 insertions(+), 6 deletions(-)

diff --git a/doc/postgres/migration.md b/doc/postgres/migration.md
index e9d24d2..d1f7d27 100644
--- a/doc/postgres/migration.md
+++ b/doc/postgres/migration.md
@@ -210,7 +210,7 @@ sudo -u postgres pg_list_replication_subscriptions
 
 #### 11.1 Stop writes on old primary
 
-(app maintenance mode, stop app/daemons)
+Put app(s) in maintenance mode, stop the app/daemons.
 
 ---
 
@@ -242,13 +242,20 @@ sudo -u postgres pg_fix_sequences [db_name]
 
 #### 11.4 Point app to NEW_HOST
 
-* Update pg.kosmos.local in /etc/hosts on app server(s) (maybe override
-  attribute for role and converge)
-* Start app/daemons, deactivate maintenance mode
+1. Update `pg.kosmos.local` in `/etc/hosts` on app server(s). For example:
+
+   ```bash
+   export NEW_PG_PRIMARY=[private_ip]
+   bundle exec knife ssh roles:ejabberd -a knife_zero.host "sudo sed -r \"s/^[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+\s(pg.kosmos.local)/$NEW_PG_PRIMARY\t\1/\" -i /etc/hosts"
+   ```
+
+   Or override node attribute(s) if necessary and/or approporiate.
+
+2. Start the app/daemons, and deactivate maintenance mode.
 
 ---
 
-### 🧹 12. CLEANUP (NEW_HOST)
+### 🧹 12. CLEANUP NEW_HOST
 
 ```bash
 sudo -u postgres pg_drop_replication_subscriptions
@@ -256,7 +263,7 @@ sudo -u postgres pg_drop_replication_subscriptions
 
 ---
 
-### 🧹 13. CLEANUP (PRIMARY)
+### 🧹 13. CLEANUP PRIMARY
 
 TODO: Looks like slots are dropped automatically, when subscriptions are dropped
 
@@ -266,6 +273,15 @@ sudo -u postgres pg_drop_replication_publications
 
 ---
 
+### 🧹 13. CLEANUP Chef
+
+Once all apps/databases are migrated, update the role in the node
+config of the new primary to 'postgres_primary' and converge it.
+
+Also delete the old primary node config from the Chef repo.
+
+---
+
 ### ✅ DONE
 
 ---

From fddcd4899ecc8a0840e4e2a31adb9889a205c24b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?R=C3=A2u=20Cao?= <raucao@kosmos.org>
Date: Fri, 10 Apr 2026 12:57:51 +0400
Subject: [PATCH 07/18] Ignore default db for migration/management

---
 site-cookbooks/kosmos_postgresql/files/create_publications.sh   | 2 +-
 site-cookbooks/kosmos_postgresql/files/drop_publications.sh     | 2 +-
 site-cookbooks/kosmos_postgresql/files/drop_subscriptions.sh    | 2 +-
 site-cookbooks/kosmos_postgresql/files/dump_all_databases.sh    | 2 +-
 .../kosmos_postgresql/files/fix_sequences_in_all_databases.sh   | 2 +-
 site-cookbooks/kosmos_postgresql/files/list_publications.sh     | 2 +-
 site-cookbooks/kosmos_postgresql/files/list_subscriptions.sh    | 2 +-
 .../kosmos_postgresql/templates/create_subscriptions.sh.erb     | 2 +-
 8 files changed, 8 insertions(+), 8 deletions(-)

diff --git a/site-cookbooks/kosmos_postgresql/files/create_publications.sh b/site-cookbooks/kosmos_postgresql/files/create_publications.sh
index eda54ee..7b98acc 100644
--- a/site-cookbooks/kosmos_postgresql/files/create_publications.sh
+++ b/site-cookbooks/kosmos_postgresql/files/create_publications.sh
@@ -3,7 +3,7 @@ set -e
 
 echo "== Creating publication in each database =="
 
-for db in $(psql -Atqc "SELECT datname FROM pg_database WHERE datallowconn AND datname NOT IN ('template1')"); do
+for db in $(psql -Atqc "SELECT datname FROM pg_database WHERE datallowconn AND datname NOT IN ('template1','postgres')"); do
   echo "Processing DB: $db"
 
   # Create publication (idempotent)
diff --git a/site-cookbooks/kosmos_postgresql/files/drop_publications.sh b/site-cookbooks/kosmos_postgresql/files/drop_publications.sh
index 78c8b1b..834108f 100644
--- a/site-cookbooks/kosmos_postgresql/files/drop_publications.sh
+++ b/site-cookbooks/kosmos_postgresql/files/drop_publications.sh
@@ -3,7 +3,7 @@ set -e
 
 echo "== Dropping subscriptions slots and publications =="
 
-for db in $(psql -Atqc "SELECT datname FROM pg_database WHERE datallowconn AND datname NOT IN ('template1')"); do
+for db in $(psql -Atqc "SELECT datname FROM pg_database WHERE datallowconn AND datname NOT IN ('template1','postgres')"); do
   echo "Processing DB: $db"
 
   SLOT="migrate_slot_${db}"
diff --git a/site-cookbooks/kosmos_postgresql/files/drop_subscriptions.sh b/site-cookbooks/kosmos_postgresql/files/drop_subscriptions.sh
index a718eb5..424a022 100644
--- a/site-cookbooks/kosmos_postgresql/files/drop_subscriptions.sh
+++ b/site-cookbooks/kosmos_postgresql/files/drop_subscriptions.sh
@@ -3,7 +3,7 @@ set -e
 
 echo "== Dropping subscriptions =="
 
-for db in $(psql -Atqc "SELECT datname FROM pg_database WHERE datallowconn AND datname NOT IN ('template1')"); do
+for db in $(psql -Atqc "SELECT datname FROM pg_database WHERE datallowconn AND datname NOT IN ('template1','postgres')"); do
   echo "Processing DB: $db"
 
   SUB="migrate_sub_${db}"
diff --git a/site-cookbooks/kosmos_postgresql/files/dump_all_databases.sh b/site-cookbooks/kosmos_postgresql/files/dump_all_databases.sh
index 7339ab5..893787a 100644
--- a/site-cookbooks/kosmos_postgresql/files/dump_all_databases.sh
+++ b/site-cookbooks/kosmos_postgresql/files/dump_all_databases.sh
@@ -1,7 +1,7 @@
 #!/bin/bash
 cd /tmp && \
 (pg_dumpall --globals-only > globals.sql) && \
-psql -Atqc "SELECT datname FROM pg_database WHERE datallowconn AND datname NOT IN (''template1'')" | \
+psql -Atqc "SELECT datname FROM pg_database WHERE datallowconn AND datname NOT IN (''template0'',''postgres'')" | \
 xargs -I{} -P4 sh -c "
   pg_dump -Fd -j 4 -d \"{}\" -f dump_{} &&
   tar -cf - dump_{} | zstd -19 -T0 > dump_{}.tar.zst &&
diff --git a/site-cookbooks/kosmos_postgresql/files/fix_sequences_in_all_databases.sh b/site-cookbooks/kosmos_postgresql/files/fix_sequences_in_all_databases.sh
index 73b9a7d..f8fc2b2 100644
--- a/site-cookbooks/kosmos_postgresql/files/fix_sequences_in_all_databases.sh
+++ b/site-cookbooks/kosmos_postgresql/files/fix_sequences_in_all_databases.sh
@@ -3,7 +3,7 @@ set -e
 
 echo "== Fixing sequences across all databases =="
 
-for db in $(psql -Atqc "SELECT datname FROM pg_database WHERE datallowconn AND datname NOT IN ('template1')"); do
+for db in $(psql -Atqc "SELECT datname FROM pg_database WHERE datallowconn AND datname NOT IN ('template1','postgres')"); do
   echo "---- DB: $db ----"
 
   # Generate fix statements
diff --git a/site-cookbooks/kosmos_postgresql/files/list_publications.sh b/site-cookbooks/kosmos_postgresql/files/list_publications.sh
index 7459005..13ab742 100644
--- a/site-cookbooks/kosmos_postgresql/files/list_publications.sh
+++ b/site-cookbooks/kosmos_postgresql/files/list_publications.sh
@@ -1,5 +1,5 @@
 #!/bin/bash
-for db in $(psql -Atqc "SELECT datname FROM pg_database WHERE datallowconn AND datname NOT IN ('template1')"); do
+for db in $(psql -Atqc "SELECT datname FROM pg_database WHERE datallowconn AND datname NOT IN ('template1','postgres')"); do
   echo "DB: $db"
   psql -d "$db" -Atqc "SELECT pubname FROM pg_publication;"
 done
diff --git a/site-cookbooks/kosmos_postgresql/files/list_subscriptions.sh b/site-cookbooks/kosmos_postgresql/files/list_subscriptions.sh
index 30796d5..3d95308 100644
--- a/site-cookbooks/kosmos_postgresql/files/list_subscriptions.sh
+++ b/site-cookbooks/kosmos_postgresql/files/list_subscriptions.sh
@@ -1,5 +1,5 @@
 #!/bin/bash
-for db in $(psql -Atqc "SELECT datname FROM pg_database WHERE datallowconn AND datname NOT IN ('template1')"); do
+for db in $(psql -Atqc "SELECT datname FROM pg_database WHERE datallowconn AND datname NOT IN ('template1','postgres')"); do
   echo "==== DB: $db ===="
   psql -d "$db" -c "SELECT * FROM pg_stat_subscription;"
 done
diff --git a/site-cookbooks/kosmos_postgresql/templates/create_subscriptions.sh.erb b/site-cookbooks/kosmos_postgresql/templates/create_subscriptions.sh.erb
index 8d79c73..9ee5078 100644
--- a/site-cookbooks/kosmos_postgresql/templates/create_subscriptions.sh.erb
+++ b/site-cookbooks/kosmos_postgresql/templates/create_subscriptions.sh.erb
@@ -3,7 +3,7 @@ set -e
 
 echo "== Creating subscriptions for all databases =="
 
-for db in $(psql -Atqc "SELECT datname FROM pg_database WHERE datallowconn AND datname NOT IN ('template1')"); do
+for db in $(psql -Atqc "SELECT datname FROM pg_database WHERE datallowconn AND datname NOT IN ('template1','postgres')"); do
   echo "Processing DB: $db"
 
   SLOT="migrate_slot_${db}"

From a89db454d0f4be29eaf05523a572649c959a55d2 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?R=C3=A2u=20Cao?= <raucao@kosmos.org>
Date: Sat, 11 Apr 2026 14:51:51 +0400
Subject: [PATCH 08/18] Improve postgres management scripts

---
 .../files/dump_all_databases.sh                 |  2 +-
 .../files/list_subscriptions.sh                 | 17 ++++++++++++++---
 2 files changed, 15 insertions(+), 4 deletions(-)

diff --git a/site-cookbooks/kosmos_postgresql/files/dump_all_databases.sh b/site-cookbooks/kosmos_postgresql/files/dump_all_databases.sh
index 893787a..5bf1eba 100644
--- a/site-cookbooks/kosmos_postgresql/files/dump_all_databases.sh
+++ b/site-cookbooks/kosmos_postgresql/files/dump_all_databases.sh
@@ -1,7 +1,7 @@
 #!/bin/bash
 cd /tmp && \
 (pg_dumpall --globals-only > globals.sql) && \
-psql -Atqc "SELECT datname FROM pg_database WHERE datallowconn AND datname NOT IN (''template0'',''postgres'')" | \
+psql -Atqc "SELECT datname FROM pg_database WHERE datallowconn AND datname NOT IN (''template1'',''postgres'')" | \
 xargs -I{} -P4 sh -c "
   pg_dump -Fd -j 4 -d \"{}\" -f dump_{} &&
   tar -cf - dump_{} | zstd -19 -T0 > dump_{}.tar.zst &&
diff --git a/site-cookbooks/kosmos_postgresql/files/list_subscriptions.sh b/site-cookbooks/kosmos_postgresql/files/list_subscriptions.sh
index 3d95308..fbfbb2f 100644
--- a/site-cookbooks/kosmos_postgresql/files/list_subscriptions.sh
+++ b/site-cookbooks/kosmos_postgresql/files/list_subscriptions.sh
@@ -1,5 +1,16 @@
 #!/bin/bash
-for db in $(psql -Atqc "SELECT datname FROM pg_database WHERE datallowconn AND datname NOT IN ('template1','postgres')"); do
-  echo "==== DB: $db ===="
-  psql -d "$db" -c "SELECT * FROM pg_stat_subscription;"
+set -euo pipefail
+
+psql -Atqc "
+SELECT datname
+FROM pg_database
+WHERE datallowconn
+  AND datname NOT IN ('template1','postgres')
+" | while read -r db; do
+  result=$(psql -X -At -d "$db" -c "SELECT * FROM pg_stat_subscription;" 2>/dev/null || true)
+
+  if [[ -n "$result" ]]; then
+    echo "==== DB: $db ===="
+    echo "$result"
+  fi
 done

From c92f9157a5eefe7a44a432164ff492285361c029 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?R=C3=A2u=20Cao?= <raucao@kosmos.org>
Date: Sat, 11 Apr 2026 14:52:29 +0400
Subject: [PATCH 09/18] Fix method undefined in heredoc

---
 site-cookbooks/kosmos_postgresql/libraries/helpers.rb | 8 --------
 site-cookbooks/kosmos_postgresql/recipes/replica.rb   | 7 +++++--
 2 files changed, 5 insertions(+), 10 deletions(-)

diff --git a/site-cookbooks/kosmos_postgresql/libraries/helpers.rb b/site-cookbooks/kosmos_postgresql/libraries/helpers.rb
index 5de6e76..f87dda2 100644
--- a/site-cookbooks/kosmos_postgresql/libraries/helpers.rb
+++ b/site-cookbooks/kosmos_postgresql/libraries/helpers.rb
@@ -39,13 +39,5 @@ class Chef
     def postgresql_version
       node['kosmos_postgresql']['postgresql_version']
     end
-
-    def postgresql_service
-      "postgresql@#{postgresql_version}-main"
-    end
-
-    def postgresql_data_dir
-      "/var/lib/postgresql/#{postgresql_version}/main"
-    end
   end
 end
diff --git a/site-cookbooks/kosmos_postgresql/recipes/replica.rb b/site-cookbooks/kosmos_postgresql/recipes/replica.rb
index 95a037a..7d355c4 100644
--- a/site-cookbooks/kosmos_postgresql/recipes/replica.rb
+++ b/site-cookbooks/kosmos_postgresql/recipes/replica.rb
@@ -16,16 +16,19 @@ if primary.nil?
   return
 end
 
+postgresql_service_name = "postgresql@#{postgresql_version}-main"
+postgresql_data_dir = "/var/lib/postgresql/#{postgresql_version}/main"
+
 # TODO Replace pg.kosmos.local with private IP once available
 # via proper node attribute
 # https://gitea.kosmos.org/kosmos/chef/issues/263
 execute "set up replication" do
   command <<-EOF
-systemctl stop #{postgresql_service}
+systemctl stop #{postgresql_service_name}
 mv #{postgresql_data_dir} #{postgresql_data_dir}.old
 pg_basebackup -h pg.kosmos.local -U replication -D #{postgresql_data_dir} -R
 chown -R postgres:postgres #{postgresql_data_dir}
-systemctl start #{postgresql_service}
+systemctl start #{postgresql_service_name}
   EOF
   environment 'PGPASSWORD' => postgresql_data_bag_item['replication_password']
   sensitive true

From db9177c9c67b3d78a522dec9ac62f2c3c80e1be0 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?R=C3=A2u=20Cao?= <raucao@kosmos.org>
Date: Sat, 11 Apr 2026 14:53:28 +0400
Subject: [PATCH 10/18] Improve RAM usage allowance

Queries can spawn more processed and then use more RAM than
`shared_buffers`
---
 site-cookbooks/kosmos_postgresql/resources/server.rb | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/site-cookbooks/kosmos_postgresql/resources/server.rb b/site-cookbooks/kosmos_postgresql/resources/server.rb
index e6c72f7..d0ae149 100644
--- a/site-cookbooks/kosmos_postgresql/resources/server.rb
+++ b/site-cookbooks/kosmos_postgresql/resources/server.rb
@@ -44,13 +44,14 @@ action :create do
 
   shared_buffers = if node['memory']['total'].to_i / 1024 < 1024 # < 1GB RAM
                      "128MB"
-                   else # >= 1GB RAM, use 50% of total RAM
-                     "#{node['memory']['total'].to_i / 1024 / 2}MB"
+                   else # >= 1GB RAM, use 25% of total RAM
+                     "#{node['memory']['total'].to_i / 1024 / 4}MB"
                    end
 
   additional_config = {
     max_connections: 200, # default
     shared_buffers: shared_buffers,
+    work_mem: "4MB",
     unix_socket_directories: "/var/run/postgresql",
     dynamic_shared_memory_type: "posix",
     timezone: "UTC", # default is GMT

From 64d5d34d856eec64b63990aec6ad27b506d74ea8 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?R=C3=A2u=20Cao?= <raucao@kosmos.org>
Date: Sat, 11 Apr 2026 14:55:02 +0400
Subject: [PATCH 11/18] Update postgres roles

Reset to normal/final
---
 roles/postgresql_primary.rb | 21 +++++++++++----------
 roles/postgresql_replica.rb | 11 ++++++-----
 2 files changed, 17 insertions(+), 15 deletions(-)

diff --git a/roles/postgresql_primary.rb b/roles/postgresql_primary.rb
index a4fb4b8..27e1dde 100644
--- a/roles/postgresql_primary.rb
+++ b/roles/postgresql_primary.rb
@@ -1,12 +1,13 @@
 name "postgresql_primary"
 
-run_list %w(
-  kosmos_postgresql::primary
-  kosmos_postgresql::firewall
-  kosmos-akkounts::pg_db
-  kosmos-bitcoin::lndhub-go_pg_db
-  kosmos-bitcoin::nbxplorer_pg_db
-  kosmos_drone::pg_db
-  kosmos_gitea::pg_db
-  kosmos-mastodon::pg_db
-)
+run_list [
+  "kosmos_postgresql::primary",
+  "kosmos-akkounts::pg_db",
+  "kosmos-bitcoin::lndhub-go_pg_db",
+  "kosmos-bitcoin::nbxplorer_pg_db",
+  "kosmos_drone::pg_db",
+  "kosmos_gitea::pg_db",
+  "kosmos-mastodon::pg_db",
+  "kosmos_postgresql::firewall",
+  "kosmos_postgresql::management_scripts"
+]
diff --git a/roles/postgresql_replica.rb b/roles/postgresql_replica.rb
index 099291d..9050352 100644
--- a/roles/postgresql_replica.rb
+++ b/roles/postgresql_replica.rb
@@ -1,7 +1,8 @@
 name "postgresql_replica"
 
-run_list %w(
-  kosmos_postgresql::hostsfile
-  kosmos_postgresql::replica
-  kosmos_postgresql::firewall
-)
+run_list [
+  "kosmos_postgresql::hostsfile",
+  "kosmos_postgresql::replica",
+  "kosmos_postgresql::firewall",
+  "kosmos_postgresql::management_scripts"
+]

From 9de37cde96287c595c5c19e992ffcc9bf4c696fd Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?R=C3=A2u=20Cao?= <raucao@kosmos.org>
Date: Sat, 11 Apr 2026 14:55:32 +0400
Subject: [PATCH 12/18] Update doc

---
 doc/postgres/migration.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/doc/postgres/migration.md b/doc/postgres/migration.md
index d1f7d27..27915a5 100644
--- a/doc/postgres/migration.md
+++ b/doc/postgres/migration.md
@@ -246,7 +246,7 @@ sudo -u postgres pg_fix_sequences [db_name]
 
    ```bash
    export NEW_PG_PRIMARY=[private_ip]
-   bundle exec knife ssh roles:ejabberd -a knife_zero.host "sudo sed -r \"s/^[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+\s(pg.kosmos.local)/$NEW_PG_PRIMARY\t\1/\" -i /etc/hosts"
+   knife ssh roles:ejabberd -a knife_zero.host "sudo sed -r \"s/^[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+\s(pg.kosmos.local)/$NEW_PG_PRIMARY\t\1/\" -i /etc/hosts"
    ```
 
    Or override node attribute(s) if necessary and/or approporiate.

From 061880536bca3cc8ec3664a934e9ccf18253dc21 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?R=C3=A2u=20Cao?= <raucao@kosmos.org>
Date: Sat, 11 Apr 2026 14:56:15 +0400
Subject: [PATCH 13/18] Fix akkounts systemd unit stop command

Typo (not using pumactl), but we don't need to specify it to do the
right thing anyway. systemd can just send sigterm on its own.
---
 site-cookbooks/kosmos-akkounts/recipes/default.rb | 1 -
 1 file changed, 1 deletion(-)

diff --git a/site-cookbooks/kosmos-akkounts/recipes/default.rb b/site-cookbooks/kosmos-akkounts/recipes/default.rb
index 0f9f6d9..f8ec675 100644
--- a/site-cookbooks/kosmos-akkounts/recipes/default.rb
+++ b/site-cookbooks/kosmos-akkounts/recipes/default.rb
@@ -230,7 +230,6 @@ systemd_unit "akkounts.service" do
       WorkingDirectory: deploy_path,
       Environment: "RAILS_ENV=#{rails_env} SOLID_QUEUE_IN_PUMA=true",
       ExecStart: "#{bundle_path} exec puma -C config/puma.rb --pidfile #{deploy_path}/tmp/puma.pid",
-      ExecStop: "#{bundle_path} exec puma -C config/puma.rb --pidfile #{deploy_path}/tmp/puma.pid stop",
       ExecReload: "#{bundle_path} exec pumactl -F config/puma.rb --pidfile #{deploy_path}/tmp/puma.pid phased-restart",
       PIDFile: "#{deploy_path}/tmp/puma.pid",
       TimeoutSec: "10",

From d5e3d625220549bb546b292226c64cc912d0e8b8 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?R=C3=A2u=20Cao?= <raucao@kosmos.org>
Date: Sat, 11 Apr 2026 15:22:34 +0400
Subject: [PATCH 14/18] Add new postgres cluster, remove old one

---
 clients/postgres-11.json                     |  4 ++
 clients/postgres-12.json                     |  4 ++
 nodes/{postgres-6.json => postgres-11.json}  | 27 +++++----
 nodes/{postgres-10.json => postgres-12.json} | 13 ++--
 nodes/postgres-8.json                        | 62 -------------------
 nodes/postgres-9.json                        | 63 --------------------
 6 files changed, 30 insertions(+), 143 deletions(-)
 create mode 100644 clients/postgres-11.json
 create mode 100644 clients/postgres-12.json
 rename nodes/{postgres-6.json => postgres-11.json} (74%)
 rename nodes/{postgres-10.json => postgres-12.json} (85%)
 delete mode 100644 nodes/postgres-8.json
 delete mode 100644 nodes/postgres-9.json

diff --git a/clients/postgres-11.json b/clients/postgres-11.json
new file mode 100644
index 0000000..8ded2ba
--- /dev/null
+++ b/clients/postgres-11.json
@@ -0,0 +1,4 @@
+{
+  "name": "postgres-11",
+  "public_key": "-----BEGIN PUBLIC KEY-----\nMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEA1foYpuubS2ovlg3uHO12\nQ/ROZ8MpG+LkCAM46uVfPaoWwfY0vdfMsBOanHDgm9DGUCEBJZ6LPrvCvGXbpPy6\n9GSswK75zVWODblNjvvV4ueGFq4bBFwRuZNjyMlqgyzeU+srZL0ivelu5XEuGuoD\nPYCBKWYqGMz85/eMC7/tinTJtKPyOtXe/G8meji+r7gh3j+ypj/EWeKfcRDa4aGe\n/DmMCurIjjPAXFLMAA6fIqPWVfcPw4APNPE60Z92yPGsTbPu7bL54M5f7udmmu7H\nOgk1HjMAmXCuLDzTkfaxqHP+57yELg/YpXR1E93VmBeQuIBsyOFEk6AmUmA1Ib6e\nnQIDAQAB\n-----END PUBLIC KEY-----\n"
+}
\ No newline at end of file
diff --git a/clients/postgres-12.json b/clients/postgres-12.json
new file mode 100644
index 0000000..2ac32cd
--- /dev/null
+++ b/clients/postgres-12.json
@@ -0,0 +1,4 @@
+{
+  "name": "postgres-12",
+  "public_key": "-----BEGIN PUBLIC KEY-----\nMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEA1mYGrYB8keUKmXA8dhWc\ncCLzp50xR0ajSw+bWYydyRqD5wuEVKjiJu4+G9QmTVXkVgJ+AYI0Y9/WZYpDqVH6\nvLUo6BSNQaWx20q93qIdOGLy8YG3Qyznezk4l8T9u9vWZDyDpKw6gCxzikMkrXxb\n0cqOYtyud8+PtSEEMogSjOKhRURVHlVrlVH3SQO7Whke9rkiFcbXzubsK9yjkUtF\nxZafSoGorOlDsPvFTfYnkepVB+GHcgiribRYSrO+73GypC2kqMhCpWrb6a0VWsP/\nh53+q3JL3vBvdvjcv51Wpf4n6JdnXnQGn2/MdXEzw+NXgjU4/IdYtbORSbaI8F5t\nowIDAQAB\n-----END PUBLIC KEY-----\n"
+}
\ No newline at end of file
diff --git a/nodes/postgres-6.json b/nodes/postgres-11.json
similarity index 74%
rename from nodes/postgres-6.json
rename to nodes/postgres-11.json
index 59d1cdc..a01909f 100644
--- a/nodes/postgres-6.json
+++ b/nodes/postgres-11.json
@@ -1,16 +1,17 @@
 {
-  "name": "postgres-6",
+  "name": "postgres-11",
+  "chef_environment": "production",
   "normal": {
     "knife_zero": {
-      "host": "10.1.1.196"
+      "host": "10.1.1.91"
     }
   },
   "automatic": {
-    "fqdn": "postgres-6",
+    "fqdn": "postgres-11",
     "os": "linux",
-    "os_version": "5.4.0-173-generic",
-    "hostname": "postgres-6",
-    "ipaddress": "192.168.122.60",
+    "os_version": "5.15.0-1095-kvm",
+    "hostname": "postgres-11",
+    "ipaddress": "192.168.122.142",
     "roles": [
       "base",
       "kvm_guest",
@@ -21,18 +22,20 @@
       "kosmos-base::default",
       "kosmos_kvm::guest",
       "kosmos_postgresql::primary",
-      "kosmos_postgresql::firewall",
       "kosmos-akkounts::pg_db",
       "kosmos-bitcoin::lndhub-go_pg_db",
       "kosmos-bitcoin::nbxplorer_pg_db",
       "kosmos_drone::pg_db",
       "kosmos_gitea::pg_db",
       "kosmos-mastodon::pg_db",
+      "kosmos_postgresql::firewall",
+      "kosmos_postgresql::management_scripts",
       "apt::default",
       "timezone_iii::default",
       "timezone_iii::debian",
       "ntp::default",
       "ntp::apparmor",
+      "kosmos-base::journald_conf",
       "kosmos-base::systemd_emails",
       "apt::unattended-upgrades",
       "kosmos-base::firewall",
@@ -44,17 +47,17 @@
       "hostname::default"
     ],
     "platform": "ubuntu",
-    "platform_version": "20.04",
+    "platform_version": "22.04",
     "cloud": null,
     "chef_packages": {
       "chef": {
-        "version": "18.4.2",
-        "chef_root": "/opt/chef/embedded/lib/ruby/gems/3.1.0/gems/chef-18.4.2/lib",
+        "version": "18.10.17",
+        "chef_root": "/opt/chef/embedded/lib/ruby/gems/3.1.0/gems/chef-18.10.17/lib",
         "chef_effortless": null
       },
       "ohai": {
-        "version": "18.1.11",
-        "ohai_root": "/opt/chef/embedded/lib/ruby/gems/3.1.0/gems/ohai-18.1.11/lib/ohai"
+        "version": "18.2.13",
+        "ohai_root": "/opt/chef/embedded/lib/ruby/gems/3.1.0/gems/ohai-18.2.13/lib/ohai"
       }
     }
   },
diff --git a/nodes/postgres-10.json b/nodes/postgres-12.json
similarity index 85%
rename from nodes/postgres-10.json
rename to nodes/postgres-12.json
index 73cde74..d2359b2 100644
--- a/nodes/postgres-10.json
+++ b/nodes/postgres-12.json
@@ -1,17 +1,17 @@
 {
-  "name": "postgres-10",
+  "name": "postgres-12",
   "chef_environment": "production",
   "normal": {
     "knife_zero": {
-      "host": "10.1.1.176"
+      "host": "10.1.1.134"
     }
   },
   "automatic": {
-    "fqdn": "postgres-10",
+    "fqdn": "postgres-12",
     "os": "linux",
-    "os_version": "5.15.0-1095-kvm",
-    "hostname": "postgres-10",
-    "ipaddress": "192.168.122.41",
+    "os_version": "5.15.0-1096-kvm",
+    "hostname": "postgres-12",
+    "ipaddress": "192.168.122.139",
     "roles": [
       "base",
       "kvm_guest",
@@ -24,6 +24,7 @@
       "kosmos_postgresql::hostsfile",
       "kosmos_postgresql::replica",
       "kosmos_postgresql::firewall",
+      "kosmos_postgresql::management_scripts",
       "apt::default",
       "timezone_iii::default",
       "timezone_iii::debian",
diff --git a/nodes/postgres-8.json b/nodes/postgres-8.json
deleted file mode 100644
index 82d4395..0000000
--- a/nodes/postgres-8.json
+++ /dev/null
@@ -1,62 +0,0 @@
-{
-  "name": "postgres-8",
-  "chef_environment": "production",
-  "normal": {
-    "knife_zero": {
-      "host": "10.1.1.99"
-    }
-  },
-  "automatic": {
-    "fqdn": "postgres-8",
-    "os": "linux",
-    "os_version": "5.15.0-1059-kvm",
-    "hostname": "postgres-8",
-    "ipaddress": "192.168.122.100",
-    "roles": [
-      "base",
-      "kvm_guest",
-      "postgresql_replica"
-    ],
-    "recipes": [
-      "kosmos-base",
-      "kosmos-base::default",
-      "kosmos_kvm::guest",
-      "kosmos_postgresql::hostsfile",
-      "kosmos_postgresql::replica",
-      "kosmos_postgresql::firewall",
-      "apt::default",
-      "timezone_iii::default",
-      "timezone_iii::debian",
-      "ntp::default",
-      "ntp::apparmor",
-      "kosmos-base::systemd_emails",
-      "apt::unattended-upgrades",
-      "kosmos-base::firewall",
-      "kosmos-postfix::default",
-      "postfix::default",
-      "postfix::_common",
-      "postfix::_attributes",
-      "postfix::sasl_auth",
-      "hostname::default"
-    ],
-    "platform": "ubuntu",
-    "platform_version": "22.04",
-    "cloud": null,
-    "chef_packages": {
-      "chef": {
-        "version": "18.5.0",
-        "chef_root": "/opt/chef/embedded/lib/ruby/gems/3.1.0/gems/chef-18.5.0/lib",
-        "chef_effortless": null
-      },
-      "ohai": {
-        "version": "18.1.11",
-        "ohai_root": "/opt/chef/embedded/lib/ruby/gems/3.1.0/gems/ohai-18.1.11/lib/ohai"
-      }
-    }
-  },
-  "run_list": [
-    "role[base]",
-    "role[kvm_guest]",
-    "role[postgresql_replica]"
-  ]
-}
diff --git a/nodes/postgres-9.json b/nodes/postgres-9.json
deleted file mode 100644
index 529052f..0000000
--- a/nodes/postgres-9.json
+++ /dev/null
@@ -1,63 +0,0 @@
-{
-  "name": "postgres-9",
-  "chef_environment": "production",
-  "normal": {
-    "knife_zero": {
-      "host": "10.1.1.3"
-    }
-  },
-  "automatic": {
-    "fqdn": "postgres-9",
-    "os": "linux",
-    "os_version": "5.15.0-1059-kvm",
-    "hostname": "postgres-9",
-    "ipaddress": "192.168.122.64",
-    "roles": [
-      "base",
-      "kvm_guest",
-      "postgresql_replica"
-    ],
-    "recipes": [
-      "kosmos-base",
-      "kosmos-base::default",
-      "kosmos_kvm::guest",
-      "kosmos_postgresql::hostsfile",
-      "kosmos_postgresql::replica",
-      "kosmos_postgresql::firewall",
-      "apt::default",
-      "timezone_iii::default",
-      "timezone_iii::debian",
-      "ntp::default",
-      "ntp::apparmor",
-      "kosmos-base::journald_conf",
-      "kosmos-base::systemd_emails",
-      "apt::unattended-upgrades",
-      "kosmos-base::firewall",
-      "kosmos-postfix::default",
-      "postfix::default",
-      "postfix::_common",
-      "postfix::_attributes",
-      "postfix::sasl_auth",
-      "hostname::default"
-    ],
-    "platform": "ubuntu",
-    "platform_version": "22.04",
-    "cloud": null,
-    "chef_packages": {
-      "chef": {
-        "version": "18.8.54",
-        "chef_root": "/opt/chef/embedded/lib/ruby/gems/3.1.0/gems/chef-18.8.54/lib",
-        "chef_effortless": null
-      },
-      "ohai": {
-        "version": "18.2.8",
-        "ohai_root": "/opt/chef/embedded/lib/ruby/gems/3.1.0/gems/ohai-18.2.8/lib/ohai"
-      }
-    }
-  },
-  "run_list": [
-    "role[base]",
-    "role[kvm_guest]",
-    "role[postgresql_replica]"
-  ]
-}

From ac4fb0c9ca0ba13abd4a2f1d5d0310a9cc69e2c1 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?R=C3=A2u=20Cao?= <raucao@kosmos.org>
Date: Sat, 11 Apr 2026 15:23:30 +0400
Subject: [PATCH 15/18] Fix Java/Homebrew cookbook resource failing with recent
 Chef

Required by our Mastodon cookbook
---
 Berksfile      | 1 +
 Berksfile.lock | 3 ++-
 2 files changed, 3 insertions(+), 1 deletion(-)

diff --git a/Berksfile b/Berksfile
index b2dd09a..5019974 100644
--- a/Berksfile
+++ b/Berksfile
@@ -24,6 +24,7 @@ cookbook 'composer',               '~> 2.7.0'
 cookbook 'fail2ban',               '~> 7.0.4'
 cookbook 'git',                    '~> 10.0.0'
 cookbook 'golang',                 '~> 5.3.1'
+cookbook 'homebrew',               '>= 6.0.0'
 cookbook 'hostname',               '= 0.4.2'
 cookbook 'hostsfile',              '~> 3.0.1'
 cookbook 'java',                   '~> 4.3.0'
diff --git a/Berksfile.lock b/Berksfile.lock
index 0bdf0ff..1651b1b 100644
--- a/Berksfile.lock
+++ b/Berksfile.lock
@@ -8,6 +8,7 @@ DEPENDENCIES
   firewall (~> 6.2.16)
   git (~> 10.0.0)
   golang (~> 5.3.1)
+  homebrew (>= 6.0.0)
   hostname (= 0.4.2)
   hostsfile (~> 3.0.1)
   ipfs
@@ -62,7 +63,7 @@ GRAPH
   git (10.0.0)
   golang (5.3.1)
     ark (>= 6.0)
-  homebrew (5.4.1)
+  homebrew (6.0.2)
   hostname (0.4.2)
     hostsfile (>= 0.0.0)
   hostsfile (3.0.1)

From f0314e0b993fdb862f86fd9ccd9087503619de9f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?R=C3=A2u=20Cao?= <raucao@kosmos.org>
Date: Sat, 11 Apr 2026 15:34:44 +0400
Subject: [PATCH 16/18] Update vendored cookbooks

---
 cookbooks/homebrew/.markdownlint-cli2.yaml |  2 +
 cookbooks/homebrew/CHANGELOG.md            | 42 ++++++++++++++++++++
 cookbooks/homebrew/libraries/helpers.rb    | 45 ++++++----------------
 cookbooks/homebrew/metadata.json           |  4 +-
 cookbooks/homebrew/metadata.rb             |  4 +-
 cookbooks/homebrew/renovate.json           |  5 ++-
 cookbooks/homebrew/resources/cask.rb       |  1 +
 cookbooks/homebrew/resources/tap.rb        |  1 +
 8 files changed, 64 insertions(+), 40 deletions(-)

diff --git a/cookbooks/homebrew/.markdownlint-cli2.yaml b/cookbooks/homebrew/.markdownlint-cli2.yaml
index 6fa8e77..0196ac7 100644
--- a/cookbooks/homebrew/.markdownlint-cli2.yaml
+++ b/cookbooks/homebrew/.markdownlint-cli2.yaml
@@ -3,3 +3,5 @@ config:
   line-length: false # MD013
   no-duplicate-heading: false # MD024
   reference-links-images: false # MD052
+ignores:
+  - .github/copilot-instructions.md
diff --git a/cookbooks/homebrew/CHANGELOG.md b/cookbooks/homebrew/CHANGELOG.md
index d179af1..31f072e 100644
--- a/cookbooks/homebrew/CHANGELOG.md
+++ b/cookbooks/homebrew/CHANGELOG.md
@@ -2,6 +2,48 @@
 
 This file is used to list changes made in each version of the homebrew cookbook.
 
+## 6.0.2 - *2025-09-04*
+
+Standardise files with files in sous-chefs/repo-management
+
+Standardise files with files in sous-chefs/repo-management
+
+## 6.0.1 - *2025-03-24*
+
+## 6.0.0 - *2025-03-17*
+
+- Updated library call for new homebrew class name found in chef-client 18.6.2+ releases
+
+## 5.4.9 - *2024-11-18*
+
+Standardise files with files in sous-chefs/repo-management
+
+Standardise files with files in sous-chefs/repo-management
+
+Standardise files with files in sous-chefs/repo-management
+
+Standardise files with files in sous-chefs/repo-management
+
+Standardise files with files in sous-chefs/repo-management
+
+## 5.4.8 - *2024-05-07*
+
+## 5.4.7 - *2024-05-06*
+
+- Explicitly include `Which` module from `Chef` which fixes runs on 18.x clients.
+
+## 5.4.6 - *2024-05-06*
+
+## 5.4.5 - *2023-11-01*
+
+Standardise files with files in sous-chefs/repo-management
+
+## 5.4.4 - *2023-09-28*
+
+## 5.4.3 - *2023-09-04*
+
+## 5.4.2 - *2023-07-10*
+
 ## 5.4.1 - *2023-06-01*
 
 ## 5.4.0 - *2023-04-24*
diff --git a/cookbooks/homebrew/libraries/helpers.rb b/cookbooks/homebrew/libraries/helpers.rb
index 1699163..f958166 100644
--- a/cookbooks/homebrew/libraries/helpers.rb
+++ b/cookbooks/homebrew/libraries/helpers.rb
@@ -20,8 +20,9 @@
 #
 
 class HomebrewUserWrapper
-  require 'chef/mixin/homebrew_user'
-  include Chef::Mixin::HomebrewUser
+  require 'chef/mixin/homebrew'
+  include Chef::Mixin::Homebrew
+  include Chef::Mixin::Which
 end
 
 module Homebrew
@@ -59,41 +60,17 @@ module Homebrew
 
   def owner
     @owner ||= begin
-      # once we only support 14.0 we can switch this to find_homebrew_username
-      require 'etc'
-      ::Etc.getpwuid(HomebrewUserWrapper.new.find_homebrew_uid).name
-               rescue Chef::Exceptions::CannotDetermineHomebrewOwner
-                 calculate_owner
-    end.tap do |owner|
-      Chef::Log.debug("Homebrew owner is #{owner}")
-    end
-  end
-
-  private
-
-  def calculate_owner
-    owner = homebrew_owner_attr || sudo_user || current_user
-    if owner == 'root'
-      raise Chef::Exceptions::User,
-           "Homebrew owner is 'root' which is not supported. " \
-           "To set an explicit owner, please set node['homebrew']['owner']."
-    end
-    owner
-  end
-
-  def homebrew_owner_attr
-    Chef.node['homebrew']['owner']
-  end
-
-  def sudo_user
-    ENV['SUDO_USER']
-  end
-
-  def current_user
-    ENV['USER']
+                 HomebrewUserWrapper.new.find_homebrew_username
+               rescue
+                 Chef::Exceptions::CannotDetermineHomebrewPath
+               end.tap do |owner|
+                 Chef::Log.debug("Homebrew owner is #{owner}")
+               end
   end
 end unless defined?(Homebrew)
 
 class HomebrewWrapper
   include Homebrew
 end
+
+Chef::Mixin::Homebrew.include(Homebrew)
diff --git a/cookbooks/homebrew/metadata.json b/cookbooks/homebrew/metadata.json
index 8ea519a..23c78ed 100644
--- a/cookbooks/homebrew/metadata.json
+++ b/cookbooks/homebrew/metadata.json
@@ -17,13 +17,13 @@
   "recipes": {
 
   },
-  "version": "5.4.1",
+  "version": "6.0.2",
   "source_url": "https://github.com/sous-chefs/homebrew",
   "issues_url": "https://github.com/sous-chefs/homebrew/issues",
   "privacy": false,
   "chef_versions": [
     [
-      ">= 15.3"
+      ">= 18.6.2"
     ]
   ],
   "ohai_versions": [
diff --git a/cookbooks/homebrew/metadata.rb b/cookbooks/homebrew/metadata.rb
index 9af47f4..204cca8 100644
--- a/cookbooks/homebrew/metadata.rb
+++ b/cookbooks/homebrew/metadata.rb
@@ -3,9 +3,9 @@ maintainer       'Sous Chefs'
 maintainer_email 'help@sous-chefs.org'
 license          'Apache-2.0'
 description      'Install Homebrew and includes resources for working with taps and casks'
-version          '5.4.1'
+version          '6.0.2'
 supports         'mac_os_x'
 
 source_url 'https://github.com/sous-chefs/homebrew'
 issues_url 'https://github.com/sous-chefs/homebrew/issues'
-chef_version '>= 15.3'
+chef_version '>= 18.6.2'
diff --git a/cookbooks/homebrew/renovate.json b/cookbooks/homebrew/renovate.json
index 7e7a8ba..a0b29c8 100644
--- a/cookbooks/homebrew/renovate.json
+++ b/cookbooks/homebrew/renovate.json
@@ -1,9 +1,10 @@
 {
   "$schema": "https://docs.renovatebot.com/renovate-schema.json",
   "extends": ["config:base"],
-  "packageRules": [{
+  "packageRules": [
+    {
       "groupName": "Actions",
-      "matchUpdateTypes": ["patch", "pin", "digest"],
+      "matchUpdateTypes": ["minor", "patch", "pin"],
       "automerge": true,
       "addLabels": ["Release: Patch", "Skip: Announcements"]
     },
diff --git a/cookbooks/homebrew/resources/cask.rb b/cookbooks/homebrew/resources/cask.rb
index c4d2053..f66fc8f 100644
--- a/cookbooks/homebrew/resources/cask.rb
+++ b/cookbooks/homebrew/resources/cask.rb
@@ -19,6 +19,7 @@
 # limitations under the License.
 #
 
+unified_mode true
 chef_version_for_provides '< 14.0' if respond_to?(:chef_version_for_provides)
 
 property :cask_name, String, regex: %r{^[\w/-]+$}, name_property: true
diff --git a/cookbooks/homebrew/resources/tap.rb b/cookbooks/homebrew/resources/tap.rb
index 8f67124..dc7f329 100644
--- a/cookbooks/homebrew/resources/tap.rb
+++ b/cookbooks/homebrew/resources/tap.rb
@@ -19,6 +19,7 @@
 # limitations under the License.
 #
 
+unified_mode true
 chef_version_for_provides '< 14.0' if respond_to?(:chef_version_for_provides)
 
 property :tap_name, String, name_property: true, regex: %r{^[\w-]+(?:\/[\w-]+)+$}

From a3be57afbc887bb1f81915a66e8ce6ee6027ba86 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?R=C3=A2u=20Cao?= <raucao@kosmos.org>
Date: Sun, 12 Apr 2026 09:10:58 +0400
Subject: [PATCH 17/18] Fix default apt keyring dir not existing on older
 Ubuntu

Recent Chef client versions use it
---
 site-cookbooks/kosmos-base/recipes/default.rb | 56 ++++++++++---------
 1 file changed, 31 insertions(+), 25 deletions(-)

diff --git a/site-cookbooks/kosmos-base/recipes/default.rb b/site-cookbooks/kosmos-base/recipes/default.rb
index 4d32ae6..7d7b5eb 100644
--- a/site-cookbooks/kosmos-base/recipes/default.rb
+++ b/site-cookbooks/kosmos-base/recipes/default.rb
@@ -24,11 +24,17 @@
 # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
 # THE SOFTWARE.
 
-include_recipe 'apt'
-include_recipe 'timezone_iii'
-include_recipe 'ntp'
-include_recipe 'kosmos-base::journald_conf'
-include_recipe 'kosmos-base::systemd_emails'
+include_recipe "apt"
+
+directory "/etc/apt/keyrings" do
+  mode "0755"
+  action :create
+end
+
+include_recipe "timezone_iii"
+include_recipe "ntp" if node["platform"] == "ubuntu" && node["platform_version"].to_f < 24.04
+include_recipe "kosmos-base::journald_conf"
+include_recipe "kosmos-base::systemd_emails"
 
 node.override["apt"]["unattended_upgrades"]["enable"] = true
 node.override["apt"]["unattended_upgrades"]["mail_only_on_error"] = false
@@ -43,57 +49,57 @@ node.override["apt"]["unattended_upgrades"]["allowed_origins"] = [
 ]
 node.override["apt"]["unattended_upgrades"]["mail"] = "ops@kosmos.org"
 node.override["apt"]["unattended_upgrades"]["syslog_enable"] = true
-include_recipe 'apt::unattended-upgrades'
+include_recipe "apt::unattended-upgrades"
 
-package 'mailutils'
-package 'mosh'
-package 'vim'
+package "mailutils"
+package "mosh"
+package "vim"
 
 # Don't create users and rewrite the sudo config in development environment.
 # It breaks the vagrant user
 unless node.chef_environment == "development"
   # Searches data bag "users" for groups attribute "sysadmin".
   # Places returned users in Unix group "sysadmin" with GID 2300.
-  users_manage 'sysadmin' do
+  users_manage "sysadmin" do
     group_id 2300
-    action [:remove, :create]
+    action %i[remove create]
   end
 
   sudo "sysadmin" do
     groups "sysadmin"
     nopasswd true
     defaults [
-    # not default on Ubuntu, explicitely enable. Uses a minimal white list of
-    # environment variables
-    'env_reset',
-    # Send emails on unauthorized attempts
-    'mail_badpass',
-    'secure_path="/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/snap/bin"',
+      # not default on Ubuntu, explicitely enable. Uses a minimal white list of
+      # environment variables
+      "env_reset",
+      # Send emails on unauthorized attempts
+      "mail_badpass",
+      'secure_path="/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/snap/bin"'
     ]
   end
 
   include_recipe "kosmos-base::firewall"
 
-  include_recipe 'kosmos-postfix'
+  include_recipe "kosmos-postfix"
 
-  node.override['set_fqdn'] = '*'
-  include_recipe 'hostname'
+  node.override["set_fqdn"] = "*"
+  include_recipe "hostname"
 
-  package 'ca-certificates'
+  package "ca-certificates"
 
-  directory '/usr/local/share/ca-certificates/cacert' do
+  directory "/usr/local/share/ca-certificates/cacert" do
     action :create
   end
 
-  ['http://www.cacert.org/certs/root.crt', 'http://www.cacert.org/certs/class3.crt'].each do |cert|
+  ["http://www.cacert.org/certs/root.crt", "http://www.cacert.org/certs/class3.crt"].each do |cert|
     remote_file "/usr/local/share/ca-certificates/cacert/#{File.basename(cert)}" do
       source cert
       action :create_if_missing
-      notifies :run, 'execute[update-ca-certificates]', :immediately
+      notifies :run, "execute[update-ca-certificates]", :immediately
     end
   end
 
-  execute 'update-ca-certificates' do
+  execute "update-ca-certificates" do
     action :nothing
   end
 end

From 1f7a1d0909fb528415cb9738527fac242439774e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?R=C3=A2u=20Cao?= <raucao@kosmos.org>
Date: Sun, 12 Apr 2026 16:11:40 +0400
Subject: [PATCH 18/18] Remove commented lines

---
 .../kosmos_postgresql/recipes/replica_logical.rb           | 7 -------
 1 file changed, 7 deletions(-)

diff --git a/site-cookbooks/kosmos_postgresql/recipes/replica_logical.rb b/site-cookbooks/kosmos_postgresql/recipes/replica_logical.rb
index 1f3dab9..7a3645b 100644
--- a/site-cookbooks/kosmos_postgresql/recipes/replica_logical.rb
+++ b/site-cookbooks/kosmos_postgresql/recipes/replica_logical.rb
@@ -6,10 +6,3 @@
 postgresql_custom_server postgresql_version do
   role "replica_logical"
 end
-
-# primary = postgresql_primary
-#
-# if primary.nil?
-#   Chef::Log.warn("No PostgreSQL primary node found. Skipping replication setup.")
-#   return
-# end