Update vendored cookbooks

Required by our Mastodon cookbook

Berksfile

@@ -24,6 +24,7 @@ cookbook 'composer',               '~> 2.7.0'
 cookbook 'fail2ban',               '~> 7.0.4'
 cookbook 'git',                    '~> 10.0.0'
 cookbook 'golang',                 '~> 5.3.1'
+cookbook 'homebrew',               '>= 6.0.0'
 cookbook 'hostname',               '= 0.4.2'
 cookbook 'hostsfile',              '~> 3.0.1'
 cookbook 'java',                   '~> 4.3.0'

Berksfile.lock

@@ -8,6 +8,7 @@ DEPENDENCIES
   firewall (~> 6.2.16)
   git (~> 10.0.0)
   golang (~> 5.3.1)
+  homebrew (>= 6.0.0)
   hostname (= 0.4.2)
   hostsfile (~> 3.0.1)
   ipfs
@@ -62,7 +63,7 @@ GRAPH
   git (10.0.0)
   golang (5.3.1)
     ark (>= 6.0)
-  homebrew (5.4.1)
+  homebrew (6.0.2)
   hostname (0.4.2)
     hostsfile (>= 0.0.0)
   hostsfile (3.0.1)

clients/postgres-11.json

@@ -0,0 +1,4 @@
+{
+  "name": "postgres-11",
+  "public_key": "-----BEGIN PUBLIC KEY-----\nMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEA1foYpuubS2ovlg3uHO12\nQ/ROZ8MpG+LkCAM46uVfPaoWwfY0vdfMsBOanHDgm9DGUCEBJZ6LPrvCvGXbpPy6\n9GSswK75zVWODblNjvvV4ueGFq4bBFwRuZNjyMlqgyzeU+srZL0ivelu5XEuGuoD\nPYCBKWYqGMz85/eMC7/tinTJtKPyOtXe/G8meji+r7gh3j+ypj/EWeKfcRDa4aGe\n/DmMCurIjjPAXFLMAA6fIqPWVfcPw4APNPE60Z92yPGsTbPu7bL54M5f7udmmu7H\nOgk1HjMAmXCuLDzTkfaxqHP+57yELg/YpXR1E93VmBeQuIBsyOFEk6AmUmA1Ib6e\nnQIDAQAB\n-----END PUBLIC KEY-----\n"
+}

clients/postgres-12.json

@@ -0,0 +1,4 @@
+{
+  "name": "postgres-12",
+  "public_key": "-----BEGIN PUBLIC KEY-----\nMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEA1mYGrYB8keUKmXA8dhWc\ncCLzp50xR0ajSw+bWYydyRqD5wuEVKjiJu4+G9QmTVXkVgJ+AYI0Y9/WZYpDqVH6\nvLUo6BSNQaWx20q93qIdOGLy8YG3Qyznezk4l8T9u9vWZDyDpKw6gCxzikMkrXxb\n0cqOYtyud8+PtSEEMogSjOKhRURVHlVrlVH3SQO7Whke9rkiFcbXzubsK9yjkUtF\nxZafSoGorOlDsPvFTfYnkepVB+GHcgiribRYSrO+73GypC2kqMhCpWrb6a0VWsP/\nh53+q3JL3vBvdvjcv51Wpf4n6JdnXnQGn2/MdXEzw+NXgjU4/IdYtbORSbaI8F5t\nowIDAQAB\n-----END PUBLIC KEY-----\n"
+}

cookbooks/homebrew/.markdownlint-cli2.yaml

@@ -3,3 +3,5 @@ config:
   line-length: false # MD013
   no-duplicate-heading: false # MD024
   reference-links-images: false # MD052
+ignores:
+  - .github/copilot-instructions.md

cookbooks/homebrew/CHANGELOG.md

@@ -2,6 +2,48 @@
 
 This file is used to list changes made in each version of the homebrew cookbook.
 
+## 6.0.2 - *2025-09-04*
+
+Standardise files with files in sous-chefs/repo-management
+
+Standardise files with files in sous-chefs/repo-management
+
+## 6.0.1 - *2025-03-24*
+
+## 6.0.0 - *2025-03-17*
+
+- Updated library call for new homebrew class name found in chef-client 18.6.2+ releases
+
+## 5.4.9 - *2024-11-18*
+
+Standardise files with files in sous-chefs/repo-management
+
+Standardise files with files in sous-chefs/repo-management
+
+Standardise files with files in sous-chefs/repo-management
+
+Standardise files with files in sous-chefs/repo-management
+
+Standardise files with files in sous-chefs/repo-management
+
+## 5.4.8 - *2024-05-07*
+
+## 5.4.7 - *2024-05-06*
+
+- Explicitly include `Which` module from `Chef` which fixes runs on 18.x clients.
+
+## 5.4.6 - *2024-05-06*
+
+## 5.4.5 - *2023-11-01*
+
+Standardise files with files in sous-chefs/repo-management
+
+## 5.4.4 - *2023-09-28*
+
+## 5.4.3 - *2023-09-04*
+
+## 5.4.2 - *2023-07-10*
+
 ## 5.4.1 - *2023-06-01*
 
 ## 5.4.0 - *2023-04-24*

cookbooks/homebrew/libraries/helpers.rb

@@ -20,8 +20,9 @@
 #
 
 class HomebrewUserWrapper
-  require 'chef/mixin/homebrew_user'
+  require 'chef/mixin/homebrew'
-  include Chef::Mixin::HomebrewUser
+  include Chef::Mixin::Homebrew
+  include Chef::Mixin::Which
 end
 
 module Homebrew
@@ -59,41 +60,17 @@ module Homebrew
 
   def owner
     @owner ||= begin
-      # once we only support 14.0 we can switch this to find_homebrew_username
+                 HomebrewUserWrapper.new.find_homebrew_username
-      require 'etc'
+               rescue
-      ::Etc.getpwuid(HomebrewUserWrapper.new.find_homebrew_uid).name
+                 Chef::Exceptions::CannotDetermineHomebrewPath
-               rescue Chef::Exceptions::CannotDetermineHomebrewOwner
+               end.tap do |owner|
-                 calculate_owner
+                 Chef::Log.debug("Homebrew owner is #{owner}")
-    end.tap do |owner|
+               end
-      Chef::Log.debug("Homebrew owner is #{owner}")
-    end
-  end
-
-  private
-
-  def calculate_owner
-    owner = homebrew_owner_attr || sudo_user || current_user
-    if owner == 'root'
-      raise Chef::Exceptions::User,
-           "Homebrew owner is 'root' which is not supported. " \
-           "To set an explicit owner, please set node['homebrew']['owner']."
-    end
-    owner
-  end
-
-  def homebrew_owner_attr
-    Chef.node['homebrew']['owner']
-  end
-
-  def sudo_user
-    ENV['SUDO_USER']
-  end
-
-  def current_user
-    ENV['USER']
   end
 end unless defined?(Homebrew)
 
 class HomebrewWrapper
   include Homebrew
 end
+
+Chef::Mixin::Homebrew.include(Homebrew)

cookbooks/homebrew/metadata.json

@@ -17,13 +17,13 @@
   "recipes": {
 
   },
-  "version": "5.4.1",
+  "version": "6.0.2",
   "source_url": "https://github.com/sous-chefs/homebrew",
   "issues_url": "https://github.com/sous-chefs/homebrew/issues",
   "privacy": false,
   "chef_versions": [
     [
-      ">= 15.3"
+      ">= 18.6.2"
     ]
   ],
   "ohai_versions": [

cookbooks/homebrew/metadata.rb

@@ -3,9 +3,9 @@ maintainer       'Sous Chefs'
 maintainer_email 'help@sous-chefs.org'
 license          'Apache-2.0'
 description      'Install Homebrew and includes resources for working with taps and casks'
-version          '5.4.1'
+version          '6.0.2'
 supports         'mac_os_x'
 
 source_url 'https://github.com/sous-chefs/homebrew'
 issues_url 'https://github.com/sous-chefs/homebrew/issues'
-chef_version '>= 15.3'
+chef_version '>= 18.6.2'

cookbooks/homebrew/renovate.json

@@ -1,9 +1,10 @@
 {
   "$schema": "https://docs.renovatebot.com/renovate-schema.json",
   "extends": ["config:base"],
-  "packageRules": [{
+  "packageRules": [
+    {
       "groupName": "Actions",
-      "matchUpdateTypes": ["patch", "pin", "digest"],
+      "matchUpdateTypes": ["minor", "patch", "pin"],
       "automerge": true,
       "addLabels": ["Release: Patch", "Skip: Announcements"]
     },

cookbooks/homebrew/resources/cask.rb

@@ -19,6 +19,7 @@
 # limitations under the License.
 #
 
+unified_mode true
 chef_version_for_provides '< 14.0' if respond_to?(:chef_version_for_provides)
 
 property :cask_name, String, regex: %r{^[\w/-]+$}, name_property: true

cookbooks/homebrew/resources/tap.rb

@@ -19,6 +19,7 @@
 # limitations under the License.
 #
 
+unified_mode true
 chef_version_for_provides '< 14.0' if respond_to?(:chef_version_for_provides)
 
 property :tap_name, String, name_property: true, regex: %r{^[\w-]+(?:\/[\w-]+)+$}

doc/postgres/migration.md

@@ -0,0 +1,287 @@
+# Migrating PostgreSQL cluster to a new major version
+
+## Summary
+
+1. Dump from a replica
+2. Restore to fresh VM running new major version
+3. Add logical replication for delta sync from current/old primary
+4. Switch primary to new server
+5. Remove logical replication on new server
+
+## Runbook
+
+* Primary host: `PRIMARY_HOST`
+* Replica host: `REPLICA_HOST`
+* New PG14 host: `NEW_HOST`
+* PostgreSQL superuser: `postgres`
+* Running locally on each machine via `sudo -u postgres`
+
+Adjust hostnames/IPs/etc. where needed.
+
+---
+
+### 🟢 0. PRIMARY — Pre-checks
+
+```bash
+sudo -u postgres psql -c "SHOW wal_level;"
+sudo -u postgres psql -c "SHOW max_replication_slots;"
+```
+
+If needed, edit config:
+
+```bash
+sudo -u postgres vi $PGDATA/postgresql.conf
+```
+
+Ensure:
+
+```conf
+wal_level = logical
+max_replication_slots = 10
+```
+
+Restart if changed:
+
+```bash
+sudo systemctl restart postgresql
+```
+
+---
+
+### 🔵🟡 3. Create keypair for syncing dump later
+
+🔵 On NEW_HOST:
+
+```bash
+sudo mkdir -p /home/postgres/.ssh && \
+sudo chown -R postgres:postgres /home/postgres && \
+sudo chmod 700 /home/postgres/.ssh && \
+sudo -u postgres bash -c 'ssh-keygen -t ecdsa -b 256 -f /home/postgres/.ssh/id_ecdsa -N "" -C "postgres@$(hostname)"' && \
+sudo cat /home/postgres/.ssh/id_ecdsa.pub
+```
+
+Copy the public key from the above output
+
+🟡 On replica:
+
+```bash
+sudo mkdir -p /home/postgres/.ssh && \
+sudo chown -R postgres:postgres /home/postgres && \
+sudo chmod 700 /home/postgres/.ssh && \
+echo [public_key] | sudo tee /home/postgres/.ssh/authorized_keys > /dev/null && \
+sudo chmod 700 /home/postgres/.ssh
+```
+
+---
+
+### 🟢 1. PRIMARY — Create publication and replication slots
+
+```bash
+sudo -u postgres pg_create_replication_publications
+```
+
+or
+
+```bash
+sudo -u postgres pg_create_replication_publication [db_name]
+```
+
+Listing publications and slots:
+
+```bash
+sudo -u postgres pg_list_replication_publications
+sudo -u postgres pg_list_replication_slots
+```
+
+---
+
+### 🟡 3. REPLICA — Pause replication
+
+```bash
+sudo -u postgres psql -c "SELECT pg_wal_replay_pause();"
+```
+
+Verify:
+
+```bash
+sudo -u postgres psql -c "SELECT pg_is_wal_replay_paused();"
+```
+
+---
+
+### 🟡 4. REPLICA — Run dump
+
+```bash
+sudo -u postgres pg_dump_all_databases
+```
+
+or
+
+```bash
+sudo -u postgres bash -c "pg_dumpall --globals-only > /tmp/globals.sql"
+sudo -u postgres pg_dump_database [db_name]
+```
+
+---
+
+### 🟡 5. REPLICA — Resume replication
+
+```bash
+sudo -u postgres psql -c "SELECT pg_wal_replay_resume();"
+```
+
+---
+
+### 🔵 6. COPY dumps to NEW HOST
+
+From NEW_HOST:
+
+```bash
+export REPLICA_HOST=[private_ip] && \
+cd /tmp && \
+sudo -u postgres scp "postgres@$REPLICA_HOST:/tmp/globals.sql" . && \
+sudo -u postgres scp "postgres@$REPLICA_HOST:/tmp/dump_*.tar.zst" .
+```
+
+---
+
+### 🔵 7. NEW HOST (PostgreSQL 14) — Restore
+
+#### 7.1 Restore globals
+
+```bash
+sudo -u postgres psql -f /tmp/globals.sql
+```
+
+---
+
+#### 7.2 Create databases
+
+```bash
+sudo -u postgres psql -Atqc "SELECT datname FROM pg_database WHERE datallowconn AND datname NOT IN ('template1')" | \
+xargs -I{} sudo -u postgres createdb {}
+```
+
+or
+
+```bash
+sudo -u postgres createdb [db_name]
+```
+
+---
+
+#### 7.3 Restore each database
+
+```bash
+sudo -u postgres pg_restore_all_databases
+```
+
+or
+
+```bash
+sudo -u postgres pg_restore_database [db_name]
+```
+
+---
+
+### 🔵 8. NEW HOST — Create subscriptions
+
+```bash
+sudo -u postgres pg_create_replication_subscriptions
+```
+
+or
+
+```bash
+sudo -u postgres pg_create_replication_subscription [db_name]
+```
+
+---
+
+### 🔵 9. NEW HOST — Monitor replication
+
+```bash
+sudo -u postgres pg_list_replication_subscriptions
+```
+
+---
+
+### 🔴 11. CUTOVER
+
+#### 11.1 Stop writes on old primary
+
+Put app(s) in maintenance mode, stop the app/daemons.
+
+---
+
+#### 11.2 Wait for replication to catch up
+
+TODO: not the best way to check, since WAL LSNs keep increasing
+
+```bash
+sudo -u postgres psql -d [db_name] -c "SELECT * FROM pg_stat_subscription;"
+```
+
+---
+
+#### 11.3 Fix sequences
+
+Run per DB:
+
+```bash
+sudo -u postgres pg_fix_sequences_in_all_databases
+```
+
+or
+
+```bash
+sudo -u postgres pg_fix_sequences [db_name]
+```
+
+---
+
+#### 11.4 Point app to NEW_HOST
+
+1. Update `pg.kosmos.local` in `/etc/hosts` on app server(s). For example:
+
+   ```bash
+   export NEW_PG_PRIMARY=[private_ip]
+   knife ssh roles:ejabberd -a knife_zero.host "sudo sed -r \"s/^[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+\s(pg.kosmos.local)/$NEW_PG_PRIMARY\t\1/\" -i /etc/hosts"
+   ```
+
+   Or override node attribute(s) if necessary and/or approporiate.
+
+2. Start the app/daemons, and deactivate maintenance mode.
+
+---
+
+### 🧹 12. CLEANUP NEW_HOST
+
+```bash
+sudo -u postgres pg_drop_replication_subscriptions
+```
+
+---
+
+### 🧹 13. CLEANUP PRIMARY
+
+TODO: Looks like slots are dropped automatically, when subscriptions are dropped
+
+```bash
+sudo -u postgres pg_drop_replication_publications
+```
+
+---
+
+### 🧹 13. CLEANUP Chef
+
+Once all apps/databases are migrated, update the role in the node
+config of the new primary to 'postgres_primary' and converge it.
+
+Also delete the old primary node config from the Chef repo.
+
+---
+
+### ✅ DONE
+
+---

nodes/postgres-11.json

@@ -1,16 +1,17 @@
 {
-  "name": "postgres-6",
+  "name": "postgres-11",
+  "chef_environment": "production",
   "normal": {
     "knife_zero": {
-      "host": "10.1.1.196"
+      "host": "10.1.1.91"
     }
   },
   "automatic": {
-    "fqdn": "postgres-6",
+    "fqdn": "postgres-11",
     "os": "linux",
-    "os_version": "5.4.0-173-generic",
+    "os_version": "5.15.0-1095-kvm",
-    "hostname": "postgres-6",
+    "hostname": "postgres-11",
-    "ipaddress": "192.168.122.60",
+    "ipaddress": "192.168.122.142",
     "roles": [
       "base",
       "kvm_guest",
@@ -21,18 +22,20 @@
       "kosmos-base::default",
       "kosmos_kvm::guest",
       "kosmos_postgresql::primary",
-      "kosmos_postgresql::firewall",
       "kosmos-akkounts::pg_db",
       "kosmos-bitcoin::lndhub-go_pg_db",
       "kosmos-bitcoin::nbxplorer_pg_db",
       "kosmos_drone::pg_db",
       "kosmos_gitea::pg_db",
       "kosmos-mastodon::pg_db",
+      "kosmos_postgresql::firewall",
+      "kosmos_postgresql::management_scripts",
       "apt::default",
       "timezone_iii::default",
       "timezone_iii::debian",
       "ntp::default",
       "ntp::apparmor",
+      "kosmos-base::journald_conf",
       "kosmos-base::systemd_emails",
       "apt::unattended-upgrades",
       "kosmos-base::firewall",
@@ -44,17 +47,17 @@
       "hostname::default"
     ],
     "platform": "ubuntu",
-    "platform_version": "20.04",
+    "platform_version": "22.04",
     "cloud": null,
     "chef_packages": {
       "chef": {
-        "version": "18.4.2",
+        "version": "18.10.17",
-        "chef_root": "/opt/chef/embedded/lib/ruby/gems/3.1.0/gems/chef-18.4.2/lib",
+        "chef_root": "/opt/chef/embedded/lib/ruby/gems/3.1.0/gems/chef-18.10.17/lib",
         "chef_effortless": null
       },
       "ohai": {
-        "version": "18.1.11",
+        "version": "18.2.13",
-        "ohai_root": "/opt/chef/embedded/lib/ruby/gems/3.1.0/gems/ohai-18.1.11/lib/ohai"
+        "ohai_root": "/opt/chef/embedded/lib/ruby/gems/3.1.0/gems/ohai-18.2.13/lib/ohai"
       }
     }
   },

nodes/postgres-12.json

@@ -1,17 +1,17 @@
 {
-  "name": "postgres-10",
+  "name": "postgres-12",
   "chef_environment": "production",
   "normal": {
     "knife_zero": {
-      "host": "10.1.1.176"
+      "host": "10.1.1.134"
     }
   },
   "automatic": {
-    "fqdn": "postgres-10",
+    "fqdn": "postgres-12",
     "os": "linux",
-    "os_version": "5.15.0-1095-kvm",
+    "os_version": "5.15.0-1096-kvm",
-    "hostname": "postgres-10",
+    "hostname": "postgres-12",
-    "ipaddress": "192.168.122.41",
+    "ipaddress": "192.168.122.139",
     "roles": [
       "base",
       "kvm_guest",
@@ -24,6 +24,7 @@
       "kosmos_postgresql::hostsfile",
       "kosmos_postgresql::replica",
       "kosmos_postgresql::firewall",
+      "kosmos_postgresql::management_scripts",
       "apt::default",
       "timezone_iii::default",
       "timezone_iii::debian",

nodes/postgres-8.json

@@ -1,62 +0,0 @@
-{
-  "name": "postgres-8",
-  "chef_environment": "production",
-  "normal": {
-    "knife_zero": {
-      "host": "10.1.1.99"
-    }
-  },
-  "automatic": {
-    "fqdn": "postgres-8",
-    "os": "linux",
-    "os_version": "5.15.0-1059-kvm",
-    "hostname": "postgres-8",
-    "ipaddress": "192.168.122.100",
-    "roles": [
-      "base",
-      "kvm_guest",
-      "postgresql_replica"
-    ],
-    "recipes": [
-      "kosmos-base",
-      "kosmos-base::default",
-      "kosmos_kvm::guest",
-      "kosmos_postgresql::hostsfile",
-      "kosmos_postgresql::replica",
-      "kosmos_postgresql::firewall",
-      "apt::default",
-      "timezone_iii::default",
-      "timezone_iii::debian",
-      "ntp::default",
-      "ntp::apparmor",
-      "kosmos-base::systemd_emails",
-      "apt::unattended-upgrades",
-      "kosmos-base::firewall",
-      "kosmos-postfix::default",
-      "postfix::default",
-      "postfix::_common",
-      "postfix::_attributes",
-      "postfix::sasl_auth",
-      "hostname::default"
-    ],
-    "platform": "ubuntu",
-    "platform_version": "22.04",
-    "cloud": null,
-    "chef_packages": {
-      "chef": {
-        "version": "18.5.0",
-        "chef_root": "/opt/chef/embedded/lib/ruby/gems/3.1.0/gems/chef-18.5.0/lib",
-        "chef_effortless": null
-      },
-      "ohai": {
-        "version": "18.1.11",
-        "ohai_root": "/opt/chef/embedded/lib/ruby/gems/3.1.0/gems/ohai-18.1.11/lib/ohai"
-      }
-    }
-  },
-  "run_list": [
-    "role[base]",
-    "role[kvm_guest]",
-    "role[postgresql_replica]"
-  ]
-}

nodes/postgres-9.json

@@ -1,63 +0,0 @@
-{
-  "name": "postgres-9",
-  "chef_environment": "production",
-  "normal": {
-    "knife_zero": {
-      "host": "10.1.1.3"
-    }
-  },
-  "automatic": {
-    "fqdn": "postgres-9",
-    "os": "linux",
-    "os_version": "5.15.0-1059-kvm",
-    "hostname": "postgres-9",
-    "ipaddress": "192.168.122.64",
-    "roles": [
-      "base",
-      "kvm_guest",
-      "postgresql_replica"
-    ],
-    "recipes": [
-      "kosmos-base",
-      "kosmos-base::default",
-      "kosmos_kvm::guest",
-      "kosmos_postgresql::hostsfile",
-      "kosmos_postgresql::replica",
-      "kosmos_postgresql::firewall",
-      "apt::default",
-      "timezone_iii::default",
-      "timezone_iii::debian",
-      "ntp::default",
-      "ntp::apparmor",
-      "kosmos-base::journald_conf",
-      "kosmos-base::systemd_emails",
-      "apt::unattended-upgrades",
-      "kosmos-base::firewall",
-      "kosmos-postfix::default",
-      "postfix::default",
-      "postfix::_common",
-      "postfix::_attributes",
-      "postfix::sasl_auth",
-      "hostname::default"
-    ],
-    "platform": "ubuntu",
-    "platform_version": "22.04",
-    "cloud": null,
-    "chef_packages": {
-      "chef": {
-        "version": "18.8.54",
-        "chef_root": "/opt/chef/embedded/lib/ruby/gems/3.1.0/gems/chef-18.8.54/lib",
-        "chef_effortless": null
-      },
-      "ohai": {
-        "version": "18.2.8",
-        "ohai_root": "/opt/chef/embedded/lib/ruby/gems/3.1.0/gems/ohai-18.2.8/lib/ohai"
-      }
-    }
-  },
-  "run_list": [
-    "role[base]",
-    "role[kvm_guest]",
-    "role[postgresql_replica]"
-  ]
-}

roles/postgresql_primary.rb

@@ -1,12 +1,13 @@
 name "postgresql_primary"
 
-run_list %w(
+run_list [
-  kosmos_postgresql::primary
+  "kosmos_postgresql::primary",
-  kosmos_postgresql::firewall
+  "kosmos-akkounts::pg_db",
-  kosmos-akkounts::pg_db
+  "kosmos-bitcoin::lndhub-go_pg_db",
-  kosmos-bitcoin::lndhub-go_pg_db
+  "kosmos-bitcoin::nbxplorer_pg_db",
-  kosmos-bitcoin::nbxplorer_pg_db
+  "kosmos_drone::pg_db",
-  kosmos_drone::pg_db
+  "kosmos_gitea::pg_db",
-  kosmos_gitea::pg_db
+  "kosmos-mastodon::pg_db",
-  kosmos-mastodon::pg_db
+  "kosmos_postgresql::firewall",
-)
+  "kosmos_postgresql::management_scripts"
+]

roles/postgresql_replica.rb

@@ -1,7 +1,8 @@
 name "postgresql_replica"
 
-run_list %w(
+run_list [
-  kosmos_postgresql::hostsfile
+  "kosmos_postgresql::hostsfile",
-  kosmos_postgresql::replica
+  "kosmos_postgresql::replica",
-  kosmos_postgresql::firewall
+  "kosmos_postgresql::firewall",
-)
+  "kosmos_postgresql::management_scripts"
+]

roles/postgresql_replica_logical.rb

@@ -1,7 +1,8 @@
 name "postgresql_replica_logical"
 
-run_list %w(
+run_list [
-  kosmos_postgresql::hostsfile
+  "kosmos_postgresql::hostsfile",
-  kosmos_postgresql::replica_logical
+  "kosmos_postgresql::replica_logical",
-  kosmos_postgresql::firewall
+  "kosmos_postgresql::firewall",
-)
+  "kosmos_postgresql::management_scripts"
+]

site-cookbooks/kosmos-akkounts/recipes/default.rb

@@ -230,7 +230,6 @@ systemd_unit "akkounts.service" do
       WorkingDirectory: deploy_path,
       Environment: "RAILS_ENV=#{rails_env} SOLID_QUEUE_IN_PUMA=true",
       ExecStart: "#{bundle_path} exec puma -C config/puma.rb --pidfile #{deploy_path}/tmp/puma.pid",
-      ExecStop: "#{bundle_path} exec puma -C config/puma.rb --pidfile #{deploy_path}/tmp/puma.pid stop",
       ExecReload: "#{bundle_path} exec pumactl -F config/puma.rb --pidfile #{deploy_path}/tmp/puma.pid phased-restart",
       PIDFile: "#{deploy_path}/tmp/puma.pid",
       TimeoutSec: "10",

site-cookbooks/kosmos_drone/attributes/default.rb

@@ -1,2 +1,6 @@
 node.default["kosmos_drone"]["domain"] = "drone.kosmos.org"
 node.default["kosmos_drone"]["upstream_port"] = 80
+node.default["kosmos_drone"]["pg_host"] = "pg.kosmos.local"
+node.default["kosmos_drone"]["pg_port"] = 5432
+node.default["kosmos_drone"]["pg_db"] = "drone"
+node.default["kosmos_drone"]["pg_user"] = "drone"

site-cookbooks/kosmos_drone/recipes/default.rb

@@ -9,11 +9,11 @@ credentials = data_bag_item("credentials", "drone")
 drone_credentials = data_bag_item('credentials', 'drone')
 
 postgres_config = {
-  username: "drone",
+  host: node["kosmos_drone"]["pg_host"],
-  password: drone_credentials["postgresql_password"],
+  port: node["kosmos_drone"]["pg_port"],
-  host: "pg.kosmos.local",
+  database: node["kosmos_drone"]["pg_db"],
-  port: 5432,
+  username: node["kosmos_drone"]["pg_user"],
-  database: "drone"
+  password: drone_credentials["postgresql_password"]
 }
 
 directory deploy_path do

site-cookbooks/kosmos_postgresql/files/create_publication.sh

@@ -0,0 +1,31 @@
+#!/bin/bash
+set -euo pipefail
+
+DB_NAME="${1:?Usage: $0 <database_name>}"
+
+echo "== Processing DB: $DB_NAME =="
+
+# Create publication (idempotent)
+psql -d "$DB_NAME" -v ON_ERROR_STOP=1 <<'SQL'
+DO $$
+BEGIN
+  IF NOT EXISTS (
+    SELECT 1 FROM pg_publication WHERE pubname = 'migrate_pub'
+  ) THEN
+    CREATE PUBLICATION migrate_pub FOR ALL TABLES;
+  END IF;
+END
+$$;
+SQL
+
+# Create logical replication slot (idempotent-ish)
+SLOT="migrate_slot_${DB_NAME}"
+
+if ! psql -d "$DB_NAME" -Atqc "SELECT 1 FROM pg_replication_slots WHERE slot_name = '$SLOT'" | grep -q 1; then
+  echo "  Creating slot: $SLOT"
+  psql -d "$DB_NAME" -c "SELECT pg_create_logical_replication_slot('$SLOT', 'pgoutput');"
+else
+  echo "  Slot already exists: $SLOT"
+fi
+
+echo "== Done =="

site-cookbooks/kosmos_postgresql/files/create_publications.sh

@@ -3,7 +3,7 @@ set -e
 
 echo "== Creating publication in each database =="
 
-for db in $(psql -Atqc "SELECT datname FROM pg_database WHERE datallowconn AND datname NOT IN ('template0','template1')"); do
+for db in $(psql -Atqc "SELECT datname FROM pg_database WHERE datallowconn AND datname NOT IN ('template1','postgres')"); do
   echo "Processing DB: $db"
 
   # Create publication (idempotent)

site-cookbooks/kosmos_postgresql/files/drop_publications.sh

@@ -1,8 +1,9 @@
+#!/bin/bash
 set -e
 
-echo "== Dropping subscriptions slots and publications on PRIMARY =="
+echo "== Dropping subscriptions slots and publications =="
 
-for db in $(psql -Atqc "SELECT datname FROM pg_database WHERE datallowconn AND datname NOT IN ('template0','template1')"); do
+for db in $(psql -Atqc "SELECT datname FROM pg_database WHERE datallowconn AND datname NOT IN ('template1','postgres')"); do
   echo "Processing DB: $db"
 
   SLOT="migrate_slot_${db}"

site-cookbooks/kosmos_postgresql/files/drop_subscriptions.sh

@@ -1,27 +1,28 @@
+#!/usr/bin/env bash
 set -e
 
-echo "== Dropping subscriptions on PG14 =="
+echo "== Dropping subscriptions =="
 
-for db in $(psql -Atqc "SELECT datname FROM pg_database WHERE datallowconn AND datname NOT IN ('template0,'template1'')"); do
+for db in $(psql -Atqc "SELECT datname FROM pg_database WHERE datallowconn AND datname NOT IN ('template1','postgres')"); do
   echo "Processing DB: $db"
 
   SUB="migrate_sub_${db}"
 
-  # Disable first (important)
+  # Check if subscription exists
-  psql -d "$db" -c "ALTER SUBSCRIPTION $SUB DISABLE;" 2>/dev/null || true
+  EXISTS=$(psql -d "$db" -Atqc "SELECT 1 FROM pg_subscription WHERE subname = '$SUB'")
 
-  # Drop subscription if exists
+  if [ "$EXISTS" = "1" ]; then
-  psql -d "$db" -v ON_ERROR_STOP=1 <<SQL
+    echo "  Found subscription: $SUB"
-DO \$\$
+
-BEGIN
+    # Disable first (good practice)
-  IF EXISTS (
+    psql -d "$db" -c "ALTER SUBSCRIPTION $SUB DISABLE;"
-    SELECT 1 FROM pg_subscription WHERE subname = '$SUB'
+
-  ) THEN
+    # Drop it (must be top-level)
-    DROP SUBSCRIPTION $SUB;
+    psql -d "$db" -c "DROP SUBSCRIPTION $SUB;"
-  END IF;
+
-END
+  else
-\$\$;
+    echo "  No subscription: $SUB"
-SQL
+  fi
 
 done
 

site-cookbooks/kosmos_postgresql/files/dump_all_databases.sh

@@ -1,7 +1,7 @@
 #!/bin/bash
 cd /tmp && \
 (pg_dumpall --globals-only > globals.sql) && \
-psql -Atqc "SELECT datname FROM pg_database WHERE datallowconn AND datname NOT IN (''template0'')" | \
+psql -Atqc "SELECT datname FROM pg_database WHERE datallowconn AND datname NOT IN (''template1'',''postgres'')" | \
 xargs -I{} -P4 sh -c "
   pg_dump -Fd -j 4 -d \"{}\" -f dump_{} &&
   tar -cf - dump_{} | zstd -19 -T0 > dump_{}.tar.zst &&

site-cookbooks/kosmos_postgresql/files/dump_database.sh

@@ -0,0 +1,10 @@
+#!/bin/bash
+set -euo pipefail
+
+DB_NAME="${1:?Usage: $0 <database_name>}"
+
+cd /tmp
+
+pg_dump -Fd -j 4 -d "$DB_NAME" -f "dump_${DB_NAME}"
+tar -cf - "dump_${DB_NAME}" | zstd -19 -T0 > "dump_${DB_NAME}.tar.zst"
+rm -rf "dump_${DB_NAME}"

site-cookbooks/kosmos_postgresql/files/fix_sequences.sh

@@ -0,0 +1,35 @@
+#!/bin/bash
+set -e
+
+DB="$1"
+
+if [ -z "$DB" ]; then
+  echo "Usage: $0 <database>"
+  exit 1
+fi
+
+echo "== Fixing sequences in database: $DB =="
+
+SQL=$(psql -d "$DB" -Atqc "
+  SELECT
+    'SELECT setval(' ||
+    quote_literal(pg_get_serial_sequence(quote_ident(n.nspname)||'.'||quote_ident(c.relname), a.attname)) ||
+    ', COALESCE(MAX(' || quote_ident(a.attname) || '), 0) + 1, false) FROM ' ||
+    quote_ident(n.nspname)||'.'||quote_ident(c.relname) || ';'
+  FROM pg_class c
+  JOIN pg_namespace n ON n.oid = c.relnamespace
+  JOIN pg_attribute a ON a.attrelid = c.oid
+  WHERE c.relkind = 'r'
+    AND a.attnum > 0
+    AND NOT a.attisdropped
+    AND pg_get_serial_sequence(quote_ident(n.nspname)||'.'||quote_ident(c.relname), a.attname) IS NOT NULL;
+")
+
+if [ -z "$SQL" ]; then
+  echo "No sequences found in $DB"
+  exit 0
+fi
+
+echo "$SQL" | psql -d "$DB"
+
+echo "== Done =="

site-cookbooks/kosmos_postgresql/files/fix_sequences_in_all_databases.sh

@@ -0,0 +1,38 @@
+#!/bin/bash
+set -e
+
+echo "== Fixing sequences across all databases =="
+
+for db in $(psql -Atqc "SELECT datname FROM pg_database WHERE datallowconn AND datname NOT IN ('template1','postgres')"); do
+  echo "---- DB: $db ----"
+
+  # Generate fix statements
+  SQL=$(psql -d "$db" -Atqc "
+    SELECT
+      'SELECT setval(' ||
+      quote_literal(pg_get_serial_sequence(quote_ident(n.nspname)||'.'||quote_ident(c.relname), a.attname)) ||
+      ', COALESCE(MAX(' || quote_ident(a.attname) || '), 0) + 1, false) FROM ' ||
+      quote_ident(n.nspname)||'.'||quote_ident(c.relname) || ';'
+    FROM pg_class c
+    JOIN pg_namespace n ON n.oid = c.relnamespace
+    JOIN pg_attribute a ON a.attrelid = c.oid
+    WHERE c.relkind = 'r'
+      AND a.attnum > 0
+      AND NOT a.attisdropped
+      AND pg_get_serial_sequence(quote_ident(n.nspname)||'.'||quote_ident(c.relname), a.attname) IS NOT NULL;
+  ")
+
+  if [ -z "$SQL" ]; then
+    echo "No sequences found in $db"
+    continue
+  fi
+
+  echo "Fixing sequences in $db..."
+
+  # Execute generated statements
+  echo "$SQL" | psql -d "$db"
+
+done
+
+echo "== Done fixing sequences =="
+

site-cookbooks/kosmos_postgresql/files/list_publications.sh

@@ -1,5 +1,5 @@
 #!/bin/bash
-for db in $(psql -Atqc "SELECT datname FROM pg_database WHERE datallowconn"); do
+for db in $(psql -Atqc "SELECT datname FROM pg_database WHERE datallowconn AND datname NOT IN ('template1','postgres')"); do
   echo "DB: $db"
   psql -d "$db" -Atqc "SELECT pubname FROM pg_publication;"
 done

site-cookbooks/kosmos_postgresql/files/list_subscriptions.sh

@@ -1,5 +1,16 @@
 #!/bin/bash
-for db in $(psql -Atqc "SELECT datname FROM pg_database WHERE datallowconn AND datname NOT IN ('template0','template1')"); do
+set -euo pipefail
-  echo "==== DB: $db ===="
+
-  psql -d "$db" -c "SELECT * FROM pg_stat_subscription;"
+psql -Atqc "
+SELECT datname
+FROM pg_database
+WHERE datallowconn
+  AND datname NOT IN ('template1','postgres')
+" | while read -r db; do
+  result=$(psql -X -At -d "$db" -c "SELECT * FROM pg_stat_subscription;" 2>/dev/null || true)
+
+  if [[ -n "$result" ]]; then
+    echo "==== DB: $db ===="
+    echo "$result"
+  fi
 done

site-cookbooks/kosmos_postgresql/files/restore_all_databases.sh

@@ -1,8 +1,12 @@
 #!/bin/bash
+set -euo pipefail
+
 cd /tmp
+
 for f in dump_*.tar.zst; do
   db=$(echo $f | sed "s/dump_\(.*\)\.tar\.zst/\1/")
   echo "Restoring $db"
   zstd -d "$f" -c | tar -xf -
   pg_restore -j 4 -d "$db" dump_$db
+  rm -rf "dump_$db"
 done

site-cookbooks/kosmos_postgresql/files/restore_database.sh

@@ -0,0 +1,14 @@
+#!/bin/bash
+set -euo pipefail
+
+DB_NAME="${1:?Usage: $0 <database_name>}"
+
+cd /tmp
+
+FILE="dump_${DB_NAME}.tar.zst"
+DIR="dump_${DB_NAME}"
+
+echo "Restoring $DB_NAME"
+zstd -d "$FILE" -c | tar -xf -
+pg_restore -j 4 -d "$DB_NAME" "$DIR"
+rm -rf "$DIR"

site-cookbooks/kosmos_postgresql/libraries/helpers.rb

@@ -39,13 +39,5 @@ class Chef
     def postgresql_version
       node['kosmos_postgresql']['postgresql_version']
     end
-
-    def postgresql_service
-      "postgresql@#{postgresql_version}-main"
-    end
-
-    def postgresql_data_dir
-      "/var/lib/postgresql/#{postgresql_version}/main"
-    end
   end
 end

site-cookbooks/kosmos_postgresql/recipes/management_scripts.rb

@@ -0,0 +1,121 @@
+#
+# Cookbook:: kosmos_postgresql
+# Recipe:: management_scripts
+#
+
+credentials = data_bag_item('credentials', 'postgresql')
+
+cookbook_file "/usr/local/bin/pg_dump_all_databases" do
+  source "dump_all_databases.sh"
+  user "postgres"
+  group "postgres"
+  mode "0744"
+end
+
+cookbook_file "/usr/local/bin/pg_dump_database" do
+  source "dump_database.sh"
+  user "postgres"
+  group "postgres"
+  mode "0744"
+end
+
+cookbook_file "/usr/local/bin/pg_restore_all_databases" do
+  source "restore_all_databases.sh"
+  user "postgres"
+  group "postgres"
+  mode "0744"
+end
+
+cookbook_file "/usr/local/bin/pg_restore_database" do
+  source "restore_database.sh"
+  user "postgres"
+  group "postgres"
+  mode "0744"
+end
+
+cookbook_file "/usr/local/bin/pg_create_replication_publications" do
+  source "create_publications.sh"
+  user "postgres"
+  group "postgres"
+  mode "0744"
+end
+
+cookbook_file "/usr/local/bin/pg_create_replication_publication" do
+  source "create_publication.sh"
+  user "postgres"
+  group "postgres"
+  mode "0744"
+end
+
+cookbook_file "/usr/local/bin/pg_drop_replication_publications" do
+  source "drop_publications.sh"
+  user "postgres"
+  group "postgres"
+  mode "0744"
+end
+
+cookbook_file "/usr/local/bin/pg_list_replication_publications" do
+  source "list_publications.sh"
+  user "postgres"
+  group "postgres"
+  mode "0744"
+end
+
+cookbook_file "/usr/local/bin/pg_list_replication_slots" do
+  source "list_replication_slots.sh"
+  user "postgres"
+  group "postgres"
+  mode "0744"
+end
+
+template "/usr/local/bin/pg_create_replication_subscriptions" do
+  source "create_subscriptions.sh.erb"
+  user "postgres"
+  group "postgres"
+  mode "0740"
+  variables pg_host: "pg.kosmos.local",
+            pg_port: 5432,
+            pg_user: "replication",
+            pg_pass: credentials["replication_password"]
+  sensitive true
+end
+
+template "/usr/local/bin/pg_create_replication_subscription" do
+  source "create_subscription.sh.erb"
+  user "postgres"
+  group "postgres"
+  mode "0740"
+  variables pg_host: "pg.kosmos.local",
+            pg_port: 5432,
+            pg_user: "replication",
+            pg_pass: credentials["replication_password"]
+  sensitive true
+end
+
+cookbook_file "/usr/local/bin/pg_drop_replication_subscriptions" do
+  source "drop_subscriptions.sh"
+  user "postgres"
+  group "postgres"
+  mode "0744"
+end
+
+cookbook_file "/usr/local/bin/pg_list_replication_subscriptions" do
+  source "list_subscriptions.sh"
+  user "postgres"
+  group "postgres"
+  mode "0744"
+end
+
+cookbook_file "/usr/local/bin/pg_fix_sequences_in_all_databases" do
+  source "fix_sequences.sh"
+  user "postgres"
+  group "postgres"
+  mode "0744"
+end
+
+cookbook_file "/usr/local/bin/pg_fix_sequences" do
+  source "fix_sequences.sh"
+  user "postgres"
+  group "postgres"
+  mode "0744"
+end

site-cookbooks/kosmos_postgresql/recipes/primary.rb

@@ -6,38 +6,3 @@
 postgresql_custom_server postgresql_version do
   role "primary"
 end
-
-cookbook_file "/usr/local/bin/pg_dump_all_databases" do
-  source "dump_all_databases.sh"
-  user "postgres"
-  group "postgres"
-  mode "0744"
-end
-
-cookbook_file "/usr/local/bin/pg_create_replication_publications" do
-  source "create_publications.sh"
-  user "postgres"
-  group "postgres"
-  mode "0744"
-end
-
-cookbook_file "/usr/local/bin/pg_drop_replication_publications" do
-  source "drop_publications.sh"
-  user "postgres"
-  group "postgres"
-  mode "0744"
-end
-
-cookbook_file "/usr/local/bin/pg_list_replication_publications" do
-  source "list_publications.sh"
-  user "postgres"
-  group "postgres"
-  mode "0744"
-end
-
-cookbook_file "/usr/local/bin/pg_list_replication_slots" do
-  source "list_replication_slots.sh"
-  user "postgres"
-  group "postgres"
-  mode "0744"
-end

site-cookbooks/kosmos_postgresql/recipes/replica.rb

@@ -3,10 +3,6 @@
 # Recipe:: replica
 #
 
-service postgresql_service do
-  supports restart: true, status: true, reload: true
-end
-
 postgresql_custom_server postgresql_version do
   role "replica"
 end
@@ -20,13 +16,19 @@ if primary.nil?
   return
 end
 
+postgresql_service_name = "postgresql@#{postgresql_version}-main"
+postgresql_data_dir = "/var/lib/postgresql/#{postgresql_version}/main"
+
+# TODO Replace pg.kosmos.local with private IP once available
+# via proper node attribute
+# https://gitea.kosmos.org/kosmos/chef/issues/263
 execute "set up replication" do
   command <<-EOF
-systemctl stop #{postgresql_service}
+systemctl stop #{postgresql_service_name}
 mv #{postgresql_data_dir} #{postgresql_data_dir}.old
 pg_basebackup -h pg.kosmos.local -U replication -D #{postgresql_data_dir} -R
 chown -R postgres:postgres #{postgresql_data_dir}
-systemctl start #{postgresql_service}
+systemctl start #{postgresql_service_name}
   EOF
   environment 'PGPASSWORD' => postgresql_data_bag_item['replication_password']
   sensitive true

site-cookbooks/kosmos_postgresql/recipes/replica_logical.rb

@@ -3,48 +3,13 @@
 # Recipe:: replica_logical
 #
 
-service postgresql_service do
-  supports restart: true, status: true, reload: true
-end
-
 postgresql_custom_server postgresql_version do
   role "replica_logical"
 end
 
-postgresql_data_bag_item = data_bag_item('credentials', 'postgresql')
+# primary = postgresql_primary
-
+#
-primary = postgresql_primary
+# if primary.nil?
-
+#   Chef::Log.warn("No PostgreSQL primary node found. Skipping replication setup.")
-if primary.nil?
+#   return
-  Chef::Log.warn("No PostgreSQL primary node found. Skipping replication setup.")
+# end
-  return
-end
-
-template "/usr/local/bin/pg_create_replication_subscriptions" do
-  source "create_subscriptions.sh.erb"
-  user "postgres"
-  group "postgres"
-  mode "0740"
-  sensitive true
-end
-
-cookbook_file "/usr/local/bin/pg_drop_replication_subscriptions" do
-  source "drop_subscriptions.sh"
-  user "postgres"
-  group "postgres"
-  mode "0744"
-end
-
-cookbook_file "/usr/local/bin/pg_list_replication_subscriptions" do
-  source "list_subscriptions.sh"
-  user "postgres"
-  group "postgres"
-  mode "0744"
-end
-
-cookbook_file "/usr/local/bin/pg_restore_all_databases" do
-  source "restore_all_databases.sh"
-  user "postgres"
-  group "postgres"
-  mode "0744"
-end

site-cookbooks/kosmos_postgresql/resources/server.rb

@@ -44,13 +44,14 @@ action :create do
 
   shared_buffers = if node['memory']['total'].to_i / 1024 < 1024 # < 1GB RAM
                      "128MB"
-                   else # >= 1GB RAM, use 50% of total RAM
+                   else # >= 1GB RAM, use 25% of total RAM
-                     "#{node['memory']['total'].to_i / 1024 / 2}MB"
+                     "#{node['memory']['total'].to_i / 1024 / 4}MB"
                    end
 
   additional_config = {
     max_connections: 200, # default
     shared_buffers: shared_buffers,
+    work_mem: "4MB",
     unix_socket_directories: "/var/run/postgresql",
     dynamic_shared_memory_type: "posix",
     timezone: "UTC", # default is GMT
@@ -64,7 +65,7 @@ action :create do
   postgresql_server_conf "main" do
     version postgresql_version
     additional_config additional_config
-    notifies :reload, "service[#{postgresql_service}]", :delayed
+    notifies :restart, "service[#{postgresql_service}]", :delayed
   end
 
   postgresql_user "replication" do

site-cookbooks/kosmos_postgresql/templates/create_subscription.sh.erb

@@ -0,0 +1,31 @@
+#!/bin/bash
+set -euo pipefail
+
+DB_NAME="${1:?Usage: $0 <database_name>}"
+
+echo "== Processing DB: $DB_NAME =="
+
+SLOT="migrate_slot_${DB_NAME}"
+SUB="migrate_sub_${DB_NAME}"
+
+psql -d "$DB_NAME" -v ON_ERROR_STOP=1 <<SQL
+DO \$\$
+BEGIN
+  IF NOT EXISTS (
+    SELECT 1 FROM pg_subscription WHERE subname = '$SUB'
+  ) THEN
+    CREATE SUBSCRIPTION $SUB
+    CONNECTION 'host=<%= @pg_host %> port=<%= @pg_port %> dbname=$DB_NAME user=<%= @pg_user %> password=<%= @pg_pass %>'
+    PUBLICATION migrate_pub
+    WITH (
+      slot_name = '$SLOT',
+      create_slot = false,
+      copy_data = false,
+      enabled = true
+    );
+  END IF;
+END
+\$\$;
+SQL
+
+echo "== Done =="

site-cookbooks/kosmos_postgresql/templates/create_subscriptions.sh.erb

@@ -1,8 +1,9 @@
+#!/bin/bash
 set -e
 
 echo "== Creating subscriptions for all databases =="
 
-for db in $(psql -Atqc "SELECT datname FROM pg_database WHERE datallowconn AND datname NOT IN ('template0','template1')"); do
+for db in $(psql -Atqc "SELECT datname FROM pg_database WHERE datallowconn AND datname NOT IN ('template1','postgres')"); do
   echo "Processing DB: $db"
 
   SLOT="migrate_slot_${db}"