kosmos/chef
kosmos/chef
8
3
Fork 0
Code Issues 26 Pull Requests 2 Projects 2 Activity

Merge pull request 'Replace postgresql primary, add new replica' (#505) from chore/replace_postgres_primary into master

Browse Source 
Reviewed-on: #505
This commit is contained in:
Greg 2023-07-21 15:17:04 +00:00
commit 97e5a85ffd
6 changed files with 778 additions and 73 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
clients/postgres-2.json
View File

@ -1,4 +0,0 @@
{
"name": "postgres-2",
"public_key": "-----BEGIN PUBLIC KEY-----\nMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAoQmrHoW5X6jpN6cJnafW\nWKz1LpT2DvOFPeRmrlSPOPIUSVPUMZI8ojdxzmri/dzC8C4quCIYaL5ssARhe17L\no16LG3z/wdPENYONBj/oH5GRvmG1/uCgzL5+QQENk3kAWVAUb7EL3EgHNx90b2oC\nDNoHcjuoi3ZkCUXxkX7BESvw1jII4hP+HyoIqGh8TZLIAQ25/mS9aNL3TbcLYp5d\npqoM0Ruc4DHVuZTamN+zBZwNRCBz6uxMmaEBD+piYTEPJOnf7yUT7d2l/QzLftfE\nkSo80KOyrPOFdIPCpqKJyPOJZgg7fSgpR0YC6BH7I/JgIsIwUsQcMS8/ELUC5+Zg\nfwIDAQAB\n-----END PUBLIC KEY-----\n"
}

4
clients/postgres-5.json Normal file
View File

@ -0,0 +1,4 @@
{
"name": "postgres-5",
"public_key": "-----BEGIN PUBLIC KEY-----\nMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAvXZv6Gk+dhIVkTXH9hJ1\nt2oqsMSLmTUj71uPN+4j0rxCQriXa095Nle9ifJAxfwzQyKEpWKyZd1Hpyye6bL1\nwgWATZ/u5ZS4B63NhRFyDxgPlHWBBohaZBN42zeq0Y0PNGHPVGDH/zFDrpP22Q9Q\nYScsyXTauE/Yf8a/rKR5jdnoVsVVMxk0LHxka8FcM2cqVsDAcK7GqIG6epqNFY8P\nUb1P+mVxRwnkzvf1VtG212ezV/yw9uiQcUkHS+JwZMAgbC34k9iDyRmk6l4sj/Zk\nNem20ImMqdDzsrX8zEe21K+KNvpejPH9fxaNCwR8W+woBMMzqD3I7P9PbLjc70Rx\nRwIDAQAB\n-----END PUBLIC KEY-----\n"
}

63
nodes/postgres-2.json
View File

@ -1,63 +0,0 @@
{
"name": "postgres-2",
"normal": {
"knife_zero": {
"host": "10.1.1.99"
}
},
"automatic": {
"fqdn": "postgres-2",
"os": "linux",
"os_version": "5.4.0-77-generic",
"hostname": "postgres-2",
"ipaddress": "192.168.122.244",
"roles": [
"kvm_guest",
"postgresql_primary"
],
"recipes": [
"kosmos-base",
"kosmos-base::default",
"kosmos_kvm::guest",
"kosmos_postgresql::primary",
"kosmos_postgresql::firewall",
"kosmos-bitcoin::lndhub-go_pg_db",
"kosmos-bitcoin::nbxplorer_pg_db",
"kosmos_drone::pg_db",
"kosmos_gitea::pg_db",
"kosmos-mastodon::pg_db",
"apt::default",
"timezone_iii::default",
"timezone_iii::debian",
"ntp::default",
"ntp::apparmor",
"kosmos-base::systemd_emails",
"apt::unattended-upgrades",
"kosmos-base::firewall",
"kosmos-postfix::default",
"postfix::default",
"postfix::_common",
"postfix::_attributes",
"postfix::sasl_auth",
"hostname::default"
],
"platform": "ubuntu",
"platform_version": "20.04",
"cloud": null,
"chef_packages": {
"chef": {
"version": "15.15.0",
"chef_root": "/opt/chef/embedded/lib/ruby/gems/2.6.0/gems/chef-15.15.0/lib"
},
"ohai": {
"version": "15.12.0",
"ohai_root": "/opt/chef/embedded/lib/ruby/gems/2.6.0/gems/ohai-15.12.0/lib/ohai"
}
}
},
"run_list": [
"recipe[kosmos-base]",
"role[kvm_guest]",
"role[postgresql_primary]"
]
}

12
nodes/postgres-4.json
View File

@ -13,15 +13,19 @@
"ipaddress": "192.168.122.3",
"roles": [
"kvm_guest",
"postgresql_replica"
"postgresql_primary"
],
"recipes": [
"kosmos-base",
"kosmos-base::default",
"kosmos_kvm::guest",
"kosmos_postgresql::hostsfile",
"kosmos_postgresql::replica",
"kosmos_postgresql::primary",
"kosmos_postgresql::firewall",
"kosmos-bitcoin::lndhub-go_pg_db",
"kosmos-bitcoin::nbxplorer_pg_db",
"kosmos_drone::pg_db",
"kosmos_gitea::pg_db",
"kosmos-mastodon::pg_db",
"apt::default",
"timezone_iii::default",
"timezone_iii::debian",
@ -55,6 +59,6 @@
"run_list": [
"recipe[kosmos-base]",
"role[kvm_guest]",
"role[postgresql_replica]"
"role[postgresql_primary]"
]
}

760
nodes/postgres-5.json Normal file
View File

@ -0,0 +1,760 @@
{
"name": "postgres-5",
"override": {
"apt": {
"unattended_upgrades": {
"allowed_origins": [
"${distro_id}:${distro_codename}-security",
"${distro_id}:${distro_codename}-updates"
],
"mail": "ops@kosmos.org",
"syslog_enable": true
}
},
"postfix": {
"sasl": {
"smtp_sasl_user_name": "postmaster@mg.kosmos.org",
"smtp_sasl_passwd": "cce5798ca3ecce052087846d42216722"
},
"sasl_password_file": "/etc/postfix/sasl_passwd",
"main": {
"relayhost": "smtp.mailgun.org:587",
"smtp_sasl_auth_enable": "yes",
"smtp_sasl_password_maps": "hash:/etc/postfix/sasl_passwd",
"smtp_sasl_security_options": "noanonymous",
"smtp_tls_CAfile": "/etc/ssl/certs/ca-certificates.crt",
"smtpd_tls_CAfile": "/etc/ssl/certs/ca-certificates.crt"
}
},
"set_fqdn": "*"
},
"normal": {
"knife_zero": {
"host": "10.1.1.54"
}
},
"default": {
"audit": {
"inspec_backend_cache": true,
"reporter": null,
"fetcher": null,
"insecure": null,
"quiet": true,
"profiles": {
},
"inputs": {
},
"attributes": {
},
"waiver_file": null,
"json_file": {
"location": "/var/chef/compliance_reports/compliance-20230721113035.json"
},
"run_time_limit": 1.0,
"result_message_limit": 10000,
"result_include_backtrace": false,
"control_results_limit": 50,
"chef_node_attribute_enabled": true,
"compliance_phase": false,
"interval": {
"enabled": false,
"time": 1440
}
},
"apt": {
"cacher_dir": "/var/cache/apt-cacher-ng",
"cacher_interface": null,
"cacher_port": 3142,
"compiletime": false,
"compile_time_update": false,
"key_proxy": "",
"periodic_update_min_delay": 86400,
"launchpad_api_version": "1.0",
"unattended_upgrades": {
"enable": false,
"update_package_lists": true,
"allowed_origins": [
"Ubuntu focal"
],
"origins_patterns": [
],
"package_blacklist": [
],
"auto_fix_interrupted_dpkg": false,
"minimal_steps": false,
"install_on_shutdown": false,
"mail": null,
"sender": null,
"mail_only_on_error": true,
"remove_unused_dependencies": false,
"automatic_reboot": false,
"automatic_reboot_time": "now",
"dl_limit": null,
"random_sleep": null,
"syslog_enable": false,
"syslog_facility": "daemon",
"dpkg_options": [
]
},
"cacher_client": {
"cacher_server": {
}
},
"confd": {
"force_confask": false,
"force_confdef": false,
"force_confmiss": false,
"force_confnew": false,
"force_confold": false,
"install_recommends": true,
"install_suggests": false
}
},
"firewall": {
"allow_ssh": false,
"allow_winrm": false,
"allow_mosh": false,
"allow_loopback": false,
"allow_icmp": false,
"firewalld": {
"permanent": false
},
"iptables": {
"defaults": {
"policy": {
"input": "DROP",
"forward": "DROP",
"output": "ACCEPT"
},
"ruleset": {
"*filter": 1,
":INPUT DROP": 2,
":FORWARD DROP": 3,
":OUTPUT ACCEPT": 4,
"COMMIT_FILTER": 100
}
}
},
"ubuntu_iptables": false,
"redhat7_iptables": false,
"allow_established": true,
"ipv6_enabled": true,
"ufw": {
"defaults": {
"ipv6": "yes",
"manage_builtins": "no",
"ipt_sysctl": "/etc/ufw/sysctl.conf",
"ipt_modules": "nf_conntrack_ftp nf_nat_ftp nf_conntrack_netbios_ns",
"policy": {
"input": "DROP",
"output": "ACCEPT",
"forward": "DROP",
"application": "SKIP"
}
}
},
"windows": {
"defaults": {
"policy": {
"input": "blockinbound",
"output": "allowoutbound"
}
}
}
},
"hostsfile": {
"path": null
},
"hostname_cookbook": {
"hostsfile_ip": "127.0.1.1",
"hostsfile_aliases": [
],
"hostsfile_include_hostname_in_aliases": true,
"append_hostsfile_ip": true
},
"postfix": {
"mail_type": "client",
"relayhost_role": "relayhost",
"multi_environment_relay": false,
"use_procmail": false,
"use_alias_maps": false,
"use_transport_maps": false,
"use_access_maps": false,
"use_virtual_aliases": false,
"use_virtual_aliases_domains": false,
"use_relay_restrictions_maps": false,
"transports": {
},
"access": {
},
"virtual_aliases": {
},
"virtual_aliases_domains": {
},
"main_template_source": "postfix",
"master_template_source": "postfix",
"sender_canonical_map_entries": {
},
"smtp_generic_map_entries": {
},
"access_db_type": "hash",
"aliases_db_type": "hash",
"transport_db_type": "hash",
"virtual_alias_db_type": "hash",
"virtual_alias_domains_db_type": "hash",
"conf_dir": "/etc/postfix",
"aliases_db": "/etc/aliases",
"transport_db": "/etc/postfix/transport",
"access_db": "/etc/postfix/access",
"virtual_alias_db": "/etc/postfix/virtual",
"virtual_alias_domains_db": "/etc/postfix/virtual_domains",
"relay_restrictions_db": "/etc/postfix/relay_restrictions",
"main": {
"biff": "no",
"append_dot_mydomain": "no",
"myhostname": "postgres-5",
"mydomain": "postgres-5",
"myorigin": "$myhostname",
"mydestination": [
"postgres-5",
"postgres-5",
"localhost.localdomain",
"localhost"
],
"smtpd_use_tls": "yes",
"smtp_use_tls": "yes",
"smtp_sasl_auth_enable": "no",
"mailbox_size_limit": 0,
"mynetworks": null,
"inet_interfaces": "loopback-only",
"smtpd_tls_cert_file": "/etc/ssl/certs/ssl-cert-snakeoil.pem",
"smtpd_tls_key_file": "/etc/ssl/private/ssl-cert-snakeoil.key",
"smtpd_tls_CAfile": "/etc/ssl/certs/ca-certificates.crt",
"smtpd_tls_session_cache_database": "btree:${data_directory}/smtpd_scache",
"smtp_tls_CAfile": "/etc/ssl/certs/ca-certificates.crt",
"smtp_tls_session_cache_database": "btree:${data_directory}/smtp_scache",
"smtp_sasl_password_maps": "hash:/etc/postfix/sasl_passwd",
"smtp_sasl_security_options": "noanonymous",
"relayhost": "",
"maildrop_destination_recipient_limit": 1,
"cyrus_destination_recipient_limit": 1
},
"cafile": "/etc/ssl/certs/ca-certificates.crt",
"master": {
"smtp": {
"active": true,
"order": 10,
"type": "inet",
"private": false,
"chroot": false,
"command": "smtpd",
"args": [
]
},
"submission": {
"active": false,
"order": 20,
"type": "inet",
"private": false,
"chroot": false,
"command": "smtpd",
"args": [
"-o smtpd_enforce_tls=yes",
" -o smtpd_sasl_auth_enable=yes",
"-o smtpd_client_restrictions=permit_sasl_authenticated,reject"
]
},
"smtps": {
"active": false,
"order": 30,
"type": "inet",
"private": false,
"chroot": false,
"command": "smtpd",
"args": [
"-o smtpd_tls_wrappermode=yes",
"-o smtpd_sasl_auth_enable=yes",
"-o smtpd_client_restrictions=permit_sasl_authenticated,reject"
]
},
"628": {
"active": false,
"order": 40,
"type": "inet",
"private": false,
"chroot": false,
"command": "qmqpdd",
"args": [
]
},
"pickup": {
"active": true,
"order": 50,
"type": "fifo",
"private": false,
"chroot": false,
"wakeup": "60",
"maxproc": "1",
"command": "pickup",
"args": [
]
},
"cleanup": {
"active": true,
"order": 60,
"type": "unix",
"private": false,
"chroot": false,
"maxproc": "0",
"command": "cleanup",
"args": [
]
},
"qmgr": {
"active": true,
"order": 70,
"type": "fifo",
"private": false,
"chroot": false,
"wakeup": "300",
"maxproc": "1",
"command": "qmgr",
"args": [
]
},
"tlsmgr": {
"active": true,
"order": 80,
"type": "unix",
"chroot": false,
"wakeup": "1000?",
"maxproc": "1",
"command": "tlsmgr",
"args": [
]
},
"rewrite": {
"active": true,
"order": 90,
"type": "unix",
"chroot": false,
"command": "trivial-rewrite",
"args": [
]
},
"bounce": {
"active": true,
"order": 100,
"type": "unix",
"chroot": false,
"maxproc": "0",
"command": "bounce",
"args": [
]
},
"defer": {
"active": true,
"order": 110,
"type": "unix",
"chroot": false,
"maxproc": "0",
"command": "bounce",
"args": [
]
},
"trace": {
"active": true,
"order": 120,
"type": "unix",
"chroot": false,
"maxproc": "0",
"command": "bounce",
"args": [
]
},
"verify": {
"active": true,
"order": 130,
"type": "unix",
"chroot": false,
"maxproc": "1",
"command": "verify",
"args": [
]
},
"flush": {
"active": true,
"order": 140,
"type": "unix",
"private": false,
"chroot": false,
"wakeup": "1000?",
"maxproc": "0",
"command": "flush",
"args": [
]
},
"proxymap": {
"active": true,
"order": 150,
"type": "unix",
"chroot": false,
"command": "proxymap",
"args": [
]
},
"smtpunix": {
"service": "smtp",
"active": true,
"order": 160,
"type": "unix",
"chroot": false,
"maxproc": "500",
"command": "smtp",
"args": [
]
},
"relay": {
"active": true,
"comment": "When relaying mail as backup MX, disable fallback_relay to avoid MX loops",
"order": 170,
"type": "unix",
"chroot": false,
"command": "smtp",
"args": [
"-o smtp_fallback_relay="
]
},
"showq": {
"active": true,
"order": 180,
"type": "unix",
"private": false,
"chroot": false,
"command": "showq",
"args": [
]
},
"error": {
"active": true,
"order": 190,
"type": "unix",
"chroot": false,
"command": "error",
"args": [
]
},
"discard": {
"active": true,
"order": 200,
"type": "unix",
"chroot": false,
"command": "discard",
"args": [
]
},
"local": {
"active": true,
"order": 210,
"type": "unix",
"unpriv": false,
"chroot": false,
"command": "local",
"args": [
]
},
"virtual": {
"active": true,
"order": 220,
"type": "unix",
"unpriv": false,
"chroot": false,
"command": "virtual",
"args": [
]
},
"lmtp": {
"active": true,
"order": 230,
"type": "unix",
"chroot": false,
"command": "lmtp",
"args": [
]
},
"anvil": {
"active": true,
"order": 240,
"type": "unix",
"chroot": false,
"maxproc": "1",
"command": "anvil",
"args": [
]
},
"scache": {
"active": true,
"order": 250,
"type": "unix",
"chroot": false,
"maxproc": "1",
"command": "scache",
"args": [
]
},
"maildrop": {
"active": true,
"comment": "See the Postfix MAILDROP_README file for details. To main.cf will be added: maildrop_destination_recipient_limit=1",
"order": 510,
"type": "unix",
"unpriv": false,
"chroot": false,
"command": "pipe",
"args": [
"flags=DRhu user=vmail argv=/usr/local/bin/maildrop -d ${recipient}"
]
},
"old-cyrus": {
"active": false,
"comment": "The Cyrus deliver program has changed incompatibly, multiple times.",
"order": 520,
"type": "unix",
"unpriv": false,
"chroot": false,
"command": "pipe",
"args": [
"flags=R user=cyrus argv=/usr/lib/cyrus-imapd/deliver -e -m ${extension} ${user}"
]
},
"cyrus": {
"active": true,
"comment": "Cyrus 2.1.5 (Amos Gouaux). To main.cf will be added: cyrus_destination_recipient_limit=1",
"order": 530,
"type": "unix",
"unpriv": false,
"chroot": false,
"command": "pipe",
"args": [
"user=cyrus argv=/usr/lib/cyrus-imapd/deliver -e -r ${sender} -m ${extension} ${user}"
]
},
"uucp": {
"active": true,
"comment": "See the Postfix UUCP_README file for configuration details.",
"order": 540,
"type": "unix",
"unpriv": false,
"chroot": false,
"command": "pipe",
"args": [
"flags=Fqhu user=uucp argv=uux -r -n -z -a$sender - $nexthop!rmail ($recipient)"
]
},
"ifmail": {
"active": false,
"order": 550,
"type": "unix",
"unpriv": false,
"chroot": false,
"command": "pipe",
"args": [
"flags=F user=ftn argv=/usr/lib/ifmail/ifmail -r $nexthop ($recipient)"
]
},
"bsmtp": {
"active": true,
"order": 560,
"type": "unix",
"unpriv": false,
"chroot": false,
"command": "pipe",
"args": [
"flags=Fq. user=foo argv=/usr/local/sbin/bsmtp -f $sender $nexthop $recipient"
]
}
},
"aliases": {
},
"sasl_password_file": "/etc/postfix/sasl_passwd",
"sasl": {
"smtp_sasl_user_name": "",
"smtp_sasl_passwd": ""
}
},
"ntp": {
"servers": [
"0.pool.ntp.org",
"1.pool.ntp.org",
"2.pool.ntp.org",
"3.pool.ntp.org"
],
"peers": [
],
"restrictions": [
],
"tinker": {
"panic": 0,
"allan": 1500,
"dispersion": 15,
"step": 0.128,
"stepout": 900
},
"restrict_default": "kod notrap nomodify nopeer noquery",
"packages": [
"ntp"
],
"service": "ntp",
"varlibdir": "/var/lib/ntp",
"driftfile": "/var/lib/ntp/ntp.drift",
"logfile": null,
"conffile": "/etc/ntp.conf",
"statsdir": "/var/log/ntpstats/",
"conf_owner": "root",
"conf_group": "root",
"var_owner": "ntp",
"var_group": "ntp",
"leapfile": "/etc/ntp.leapseconds",
"sync_clock": false,
"sync_hw_clock": false,
"listen": null,
"listen_network": null,
"ignore": null,
"apparmor_enabled": true,
"monitor": false,
"statistics": true,
"conf_restart_immediate": false,
"keys": null,
"trustedkey": null,
"requestkey": null,
"disable_tinker_panic_on_virtualization_guest": true,
"peer": {
"key": null,
"use_iburst": true,
"use_burst": false,
"minpoll": 6,
"maxpoll": 10
},
"server": {
"prefer": "",
"use_iburst": true,
"use_burst": false,
"minpoll": 6,
"maxpoll": 10
},
"orphan": {
"enabled": false,
"stratum": 5
},
"localhost": {
"noquery": false
},
"use_cmos": false
},
"timezone_iii": {
"timezone": "Etc/UTC",
"tzdata_dir": "/usr/share/zoneinfo",
"localtime_path": "/etc/localtime",
"use_symlink": false
},
"kosmos_kvm": {
"host": {
"qemu_base_image": {
"url": "https://cloud-images.ubuntu.com/releases/focal/release-20230506/ubuntu-20.04-server-cloudimg-amd64-disk-kvm.img",
"checksum": "27d2b91fd2b715729d739e2a3155dce70d1aaae4f05c177f338b9d4b60be638c",
"path": "/var/lib/libvirt/images/base/ubuntu-20.04-server-cloudimg-amd64-disk-kvm-20230506.qcow2"
}
},
"backup": {
"schedule": "daily"
}
}
},
"automatic": {
"fqdn": "postgres-5",
"os": "linux",
"os_version": "5.4.0-153-generic",
"hostname": "postgres-5",
"ipaddress": "192.168.122.211",
"roles": [
"base",
"kvm_guest"
],
"recipes": [
"kosmos-base",
"kosmos-base::default",
"kosmos_kvm::guest",
"apt::default",
"timezone_iii::default",
"timezone_iii::debian",
"ntp::default",
"ntp::apparmor",
"kosmos-base::systemd_emails",
"apt::unattended-upgrades",
"kosmos-base::firewall",
"kosmos-postfix::default",
"postfix::default",
"postfix::_common",
"postfix::_attributes",
"postfix::sasl_auth",
"hostname::default"
],
"platform": "ubuntu",
"platform_version": "20.04",
"cloud": null,
"chef_packages": {
"chef": {
"version": "18.2.7",
"chef_root": "/opt/chef/embedded/lib/ruby/gems/3.1.0/gems/chef-18.2.7/lib",
"chef_effortless": null
},
"ohai": {
"version": "18.1.4",
"ohai_root": "/opt/chef/embedded/lib/ruby/gems/3.1.0/gems/ohai-18.1.4/lib/ohai"
}
}
},
"run_list": [
"role[base]",
"role[kvm_guest]",
"role[postgresql_replica]"
]
}

8
scripts/postgresql/switch_primary.sh
View File

@ -13,7 +13,11 @@ new_primary_ip=`bundle exec knife node show $new_primary_hostname -a knife_zero.
echo "Switching primary to $new_primary_hostname ($new_primary_ip)"
bundle exec knife ssh name:$new_primary_hostname -a knife_zero.host "sudo su - postgres -c '/usr/lib/postgresql/12/bin/pg_ctl promote -D /var/lib/postgresql/12/main'"
echo "Replacing pg.kosmos.local host entries with $new_primary_ip"
bundle exec knife ssh roles:postgresql_client -a knife_zero.host "sudo sed -r \"s/^[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+\s(pg.kosmos.local)/$new_primary_ip\t\1/\" -i /etc/hosts"
echo "Stopping postgresql on $old_primary_hostname"
bundle exec knife ssh name:$old_primary_hostname -a knife_zero.host "sudo systemctl stop postgresql@12-main.service"
echo "Replacing pg.kosmos.local host entries with $new_primary_ip"
bundle exec knife ssh roles:postgresql_client -a knife_zero.host "sudo sed -r \"s/^[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+\s(pg.kosmos.local)/$new_primary_ip\t\1/\" -i /etc/hosts"
# TODO
# 1. Change roles in node configs
# 2. Converge new primary