Rename postgres cookbook, deploy new replica

fixes #361
closes #330

nodes/akkounts-1.json

@@ -18,7 +18,7 @@
     "recipes": [
       "kosmos-base",
       "kosmos-base::default",
-      "kosmos-postgresql::hostsfile",
+      "kosmos_postgresql::hostsfile",
       "kosmos-akkounts",
       "kosmos-akkounts::default",
       "kosmos-akkounts::nginx",

nodes/postgres-2.json

@@ -8,17 +8,17 @@
   "automatic": {
     "fqdn": "postgres-2",
     "os": "linux",
-    "os_version": "5.4.0-64-generic",
+    "os_version": "5.4.0-77-generic",
     "hostname": "postgres-2",
     "ipaddress": "192.168.122.244",
     "roles": [
-      "postgresql_replica"
+      "postgresql_primary"
     ],
     "recipes": [
       "kosmos-base",
       "kosmos-base::default",
-      "kosmos-postgresql::replica",
+      "kosmos_postgresql::primary",
-      "kosmos-postgresql::firewall",
+      "kosmos_postgresql::firewall",
       "apt::default",
       "timezone_iii::default",
       "timezone_iii::debian",

roles/postgresql_client.rb

@@ -3,5 +3,5 @@
 name "postgresql_client"
 
 run_list %w(
-  kosmos-postgresql::hostsfile
+  kosmos_postgresql::hostsfile
 )

roles/postgresql_primary.rb

@@ -1,6 +1,6 @@
 name "postgresql_primary"
 
 run_list %w(
-  kosmos-postgresql::primary
+  kosmos_postgresql::primary
-  kosmos-postgresql::firewall
+  kosmos_postgresql::firewall
 )

roles/postgresql_replica.rb

@@ -1,7 +1,7 @@
 name "postgresql_replica"
 
 run_list %w(
-  kosmos-postgresql::hostsfile
+  kosmos_postgresql::hostsfile
-  kosmos-postgresql::replica
+  kosmos_postgresql::replica
-  kosmos-postgresql::firewall
+  kosmos_postgresql::firewall
 )

site-cookbooks/kosmos-postgresql/.gitignore

@@ -1,22 +0,0 @@
-.vagrant
-*~
-*#
-.#*
-\#*#
-.*.sw[a-z]
-*.un~
-
-# Bundler
-Gemfile.lock
-gems.locked
-bin/*
-.bundle/*
-
-# test kitchen
-.kitchen/
-.kitchen.local.yml
-
-# Chef
-Berksfile.lock
-.zero-knife.rb
-Policyfile.lock.json

site-cookbooks/kosmos-postgresql/Berksfile

@@ -1,4 +0,0 @@
-# frozen_string_literal: true
-source 'https://supermarket.chef.io'
-
-metadata

site-cookbooks/kosmos-postgresql/CHANGELOG.md

@@ -1,5 +0,0 @@
-# kosmos-postgresql CHANGELOG
-
-# 0.1.0
-
-Initial release.

site-cookbooks/kosmos-postgresql/LICENSE

@@ -1,20 +0,0 @@
-Copyright (c) 2019-2020 Kosmos Developers
-
-Permission is hereby granted, free of charge, to any person obtaining
-a copy of this software and associated documentation files (the
-"Software"), to deal in the Software without restriction, including
-without limitation the rights to use, copy, modify, merge, publish,
-distribute, sublicense, and/or sell copies of the Software, and to
-permit persons to whom the Software is furnished to do so, subject to
-the following conditions:
-
-The above copyright notice and this permission notice shall be
-included in all copies or substantial portions of the Software.
-
-THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
-EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
-MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
-NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
-LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
-OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
-WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

site-cookbooks/kosmos-postgresql/README.md

@@ -1,57 +0,0 @@
-# kosmos-postgresql
-
-## Usage
-
-### On the primary:
-
-Set the `postgresql_primary` role on the node
-
-### On the replica:
-
-Add the `postgresql_replica` role to the node's run list. Run Chef on the node
-a first time.
-After the initial Chef run on the replica, run Chef on the primary to add the
-firewall rules and PostgreSQL access rules, then run Chef again on the replica
-to set up replication.
-
-## Caveat
-
-[`firewall_rules`](https://github.com/chef-cookbooks/firewall/issues/134) and
-[`postgresql_access`](https://github.com/sous-chefs/postgresql/issues/648) are
-declared in recipes, not resources because of the way custom resources
-work currently in Chef. See the `default.rb` and `replica.rb` recipes.
-
-The primary gives access to the `replication` db to the `replication` user
-connecting from a replica, and replicas to the primary. For more information
-about PostgreSQL client authentication, see the
-[official docs](https://www.postgresql.org/docs/12/auth-pg-hba-conf.html)
-
-The primary opens up the PostgreSQL port (5432 TCP) to replicas, and replicas
-to the primary.
-
-## TLS self-signed certificate
-
-A wildcard (`*.kosmos.org` certificate) was generated with the following
-commands:
-
-```
-openssl req -new -nodes -text -out root.csr -keyout root.key \
-  -subj "/CN=root.kosmos.org"
-chmod og-rwx root.key
-openssl x509 -req -in root.csr -text -days 3650 \
-  -extfile /etc/ssl/openssl.cnf -extensions v3_ca \
-  -signkey root.key -out root.crt
-openssl req -new -nodes -text -out server.csr \
-  -keyout server.key -subj "/CN=*.kosmos.org"
-chmod og-rwx server.key
-openssl x509 -req -in server.csr -text -days 1825 \
-  -CA root.crt -CAkey root.key -CAcreateserial \
-  -out server.crt
-```
-
-It is valid until May 12 2025.
-
-The content of `server.crt`, `server.key` and `root.crt` an stored in the
-`postgresql` encrypted data bag. The root key is stored in LastPass
-("Self-signed TLS root certificate"). `server.crt` & `server.key` are used by
-the PostgreSQL server.

site-cookbooks/kosmos-postgresql/attributes/default.rb

@@ -1,3 +0,0 @@
-# This is set to false by default, and set to true in the server resource
-# for replicas.
-node.default['kosmos-postgresql']['ready_to_set_up_replica'] = false

site-cookbooks/kosmos-postgresql/chefignore

@@ -1,104 +0,0 @@
-# Put files/directories that should be ignored in this file when uploading
-# to a chef-server or supermarket.
-# Lines that start with '# ' are comments.
-
-# OS generated files #
-######################
-.DS_Store
-Icon?
-nohup.out
-ehthumbs.db
-Thumbs.db
-
-# SASS #
-########
-.sass-cache
-
-# EDITORS #
-###########
-\#*
-.#*
-*~
-*.sw[a-z]
-*.bak
-REVISION
-TAGS*
-tmtags
-*_flymake.*
-*_flymake
-*.tmproj
-.project
-.settings
-mkmf.log
-
-## COMPILED ##
-##############
-a.out
-*.o
-*.pyc
-*.so
-*.com
-*.class
-*.dll
-*.exe
-*/rdoc/
-
-# Testing #
-###########
-.watchr
-.rspec
-spec/*
-spec/fixtures/*
-test/*
-features/*
-examples/*
-Guardfile
-Procfile
-.kitchen*
-kitchen.yml*
-.rubocop.yml
-spec/*
-Rakefile
-.travis.yml
-.foodcritic
-.codeclimate.yml
-
-# SCM #
-#######
-.git
-*/.git
-.gitignore
-.gitmodules
-.gitconfig
-.gitattributes
-.svn
-*/.bzr/*
-*/.hg/*
-*/.svn/*
-
-# Berkshelf #
-#############
-Berksfile
-Berksfile.lock
-cookbooks/*
-tmp
-
-# Bundler #
-###########
-vendor/*
-
-# Policyfile #
-##############
-Policyfile.rb
-Policyfile.lock.json
-
-# Cookbooks #
-#############
-CONTRIBUTING*
-CHANGELOG*
-TESTING*
-
-# Vagrant #
-###########
-.vagrant
-Vagrantfile

site-cookbooks/kosmos-postgresql/libraries/helpers.rb

@@ -1,45 +0,0 @@
-class Chef
-  class Recipe
-    def postgresql_primary
-      postgresql_primary = search(:node, "role:postgresql_primary AND chef_environment:#{node.chef_environment}").first
-
-      unless postgresql_primary.nil?
-        primary_ip = ip_for(postgresql_primary)
-
-        { hostname: postgresql_primary[:hostname], ipaddress: primary_ip }
-      end
-    end
-
-    def postgresql_replicas
-      postgresql_replicas = []
-
-      search(:node, "role:postgresql_replica AND chef_environment:#{node.chef_environment}").each do |replica|
-        replica_ip = ip_for(replica)
-
-        postgresql_replicas << { hostname: replica[:hostname], ipaddress: replica_ip }
-      end
-
-      postgresql_replicas
-    end
-
-    def ip_for(server_node)
-      if node.chef_environment == "development"
-        server_node['network']['interfaces']['eth1']['routes'].first['src']
-      else
-        # If the server has a private Zerotier IP, use it
-        if server_node['knife_zero'] && server_node['knife_zero']['host'] && \
-            server_node['knife_zero']['host'].start_with?("10.1.1.")
-          server_node['knife_zero']['host']
-        else
-          server_node['ipaddress']
-        end
-      end
-    end
-
-    def postgresql_service_name
-      postgresql_version = "12"
-
-      "postgresql@#{postgresql_version}-main"
-    end
-  end
-end

site-cookbooks/kosmos-postgresql/metadata.rb

@@ -1,25 +0,0 @@
-name 'kosmos-postgresql'
-maintainer 'Kosmos'
-maintainer_email 'ops@5apps.com'
-license 'MIT'
-description 'Installs/Configures kosmos-postgresql'
-long_description 'Installs/Configures kosmos-postgresql'
-version '0.1.0'
-chef_version '>= 12.14' if respond_to?(:chef_version)
-
-# The `issues_url` points to the location where issues for this cookbook are
-# tracked.  A `View Issues` link will be displayed on this cookbook's page when
-# uploaded to a Supermarket.
-#
-# issues_url 'https://github.com/<insert_org_here>/kosmos-postgresql/issues'
-
-# The `source_url` points to the development repository for this cookbook.  A
-# `View Source` link will be displayed on this cookbook's page when uploaded to
-# a Supermarket.
-#
-# source_url 'https://github.com/<insert_org_here>/kosmos-postgresql'
-
-depends "postgresql", ">= 7.0.0"
-depends "build-essential"
-depends "kosmos_encfs"
-depends "hostsfile"

site-cookbooks/kosmos-postgresql/recipes/firewall.rb

@@ -1,15 +0,0 @@
-#
-# Cookbook:: kosmos-postgresql
-# Recipe:: firewall
-#
-
-unless node.chef_environment == "development"
-  include_recipe "kosmos-base::firewall"
-
-  firewall_rule "postgresql zerotier members" do
-    port     5432
-    protocol :tcp
-    command  :allow
-    source   "10.1.1.0/24"
-  end
-end

site-cookbooks/kosmos-postgresql/recipes/hostsfile.rb

@@ -1,16 +0,0 @@
-#
-# Cookbook:: kosmos-postgresql
-# Recipe:: hostsfile
-#
-
-begin
-primary_ip = postgresql_primary[:ipaddress]
-rescue NoMethodError
-end
-
-unless primary_ip.nil?
-  hostsfile_entry primary_ip do
-    hostname  "pg.kosmos.local"
-    unique    true
-  end
-end

site-cookbooks/kosmos-postgresql/recipes/primary.rb

@@ -1,33 +0,0 @@
-#
-# Cookbook:: kosmos-postgresql
-# Recipe:: primary
-#
-
-postgresql_version = "12"
-postgresql_service = "postgresql@#{postgresql_version}-main"
-
-service postgresql_service do
-  supports restart: true, status: true, reload: true
-end
-
-postgresql_custom_server postgresql_version do
-  role "primary"
-end
-
-postgresql_access "zerotier members" do
-  access_type "host"
-  access_db "all"
-  access_user "all"
-  access_addr "10.1.1.0/24"
-  access_method "md5"
-  notifies :reload, "service[#{postgresql_service}]", :immediately
-end
-
-postgresql_access "zerotier members replication" do
-  access_type "host"
-  access_db "replication"
-  access_user "replication"
-  access_addr "10.1.1.0/24"
-  access_method "md5"
-  notifies :reload, "service[#{postgresql_service}]", :immediately
-end

site-cookbooks/kosmos-postgresql/recipes/replica.rb

@@ -1,56 +0,0 @@
-#
-# Cookbook:: kosmos-postgresql
-# Recipe:: replica
-#
-
-postgresql_version = "12"
-postgresql_service = "postgresql@#{postgresql_version}-main"
-
-postgresql_custom_server postgresql_version do
-  role "replica"
-end
-
-service postgresql_service do
-  supports restart: true, status: true, reload: true
-end
-
-postgresql_data_bag_item = data_bag_item('credentials', 'postgresql')
-
-primary = postgresql_primary
-
-unless primary.nil?
-  # TODO
-  postgresql_data_dir = "/var/lib/postgresql/#{postgresql_version}/main"
-
-  # FIXME get zerotier IP
-  execute "set up replication" do
-    command <<-EOF
-systemctl stop #{postgresql_service}
-mv #{postgresql_data_dir} #{postgresql_data_dir}.old
-pg_basebackup -h pg.kosmos.local -U replication -D #{postgresql_data_dir} -R
-chown -R postgres:postgres #{postgresql_data_dir}
-systemctl start #{postgresql_service}
-    EOF
-    environment 'PGPASSWORD' => postgresql_data_bag_item['replication_password']
-    sensitive true
-    not_if { ::File.exist? "#{postgresql_data_dir}/standby.signal" }
-  end
-
-  postgresql_access "zerotier members" do
-    access_type "host"
-    access_db "all"
-    access_user "all"
-    access_addr "10.1.1.0/24"
-    access_method "md5"
-    notifies :reload, "service[#{postgresql_service}]", :immediately
-  end
-
-  postgresql_access "zerotier members replication" do
-    access_type "host"
-    access_db "replication"
-    access_user "replication"
-    access_addr "10.1.1.0/24"
-    access_method "md5"
-    notifies :reload, "service[#{postgresql_service}]", :immediately
-  end
-end

site-cookbooks/kosmos-postgresql/resources/server.rb

@@ -1,77 +0,0 @@
-resource_name :postgresql_custom_server
-
-property :postgresql_version, String, required: true, name_property: true
-property :role, String, required: true # Can be primary or replica
-
-action :create do
-  postgresql_version = new_resource.postgresql_version
-  postgresql_data_dir = "/var/lib/postgresql/#{postgresql_version}/main"
-  postgresql_service = "postgresql@#{postgresql_version}-main"
-  postgresql_credentials = data_bag_item('credentials', 'postgresql')
-
-  build_essential do
-    compile_time true
-  end
-
-  package("libpq-dev") { action :nothing }.run_action(:install)
-
-  chef_gem 'pg' do
-    compile_time true
-  end
-
-  user "postgres" do
-    manage_home false
-  end
-
-  postgresql_server_install "main" do
-    version postgresql_version
-    setup_repo true
-    password postgresql_credentials['server_password']
-    action :install
-  end
-
-  service postgresql_service do
-    supports restart: true, status: true, reload: true
-    action [:enable, :start]
-  end
-
-  # This service is a dependency that will auto-start our cluster service on
-  # boot if it's enabled, so we disable it explicitly
-  service "postgresql" do
-    action :disable
-  end
-
-  shared_buffers = if node['memory']['total'].to_i / 1024 < 1024 # > 1GB RAM
-                     "128MB"
-                   else # >= 1GB RAM, use 25% of total RAM
-                     "#{node['memory']['total'].to_i / 1024 / 4}MB"
-                   end
-
-  additional_config = {
-    max_connections: 100, # default
-    shared_buffers: shared_buffers,
-    unix_socket_directories: "/var/run/postgresql",
-    dynamic_shared_memory_type: "posix",
-    timezone: "UTC", # default is GMT
-    listen_addresses: "0.0.0.0"
-  }
-
-  additional_config[:promote_trigger_file] = "#{postgresql_data_dir}/failover.trigger"
-
-  postgresql_server_conf "main" do
-    version postgresql_version
-    additional_config additional_config
-    notifies :reload, "service[#{postgresql_service}]", :delayed
-  end
-
-  postgresql_user "replication" do
-    action :create
-    replication true
-    password postgresql_credentials['replication_password']
-  end
-end
-
-action_class do
-  # to use the data_dir helper
-  include PostgresqlCookbook::Helpers
-end

site-cookbooks/kosmos_kvm/recipes/host.rb

@@ -2,34 +2,13 @@
 # Cookbook:: kosmos_kvm
 # Recipe:: host
 #
-# The MIT License (MIT)
-#
-# Copyright:: 2020, Kosmos Developers
-#
-# Permission is hereby granted, free of charge, to any person obtaining a copy
-# of this software and associated documentation files (the "Software"), to deal
-# in the Software without restriction, including without limitation the rights
-# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
-# copies of the Software, and to permit persons to whom the Software is
-# furnished to do so, subject to the following conditions:
-#
-# The above copyright notice and this permission notice shall be included in
-# all copies or substantial portions of the Software.
-#
-# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
-# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
-# THE SOFTWARE.
 
 package %w(virtinst libvirt-daemon-system)
 
 directory "/var/lib/libvirt/images/base" do
   recursive true
   owner "libvirt-qemu"
-  group "root"
+  group "kvm"
   mode "0750"
 end
 
@@ -37,7 +16,7 @@ end
 remote_file "/var/lib/libvirt/images/base/ubuntu-20.04-server-cloudimg-amd64-disk-kvm.qcow2" do
   source "http://cloud-images.ubuntu.com/releases/focal/release/ubuntu-20.04-server-cloudimg-amd64-disk-kvm.img"
   owner "libvirt-qemu"
-  group "root"
+  group "kvm"
   mode "0640"
 end