Add a custom resource to set up PostgreSQL 12

Supports both primary and replica. The access rules and firewall have to
be set up outside of the custom resource, so they are part of the
recipes instead

Refs #160

site-cookbooks/kosmos-postgresql/README.md

@@ -1,4 +1,41 @@
 # kosmos-postgresql
 
-TODO: Enter the cookbook description here.
+## Custom resources
 
+### `postgresql_custom_server`
+
+Usage:
+
+When the `tls` attribute is set to true, a TLS certificate for the FQDN
+(`node['fqdn']`, for example `andromeda.kosmos.org`) is generated using Let's
+Encrypt and copied to the PostgreSQL data directory and added to the
+`postgresql.conf` file
+
+#### On the primary:
+
+```ruby
+postgresql_custom_server "12" do
+  role "primary"
+  tls true
+end
+```
+
+#### On a replica:
+
+```ruby
+postgresql_custom_server "12" do
+  role "primary"
+  tls true
+end
+```
+
+After the initial Chef run on the replica, run Chef on the primary to add the
+firewall rules and PostgreSQL access rules, then run Chef again on the replica
+to set up replication.
+
+#### Caveat
+
+[`firewall_rules`](https://github.com/chef-cookbooks/firewall/issues/134) and
+[`postgresql_access`](https://github.com/sous-chefs/postgresql/issues/648) need
+to be declared in recipes, not resources because of the way custom resources
+work currently in Chef

site-cookbooks/kosmos-postgresql/attributes/default.rb

@@ -0,0 +1,3 @@
+# This is set to false by default, and set to true in the server resource
+# for replicas.
+node.default['kosmos-postgresql']['ready_to_set_up_replica'] = false

site-cookbooks/kosmos-postgresql/libraries/helpers.rb

@@ -0,0 +1,33 @@
+class Chef
+  class Recipe
+    def postgresql_primary
+      postgresql_primary = search(:node, "role:postgresql_primary AND chef_environment:#{node.chef_environment}").first
+
+      unless postgresql_primary.nil?
+        primary_ip = ip_for(postgresql_primary)
+
+        { hostname: postgresql_primary[:hostname], ipaddress: primary_ip }
+      end
+    end
+
+    def postgresql_replicas
+      postgresql_replicas = []
+
+      search(:node, "role:postgresql_replica AND chef_environment:#{node.chef_environment}").each do |replica|
+        replica_ip = ip_for(replica)
+
+        postgresql_replicas << { hostname: replica[:hostname], ipaddress: replica_ip }
+      end
+
+      postgresql_replicas
+    end
+
+    def ip_for(server_node)
+      if node.chef_environment == "development"
+        server_node['network']['interfaces']['eth1']['routes'].first['src']
+      else
+        server_node['ipaddress']
+      end
+    end
+  end
+end

site-cookbooks/kosmos-postgresql/metadata.rb

@@ -21,3 +21,5 @@ chef_version '>= 12.14' if respond_to?(:chef_version)
 
 depends "postgresql", ">= 7.0.0"
 depends "build-essential"
+depends "kosmos-base"
+depends "kosmos-nginx"

site-cookbooks/kosmos-postgresql/recipes/default.rb

@@ -24,28 +24,30 @@
 # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
 # THE SOFTWARE.
 
-return if platform?('ubuntu') && node[:platform_version].to_f < 18.04
+postgresql_version = "12"
+postgresql_service = "postgresql@#{postgresql_version}-main"
 
-node.override['build-essential']['compile_time'] = true
-include_recipe 'build-essential::default'
-
-package("libpq-dev") { action :nothing }.run_action(:install)
-
-chef_gem 'pg' do
-  compile_time true
+postgresql_custom_server postgresql_version do
+  role "primary"
+  tls true unless node.chef_environment == "development"
 end
 
-postgresql_data_bag_item = data_bag_item('credentials', 'postgresql')
-
-postgresql_server_install "main" do
-  version "10"
-  setup_repo false
-  password postgresql_data_bag_item['server_password']
-  action :install
+service postgresql_service do
+  supports restart: true, status: true, reload: true
+  action [:enable]
 end
 
-postgresql_client_install "main" do
-  version "10"
-  setup_repo false
-  action :install
+postgresql_replicas.each do |replica|
+  postgresql_access "#{replica[:hostname]} replication" do
+    access_type "host"
+    access_db "replication"
+    access_user "replication"
+    access_addr "#{replica[:ipaddress]}/32"
+    access_method "md5"
+    # notification does not work, as postgresql_access always says the
+    # resource was already up to date
+    notifies :reload, "service[#{postgresql_service}]", :immediately
+  end
 end
+
+include_recipe "kosmos-postgresql::firewall"

site-cookbooks/kosmos-postgresql/recipes/firewall.rb

@@ -0,0 +1,40 @@
+#
+# Cookbook:: kosmos-postgresql
+# Recipe:: firewall
+#
+# The MIT License (MIT)
+#
+# Copyright:: 2019, Kosmos Developers
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+# THE SOFTWARE.
+
+# FIXME: The firewall recipe do not work in the custom resource, so the code
+# lives here for now. The issue is described here, but I think messing with the
+# run context is confusing:
+#
+# https://github.com/chef-cookbooks/firewall/issues/134
+unless node.chef_environment == "development"
+  include_recipe "firewall"
+
+  firewall_rule "postgresql" do
+    port     5432
+    protocol :tcp
+    command  :allow
+  end
+end

site-cookbooks/kosmos-postgresql/recipes/replica.rb

@@ -0,0 +1,76 @@
+#
+# Cookbook:: kosmos-postgresql
+# Recipe:: replica
+#
+# The MIT License (MIT)
+#
+# Copyright:: 2019, Kosmos Developers
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+# THE SOFTWARE.
+
+postgresql_version = "12"
+postgresql_service = "postgresql@#{postgresql_version}-main"
+
+postgresql_custom_server postgresql_version do
+  role "replica"
+  tls true unless node.chef_environment == "development"
+end
+
+service postgresql_service do
+  supports restart: true, status: true, reload: true
+  action [:enable]
+end
+
+postgresql_data_bag_item = data_bag_item('credentials', 'postgresql')
+
+primary = postgresql_primary
+
+unless primary.nil?
+  postgresql_data_dir = "/var/lib/postgresql/#{postgresql_version}/main"
+
+  if node['kosmos-postgresql']['ready_to_set_up_replica']
+    execute "set up replication" do
+      command <<-EOF
+systemctl stop #{postgresql_service}
+mv #{postgresql_data_dir} #{postgresql_data_dir}.old
+PGPASSWORD=#{postgresql_data_bag_item['replication_password']} pg_basebackup -h #{primary[:ipaddress]} -U replication -D #{postgresql_data_dir} -R
+chown -R postgres:postgres #{postgresql_data_dir}
+systemctl start #{postgresql_service}
+      EOF
+      sensitive true
+      not_if { ::File.exist? "#{postgresql_data_dir}/standby.signal" }
+    end
+  end
+
+  postgresql_access "replication" do
+    access_type "host"
+    access_db "replication"
+    access_user "replication"
+    access_addr "#{primary[:ipaddress]}/32"
+    access_method "md5"
+    # notification does not work, as postgresql_access always says the
+    # resource was already up to date
+    notifies :reload, "service[#{postgresql_service}]", :immediately
+  end
+
+  # On the next Chef run the replica will be set up
+  node.normal['kosmos-postgresql']['ready_to_set_up_replica'] = true
+end
+
+include_recipe "kosmos-postgresql::firewall"

site-cookbooks/kosmos-postgresql/resources/server.rb

@@ -0,0 +1,126 @@
+resource_name :postgresql_custom_server
+
+property :postgresql_version, String, required: true, name_property: true
+property :role, String, required: true # Can be primary or replica
+property :tls, [TrueClass, FalseClass], default: false
+
+action :create do
+  postgresql_version = new_resource.postgresql_version
+  postgresql_data_dir = data_dir(postgresql_version)
+  postgresql_service = "postgresql@#{postgresql_version}-main"
+
+  node.override['build-essential']['compile_time'] = true
+  include_recipe 'build-essential::default'
+
+  package("libpq-dev") { action :nothing }.run_action(:install)
+
+  chef_gem 'pg' do
+    compile_time true
+  end
+
+  postgresql_data_bag_item = data_bag_item('credentials', 'postgresql')
+
+  postgresql_server_install "main" do
+    version postgresql_version
+    setup_repo true
+    password postgresql_data_bag_item['server_password']
+    action :install
+  end
+
+  service postgresql_service do
+    supports restart: true, status: true, reload: true
+    # action [:enable, :start]
+  end
+
+  postgresql_client_install "main" do
+    version postgresql_version
+    setup_repo true
+    action :install
+  end
+
+  postgresql_user "replication" do
+    action :create
+    replication true
+    password postgresql_data_bag_item['replication_password']
+  end
+
+  shared_buffers = if node['memory']['total'].to_i / 1024 < 1024 # > 1GB RAM
+                     "128MB"
+                   else # >= 1GB RAM, use 25% of total RAM
+                     "#{node['memory']['total'].to_i / 1024 / 4}MB"
+                   end
+
+  additional_config = {
+    max_connections: 100, # default
+    shared_buffers: shared_buffers,
+    unix_socket_directories: "/var/run/postgresql",
+    dynamic_shared_memory_type: "posix",
+    timezone: "UTC", # default is GMT
+    listen_addresses: "0.0.0.0",
+  }
+
+  if new_resource.role == "replica"
+    additional_config[:promote_trigger_file] = "#{postgresql_data_dir}/failover.trigger"
+  end
+
+  if new_resource.tls
+    include_recipe "kosmos-nginx"
+    include_recipe "kosmos-base::letsencrypt"
+
+    domain = node[:fqdn]
+
+    postgresql_post_hook = <<-EOF
+#!/usr/bin/env bash
+
+set -e
+
+# Copy the postgresql certificate and restart the server if it has been renewed
+# This is necessary because the postgresql user doesn't have access to the
+# letsencrypt live folder
+for domain in $RENEWED_DOMAINS; do
+  case $domain in
+  #{domain})
+    cp "${RENEWED_LINEAGE}/privkey.pem" #{postgresql_data_dir}/#{domain}.key
+    cp "${RENEWED_LINEAGE}/fullchain.pem" #{postgresql_data_dir}/#{domain}.crt
+    chown postgres:postgres #{postgresql_data_dir}/#{domain}.*
+    chmod 600 #{postgresql_data_dir}/#{domain}.*
+    systemctl reload #{postgresql_service}
+    ;;
+  esac
+done
+    EOF
+
+    # This hook will be executed by certbot after every successful certificate
+    # creation or renewal
+    file "/etc/letsencrypt/renewal-hooks/post/postgresql" do
+      content postgresql_post_hook
+      mode 0755
+      owner "root"
+      group "root"
+    end
+
+    template "#{node['nginx']['dir']}/sites-available/#{domain}" do
+      source 'nginx_conf_empty.erb'
+      owner node["nginx"]["user"]
+      mode 0640
+      notifies :reload, 'service[nginx]', :delayed
+    end
+
+    nginx_certbot_site domain
+
+    additional_config[:ssl]           = "on"
+    additional_config[:ssl_cert_file] = "#{postgresql_data_dir}/#{domain}.crt"
+    additional_config[:ssl_key_file]  = "#{postgresql_data_dir}/#{domain}.key"
+  end
+
+  postgresql_server_conf "main" do
+    version postgresql_version
+    additional_config additional_config
+    notifies :reload, "service[#{postgresql_service}]"
+  end
+end
+
+action_class do
+  # to use the data_dir helper
+  include PostgresqlCookbook::Helpers
+end