Add missing files

This commit is contained in:
Basti 2021-11-29 13:12:17 -06:00
parent ecdc41a54f
commit 91ffe75bc1
Signed by untrusted user: basti
GPG Key ID: 9F88009D31D99C72
16 changed files with 544 additions and 0 deletions

4
clients/postgres-4.json Normal file
View File

@ -0,0 +1,4 @@
{
"name": "postgres-4",
"public_key": "-----BEGIN PUBLIC KEY-----\nMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAu6fPxOZeKloF/EgYvU0k\nOwv8bJjsCQcWaMTPle5//mRTszA6PM2z9RI+Mfr45qxTlsL9pQY8WJOWF6QOK31x\nszuqcr7oOjtAhrLI8f/oNDEDjcx325FqG9gNKQEAD7d4zodh+PhDe6x7GIyIS7lG\nIcD5Zre9iDwv8FGLR+5GLqS8SJOPL/wJkQ8w+N0f8YDFw81kiTta5NLhAx3fMDs0\n2kmoNlbmKlNZTtLjCfCV+/pa9oY6wycjck3GvobiFE/4cWaNkeGlPc+uAwlfmrOv\nHy0tq1XBX/BCvE5kMXmhnMT23JXjm2s2PgCLgEVGAXilXk/T597KDm+z4oBpAQma\nnQIDAQAB\n-----END PUBLIC KEY-----\n"
}

57
nodes/postgres-4.json Normal file
View File

@ -0,0 +1,57 @@
{
"name": "postgres-4",
"normal": {
"knife_zero": {
"host": "10.1.1.107"
}
},
"automatic": {
"fqdn": "postgres-4",
"os": "linux",
"os_version": "5.4.0-91-generic",
"hostname": "postgres-4",
"ipaddress": "192.168.122.3",
"roles": [
"postgresql_replica"
],
"recipes": [
"kosmos-base",
"kosmos-base::default",
"kosmos_postgresql::hostsfile",
"kosmos_postgresql::replica",
"kosmos_postgresql::firewall",
"apt::default",
"timezone_iii::default",
"timezone_iii::debian",
"ntp::default",
"ntp::apparmor",
"kosmos-base::systemd_emails",
"apt::unattended-upgrades",
"kosmos-base::firewall",
"kosmos-postfix::default",
"postfix::default",
"postfix::_common",
"postfix::_attributes",
"postfix::sasl_auth",
"hostname::default"
],
"platform": "ubuntu",
"platform_version": "20.04",
"cloud": null,
"chef_packages": {
"chef": {
"version": "17.7.29",
"chef_root": "/opt/chef/embedded/lib/ruby/gems/3.0.0/gems/chef-17.7.29/lib",
"chef_effortless": null
},
"ohai": {
"version": "17.7.8",
"ohai_root": "/opt/chef/embedded/lib/ruby/gems/3.0.0/gems/ohai-17.7.8/lib/ohai"
}
}
},
"run_list": [
"recipe[kosmos-base]",
"role[postgresql_replica]"
]
}

View File

@ -0,0 +1,22 @@
.vagrant
*~
*#
.#*
\#*#
.*.sw[a-z]
*.un~
# Bundler
Gemfile.lock
gems.locked
bin/*
.bundle/*
# test kitchen
.kitchen/
.kitchen.local.yml
# Chef
Berksfile.lock
.zero-knife.rb
Policyfile.lock.json

View File

@ -0,0 +1,4 @@
# frozen_string_literal: true
source 'https://supermarket.chef.io'
metadata

View File

@ -0,0 +1,5 @@
# kosmos_postgresql CHANGELOG
# 0.1.0
Initial release.

View File

@ -0,0 +1,20 @@
Copyright (c) 2019-2020 Kosmos Developers
Permission is hereby granted, free of charge, to any person obtaining
a copy of this software and associated documentation files (the
"Software"), to deal in the Software without restriction, including
without limitation the rights to use, copy, modify, merge, publish,
distribute, sublicense, and/or sell copies of the Software, and to
permit persons to whom the Software is furnished to do so, subject to
the following conditions:
The above copyright notice and this permission notice shall be
included in all copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

View File

@ -0,0 +1,57 @@
# kosmos_postgresql
## Usage
### On the primary:
Set the `postgresql_primary` role on the node
### On the replica:
Add the `postgresql_replica` role to the node's run list. Run Chef on the node
a first time.
After the initial Chef run on the replica, run Chef on the primary to add the
firewall rules and PostgreSQL access rules, then run Chef again on the replica
to set up replication.
## Caveat
[`firewall_rules`](https://github.com/chef-cookbooks/firewall/issues/134) and
[`postgresql_access`](https://github.com/sous-chefs/postgresql/issues/648) are
declared in recipes, not resources because of the way custom resources
work currently in Chef. See the `default.rb` and `replica.rb` recipes.
The primary gives access to the `replication` db to the `replication` user
connecting from a replica, and replicas to the primary. For more information
about PostgreSQL client authentication, see the
[official docs](https://www.postgresql.org/docs/12/auth-pg-hba-conf.html)
The primary opens up the PostgreSQL port (5432 TCP) to replicas, and replicas
to the primary.
## TLS self-signed certificate
A wildcard (`*.kosmos.org` certificate) was generated with the following
commands:
```
openssl req -new -nodes -text -out root.csr -keyout root.key \
-subj "/CN=root.kosmos.org"
chmod og-rwx root.key
openssl x509 -req -in root.csr -text -days 3650 \
-extfile /etc/ssl/openssl.cnf -extensions v3_ca \
-signkey root.key -out root.crt
openssl req -new -nodes -text -out server.csr \
-keyout server.key -subj "/CN=*.kosmos.org"
chmod og-rwx server.key
openssl x509 -req -in server.csr -text -days 1825 \
-CA root.crt -CAkey root.key -CAcreateserial \
-out server.crt
```
It is valid until May 12 2025.
The content of `server.crt`, `server.key` and `root.crt` an stored in the
`postgresql` encrypted data bag. The root key is stored in LastPass
("Self-signed TLS root certificate"). `server.crt` & `server.key` are used by
the PostgreSQL server.

View File

@ -0,0 +1,3 @@
# This is set to false by default, and set to true in the server resource
# for replicas.
node.default['kosmos_postgresql']['ready_to_set_up_replica'] = false

View File

@ -0,0 +1,104 @@
# Put files/directories that should be ignored in this file when uploading
# to a chef-server or supermarket.
# Lines that start with '# ' are comments.
# OS generated files #
######################
.DS_Store
Icon?
nohup.out
ehthumbs.db
Thumbs.db
# SASS #
########
.sass-cache
# EDITORS #
###########
\#*
.#*
*~
*.sw[a-z]
*.bak
REVISION
TAGS*
tmtags
*_flymake.*
*_flymake
*.tmproj
.project
.settings
mkmf.log
## COMPILED ##
##############
a.out
*.o
*.pyc
*.so
*.com
*.class
*.dll
*.exe
*/rdoc/
# Testing #
###########
.watchr
.rspec
spec/*
spec/fixtures/*
test/*
features/*
examples/*
Guardfile
Procfile
.kitchen*
kitchen.yml*
.rubocop.yml
spec/*
Rakefile
.travis.yml
.foodcritic
.codeclimate.yml
# SCM #
#######
.git
*/.git
.gitignore
.gitmodules
.gitconfig
.gitattributes
.svn
*/.bzr/*
*/.hg/*
*/.svn/*
# Berkshelf #
#############
Berksfile
Berksfile.lock
cookbooks/*
tmp
# Bundler #
###########
vendor/*
# Policyfile #
##############
Policyfile.rb
Policyfile.lock.json
# Cookbooks #
#############
CONTRIBUTING*
CHANGELOG*
TESTING*
# Vagrant #
###########
.vagrant
Vagrantfile

View File

@ -0,0 +1,45 @@
class Chef
class Recipe
def postgresql_primary
postgresql_primary = search(:node, "role:postgresql_primary AND chef_environment:#{node.chef_environment}").first
unless postgresql_primary.nil?
primary_ip = ip_for(postgresql_primary)
{ hostname: postgresql_primary[:hostname], ipaddress: primary_ip }
end
end
def postgresql_replicas
postgresql_replicas = []
search(:node, "role:postgresql_replica AND chef_environment:#{node.chef_environment}").each do |replica|
replica_ip = ip_for(replica)
postgresql_replicas << { hostname: replica[:hostname], ipaddress: replica_ip }
end
postgresql_replicas
end
def ip_for(server_node)
if node.chef_environment == "development"
server_node['network']['interfaces']['eth1']['routes'].first['src']
else
# If the server has a private Zerotier IP, use it
if server_node['knife_zero'] && server_node['knife_zero']['host'] && \
server_node['knife_zero']['host'].start_with?("10.1.1.")
server_node['knife_zero']['host']
else
server_node['ipaddress']
end
end
end
def postgresql_service_name
postgresql_version = "12"
"postgresql@#{postgresql_version}-main"
end
end
end

View File

@ -0,0 +1,25 @@
name 'kosmos_postgresql'
maintainer 'Kosmos'
maintainer_email 'ops@5apps.com'
license 'MIT'
description 'Installs/Configures kosmos_postgresql'
long_description 'Installs/Configures kosmos_postgresql'
version '0.1.0'
chef_version '>= 12.14' if respond_to?(:chef_version)
# The `issues_url` points to the location where issues for this cookbook are
# tracked. A `View Issues` link will be displayed on this cookbook's page when
# uploaded to a Supermarket.
#
# issues_url 'https://github.com/<insert_org_here>/kosmos_postgresql/issues'
# The `source_url` points to the development repository for this cookbook. A
# `View Source` link will be displayed on this cookbook's page when uploaded to
# a Supermarket.
#
# source_url 'https://github.com/<insert_org_here>/kosmos_postgresql'
depends "postgresql", ">= 7.0.0"
depends "build-essential"
depends "kosmos_encfs"
depends "hostsfile"

View File

@ -0,0 +1,15 @@
#
# Cookbook:: kosmos_postgresql
# Recipe:: firewall
#
unless node.chef_environment == "development"
include_recipe "kosmos-base::firewall"
firewall_rule "postgresql zerotier members" do
port 5432
protocol :tcp
command :allow
source "10.1.1.0/24"
end
end

View File

@ -0,0 +1,16 @@
#
# Cookbook:: kosmos_postgresql
# Recipe:: hostsfile
#
begin
primary_ip = postgresql_primary[:ipaddress]
rescue NoMethodError
end
unless primary_ip.nil?
hostsfile_entry primary_ip do
hostname "pg.kosmos.local"
unique true
end
end

View File

@ -0,0 +1,33 @@
#
# Cookbook:: kosmos_postgresql
# Recipe:: primary
#
postgresql_version = "12"
postgresql_service = "postgresql@#{postgresql_version}-main"
service postgresql_service do
supports restart: true, status: true, reload: true
end
postgresql_custom_server postgresql_version do
role "primary"
end
postgresql_access "zerotier members" do
access_type "host"
access_db "all"
access_user "all"
access_addr "10.1.1.0/24"
access_method "md5"
notifies :reload, "service[#{postgresql_service}]", :immediately
end
postgresql_access "zerotier members replication" do
access_type "host"
access_db "replication"
access_user "replication"
access_addr "10.1.1.0/24"
access_method "md5"
notifies :reload, "service[#{postgresql_service}]", :immediately
end

View File

@ -0,0 +1,56 @@
#
# Cookbook:: kosmos_postgresql
# Recipe:: replica
#
postgresql_version = "12"
postgresql_service = "postgresql@#{postgresql_version}-main"
postgresql_custom_server postgresql_version do
role "replica"
end
service postgresql_service do
supports restart: true, status: true, reload: true
end
postgresql_data_bag_item = data_bag_item('credentials', 'postgresql')
primary = postgresql_primary
unless primary.nil?
# TODO
postgresql_data_dir = "/var/lib/postgresql/#{postgresql_version}/main"
# FIXME get zerotier IP
execute "set up replication" do
command <<-EOF
systemctl stop #{postgresql_service}
mv #{postgresql_data_dir} #{postgresql_data_dir}.old
pg_basebackup -h pg.kosmos.local -U replication -D #{postgresql_data_dir} -R
chown -R postgres:postgres #{postgresql_data_dir}
systemctl start #{postgresql_service}
EOF
environment 'PGPASSWORD' => postgresql_data_bag_item['replication_password']
sensitive true
not_if { ::File.exist? "#{postgresql_data_dir}/standby.signal" }
end
postgresql_access "zerotier members" do
access_type "host"
access_db "all"
access_user "all"
access_addr "10.1.1.0/24"
access_method "md5"
notifies :reload, "service[#{postgresql_service}]", :immediately
end
postgresql_access "zerotier members replication" do
access_type "host"
access_db "replication"
access_user "replication"
access_addr "10.1.1.0/24"
access_method "md5"
notifies :reload, "service[#{postgresql_service}]", :immediately
end
end

View File

@ -0,0 +1,78 @@
resource_name :postgresql_custom_server
provides :postgresql_custom_server
property :postgresql_version, String, required: true, name_property: true
property :role, String, required: true # Can be primary or replica
action :create do
postgresql_version = new_resource.postgresql_version
postgresql_data_dir = "/var/lib/postgresql/#{postgresql_version}/main"
postgresql_service = "postgresql@#{postgresql_version}-main"
postgresql_credentials = data_bag_item('credentials', 'postgresql')
build_essential do
compile_time true
end
package("libpq-dev") { action :nothing }.run_action(:install)
chef_gem 'pg' do
compile_time true
end
user "postgres" do
manage_home false
end
postgresql_server_install "main" do
version postgresql_version
setup_repo true
password postgresql_credentials['server_password']
action :install
end
service postgresql_service do
supports restart: true, status: true, reload: true
action [:enable, :start]
end
# This service is a dependency that will auto-start our cluster service on
# boot if it's enabled, so we disable it explicitly
service "postgresql" do
action :disable
end
shared_buffers = if node['memory']['total'].to_i / 1024 < 1024 # < 1GB RAM
"128MB"
else # >= 1GB RAM, use 50% of total RAM
"#{node['memory']['total'].to_i / 1024 / 2}MB"
end
additional_config = {
max_connections: 200, # default
shared_buffers: shared_buffers,
unix_socket_directories: "/var/run/postgresql",
dynamic_shared_memory_type: "posix",
timezone: "UTC", # default is GMT
listen_addresses: "0.0.0.0"
}
additional_config[:promote_trigger_file] = "#{postgresql_data_dir}/failover.trigger"
postgresql_server_conf "main" do
version postgresql_version
additional_config additional_config
notifies :reload, "service[#{postgresql_service}]", :delayed
end
postgresql_user "replication" do
action :create
replication true
password postgresql_credentials['replication_password']
end
end
action_class do
# to use the data_dir helper
include PostgresqlCookbook::Helpers
end