diff --git a/nodes/akkounts-1.json b/nodes/akkounts-1.json index 93a1617..1168a2d 100644 --- a/nodes/akkounts-1.json +++ b/nodes/akkounts-1.json @@ -18,7 +18,7 @@ "recipes": [ "kosmos-base", "kosmos-base::default", - "kosmos-postgresql::hostsfile", + "kosmos_postgresql::hostsfile", "kosmos-akkounts", "kosmos-akkounts::default", "kosmos-akkounts::nginx", diff --git a/nodes/postgres-2.json b/nodes/postgres-2.json index cd48a8c..89bcc85 100644 --- a/nodes/postgres-2.json +++ b/nodes/postgres-2.json @@ -8,17 +8,17 @@ "automatic": { "fqdn": "postgres-2", "os": "linux", - "os_version": "5.4.0-64-generic", + "os_version": "5.4.0-77-generic", "hostname": "postgres-2", "ipaddress": "192.168.122.244", "roles": [ - "postgresql_replica" + "postgresql_primary" ], "recipes": [ "kosmos-base", "kosmos-base::default", - "kosmos-postgresql::replica", - "kosmos-postgresql::firewall", + "kosmos_postgresql::primary", + "kosmos_postgresql::firewall", "apt::default", "timezone_iii::default", "timezone_iii::debian", @@ -52,4 +52,4 @@ "recipe[kosmos-base]", "role[postgresql_primary]" ] -} +} \ No newline at end of file diff --git a/roles/postgresql_client.rb b/roles/postgresql_client.rb index 96b5418..f2fbb71 100644 --- a/roles/postgresql_client.rb +++ b/roles/postgresql_client.rb @@ -3,5 +3,5 @@ name "postgresql_client" run_list %w( - kosmos-postgresql::hostsfile + kosmos_postgresql::hostsfile ) diff --git a/roles/postgresql_primary.rb b/roles/postgresql_primary.rb index ba5e5be..58ef4b7 100644 --- a/roles/postgresql_primary.rb +++ b/roles/postgresql_primary.rb @@ -1,6 +1,6 @@ name "postgresql_primary" run_list %w( - kosmos-postgresql::primary - kosmos-postgresql::firewall + kosmos_postgresql::primary + kosmos_postgresql::firewall ) diff --git a/roles/postgresql_replica.rb b/roles/postgresql_replica.rb index 6d73f31..099291d 100644 --- a/roles/postgresql_replica.rb +++ b/roles/postgresql_replica.rb @@ -1,7 +1,7 @@ name "postgresql_replica" run_list %w( - kosmos-postgresql::hostsfile - kosmos-postgresql::replica - kosmos-postgresql::firewall + kosmos_postgresql::hostsfile + kosmos_postgresql::replica + kosmos_postgresql::firewall ) diff --git a/site-cookbooks/kosmos-postgresql/.gitignore b/site-cookbooks/kosmos-postgresql/.gitignore deleted file mode 100644 index 13e41c4..0000000 --- a/site-cookbooks/kosmos-postgresql/.gitignore +++ /dev/null @@ -1,22 +0,0 @@ -.vagrant -*~ -*# -.#* -\#*# -.*.sw[a-z] -*.un~ - -# Bundler -Gemfile.lock -gems.locked -bin/* -.bundle/* - -# test kitchen -.kitchen/ -.kitchen.local.yml - -# Chef -Berksfile.lock -.zero-knife.rb -Policyfile.lock.json diff --git a/site-cookbooks/kosmos-postgresql/Berksfile b/site-cookbooks/kosmos-postgresql/Berksfile deleted file mode 100644 index 0656a99..0000000 --- a/site-cookbooks/kosmos-postgresql/Berksfile +++ /dev/null @@ -1,4 +0,0 @@ -# frozen_string_literal: true -source 'https://supermarket.chef.io' - -metadata diff --git a/site-cookbooks/kosmos-postgresql/CHANGELOG.md b/site-cookbooks/kosmos-postgresql/CHANGELOG.md deleted file mode 100644 index 20e9a6a..0000000 --- a/site-cookbooks/kosmos-postgresql/CHANGELOG.md +++ /dev/null @@ -1,5 +0,0 @@ -# kosmos-postgresql CHANGELOG - -# 0.1.0 - -Initial release. diff --git a/site-cookbooks/kosmos-postgresql/LICENSE b/site-cookbooks/kosmos-postgresql/LICENSE deleted file mode 100644 index c150a1f..0000000 --- a/site-cookbooks/kosmos-postgresql/LICENSE +++ /dev/null @@ -1,20 +0,0 @@ -Copyright (c) 2019-2020 Kosmos Developers - -Permission is hereby granted, free of charge, to any person obtaining -a copy of this software and associated documentation files (the -"Software"), to deal in the Software without restriction, including -without limitation the rights to use, copy, modify, merge, publish, -distribute, sublicense, and/or sell copies of the Software, and to -permit persons to whom the Software is furnished to do so, subject to -the following conditions: - -The above copyright notice and this permission notice shall be -included in all copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, -EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF -MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND -NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE -LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION -OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION -WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. diff --git a/site-cookbooks/kosmos-postgresql/README.md b/site-cookbooks/kosmos-postgresql/README.md deleted file mode 100644 index 2541424..0000000 --- a/site-cookbooks/kosmos-postgresql/README.md +++ /dev/null @@ -1,57 +0,0 @@ -# kosmos-postgresql - -## Usage - -### On the primary: - -Set the `postgresql_primary` role on the node - -### On the replica: - -Add the `postgresql_replica` role to the node's run list. Run Chef on the node -a first time. -After the initial Chef run on the replica, run Chef on the primary to add the -firewall rules and PostgreSQL access rules, then run Chef again on the replica -to set up replication. - -## Caveat - -[`firewall_rules`](https://github.com/chef-cookbooks/firewall/issues/134) and -[`postgresql_access`](https://github.com/sous-chefs/postgresql/issues/648) are -declared in recipes, not resources because of the way custom resources -work currently in Chef. See the `default.rb` and `replica.rb` recipes. - -The primary gives access to the `replication` db to the `replication` user -connecting from a replica, and replicas to the primary. For more information -about PostgreSQL client authentication, see the -[official docs](https://www.postgresql.org/docs/12/auth-pg-hba-conf.html) - -The primary opens up the PostgreSQL port (5432 TCP) to replicas, and replicas -to the primary. - -## TLS self-signed certificate - -A wildcard (`*.kosmos.org` certificate) was generated with the following -commands: - -``` -openssl req -new -nodes -text -out root.csr -keyout root.key \ - -subj "/CN=root.kosmos.org" -chmod og-rwx root.key -openssl x509 -req -in root.csr -text -days 3650 \ - -extfile /etc/ssl/openssl.cnf -extensions v3_ca \ - -signkey root.key -out root.crt -openssl req -new -nodes -text -out server.csr \ - -keyout server.key -subj "/CN=*.kosmos.org" -chmod og-rwx server.key -openssl x509 -req -in server.csr -text -days 1825 \ - -CA root.crt -CAkey root.key -CAcreateserial \ - -out server.crt -``` - -It is valid until May 12 2025. - -The content of `server.crt`, `server.key` and `root.crt` an stored in the -`postgresql` encrypted data bag. The root key is stored in LastPass -("Self-signed TLS root certificate"). `server.crt` & `server.key` are used by -the PostgreSQL server. diff --git a/site-cookbooks/kosmos-postgresql/attributes/default.rb b/site-cookbooks/kosmos-postgresql/attributes/default.rb deleted file mode 100644 index dec530f..0000000 --- a/site-cookbooks/kosmos-postgresql/attributes/default.rb +++ /dev/null @@ -1,3 +0,0 @@ -# This is set to false by default, and set to true in the server resource -# for replicas. -node.default['kosmos-postgresql']['ready_to_set_up_replica'] = false diff --git a/site-cookbooks/kosmos-postgresql/chefignore b/site-cookbooks/kosmos-postgresql/chefignore deleted file mode 100644 index 4439807..0000000 --- a/site-cookbooks/kosmos-postgresql/chefignore +++ /dev/null @@ -1,104 +0,0 @@ -# Put files/directories that should be ignored in this file when uploading -# to a chef-server or supermarket. -# Lines that start with '# ' are comments. - -# OS generated files # -###################### -.DS_Store -Icon? -nohup.out -ehthumbs.db -Thumbs.db - -# SASS # -######## -.sass-cache - -# EDITORS # -########### -\#* -.#* -*~ -*.sw[a-z] -*.bak -REVISION -TAGS* -tmtags -*_flymake.* -*_flymake -*.tmproj -.project -.settings -mkmf.log - -## COMPILED ## -############## -a.out -*.o -*.pyc -*.so -*.com -*.class -*.dll -*.exe -*/rdoc/ - -# Testing # -########### -.watchr -.rspec -spec/* -spec/fixtures/* -test/* -features/* -examples/* -Guardfile -Procfile -.kitchen* -kitchen.yml* -.rubocop.yml -spec/* -Rakefile -.travis.yml -.foodcritic -.codeclimate.yml - -# SCM # -####### -.git -*/.git -.gitignore -.gitmodules -.gitconfig -.gitattributes -.svn -*/.bzr/* -*/.hg/* -*/.svn/* - -# Berkshelf # -############# -Berksfile -Berksfile.lock -cookbooks/* -tmp - -# Bundler # -########### -vendor/* - -# Policyfile # -############## -Policyfile.rb -Policyfile.lock.json - -# Cookbooks # -############# -CONTRIBUTING* -CHANGELOG* -TESTING* - -# Vagrant # -########### -.vagrant -Vagrantfile diff --git a/site-cookbooks/kosmos-postgresql/libraries/helpers.rb b/site-cookbooks/kosmos-postgresql/libraries/helpers.rb deleted file mode 100644 index 7d3c397..0000000 --- a/site-cookbooks/kosmos-postgresql/libraries/helpers.rb +++ /dev/null @@ -1,45 +0,0 @@ -class Chef - class Recipe - def postgresql_primary - postgresql_primary = search(:node, "role:postgresql_primary AND chef_environment:#{node.chef_environment}").first - - unless postgresql_primary.nil? - primary_ip = ip_for(postgresql_primary) - - { hostname: postgresql_primary[:hostname], ipaddress: primary_ip } - end - end - - def postgresql_replicas - postgresql_replicas = [] - - search(:node, "role:postgresql_replica AND chef_environment:#{node.chef_environment}").each do |replica| - replica_ip = ip_for(replica) - - postgresql_replicas << { hostname: replica[:hostname], ipaddress: replica_ip } - end - - postgresql_replicas - end - - def ip_for(server_node) - if node.chef_environment == "development" - server_node['network']['interfaces']['eth1']['routes'].first['src'] - else - # If the server has a private Zerotier IP, use it - if server_node['knife_zero'] && server_node['knife_zero']['host'] && \ - server_node['knife_zero']['host'].start_with?("10.1.1.") - server_node['knife_zero']['host'] - else - server_node['ipaddress'] - end - end - end - - def postgresql_service_name - postgresql_version = "12" - - "postgresql@#{postgresql_version}-main" - end - end -end diff --git a/site-cookbooks/kosmos-postgresql/metadata.rb b/site-cookbooks/kosmos-postgresql/metadata.rb deleted file mode 100644 index 1b031b2..0000000 --- a/site-cookbooks/kosmos-postgresql/metadata.rb +++ /dev/null @@ -1,25 +0,0 @@ -name 'kosmos-postgresql' -maintainer 'Kosmos' -maintainer_email 'ops@5apps.com' -license 'MIT' -description 'Installs/Configures kosmos-postgresql' -long_description 'Installs/Configures kosmos-postgresql' -version '0.1.0' -chef_version '>= 12.14' if respond_to?(:chef_version) - -# The `issues_url` points to the location where issues for this cookbook are -# tracked. A `View Issues` link will be displayed on this cookbook's page when -# uploaded to a Supermarket. -# -# issues_url 'https://github.com//kosmos-postgresql/issues' - -# The `source_url` points to the development repository for this cookbook. A -# `View Source` link will be displayed on this cookbook's page when uploaded to -# a Supermarket. -# -# source_url 'https://github.com//kosmos-postgresql' - -depends "postgresql", ">= 7.0.0" -depends "build-essential" -depends "kosmos_encfs" -depends "hostsfile" diff --git a/site-cookbooks/kosmos-postgresql/recipes/firewall.rb b/site-cookbooks/kosmos-postgresql/recipes/firewall.rb deleted file mode 100644 index 7b9b380..0000000 --- a/site-cookbooks/kosmos-postgresql/recipes/firewall.rb +++ /dev/null @@ -1,15 +0,0 @@ -# -# Cookbook:: kosmos-postgresql -# Recipe:: firewall -# - -unless node.chef_environment == "development" - include_recipe "kosmos-base::firewall" - - firewall_rule "postgresql zerotier members" do - port 5432 - protocol :tcp - command :allow - source "10.1.1.0/24" - end -end diff --git a/site-cookbooks/kosmos-postgresql/recipes/hostsfile.rb b/site-cookbooks/kosmos-postgresql/recipes/hostsfile.rb deleted file mode 100644 index 265f563..0000000 --- a/site-cookbooks/kosmos-postgresql/recipes/hostsfile.rb +++ /dev/null @@ -1,16 +0,0 @@ -# -# Cookbook:: kosmos-postgresql -# Recipe:: hostsfile -# - -begin -primary_ip = postgresql_primary[:ipaddress] -rescue NoMethodError -end - -unless primary_ip.nil? - hostsfile_entry primary_ip do - hostname "pg.kosmos.local" - unique true - end -end diff --git a/site-cookbooks/kosmos-postgresql/recipes/primary.rb b/site-cookbooks/kosmos-postgresql/recipes/primary.rb deleted file mode 100644 index b3a7534..0000000 --- a/site-cookbooks/kosmos-postgresql/recipes/primary.rb +++ /dev/null @@ -1,33 +0,0 @@ -# -# Cookbook:: kosmos-postgresql -# Recipe:: primary -# - -postgresql_version = "12" -postgresql_service = "postgresql@#{postgresql_version}-main" - -service postgresql_service do - supports restart: true, status: true, reload: true -end - -postgresql_custom_server postgresql_version do - role "primary" -end - -postgresql_access "zerotier members" do - access_type "host" - access_db "all" - access_user "all" - access_addr "10.1.1.0/24" - access_method "md5" - notifies :reload, "service[#{postgresql_service}]", :immediately -end - -postgresql_access "zerotier members replication" do - access_type "host" - access_db "replication" - access_user "replication" - access_addr "10.1.1.0/24" - access_method "md5" - notifies :reload, "service[#{postgresql_service}]", :immediately -end diff --git a/site-cookbooks/kosmos-postgresql/recipes/replica.rb b/site-cookbooks/kosmos-postgresql/recipes/replica.rb deleted file mode 100644 index 6525948..0000000 --- a/site-cookbooks/kosmos-postgresql/recipes/replica.rb +++ /dev/null @@ -1,56 +0,0 @@ -# -# Cookbook:: kosmos-postgresql -# Recipe:: replica -# - -postgresql_version = "12" -postgresql_service = "postgresql@#{postgresql_version}-main" - -postgresql_custom_server postgresql_version do - role "replica" -end - -service postgresql_service do - supports restart: true, status: true, reload: true -end - -postgresql_data_bag_item = data_bag_item('credentials', 'postgresql') - -primary = postgresql_primary - -unless primary.nil? - # TODO - postgresql_data_dir = "/var/lib/postgresql/#{postgresql_version}/main" - - # FIXME get zerotier IP - execute "set up replication" do - command <<-EOF -systemctl stop #{postgresql_service} -mv #{postgresql_data_dir} #{postgresql_data_dir}.old -pg_basebackup -h pg.kosmos.local -U replication -D #{postgresql_data_dir} -R -chown -R postgres:postgres #{postgresql_data_dir} -systemctl start #{postgresql_service} - EOF - environment 'PGPASSWORD' => postgresql_data_bag_item['replication_password'] - sensitive true - not_if { ::File.exist? "#{postgresql_data_dir}/standby.signal" } - end - - postgresql_access "zerotier members" do - access_type "host" - access_db "all" - access_user "all" - access_addr "10.1.1.0/24" - access_method "md5" - notifies :reload, "service[#{postgresql_service}]", :immediately - end - - postgresql_access "zerotier members replication" do - access_type "host" - access_db "replication" - access_user "replication" - access_addr "10.1.1.0/24" - access_method "md5" - notifies :reload, "service[#{postgresql_service}]", :immediately - end -end diff --git a/site-cookbooks/kosmos-postgresql/resources/server.rb b/site-cookbooks/kosmos-postgresql/resources/server.rb deleted file mode 100644 index f71520e..0000000 --- a/site-cookbooks/kosmos-postgresql/resources/server.rb +++ /dev/null @@ -1,77 +0,0 @@ -resource_name :postgresql_custom_server - -property :postgresql_version, String, required: true, name_property: true -property :role, String, required: true # Can be primary or replica - -action :create do - postgresql_version = new_resource.postgresql_version - postgresql_data_dir = "/var/lib/postgresql/#{postgresql_version}/main" - postgresql_service = "postgresql@#{postgresql_version}-main" - postgresql_credentials = data_bag_item('credentials', 'postgresql') - - build_essential do - compile_time true - end - - package("libpq-dev") { action :nothing }.run_action(:install) - - chef_gem 'pg' do - compile_time true - end - - user "postgres" do - manage_home false - end - - postgresql_server_install "main" do - version postgresql_version - setup_repo true - password postgresql_credentials['server_password'] - action :install - end - - service postgresql_service do - supports restart: true, status: true, reload: true - action [:enable, :start] - end - - # This service is a dependency that will auto-start our cluster service on - # boot if it's enabled, so we disable it explicitly - service "postgresql" do - action :disable - end - - shared_buffers = if node['memory']['total'].to_i / 1024 < 1024 # > 1GB RAM - "128MB" - else # >= 1GB RAM, use 25% of total RAM - "#{node['memory']['total'].to_i / 1024 / 4}MB" - end - - additional_config = { - max_connections: 100, # default - shared_buffers: shared_buffers, - unix_socket_directories: "/var/run/postgresql", - dynamic_shared_memory_type: "posix", - timezone: "UTC", # default is GMT - listen_addresses: "0.0.0.0" - } - - additional_config[:promote_trigger_file] = "#{postgresql_data_dir}/failover.trigger" - - postgresql_server_conf "main" do - version postgresql_version - additional_config additional_config - notifies :reload, "service[#{postgresql_service}]", :delayed - end - - postgresql_user "replication" do - action :create - replication true - password postgresql_credentials['replication_password'] - end -end - -action_class do - # to use the data_dir helper - include PostgresqlCookbook::Helpers -end diff --git a/site-cookbooks/kosmos_kvm/recipes/host.rb b/site-cookbooks/kosmos_kvm/recipes/host.rb index a89f7eb..1d611bd 100644 --- a/site-cookbooks/kosmos_kvm/recipes/host.rb +++ b/site-cookbooks/kosmos_kvm/recipes/host.rb @@ -2,34 +2,13 @@ # Cookbook:: kosmos_kvm # Recipe:: host # -# The MIT License (MIT) -# -# Copyright:: 2020, Kosmos Developers -# -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in -# all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -# THE SOFTWARE. package %w(virtinst libvirt-daemon-system) directory "/var/lib/libvirt/images/base" do recursive true owner "libvirt-qemu" - group "root" + group "kvm" mode "0750" end @@ -37,7 +16,7 @@ end remote_file "/var/lib/libvirt/images/base/ubuntu-20.04-server-cloudimg-amd64-disk-kvm.qcow2" do source "http://cloud-images.ubuntu.com/releases/focal/release/ubuntu-20.04-server-cloudimg-amd64-disk-kvm.img" owner "libvirt-qemu" - group "root" + group "kvm" mode "0640" end