From d3c22ea78764f3744c3a99ebd9fe90bc58c4772d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A2u=20Cao?= Date: Thu, 25 Jan 2024 16:26:41 +0300 Subject: [PATCH 01/12] Add liquor_cabinet cookbook --- site-cookbooks/liquor_cabinet/.gitignore | 25 ++++ site-cookbooks/liquor_cabinet/CHANGELOG.md | 7 + site-cookbooks/liquor_cabinet/LICENSE | 21 +++ site-cookbooks/liquor_cabinet/README.md | 6 + .../liquor_cabinet/attributes/default.rb | 24 +++ site-cookbooks/liquor_cabinet/chefignore | 115 +++++++++++++++ site-cookbooks/liquor_cabinet/kitchen.yml | 37 +++++ site-cookbooks/liquor_cabinet/metadata.rb | 12 ++ .../liquor_cabinet/recipes/default.rb | 139 ++++++++++++++++++ .../liquor_cabinet/templates/config.yml.erb | 12 ++ .../liquor_cabinet/templates/rainbows.rb.erb | 32 ++++ 11 files changed, 430 insertions(+) create mode 100644 site-cookbooks/liquor_cabinet/.gitignore create mode 100644 site-cookbooks/liquor_cabinet/CHANGELOG.md create mode 100644 site-cookbooks/liquor_cabinet/LICENSE create mode 100644 site-cookbooks/liquor_cabinet/README.md create mode 100644 site-cookbooks/liquor_cabinet/attributes/default.rb create mode 100644 site-cookbooks/liquor_cabinet/chefignore create mode 100644 site-cookbooks/liquor_cabinet/kitchen.yml create mode 100644 site-cookbooks/liquor_cabinet/metadata.rb create mode 100644 site-cookbooks/liquor_cabinet/recipes/default.rb create mode 100644 site-cookbooks/liquor_cabinet/templates/config.yml.erb create mode 100644 site-cookbooks/liquor_cabinet/templates/rainbows.rb.erb diff --git a/site-cookbooks/liquor_cabinet/.gitignore b/site-cookbooks/liquor_cabinet/.gitignore new file mode 100644 index 0000000..f1e57b8 --- /dev/null +++ b/site-cookbooks/liquor_cabinet/.gitignore @@ -0,0 +1,25 @@ +.vagrant +*~ +*# +.#* +\#*# +.*.sw[a-z] +*.un~ + +# Bundler +Gemfile.lock +gems.locked +bin/* +.bundle/* + +# test kitchen +.kitchen/ +kitchen.local.yml + +# Chef Infra +Berksfile.lock +.zero-knife.rb +Policyfile.lock.json + +.idea/ + diff --git a/site-cookbooks/liquor_cabinet/CHANGELOG.md b/site-cookbooks/liquor_cabinet/CHANGELOG.md new file mode 100644 index 0000000..feaf621 --- /dev/null +++ b/site-cookbooks/liquor_cabinet/CHANGELOG.md @@ -0,0 +1,7 @@ +# liquor_cabinet CHANGELOG + +This file is used to list changes made in each version of the liquor_cabinet cookbook. + +## 0.1.0 + +Initial release. diff --git a/site-cookbooks/liquor_cabinet/LICENSE b/site-cookbooks/liquor_cabinet/LICENSE new file mode 100644 index 0000000..a00749b --- /dev/null +++ b/site-cookbooks/liquor_cabinet/LICENSE @@ -0,0 +1,21 @@ +The MIT License (MIT) + +Copyright (c) 2024 Kosmos Developers + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. diff --git a/site-cookbooks/liquor_cabinet/README.md b/site-cookbooks/liquor_cabinet/README.md new file mode 100644 index 0000000..1f7bc27 --- /dev/null +++ b/site-cookbooks/liquor_cabinet/README.md @@ -0,0 +1,6 @@ +# liquor_cabinet + +Installs/configures the [Liquor Cabinet][1] [remoteStorage][2] API server. + +[1]: https://gitea.kosmos.org/5apps/liquor-cabinet +[2]: https://remotestorage.io diff --git a/site-cookbooks/liquor_cabinet/attributes/default.rb b/site-cookbooks/liquor_cabinet/attributes/default.rb new file mode 100644 index 0000000..6e88013 --- /dev/null +++ b/site-cookbooks/liquor_cabinet/attributes/default.rb @@ -0,0 +1,24 @@ +node.default['liquor-cabinet']['ruby']['version'] = "3.1.4" +node.default['liquor-cabinet']['repo'] = 'https://gitea.kosmos.org/5apps/liquor-cabinet.git' +node.default['liquor-cabinet']['revision'] = 'master' +node.default['liquor-cabinet']['redis_server_role'] = 'redis_server' +node.default['liquor-cabinet']['redis_port'] = 6379 +node.default['liquor-cabinet']['redis_db'] = 1 +node.default['liquor-cabinet']['s3_endpoint'] = nil +node.default['liquor-cabinet']['s3_region'] = nil +node.default['liquor-cabinet']['s3_bucket'] = nil +node.default['liquor-cabinet']['ufw_source_allowed'] = nil +node.default['liquor-cabinet']['maintenance_mode_enabled'] = false +node.default['liquor-cabinet']['rainbows'] = { + 'port' => 3000, + 'preload_app' => true, + 'timeout' => 60, + 'worker_processes' => node['cpu']['total'], + 'worker_connections' => 100, + 'client_header_buffer_size' => 1024, + 'client_max_body_size' => 104857600, + 'client_max_header_size' => 114688, + 'copy_stream' => 'IO', + 'keepalive_requests' => 100, + 'keepalive_timeout' => 5 +} diff --git a/site-cookbooks/liquor_cabinet/chefignore b/site-cookbooks/liquor_cabinet/chefignore new file mode 100644 index 0000000..cc170ea --- /dev/null +++ b/site-cookbooks/liquor_cabinet/chefignore @@ -0,0 +1,115 @@ +# Put files/directories that should be ignored in this file when uploading +# to a Chef Infra Server or Supermarket. +# Lines that start with '# ' are comments. + +# OS generated files # +###################### +.DS_Store +ehthumbs.db +Icon? +nohup.out +Thumbs.db +.envrc + +# EDITORS # +########### +.#* +.project +.settings +*_flymake +*_flymake.* +*.bak +*.sw[a-z] +*.tmproj +*~ +\#* +REVISION +TAGS* +tmtags +.vscode +.editorconfig + +## COMPILED ## +############## +*.class +*.com +*.dll +*.exe +*.o +*.pyc +*.so +*/rdoc/ +a.out +mkmf.log + +# Testing # +########### +.circleci/* +.codeclimate.yml +.delivery/* +.foodcritic +.kitchen* +.mdlrc +.overcommit.yml +.rspec +.rubocop.yml +.travis.yml +.watchr +.yamllint +azure-pipelines.yml +Dangerfile +examples/* +features/* +Guardfile +kitchen.yml* +mlc_config.json +Procfile +Rakefile +spec/* +test/* + +# SCM # +####### +.git +.gitattributes +.gitconfig +.github/* +.gitignore +.gitkeep +.gitmodules +.svn +*/.bzr/* +*/.git +*/.hg/* +*/.svn/* + +# Berkshelf # +############# +Berksfile +Berksfile.lock +cookbooks/* +tmp + +# Bundler # +########### +vendor/* +Gemfile +Gemfile.lock + +# Policyfile # +############## +Policyfile.rb +Policyfile.lock.json + +# Documentation # +############# +CODE_OF_CONDUCT* +CONTRIBUTING* +documentation/* +TESTING* +UPGRADING* + +# Vagrant # +########### +.vagrant +Vagrantfile diff --git a/site-cookbooks/liquor_cabinet/kitchen.yml b/site-cookbooks/liquor_cabinet/kitchen.yml new file mode 100644 index 0000000..e309743 --- /dev/null +++ b/site-cookbooks/liquor_cabinet/kitchen.yml @@ -0,0 +1,37 @@ +--- +driver: + name: dokken + privileged: true # allows systemd services to start + +provisioner: + name: dokken + +transport: + name: dokken + +verifier: + name: inspec + +platforms: + # @see https://github.com/chef-cookbooks/testing_examples/blob/main/kitchen.dokken.yml + # @see https://hub.docker.com/u/dokken + - name: ubuntu-20.04 + driver: + image: dokken/ubuntu-20.04 + pid_one_command: /bin/systemd + intermediate_instructions: + - RUN /usr/bin/apt-get update + + - name: centos-8 + driver: + image: dokken/centos-8 + pid_one_command: /usr/lib/systemd/systemd + +suites: + - name: default + run_list: + - recipe[liquor_cabinet::default] + verifier: + inspec_tests: + - test/integration/default + attributes: diff --git a/site-cookbooks/liquor_cabinet/metadata.rb b/site-cookbooks/liquor_cabinet/metadata.rb new file mode 100644 index 0000000..bd24a4c --- /dev/null +++ b/site-cookbooks/liquor_cabinet/metadata.rb @@ -0,0 +1,12 @@ +name 'liquor_cabinet' +maintainer 'Kosmos Developers' +maintainer_email 'ops@kosmos.org' +license 'MIT' +description 'Installs/configures the Liquor Cabinet remoteStorage API server' +version '0.1.0' +chef_version '>= 18.2' +issues_url 'https://gitea.kosmos.org/kosmos/chef/issues' +# source_url 'https://gitea.kosmos.org/kosmos/chef' + +depends 'firewall' +depends "ruby_build" diff --git a/site-cookbooks/liquor_cabinet/recipes/default.rb b/site-cookbooks/liquor_cabinet/recipes/default.rb new file mode 100644 index 0000000..ebf6b0e --- /dev/null +++ b/site-cookbooks/liquor_cabinet/recipes/default.rb @@ -0,0 +1,139 @@ +# +# Cookbook:: liquor_cabinet +# Recipe:: default +# + +app_name = "liquor-cabinet" +deploy_user = "deploy" +deploy_group = "deploy" +deploy_path = "/opt/#{app_name}" +credentials = Chef::EncryptedDataBagItem.load('credentials', app_name) + +ruby_version = node[app_name]['ruby']['version'] +ruby_path = "/opt/ruby_build/builds/#{ruby_version}" +bundle_path = "#{ruby_path}/bin/bundle" +rack_env = node.chef_environment == "production" ? "production" : "development" + +ruby_build_install 'v20231225' +ruby_build_definition ruby_version do + prefix_path ruby_path +end + +group deploy_group + +user deploy_user do + group deploy_group + manage_home true + shell "/bin/bash" +end + +directory deploy_path do + owner deploy_user + group deploy_group + mode '0750' +end + +redis_server_role = node[app_name]['redis_server_role'] +redis_host = search(:node, "role:#{redis_server_role}").first['knife_zero']['host'] rescue nil +if redis_host.nil? + Chef::Log.warn("No node found with '#{redis_server_role}' role. Stopping here.") + return +end + +git deploy_path do + repository node[app_name]['repo'] + revision node[app_name]['revision'] + user deploy_user + group deploy_group + notifies :restart, "service[#{app_name}]", :delayed +end + +directory "#{deploy_path}/tmp" do + owner deploy_user + group deploy_group + mode 0750 +end + +execute "bundle install" do + user deploy_user + cwd deploy_path + command "#{bundle_path} install --without development,test --deployment" +end + +template "#{deploy_path}/config.yml.erb" do + source 'config.yml.erb' + owner deploy_user + group deploy_group + mode '0600' + sensitive true + variables environment: rack_env, + redis_host: redis_host, + redis_port: node[app_name]['redis_port'], + redis_db: node[app_name]['redis_db'], + s3_endpoint: node[app_name]['s3_endpoint'], + s3_region: node[app_name]['s3_region'], + s3_bucket: node[app_name]['s3_bucket'], + s3_access_key: credentials['s3_access_key'], + s3_secret_key: credentials['s3_secret_key'], + maintenance_mode_enabled: node[app_name]['maintenance_mode_enabled'] + # TODO sentry_dsn: credentials['sentry_dsn'] + notifies :restart, "service[#{app_name}]", :delayed +end + +directory '/etc/rainbows' do + owner deploy_user + group deploy_group + mode '0750' +end + +template "/etc/rainbows/#{app_name}.rb" do + source 'rainbows.rb.erb' + owner deploy_user + group deploy_group + mode '0640' + variables user: deploy_user, + group: deploy_group, + app_name: app_name, + working_directory: deploy_path, + config: node[app_name]['rainbows'] + notifies :restart, "service[#{app_name}]", :delayed +end + +systemd_unit "#{app_name}.service" do + content({ + Unit: { + Description: "Liquor Cabinet remoteStorage HTTP API", + Documentation: ["https://gitea.kosmos.org/5apps/liquor-cabinet"], + After: "syslog.target network.target" + }, + Service: { + Type: "simple", + User: deploy_user, + WorkingDirectory: deploy_path, + Environment: "RACK_ENV=#{rack_env}", + ExecStart: "#{bundle_path} exec rainbows -c /etc/rainbows/#{app_name}.rb -E #{rack_env}", + PIDFile: "#{deploy_path}/tmp/rainbows.pid", + TimeoutSec: "10", + Restart: "on-failure", + }, + Install: { + WantedBy: "multi-user.target" + } + }) + verify false + triggers_reload true + action [:create, :enable] +end + +service app_name do + action [:enable, :start] +end + +if node[app_name]['ufw_source_allowed'] + firewall_rule app_name do + command :allow + protocol :tcp + port node[app_name]['rainbows']['port'] + source node[app_name]['ufw_source_allowed'] + end +end diff --git a/site-cookbooks/liquor_cabinet/templates/config.yml.erb b/site-cookbooks/liquor_cabinet/templates/config.yml.erb new file mode 100644 index 0000000..615d28d --- /dev/null +++ b/site-cookbooks/liquor_cabinet/templates/config.yml.erb @@ -0,0 +1,12 @@ +<%= @environment %>: + maintenance: <%= @maintenance_mode_enabled %> + redis: + host: <%= @redis_host %> + port: <%= @redis_port %> + db: <%= @redis_db %> + s3: + endpoint: <%= @s3_endpoint %> + region: <%= @s3_region %> + bucket: <%= @s3_bucket %> + access_key_id: <%= @s3_access_key %> + secret_key_id: <%= @s3_secret_key %> diff --git a/site-cookbooks/liquor_cabinet/templates/rainbows.rb.erb b/site-cookbooks/liquor_cabinet/templates/rainbows.rb.erb new file mode 100644 index 0000000..4cf5125 --- /dev/null +++ b/site-cookbooks/liquor_cabinet/templates/rainbows.rb.erb @@ -0,0 +1,32 @@ +## +# Rainbows config at /etc/rainbows/<%= @app_name %>.rb +# Managed by Chef - Local changes will be overwritten by Chef runs +## + +# What ports/sockets to listen on, and what options for them. +listen "<%= @config['port'] %>", { tcp_nodelay: true, backlog: 100 } + +# What the timeout for killing busy workers is, in seconds +timeout <%= @config['timeout'] %> + +# Whether the app should be pre-loaded +preload_app <%= @config['preload_app'] %> + +# How many worker processes +worker_processes <%= @config['worker_processes'] %> + +# Run forked children as specified user/group +user "<%= @user %>", "<%= @group %>" + +pid "<%= @working_directory %>/tmp/rainbows.pid" + +Rainbows! do + use :ThreadSpawn + client_header_buffer_size <%= @config['client_header_buffer_size'] %> + client_max_body_size <%= @config['client_max_body_size'] %> + client_max_header_size <%= @config['client_max_header_size'] %> + copy_stream <%= @config['copy_stream'] %> + keepalive_requests <%= @config['keepalive_requests'] %> + keepalive_timeout <%= @config['keepalive_timeout'] %> + worker_connections <%= @config['worker_connections'] %> +end From 211a613e5c21b64dcf236197ef99f3dfa17f2077 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A2u=20Cao?= Date: Thu, 25 Jan 2024 16:27:31 +0300 Subject: [PATCH 02/12] Add liquor-cabinet configs, deploy to production --- data_bags/credentials/liquor-cabinet.json | 17 +++++++++++++++++ environments/production.json | 6 ++++++ nodes/lq-1.json | 19 +++++++++++++++---- nodes/lq-2.json | 19 +++++++++++++++---- roles/liquor_cabinet.rb | 5 +++++ 5 files changed, 58 insertions(+), 8 deletions(-) create mode 100644 data_bags/credentials/liquor-cabinet.json create mode 100644 roles/liquor_cabinet.rb diff --git a/data_bags/credentials/liquor-cabinet.json b/data_bags/credentials/liquor-cabinet.json new file mode 100644 index 0000000..81dc301 --- /dev/null +++ b/data_bags/credentials/liquor-cabinet.json @@ -0,0 +1,17 @@ +{ + "id": "liquor-cabinet", + "s3_access_key": { + "encrypted_data": "TKYUWVboQZUKvw4bqrKsL28dH2DGR5iDBQclAwm5I7GqkxFfkG2d91qLv+BA\n", + "iv": "B8YYzXeFGxMG34WI\n", + "auth_tag": "HOIfcpJOFYIVvf5o8lk4mg==\n", + "version": 3, + "cipher": "aes-256-gcm" + }, + "s3_secret_key": { + "encrypted_data": "GRqGJkGJ/f0zQVtO0r9TcXBqlpnfC5PiwTZK8QmsqEhzQI6U67NAf62QqTgl\nGVI1h8G5ITgC3l0xVhcvH6m2bcs9fjNzFIqnhoZhzGwEt51A5Zk=\n", + "iv": "UAlmoUWLedpd79xa\n", + "auth_tag": "2F/EJhY5/59dtFFwkd106A==\n", + "version": 3, + "cipher": "aes-256-gcm" + } +} \ No newline at end of file diff --git a/environments/production.json b/environments/production.json index ca1912e..145ec3c 100644 --- a/environments/production.json +++ b/environments/production.json @@ -77,6 +77,12 @@ "mastodon.w7nooprauv6yrnhzh2ajpcnj3doinked2aaztlwfyt6u6pva2qdxqhid.onion" ] }, + "liquor-cabinet": { + "ufw_source_allowed": "10.1.1.0/24", + "s3_endpoint": "http://localhost:3900", + "s3_region": "garage", + "s3_bucket": "rs-kosmos" + }, "mediawiki": { "url": "https://wiki.kosmos.org" }, diff --git a/nodes/lq-1.json b/nodes/lq-1.json index c969a94..e3b0a26 100644 --- a/nodes/lq-1.json +++ b/nodes/lq-1.json @@ -1,5 +1,6 @@ { "name": "lq-1", + "chef_environment": "production", "normal": { "knife_zero": { "host": "10.1.1.87" @@ -8,17 +9,24 @@ "automatic": { "fqdn": "lq-1", "os": "linux", - "os_version": "5.4.0-1090-kvm", + "os_version": "5.4.0-1104-kvm", "hostname": "lq-1", "ipaddress": "192.168.122.158", "roles": [ "base", - "kvm_guest" + "kvm_guest", + "garage_gateway", + "liquor_cabinet" ], "recipes": [ "kosmos-base", "kosmos-base::default", "kosmos_kvm::guest", + "kosmos_garage", + "kosmos_garage::default", + "kosmos_garage::firewall_rpc", + "liquor_cabinet", + "liquor_cabinet::default", "apt::default", "timezone_iii::default", "timezone_iii::debian", @@ -32,7 +40,8 @@ "postfix::_common", "postfix::_attributes", "postfix::sasl_auth", - "hostname::default" + "hostname::default", + "firewall::default" ], "platform": "ubuntu", "platform_version": "20.04", @@ -51,6 +60,8 @@ }, "run_list": [ "role[base]", - "role[kvm_guest]" + "role[kvm_guest]", + "role[garage_gateway]", + "role[liquor_cabinet]" ] } \ No newline at end of file diff --git a/nodes/lq-2.json b/nodes/lq-2.json index 8174bc6..65a82d5 100644 --- a/nodes/lq-2.json +++ b/nodes/lq-2.json @@ -1,5 +1,6 @@ { "name": "lq-2", + "chef_environment": "production", "normal": { "knife_zero": { "host": "10.1.1.188" @@ -8,17 +9,24 @@ "automatic": { "fqdn": "lq-2", "os": "linux", - "os_version": "5.4.0-1090-kvm", + "os_version": "5.4.0-1104-kvm", "hostname": "lq-2", "ipaddress": "192.168.122.47", "roles": [ "base", - "kvm_guest" + "kvm_guest", + "garage_gateway", + "liquor_cabinet" ], "recipes": [ "kosmos-base", "kosmos-base::default", "kosmos_kvm::guest", + "kosmos_garage", + "kosmos_garage::default", + "kosmos_garage::firewall_rpc", + "liquor_cabinet", + "liquor_cabinet::default", "apt::default", "timezone_iii::default", "timezone_iii::debian", @@ -32,7 +40,8 @@ "postfix::_common", "postfix::_attributes", "postfix::sasl_auth", - "hostname::default" + "hostname::default", + "firewall::default" ], "platform": "ubuntu", "platform_version": "20.04", @@ -51,6 +60,8 @@ }, "run_list": [ "role[base]", - "role[kvm_guest]" + "role[kvm_guest]", + "role[garage_gateway]", + "role[liquor_cabinet]" ] } \ No newline at end of file diff --git a/roles/liquor_cabinet.rb b/roles/liquor_cabinet.rb new file mode 100644 index 0000000..2651a6a --- /dev/null +++ b/roles/liquor_cabinet.rb @@ -0,0 +1,5 @@ +name "liquor_cabinet" + +run_list %w( + liquor_cabinet::default +) From 715fdbc2ba5c4b7fcf45842cc3edd8a93f9b2111 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A2u=20Cao?= Date: Thu, 25 Jan 2024 17:40:42 +0300 Subject: [PATCH 03/12] Node-attribute ALL THE THINGS --- site-cookbooks/liquor_cabinet/attributes/default.rb | 6 +++++- site-cookbooks/liquor_cabinet/recipes/default.rb | 8 ++++---- 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/site-cookbooks/liquor_cabinet/attributes/default.rb b/site-cookbooks/liquor_cabinet/attributes/default.rb index 6e88013..467aebb 100644 --- a/site-cookbooks/liquor_cabinet/attributes/default.rb +++ b/site-cookbooks/liquor_cabinet/attributes/default.rb @@ -1,6 +1,9 @@ -node.default['liquor-cabinet']['ruby']['version'] = "3.1.4" +node.default['liquor-cabinet']['app_name'] = "liquor-cabinet" +node.default['liquor-cabinet']['user'] = "deploy" +node.default['liquor-cabinet']['group'] = "deploy" node.default['liquor-cabinet']['repo'] = 'https://gitea.kosmos.org/5apps/liquor-cabinet.git' node.default['liquor-cabinet']['revision'] = 'master' +node.default['liquor-cabinet']['deploy_path'] = "/opt/#{node['liquor-cabinet']['app_name']}" node.default['liquor-cabinet']['redis_server_role'] = 'redis_server' node.default['liquor-cabinet']['redis_port'] = 6379 node.default['liquor-cabinet']['redis_db'] = 1 @@ -9,6 +12,7 @@ node.default['liquor-cabinet']['s3_region'] = nil node.default['liquor-cabinet']['s3_bucket'] = nil node.default['liquor-cabinet']['ufw_source_allowed'] = nil node.default['liquor-cabinet']['maintenance_mode_enabled'] = false +node.default['liquor-cabinet']['ruby']['version'] = "3.1.4" node.default['liquor-cabinet']['rainbows'] = { 'port' => 3000, 'preload_app' => true, diff --git a/site-cookbooks/liquor_cabinet/recipes/default.rb b/site-cookbooks/liquor_cabinet/recipes/default.rb index ebf6b0e..39687cd 100644 --- a/site-cookbooks/liquor_cabinet/recipes/default.rb +++ b/site-cookbooks/liquor_cabinet/recipes/default.rb @@ -3,10 +3,10 @@ # Recipe:: default # -app_name = "liquor-cabinet" -deploy_user = "deploy" -deploy_group = "deploy" -deploy_path = "/opt/#{app_name}" +app_name = node['liquor-cabinet']['app_name'] +deploy_user = node[app_name]['user'] +deploy_group = node[app_name]['group'] +deploy_path = node[app_name]['deploy_path'] credentials = Chef::EncryptedDataBagItem.load('credentials', app_name) ruby_version = node[app_name]['ruby']['version'] From a2ec41b68a67bd06a6bdd6f6ee52425355908eca Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A2u=20Cao?= Date: Thu, 25 Jan 2024 17:42:56 +0300 Subject: [PATCH 04/12] Add/use kosmos_liquor-cabinet cookbook --- nodes/lq-1.json | 7 +- roles/liquor_cabinet.rb | 2 +- .../kosmos_liquor-cabinet/.gitignore | 25 ++++ .../kosmos_liquor-cabinet/CHANGELOG.md | 7 ++ site-cookbooks/kosmos_liquor-cabinet/LICENSE | 21 ++++ .../kosmos_liquor-cabinet/README.md | 7 ++ .../attributes/default.rb | 0 .../kosmos_liquor-cabinet/chefignore | 115 ++++++++++++++++++ .../kosmos_liquor-cabinet/kitchen.yml | 37 ++++++ .../kosmos_liquor-cabinet/metadata.rb | 12 ++ .../kosmos_liquor-cabinet/recipes/default.rb | 6 + 11 files changed, 235 insertions(+), 4 deletions(-) create mode 100644 site-cookbooks/kosmos_liquor-cabinet/.gitignore create mode 100644 site-cookbooks/kosmos_liquor-cabinet/CHANGELOG.md create mode 100644 site-cookbooks/kosmos_liquor-cabinet/LICENSE create mode 100644 site-cookbooks/kosmos_liquor-cabinet/README.md create mode 100644 site-cookbooks/kosmos_liquor-cabinet/attributes/default.rb create mode 100644 site-cookbooks/kosmos_liquor-cabinet/chefignore create mode 100644 site-cookbooks/kosmos_liquor-cabinet/kitchen.yml create mode 100644 site-cookbooks/kosmos_liquor-cabinet/metadata.rb create mode 100644 site-cookbooks/kosmos_liquor-cabinet/recipes/default.rb diff --git a/nodes/lq-1.json b/nodes/lq-1.json index e3b0a26..574ec90 100644 --- a/nodes/lq-1.json +++ b/nodes/lq-1.json @@ -25,8 +25,8 @@ "kosmos_garage", "kosmos_garage::default", "kosmos_garage::firewall_rpc", - "liquor_cabinet", - "liquor_cabinet::default", + "kosmos_liquor-cabinet", + "kosmos_liquor-cabinet::default", "apt::default", "timezone_iii::default", "timezone_iii::debian", @@ -41,7 +41,8 @@ "postfix::_attributes", "postfix::sasl_auth", "hostname::default", - "firewall::default" + "firewall::default", + "liquor_cabinet::default" ], "platform": "ubuntu", "platform_version": "20.04", diff --git a/roles/liquor_cabinet.rb b/roles/liquor_cabinet.rb index 2651a6a..9183055 100644 --- a/roles/liquor_cabinet.rb +++ b/roles/liquor_cabinet.rb @@ -1,5 +1,5 @@ name "liquor_cabinet" run_list %w( - liquor_cabinet::default + kosmos_liquor-cabinet::default ) diff --git a/site-cookbooks/kosmos_liquor-cabinet/.gitignore b/site-cookbooks/kosmos_liquor-cabinet/.gitignore new file mode 100644 index 0000000..f1e57b8 --- /dev/null +++ b/site-cookbooks/kosmos_liquor-cabinet/.gitignore @@ -0,0 +1,25 @@ +.vagrant +*~ +*# +.#* +\#*# +.*.sw[a-z] +*.un~ + +# Bundler +Gemfile.lock +gems.locked +bin/* +.bundle/* + +# test kitchen +.kitchen/ +kitchen.local.yml + +# Chef Infra +Berksfile.lock +.zero-knife.rb +Policyfile.lock.json + +.idea/ + diff --git a/site-cookbooks/kosmos_liquor-cabinet/CHANGELOG.md b/site-cookbooks/kosmos_liquor-cabinet/CHANGELOG.md new file mode 100644 index 0000000..bc1846e --- /dev/null +++ b/site-cookbooks/kosmos_liquor-cabinet/CHANGELOG.md @@ -0,0 +1,7 @@ +# kosmos_liquor-cabinet CHANGELOG + +This file is used to list changes made in each version of the kosmos_liquor-cabinet cookbook. + +## 0.1.0 + +Initial release. diff --git a/site-cookbooks/kosmos_liquor-cabinet/LICENSE b/site-cookbooks/kosmos_liquor-cabinet/LICENSE new file mode 100644 index 0000000..a00749b --- /dev/null +++ b/site-cookbooks/kosmos_liquor-cabinet/LICENSE @@ -0,0 +1,21 @@ +The MIT License (MIT) + +Copyright (c) 2024 Kosmos Developers + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. diff --git a/site-cookbooks/kosmos_liquor-cabinet/README.md b/site-cookbooks/kosmos_liquor-cabinet/README.md new file mode 100644 index 0000000..5c0b08f --- /dev/null +++ b/site-cookbooks/kosmos_liquor-cabinet/README.md @@ -0,0 +1,7 @@ +# kosmos_liquor-cabinet + +Installs/configures the [Liquor Cabinet][1] [remoteStorage][2] API server and +reverse proxy. + +[1]: https://gitea.kosmos.org/5apps/liquor-cabinet +[2]: https://remotestorage.io diff --git a/site-cookbooks/kosmos_liquor-cabinet/attributes/default.rb b/site-cookbooks/kosmos_liquor-cabinet/attributes/default.rb new file mode 100644 index 0000000..e69de29 diff --git a/site-cookbooks/kosmos_liquor-cabinet/chefignore b/site-cookbooks/kosmos_liquor-cabinet/chefignore new file mode 100644 index 0000000..cc170ea --- /dev/null +++ b/site-cookbooks/kosmos_liquor-cabinet/chefignore @@ -0,0 +1,115 @@ +# Put files/directories that should be ignored in this file when uploading +# to a Chef Infra Server or Supermarket. +# Lines that start with '# ' are comments. + +# OS generated files # +###################### +.DS_Store +ehthumbs.db +Icon? +nohup.out +Thumbs.db +.envrc + +# EDITORS # +########### +.#* +.project +.settings +*_flymake +*_flymake.* +*.bak +*.sw[a-z] +*.tmproj +*~ +\#* +REVISION +TAGS* +tmtags +.vscode +.editorconfig + +## COMPILED ## +############## +*.class +*.com +*.dll +*.exe +*.o +*.pyc +*.so +*/rdoc/ +a.out +mkmf.log + +# Testing # +########### +.circleci/* +.codeclimate.yml +.delivery/* +.foodcritic +.kitchen* +.mdlrc +.overcommit.yml +.rspec +.rubocop.yml +.travis.yml +.watchr +.yamllint +azure-pipelines.yml +Dangerfile +examples/* +features/* +Guardfile +kitchen.yml* +mlc_config.json +Procfile +Rakefile +spec/* +test/* + +# SCM # +####### +.git +.gitattributes +.gitconfig +.github/* +.gitignore +.gitkeep +.gitmodules +.svn +*/.bzr/* +*/.git +*/.hg/* +*/.svn/* + +# Berkshelf # +############# +Berksfile +Berksfile.lock +cookbooks/* +tmp + +# Bundler # +########### +vendor/* +Gemfile +Gemfile.lock + +# Policyfile # +############## +Policyfile.rb +Policyfile.lock.json + +# Documentation # +############# +CODE_OF_CONDUCT* +CONTRIBUTING* +documentation/* +TESTING* +UPGRADING* + +# Vagrant # +########### +.vagrant +Vagrantfile diff --git a/site-cookbooks/kosmos_liquor-cabinet/kitchen.yml b/site-cookbooks/kosmos_liquor-cabinet/kitchen.yml new file mode 100644 index 0000000..bd2abe1 --- /dev/null +++ b/site-cookbooks/kosmos_liquor-cabinet/kitchen.yml @@ -0,0 +1,37 @@ +--- +driver: + name: dokken + privileged: true # allows systemd services to start + +provisioner: + name: dokken + +transport: + name: dokken + +verifier: + name: inspec + +platforms: + # @see https://github.com/chef-cookbooks/testing_examples/blob/main/kitchen.dokken.yml + # @see https://hub.docker.com/u/dokken + - name: ubuntu-20.04 + driver: + image: dokken/ubuntu-20.04 + pid_one_command: /bin/systemd + intermediate_instructions: + - RUN /usr/bin/apt-get update + + - name: centos-8 + driver: + image: dokken/centos-8 + pid_one_command: /usr/lib/systemd/systemd + +suites: + - name: default + run_list: + - recipe[kosmos_liquor-cabinet::default] + verifier: + inspec_tests: + - test/integration/default + attributes: diff --git a/site-cookbooks/kosmos_liquor-cabinet/metadata.rb b/site-cookbooks/kosmos_liquor-cabinet/metadata.rb new file mode 100644 index 0000000..1ab8e56 --- /dev/null +++ b/site-cookbooks/kosmos_liquor-cabinet/metadata.rb @@ -0,0 +1,12 @@ +name 'kosmos_liquor-cabinet' +maintainer 'Kosmos Developers' +maintainer_email 'ops@kosmos.org' +license 'MIT' +description 'Installs/configures Liquor Cabinet API and reverse proxy' +version '0.1.0' +chef_version '>= 18.2' +issues_url 'https://gitea.kosmos.org/kosmos/chef/issues' +# source_url 'https://gitea.kosmos.org/kosmos/chef' + +depends 'liquor_cabinet' +depends 'kosmos_openresty' diff --git a/site-cookbooks/kosmos_liquor-cabinet/recipes/default.rb b/site-cookbooks/kosmos_liquor-cabinet/recipes/default.rb new file mode 100644 index 0000000..1e6d8f8 --- /dev/null +++ b/site-cookbooks/kosmos_liquor-cabinet/recipes/default.rb @@ -0,0 +1,6 @@ +# +# Cookbook:: kosmos_liquor-cabinet +# Recipe:: default +# + +include_recipe 'liquor_cabinet' From 6114f0f799f4f0994221c309a83f9a38bfc84ff3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A2u=20Cao?= Date: Fri, 26 Jan 2024 08:15:53 +0300 Subject: [PATCH 05/12] Add liquor-cabinet proxy recipe --- .../attributes/default.rb | 4 + .../kosmos_liquor-cabinet/recipes/nginx.rb | 30 +++++++ .../templates/nginx_conf_liquor-cabinet.erb | 79 +++++++++++++++++++ 3 files changed, 113 insertions(+) create mode 100644 site-cookbooks/kosmos_liquor-cabinet/recipes/nginx.rb create mode 100644 site-cookbooks/kosmos_liquor-cabinet/templates/nginx_conf_liquor-cabinet.erb diff --git a/site-cookbooks/kosmos_liquor-cabinet/attributes/default.rb b/site-cookbooks/kosmos_liquor-cabinet/attributes/default.rb index e69de29..6c4b547 100644 --- a/site-cookbooks/kosmos_liquor-cabinet/attributes/default.rb +++ b/site-cookbooks/kosmos_liquor-cabinet/attributes/default.rb @@ -0,0 +1,4 @@ +node.default['liquor-cabinet']['app_server_role'] = 'liquor_cabinet' +node.default['liquor-cabinet']['max_upload_size'] = 100 # MB +node.default['liquor-cabinet']['server_name'] = 'storage.example.com' +node.default['liquor-cabinet']['root_redirect_url'] = 'https://example.com/storage' diff --git a/site-cookbooks/kosmos_liquor-cabinet/recipes/nginx.rb b/site-cookbooks/kosmos_liquor-cabinet/recipes/nginx.rb new file mode 100644 index 0000000..e3c1b6b --- /dev/null +++ b/site-cookbooks/kosmos_liquor-cabinet/recipes/nginx.rb @@ -0,0 +1,30 @@ +# +# Cookbook:: kosmos_liquor-cabinet +# Recipe:: nginx +# + +app_name = node['liquor-cabinet']['app_name'] +domain = node[app_name]['domain'] + +tls_cert_for domain do + auth "gandi_dns" + action :create +end + +upstream_hosts = [] +search(:node, "role:#{node[app_name]['app_server_role']}").each do |node| + upstream_hosts << node["knife_zero"]["host"] +end +upstream_hosts.push("localhost") if upstream_hosts.empty? + +openresty_site domain do + template "nginx_conf_liquor-cabinet.erb" + variables app_name: app_name, + server_name: domain, + root_redirect_url: node[app_name]['root_redirect_url'], + max_upload_size: node['liquor-cabinet']['max_upload_size'], + upstream_hosts: upstream_hosts, + upstream_port: node[app_name]['rainbows']['port'], + ssl_cert: "/etc/letsencrypt/live/#{domain}/fullchain.pem", + ssl_key: "/etc/letsencrypt/live/#{domain}/privkey.pem" +end diff --git a/site-cookbooks/kosmos_liquor-cabinet/templates/nginx_conf_liquor-cabinet.erb b/site-cookbooks/kosmos_liquor-cabinet/templates/nginx_conf_liquor-cabinet.erb new file mode 100644 index 0000000..870713b --- /dev/null +++ b/site-cookbooks/kosmos_liquor-cabinet/templates/nginx_conf_liquor-cabinet.erb @@ -0,0 +1,79 @@ +# +# Generated by Chef +# +upstream _<%= @app_name %> { +<% @upstream_hosts.each do |host| -%> + server <%= host %>:<%= @upstream_port %>; +<% end -%> +} + +# TODO use cookbook attribute when enabling +# variables_hash_max_size 2048; + +server { + listen 80; + listen [::]:80; + server_name <%= @server_name %>; + # Redirect to https + location / { + return 301 https://<%= @server_name %>$request_uri; + } +} + +server { + listen 443 ssl http2; + listen [::]:443 ssl http2; + server_name <%= @server_name %>; + + access_log <%= node[:nginx][:log_dir] %>/<%= @app_name %>.access.log; # TODO json_liquor_cabinet; + error_log <%= node[:nginx][:log_dir] %>/<%= @app_name %>.error.log warn; + + add_header Strict-Transport-Security "max-age=15768000; includeSubDomains; preload"; + + # TODO + # log_by_lua_file "<%= @log_by_lua_file %>"; + + # We need strong ETags, disable compression + gzip off; + # brotli off; + # pagespeed off; + + # Set a large maximum upload size + client_max_body_size <%= @max_upload_size %>m; + + # TODO + # Use rate limiting (the zone is defined in + # /etc/nginx/conf.d/rate_limiting.conf) + # limit_req zone=per_ip burst=5000; + + location = / { + return 301 <%= @root_redirect_url %>; + } + + location / { + try_files $uri @proxy; + } + + location @proxy { + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto https; + proxy_set_header Host $http_host; + + proxy_redirect off; + + proxy_buffering on; + # Increase number of buffers. Default is 8 + proxy_buffers 1024 8k; + + # Needed for big uploads + proxy_read_timeout 180s; + proxy_send_timeout 180s; + + proxy_pass http://_<%= @app_name %>; + + proxy_next_upstream error timeout http_502 http_500; + } + + ssl_certificate <%= @ssl_cert %>; + ssl_certificate_key <%= @ssl_key %>; +} From e10e54c12aec2c2c41dbb825c1e97ac54cd7e8cc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A2u=20Cao?= Date: Fri, 26 Jan 2024 08:16:26 +0300 Subject: [PATCH 06/12] Deploy liquor-cabinet proxy to production --- environments/production.json | 4 +++- nodes/draco.kosmos.org.json | 1 + nodes/fornax.kosmos.org.json | 1 + roles/openresty_proxy.rb | 1 + 4 files changed, 6 insertions(+), 1 deletion(-) diff --git a/environments/production.json b/environments/production.json index 145ec3c..b924d30 100644 --- a/environments/production.json +++ b/environments/production.json @@ -81,7 +81,9 @@ "ufw_source_allowed": "10.1.1.0/24", "s3_endpoint": "http://localhost:3900", "s3_region": "garage", - "s3_bucket": "rs-kosmos" + "s3_bucket": "rs-kosmos", + "domain": "storage.kosmos.org", + "root_redirect_url": "https://accounts.kosmos.org" }, "mediawiki": { "url": "https://wiki.kosmos.org" diff --git a/nodes/draco.kosmos.org.json b/nodes/draco.kosmos.org.json index 9854a65..34262f1 100644 --- a/nodes/draco.kosmos.org.json +++ b/nodes/draco.kosmos.org.json @@ -52,6 +52,7 @@ "kosmos_garage::nginx_s3", "kosmos_gitea::nginx", "kosmos_gitea::nginx_ssh", + "kosmos_liquor-cabinet::nginx", "kosmos_rsk::nginx_testnet", "kosmos_rsk::nginx_mainnet", "kosmos_website", diff --git a/nodes/fornax.kosmos.org.json b/nodes/fornax.kosmos.org.json index d304aa5..8744139 100644 --- a/nodes/fornax.kosmos.org.json +++ b/nodes/fornax.kosmos.org.json @@ -45,6 +45,7 @@ "kosmos_garage::nginx_s3", "kosmos_gitea::nginx", "kosmos_gitea::nginx_ssh", + "kosmos_liquor-cabinet::nginx", "kosmos_rsk::nginx_testnet", "kosmos_rsk::nginx_mainnet", "kosmos_website", diff --git a/roles/openresty_proxy.rb b/roles/openresty_proxy.rb index 2ecd45e..6779898 100644 --- a/roles/openresty_proxy.rb +++ b/roles/openresty_proxy.rb @@ -26,6 +26,7 @@ production_run_list = %w( kosmos_garage::nginx_s3 kosmos_gitea::nginx kosmos_gitea::nginx_ssh + kosmos_liquor-cabinet::nginx kosmos_rsk::nginx_testnet kosmos_rsk::nginx_mainnet kosmos_website::default From 7ab83d3d82876b8802ae909ea1a5aadf604317e7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A2u=20Cao?= Date: Fri, 26 Jan 2024 09:00:25 +0300 Subject: [PATCH 07/12] Section header comments for ENV vars Improve readability of recipe --- .../kosmos-akkounts/recipes/default.rb | 36 +++++++++++++++++++ 1 file changed, 36 insertions(+) diff --git a/site-cookbooks/kosmos-akkounts/recipes/default.rb b/site-cookbooks/kosmos-akkounts/recipes/default.rb index e80f4ba..af41c51 100644 --- a/site-cookbooks/kosmos-akkounts/recipes/default.rb +++ b/site-cookbooks/kosmos-akkounts/recipes/default.rb @@ -69,17 +69,33 @@ if webhooks_allowed_ips.length > 0 env[:webhooks_allowed_ips] = webhooks_allowed_ips end +# +# BTCPay Server +# + if btcpay_host env[:btcpay_api_url] = "http://#{btcpay_host}:23001/api/v1" env[:btcpay_store_id] = node['akkounts']['btcpay']['store_id'] env[:btcpay_auth_token] = credentials["btcpay_auth_token"] end +# +# Discourse +# + env[:discourse_public_url] = "https://#{node['discourse']['domain']}" env[:discourse_connect_secret] = credentials['discourse_connect_secret'] +# +# Drone CI +# + env[:droneci_public_url] = node["droneci"]["public_url"] +# +# ejabberd +# + ejabberd_private_ip_addresses = [] search(:node, "role:ejabberd").each do |node| ejabberd_private_ip_addresses << node["knife_zero"]["host"] @@ -101,8 +117,16 @@ if ejabberd_private_ip_addresses.size > 0 env[:ejabberd_admin_url] = node['akkounts']['ejabberd']['admin_url'] end +# +# Gitea +# + env[:gitea_public_url] = "https://#{node['gitea']['domain']}" +# +# lndhub.go +# + if lndhub_host node.override["akkounts"]["lndhub"]["api_url"] = "http://#{lndhub_host}:3026" env[:lndhub_legacy_api_url] = node["akkounts"]["lndhub"]["api_url"] @@ -119,10 +143,22 @@ if lndhub_host end end +# +# Mastodon +# + env[:mastodon_public_url] = "https://#{node['kosmos-mastodon']['domain']}" +# +# MediaWiki +# + env[:mediawiki_public_url] = node['mediawiki']['url'] +# +# Akkounts Deployment +# + systemd_unit "akkounts.service" do content({ Unit: { From 98543f3e7d0b92a2e3d701b7f62c184c4776e254 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A2u=20Cao?= Date: Fri, 26 Jan 2024 09:01:55 +0300 Subject: [PATCH 08/12] Configure RS integration for akkounts --- environments/production.json | 2 ++ site-cookbooks/kosmos-akkounts/recipes/default.rb | 13 +++++++++++++ 2 files changed, 15 insertions(+) diff --git a/environments/production.json b/environments/production.json index 0b458e8..e80e3b7 100644 --- a/environments/production.json +++ b/environments/production.json @@ -82,6 +82,8 @@ }, "liquor-cabinet": { "ufw_source_allowed": "10.1.1.0/24", + "redis_port": 6379, + "redis_db": 1, "s3_endpoint": "http://localhost:3900", "s3_region": "garage", "s3_bucket": "rs-kosmos", diff --git a/site-cookbooks/kosmos-akkounts/recipes/default.rb b/site-cookbooks/kosmos-akkounts/recipes/default.rb index af41c51..558dc5b 100644 --- a/site-cookbooks/kosmos-akkounts/recipes/default.rb +++ b/site-cookbooks/kosmos-akkounts/recipes/default.rb @@ -155,6 +155,19 @@ env[:mastodon_public_url] = "https://#{node['kosmos-mastodon']['domain']}" env[:mediawiki_public_url] = node['mediawiki']['url'] +# +# remoteStorage / Liquor Cabinet +# + +env[:rs_storage_url] = "https://#{node['liquor-cabinet']['domain']}" + +rs_redis_host = search(:node, "role:redis_server").first["knife_zero"]["host"] rescue nil +rs_redis_port = node['liquor-cabinet']['redis_port'] +rs_redis_db = node['liquor-cabinet']['redis_db'] +if rs_redis_host + env[:rs_redis_url] = "redis://#{rs_redis_host}:#{rs_redis_port}/#{rs_redis_db}" +end + # # Akkounts Deployment # From 5c1c63f94e73d0f82ff1831f872bca79f274f92d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A2u=20Cao?= Date: Fri, 26 Jan 2024 10:40:36 +0300 Subject: [PATCH 09/12] Add S3/Garage config for akkounts --- data_bags/credentials/akkounts.json | 56 ++++++++++++------- environments/production.json | 5 +- nodes/akkounts-1.json | 6 ++ .../kosmos-akkounts/attributes/default.rb | 6 ++ .../kosmos-akkounts/recipes/default.rb | 14 +++++ 5 files changed, 64 insertions(+), 23 deletions(-) diff --git a/data_bags/credentials/akkounts.json b/data_bags/credentials/akkounts.json index d9461ef..6c754de 100644 --- a/data_bags/credentials/akkounts.json +++ b/data_bags/credentials/akkounts.json @@ -1,51 +1,65 @@ { "id": "akkounts", "postgresql_username": { - "encrypted_data": "/Idxzq83imf6o6pbmFAk7bgxg69N7/1KNhgj\n", - "iv": "34BrmVmlxzuA7IJG\n", - "auth_tag": "VyLpWDshrOd417ZiY3432w==\n", + "encrypted_data": "l00Lmdbl5xNq07XU4XmcnRxXsIJaYyMQQ6xI\n", + "iv": "yxvL6hKwlVWmdMzl\n", + "auth_tag": "mMCV9ewJW/0TfVE76WBSZw==\n", "version": 3, "cipher": "aes-256-gcm" }, "postgresql_password": { - "encrypted_data": "XqEmt+yu7mB6vBOUCT/5AtIptdUamfniz+PrFYCP0A==\n", - "iv": "2XdVUHkeeS1LHzMx\n", - "auth_tag": "mq0v9ikHD7pxTUrGO+VF9A==\n", + "encrypted_data": "Q6xWsH6bmI1GfMzme3mBRYrt3XmDwFJ7E4FjYg2Rrw==\n", + "iv": "jcQmuT7Jz3g3XE8d\n", + "auth_tag": "nNMvf9UmP6ikf1BW93QZIw==\n", "version": 3, "cipher": "aes-256-gcm" }, "sentry_dsn": { - "encrypted_data": "u82JsPq5HvQRE2eWIbVp73LdqffyuTTylbURtM7XRJ6AXyKp1WD/iwVhNnL7\n/NKSWR24/u63WJCP4rXpW7293ZRU5UW/W3GwlOjNtbdxcaQ=\n", - "iv": "0GIV8v92dh4+Ma/Z\n", - "auth_tag": "XbuxPIZ5VxuMjw/f+usCgA==\n", + "encrypted_data": "V7cqlH2baN1Ix/ggQFeo9PY6dNKKpnDECaB1cO3XuCfy74oN2ot44nbpCQTA\nUl0+1LQv/qNn/L4gmJkqZfdIXZQqhR+iTc06UJxe3aTKJDw=\n", + "iv": "HJtdKYcApwaxhTXI\n", + "auth_tag": "qyIYK9h6nciJTFXBWOjVOA==\n", "version": 3, "cipher": "aes-256-gcm" }, "rails_master_key": { - "encrypted_data": "31N79um4TTD0tuDurrZVztoSv0sxZ70paV7AhD8P4+lX8kUkfhiugCbdhst0\n12YP5v/8\n", - "iv": "l4qanaerdou8AApw\n", - "auth_tag": "yvkcM4on1EMm1LhmmZ+O+g==\n", + "encrypted_data": "KAl2Kgq1TXjOm4TNxGwZkPwJeOSNLbLLKiRdb4fTyBFfUhIGGeCS9VvV9kIb\n9sQZ6HLU\n", + "iv": "BBPvDNs6nBXDti5I\n", + "auth_tag": "yjM/0nyUwt+5SSGuLC5qWA==\n", "version": 3, "cipher": "aes-256-gcm" }, "discourse_connect_secret": { - "encrypted_data": "Ebs8KVEA0r4nFxYNjxxZFUWndxwoKes/9ihEgqgKLN76t6yzCUONeJZBMl0G\nXLdI8A==\n", - "iv": "ob8KBWeoHXFlZ7Nk\n", - "auth_tag": "motppQbVEhg6qyKRYpqctA==\n", + "encrypted_data": "YHkZGzXeK3nDHaXt3JKmGtCcvMfgvv3yHbvS2C+CLKagOIOe+0+2/CiNuh4U\nxO1Pug==\n", + "iv": "SnUxDpIMQum8ySfN\n", + "auth_tag": "Ny6I+3EoCA1s74JLjjbbyQ==\n", "version": 3, "cipher": "aes-256-gcm" }, "lndhub_admin_token": { - "encrypted_data": "I2hSF6X9L3OWbet5QWzrCyA3XyGFhFBgHh/uFr5dQ3RB\n", - "iv": "Kr8u2j5napFSamYc\n", - "auth_tag": "t93UNWomf+6WaZF7VVzTeQ==\n", + "encrypted_data": "dJHxB80Enwkm+2aNuIrp7lILAy2J5tQaChPJCl/BHwMo\n", + "iv": "zHLtD1jTIwvjMt1l\n", + "auth_tag": "IC0adEzsS5YF5YHqabWw2A==\n", "version": 3, "cipher": "aes-256-gcm" }, "btcpay_auth_token": { - "encrypted_data": "0qesJ5KMvU2DlKdz7lExJWq0X9XYjpsqw61kLXWw4UNYwpNxPyFJSjbR9yKh\ntu0zMdtMB9Vur9izWBY=\n", - "iv": "gw2oAyeF2Kuvb3Em\n", - "auth_tag": "zMtos/E3e3XXeTlAY7o0lg==\n", + "encrypted_data": "YbM0HvgIijluKQBcgfKn6hmWvdbhr0ijR1xKc+BRZCZJsRaJBHTjCbwhH8T9\nVnBESruyjhxphtBetcc=\n", + "iv": "3107v/c2Tonx6/cP\n", + "auth_tag": "jnO9fvoXJW5gbDMRjkdMPA==\n", + "version": 3, + "cipher": "aes-256-gcm" + }, + "s3_access_key": { + "encrypted_data": "PFjQKe1us12SNHlReQ4f0qctulPp4d2F3t5t+AGocp87PS/kZx77rtHQtruK\n", + "iv": "BGD8+XchqwPmhhwi\n", + "auth_tag": "XefaZKCVs8hotszALN+kxQ==\n", + "version": 3, + "cipher": "aes-256-gcm" + }, + "s3_secret_key": { + "encrypted_data": "ziO35x8P1YMaSeenMNQoTWug62b5ZVLFlkMlJEFGnYjHK5qTAn6ir06WnMJC\n0zErzTZsPpcr7KpE/ipWgWHRy7qVbGnd6iVO4t9tf5NjiU2OXfA=\n", + "iv": "S3syCCxh2m+mylLu\n", + "auth_tag": "ZMkyBqXMXr3K3LGqxWvbtA==\n", "version": 3, "cipher": "aes-256-gcm" } diff --git a/environments/production.json b/environments/production.json index e80e3b7..12033e7 100644 --- a/environments/production.json +++ b/environments/production.json @@ -43,8 +43,9 @@ "s3_web_root_domain": "web.s3.kosmos.org", "s3_web_domains": [ "media.kosmos.chat", - "s3.kosmos.social", - "s3.community.kosmos.org" + "s3.accounts.kosmos.org", + "s3.community.kosmos.org", + "s3.kosmos.social" ], "xmpp_upload_bucket": "kosmos-xmpp-uploads" }, diff --git a/nodes/akkounts-1.json b/nodes/akkounts-1.json index a183843..e97d59c 100644 --- a/nodes/akkounts-1.json +++ b/nodes/akkounts-1.json @@ -17,6 +17,7 @@ "kvm_guest", "ldap_client", "sentry_client", + "garage_gateway", "akkounts", "postgresql_client" ], @@ -26,6 +27,9 @@ "kosmos_kvm::guest", "kosmos-dirsrv::hostsfile", "kosmos_sentry::client", + "kosmos_garage", + "kosmos_garage::default", + "kosmos_garage::firewall_rpc", "kosmos_postgresql::hostsfile", "kosmos-akkounts", "kosmos-akkounts::default", @@ -43,6 +47,7 @@ "postfix::_attributes", "postfix::sasl_auth", "hostname::default", + "firewall::default", "redisio::default", "redisio::_install_prereqs", "redisio::install", @@ -76,6 +81,7 @@ "role[kvm_guest]", "role[ldap_client]", "role[sentry_client]", + "role[garage_gateway]", "role[akkounts]" ] } \ No newline at end of file diff --git a/site-cookbooks/kosmos-akkounts/attributes/default.rb b/site-cookbooks/kosmos-akkounts/attributes/default.rb index 0f4ca0c..0fc749e 100644 --- a/site-cookbooks/kosmos-akkounts/attributes/default.rb +++ b/site-cookbooks/kosmos-akkounts/attributes/default.rb @@ -19,3 +19,9 @@ node.default['akkounts']['lndhub']['api_url'] = nil node.default['akkounts']['lndhub']['public_url'] = nil node.default['akkounts']['lndhub']['public_key'] = nil node.default['akkounts']['lndhub']['postgres_db'] = 'lndhub' + +node.default['akkounts']['s3_enabled'] = true +node.default['akkounts']['s3_endpoint'] = "http://localhost:3900" +node.default['akkounts']['s3_region'] = "garage" +node.default['akkounts']['s3_bucket'] = "akkounts-production" +node.default['akkounts']['s3_alias_host'] = "https://s3.accounts.kosmos.org" diff --git a/site-cookbooks/kosmos-akkounts/recipes/default.rb b/site-cookbooks/kosmos-akkounts/recipes/default.rb index 558dc5b..9adc442 100644 --- a/site-cookbooks/kosmos-akkounts/recipes/default.rb +++ b/site-cookbooks/kosmos-akkounts/recipes/default.rb @@ -168,6 +168,20 @@ if rs_redis_host env[:rs_redis_url] = "redis://#{rs_redis_host}:#{rs_redis_port}/#{rs_redis_db}" end +# +# S3 +# + +if node['akkounts']['s3_enabled'] + env[:s3_enabled] = true + env[:s3_endpoint] = node['akkounts']['s3_endpoint'] + env[:s3_region] = node['akkounts']['s3_region'] + env[:s3_bucket] = node['akkounts']['s3_bucket'] + env[:s3_alias_host] = node['akkounts']['s3_alias_host'] + env[:s3_access_key] = credentials['s3_access_key'] + env[:s3_secret_key] = credentials['s3_secret_key'] +end + # # Akkounts Deployment # From f3529015829ec6a57ef85acc66ed25ddc8a64deb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A2u=20Cao?= Date: Mon, 29 Jan 2024 14:17:15 +0200 Subject: [PATCH 10/12] Consolidate akkounts well-known proxying, add Webfinger and Nostr refs #158 --- .../kosmos_website/templates/nginx_conf_website.erb | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) diff --git a/site-cookbooks/kosmos_website/templates/nginx_conf_website.erb b/site-cookbooks/kosmos_website/templates/nginx_conf_website.erb index 3432221..5ae24c1 100644 --- a/site-cookbooks/kosmos_website/templates/nginx_conf_website.erb +++ b/site-cookbooks/kosmos_website/templates/nginx_conf_website.erb @@ -18,15 +18,8 @@ server { ssl_certificate <%= @ssl_cert %>; ssl_certificate_key <%= @ssl_key %>; - location /.well-known/lnurlp/ { + location ~ ^/.well-known/(webfinger|nostr|lnurlp|keysend) { proxy_ssl_server_name on; - rewrite /.well-known/lnurlp/([^/]+) /lnurlpay/$1@kosmos.org break; - proxy_pass https://accounts.kosmos.org; - } - - location /.well-known/keysend/ { - proxy_ssl_server_name on; - rewrite /.well-known/keysend/([^/]+) /keysend/$1@kosmos.org break; proxy_pass https://accounts.kosmos.org; } } From bcaee3bb21219251f6b8c2eeb00b01813db97bd4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A2u=20Cao?= Date: Mon, 29 Jan 2024 18:15:36 +0200 Subject: [PATCH 11/12] Fix ActiveStorage not liking the local S3 endpoint --- site-cookbooks/kosmos-akkounts/attributes/default.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/site-cookbooks/kosmos-akkounts/attributes/default.rb b/site-cookbooks/kosmos-akkounts/attributes/default.rb index 0fc749e..635cf40 100644 --- a/site-cookbooks/kosmos-akkounts/attributes/default.rb +++ b/site-cookbooks/kosmos-akkounts/attributes/default.rb @@ -21,7 +21,7 @@ node.default['akkounts']['lndhub']['public_key'] = nil node.default['akkounts']['lndhub']['postgres_db'] = 'lndhub' node.default['akkounts']['s3_enabled'] = true -node.default['akkounts']['s3_endpoint'] = "http://localhost:3900" +node.default['akkounts']['s3_endpoint'] = "https://s3.kosmos.org" node.default['akkounts']['s3_region'] = "garage" node.default['akkounts']['s3_bucket'] = "akkounts-production" node.default['akkounts']['s3_alias_host'] = "https://s3.accounts.kosmos.org" From 5f7701c2888226c9057e237c2e87765364ffdf7d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A2u=20Cao?= Date: Mon, 29 Jan 2024 18:16:07 +0200 Subject: [PATCH 12/12] Fix missing listen IP for storage proxy --- .../templates/nginx_conf_liquor-cabinet.erb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/site-cookbooks/kosmos_liquor-cabinet/templates/nginx_conf_liquor-cabinet.erb b/site-cookbooks/kosmos_liquor-cabinet/templates/nginx_conf_liquor-cabinet.erb index 870713b..fb4602e 100644 --- a/site-cookbooks/kosmos_liquor-cabinet/templates/nginx_conf_liquor-cabinet.erb +++ b/site-cookbooks/kosmos_liquor-cabinet/templates/nginx_conf_liquor-cabinet.erb @@ -11,7 +11,7 @@ upstream _<%= @app_name %> { # variables_hash_max_size 2048; server { - listen 80; + listen <%= "#{node['openresty']['listen_ip']}:" if node['openresty']['listen_ip'] %>80; listen [::]:80; server_name <%= @server_name %>; # Redirect to https @@ -21,7 +21,7 @@ server { } server { - listen 443 ssl http2; + listen <%= "#{node['openresty']['listen_ip']}:" if node['openresty']['listen_ip'] %>443 ssl http2; listen [::]:443 ssl http2; server_name <%= @server_name %>;