Migrate S3 backups from AWS, fix automatic cleanups

The cleanups were broken in that every single archive was also copied to
a shared folder and never deleted from there.

Co-authored-by: Greg Karékinian <greg@karekinian.com>

data_bags/credentials/backup.json

@@ -1,27 +1,38 @@
 {
   "id": "backup",
   "s3_access_key_id": {
-    "encrypted_data": "emGNH4v7TTEh05Go/DsI3k7CFnaK4p/4JxodC4BYpyWw47/Z3dsuRMu4vXM3\n3YLH\n",
-    "iv": "Dau+ekb3UTYdl8w3fQKVcA==\n",
-    "version": 1,
-    "cipher": "aes-256-cbc"
+    "encrypted_data": "245TrPvuoBRRTimhbt6qqsFb+JnnD377sPt1pguJy7Q2BXOy/jrX0wyMt+cP\nuA==\n",
+    "iv": "ylmRxSRO3AA4MSJN\n",
+    "auth_tag": "45tBcYZowPLrbv4Zu2P0Fw==\n",
+    "version": 3,
+    "cipher": "aes-256-gcm"
   },
   "s3_secret_access_key": {
-    "encrypted_data": "Mxyly86JxrWUbubbSiqPdRosChzfI1Q8eBEG4n+2B9JJG4yExltO5Wc5kgSs\nX01MPXAc+PGLm+J9MngUtypo/g==\n",
-    "iv": "WRhBJGiuScYYsUsoT5j/UA==\n",
-    "version": 1,
-    "cipher": "aes-256-cbc"
+    "encrypted_data": "jDIOjlBzTkBUzpj243T6KnBuH0qwyW7BUFMcqllljFSzxs7K8wYJOUreNbOP\ny8OpDWAuO0H4O4LuFMJXeM8=\n",
+    "iv": "PzvZr37EkJqz6JtM\n",
+    "auth_tag": "e3XW8oHVgmYibv/IBzj0yA==\n",
+    "version": 3,
+    "cipher": "aes-256-gcm"
+  },
+  "s3_endpoint": {
+    "encrypted_data": "ErJIEChxrreW7WKEwRtuP2MyYlsZRtqLdGa/x5QY58qgO036FgR3Hs2Z3yce\n",
+    "iv": "HOSAOgUjO7XGwk50\n",
+    "auth_tag": "XE1bwMIXHHE72V9K2KOLnw==\n",
+    "version": 3,
+    "cipher": "aes-256-gcm"
   },
   "s3_region": {
-    "encrypted_data": "2ZGxu0tVzKNfx3K1Wleg0SAwGaPkHCi/XfKpJ+J7q40=\n",
-    "iv": "CNTZW2SEIgfw+IyzGI3TzQ==\n",
-    "version": 1,
-    "cipher": "aes-256-cbc"
+    "encrypted_data": "8cNSaYu7HH95ftG66lFdUIPZD7soz907CPA=\n",
+    "iv": "pU21ulF75y/SIs3x\n",
+    "auth_tag": "7WQQCbSbB2GybjY+C+5IvQ==\n",
+    "version": 3,
+    "cipher": "aes-256-gcm"
   },
   "encryption_password": {
-    "encrypted_data": "tsBWKBwhQFfEAM0EWMPtljSbqU1c5mOJXPjYJjNT5RUFhPlqa7gsE8aJbs+D\nSPKjAQ62j+iHeqCk9mE9CCkgBA==\n",
-    "iv": "uq5YAXuq2ynRLv9EIWoCFA==\n",
-    "version": 1,
-    "cipher": "aes-256-cbc"
+    "encrypted_data": "l23CiIO2s1fIRn0NdoWZ+wK+Zhx3hCYDHf4ypjqMRekZ7xqafvXHHuogD5aj\npxYUKloH\n",
+    "iv": "Dzx83eP9L7Jqqidh\n",
+    "auth_tag": "UVn5XA5Tgsikc1GdOt1MUQ==\n",
+    "version": 3,
+    "cipher": "aes-256-gcm"
   }
 }

nodes/gitea-2.json

@@ -32,6 +32,7 @@
       "kosmos_postgresql::hostsfile",
       "kosmos_gitea",
       "kosmos_gitea::default",
+      "kosmos_gitea::backup",
       "kosmos_gitea::act_runner",
       "apt::default",
       "timezone_iii::default",
@@ -47,7 +48,9 @@
       "postfix::_attributes",
       "postfix::sasl_auth",
       "hostname::default",
-      "firewall::default"
+      "firewall::default",
+      "backup::default",
+      "logrotate::default"
     ],
     "platform": "ubuntu",
     "platform_version": "20.04",

roles/gitea.rb

@@ -3,4 +3,5 @@ name "gitea"
 run_list %w(
   role[postgresql_client]
   kosmos_gitea::default
+  kosmos_gitea::backup
 )

site-cookbooks/backup/attributes/default.rb

@@ -42,5 +42,5 @@ default['backup']['orbit']['keep'] = 10
 default['backup']['cron']['hour'] = "05"
 default['backup']['cron']['minute'] = "7"
 
-default['backup']['s3']['keep'] = 15
-default['backup']['s3']['bucket'] = "kosmos-dev-backups"
+default['backup']['s3']['keep'] = 10
+default['backup']['s3']['bucket'] = "kosmos-backups"

site-cookbooks/backup/recipes/default.rb

@@ -28,6 +28,7 @@ template "#{backup_dir}/config.rb" do
   sensitive true
   variables s3_access_key_id: backup_data["s3_access_key_id"],
             s3_secret_access_key: backup_data["s3_secret_access_key"],
+            s3_endpoint: backup_data["s3_endpoint"],
             s3_region: backup_data["s3_region"],
             encryption_password: backup_data["encryption_password"],
             mail_from: "backups@kosmos.org",

site-cookbooks/backup/templates/default/config.rb.erb

@@ -23,6 +23,10 @@ Storage::S3.defaults do |s3|
   s3.secret_access_key = "<%= @s3_secret_access_key %>"
   s3.region            = "<%= @s3_region %>"
   s3.bucket            = "<%= node['backup']['s3']['bucket'] %>"
+  s3.fog_options       = {
+    endpoint: "<%= @s3_endpoint %>",
+    aws_signature_version: 2
+  }
 end
 
 Encryptor::OpenSSL.defaults do |encryption|
@@ -88,7 +92,6 @@ end
 
 preconfigure 'KosmosBackup' do
   split_into_chunks_of 250 # megabytes
-  store_with S3
   compress_with Bzip2
   encrypt_with OpenSSL
   notify_by Mail do |mail|

site-cookbooks/kosmos-bitcoin/recipes/lnd-scb-s3.rb

@@ -10,12 +10,14 @@ include_recipe "kosmos-bitcoin::aws-client"
 package "inotify-tools"
 
 backup_script_path = "/opt/lnd-channel-backup-s3.sh"
+backup_credentials = Chef::EncryptedDataBagItem.load('credentials', 'backup')
 
 template backup_script_path do
   source "lnd-channel-backup-s3.sh.erb"
   mode '0740'
   variables lnd_dir: node['lnd']['lnd_dir'],
             bitcoin_network: node['bitcoin']['network'],
+            s3_endpoint: backup_credentials['s3_endpoint'],
             s3_bucket: node['backup']['s3']['bucket'],
             s3_scb_dir: "#{node['name']}/lnd/#{node['bitcoin']['network']}"
   notifies :restart, "systemd_unit[lnd-channel-backup.service]", :delayed

site-cookbooks/kosmos-bitcoin/templates/lnd-channel-backup-s3.sh.erb

@@ -3,5 +3,5 @@ set -xe -o pipefail
 
 while true; do
   inotifywait <%= @lnd_dir %>/data/chain/bitcoin/<%= @bitcoin_network %>/channel.backup
-  aws s3 cp <%= @lnd_dir %>/data/chain/bitcoin/<%= @bitcoin_network %>/channel.backup "s3://<%= @s3_bucket %>/<%= @s3_scb_dir %>/channel.backup"
+  aws --endpoint <%= @s3_endpoint %> s3 cp <%= @lnd_dir %>/data/chain/bitcoin/<%= @bitcoin_network %>/channel.backup "s3://<%= @s3_bucket %>/<%= @s3_scb_dir %>/channel.backup"
 done

site-cookbooks/kosmos-mastodon/recipes/backup.rb

@@ -6,13 +6,12 @@
 postgresql_data_bag_item = data_bag_item('credentials', 'postgresql')
 
 unless node.chef_environment == "development"
-  unless node["backup"]["postgresql"]["databases"].keys.include? 'mastodon'
-    node.override["backup"]["postgresql"]["host"] = "pg.kosmos.local"
-    node.override["backup"]["postgresql"]["databases"]["mastodon"] = {
-      username: "mastodon",
-      password: postgresql_data_bag_item['mastodon_user_password']
-    }
-  end
+  node.override['backup']['s3']['keep'] = 1
+  node.override["backup"]["postgresql"]["host"] = "pg.kosmos.local"
+  node.override["backup"]["postgresql"]["databases"]["mastodon"] = {
+    username: "mastodon",
+    password: postgresql_data_bag_item['mastodon_user_password']
+  }
 
   include_recipe "backup"
 end

site-cookbooks/kosmos_gitea/recipes/backup.rb

@@ -8,5 +8,6 @@
 unless node.chef_environment == "development"
   # backup the data dir and the config files
   node.override["backup"]["archives"]["gitea"] = [node["gitea"]["working_directory"]]
+  node.override['backup']['s3']['keep'] = 2
   include_recipe "backup"
 end