Upgrade Mastodon to 4.7

Bump the production branch to 4.7 and adjust for the changes between 4.3 and 4.7:

- Node 24.21.0 and Ruby 4.0.7 (via ruby-build v20260924)
- Redis 7.4.11 (Mastodon 4.5 requires >= 7.0)
- Replace ImageMagick with libvips (required since 4.6) and libidn11 with libidn
- Split database migrations into pre-/post-deployment phases and rebuild
  the Elasticsearch accounts index mappings (required since 4.4)
- Remove the OTP_SECRET environment variable (removed in 4.4)
- Disable email subscriptions (new optional feature in 4.6) and force the
  default locale (DEFAULT_LOCALE no longer overrides it since 4.4)
- Add the new fasp Sidekiq queue
- Enable corepack for yarn instead of the removed no-arg 'corepack prepare'

site-cookbooks/kosmos-mastodon/attributes/default.rb

@@ -1,5 +1,5 @@
 node.default["kosmos-mastodon"]["repo"]              = "https://gitea.kosmos.org/kosmos/mastodon.git"
-node.default["kosmos-mastodon"]["revision"]          = "production-4.3"
+node.default["kosmos-mastodon"]["revision"]          = "production-4.7"
 node.default["kosmos-mastodon"]["directory"]         = "/opt/mastodon"
 node.default["kosmos-mastodon"]["bind_ip"]           = "127.0.0.1"
 node.default["kosmos-mastodon"]["app_port"]          = 3000
@@ -25,6 +25,12 @@ node.default["kosmos-mastodon"]["sso_account_reset_password_url"] = "https://acc
 node.default["kosmos-mastodon"]["sso_account_resend_confirmation_url"] = "https://accounts.kosmos.org/users/confirmation/new"
 
 node.default["kosmos-mastodon"]["default_locale"] = "en"
+# From Mastodon 4.4 on, DEFAULT_LOCALE no longer overrides the browser language
+# of unauthenticated users unless this is set to true.
+node.default["kosmos-mastodon"]["force_default_locale"] = true
+# Mastodon 4.6 introduced optional email subscriptions which can cause
+# additional outgoing mail/costs. Disabled by default.
+node.default["kosmos-mastodon"]["disable_email_subscriptions"] = true
 node.default["kosmos-mastodon"]["libre_translate_endpoint"] = nil
 
-node.override["redisio"]["version"] = "6.2.6"
+node.override["redisio"]["version"] = "7.4.11"

site-cookbooks/kosmos-mastodon/recipes/default.rb

@@ -3,7 +3,7 @@
 # Recipe:: default
 #
 
-node.override["kosmos_nodejs"]["version"] = "18.20.8"
+node.override["kosmos_nodejs"]["version"] = "24.21.0"
 
 include_recipe "kosmos-nodejs"
 include_recipe "java"
@@ -67,16 +67,17 @@ user mastodon_user do
   home mastodon_path
 end
 
-package %w(build-essential imagemagick ffmpeg libxml2-dev libxslt1-dev file git
-           curl pkg-config libprotobuf-dev protobuf-compiler libidn11
-           libidn11-dev libjemalloc2 libpq-dev)
+# Mastodon 4.6 dropped ImageMagick in favor of libvips and requires libvips >= 8.13
+package %w(build-essential ffmpeg libxml2-dev libxslt1-dev file git
+           curl pkg-config libprotobuf-dev protobuf-compiler libidn-dev
+           libjemalloc2 libpq-dev libvips-dev)
 
-ruby_version = "3.3.5"
+ruby_version = "4.0.7"
 
 ruby_path = "/opt/ruby_build/builds/#{ruby_version}"
 bundle_path = "#{ruby_path}/bin/bundle"
 
-ruby_build_install 'v20231025'
+ruby_build_install 'v20260924'
 ruby_build_definition ruby_version do
   prefix_path ruby_path
 end
@@ -142,7 +143,7 @@ deploy_env = {
   "HOME" => mastodon_path,
   "RAILS_ENV" => rails_env,
   "NODE_ENV" => rails_env,
-  "SKIP_POST_DEPLOYMENT_MIGRATIONS" => "true"
+  "COREPACK_ENABLE_DOWNLOAD_PROMPT" => "0"
 }
 
 git mastodon_path do
@@ -151,17 +152,14 @@ git mastodon_path do
 
   repository node["kosmos-mastodon"]["repo"]
   revision   node["kosmos-mastodon"]["revision"]
-  # Restart services on deployments
+  # Restart services (and run post-deployment migrations) on deployments
   notifies :run, "execute[restart mastodon services]", :delayed
 end
 
 execute "restart mastodon services" do
-  command "true"
+  command "systemctl restart mastodon-web mastodon-sidekiq mastodon-sidekiq-scheduler mastodon-streaming"
   action :nothing
-  notifies :restart, "service[mastodon-web]", :delayed
-  notifies :restart, "service[mastodon-sidekiq]", :delayed
-  notifies :restart, "service[mastodon-sidekiq-scheduler]", :delayed
-  notifies :restart, "service[mastodon-streaming]", :delayed
+  notifies :run, "execute[rake db:migrate (post-deployment)]", :immediately
 end
 
 credentials = data_bag_item('credentials', 'mastodon')
@@ -195,7 +193,6 @@ template "#{mastodon_path}/.env.#{rails_env}" do
             active_record_encryption_primary_key: credentials["active_record_encryption_primary_key"],
             paperclip_secret: credentials['paperclip_secret'],
             secret_key_base: credentials['secret_key_base'],
-            otp_secret: credentials['otp_secret'],
             ldap: ldap_config,
             smtp_login: credentials['smtp_user_name'],
             smtp_password: credentials['smtp_password'],
@@ -214,23 +211,32 @@ template "#{mastodon_path}/.env.#{rails_env}" do
             sso_account_reset_password_url: node["kosmos-mastodon"]["sso_account_reset_password_url"],
             sso_account_resend_confirmation_url: node["kosmos-mastodon"]["sso_account_resend_confirmation_url"],
             default_locale: node["kosmos-mastodon"]["default_locale"],
+            force_default_locale: node["kosmos-mastodon"]["force_default_locale"],
+            disable_email_subscriptions: node["kosmos-mastodon"]["disable_email_subscriptions"],
             allowed_private_addresses: node["kosmos-mastodon"]["allowed_private_addresses"],
             libre_translate_endpoint: node["kosmos-mastodon"]["libre_translate_endpoint"]
   notifies :run, "execute[restart mastodon services]", :delayed
 end
 
 execute "bundle install" do
-  environment deploy_env
+  environment deploy_env.merge("BUNDLE_BUILD__CHARLOCK_HOLMES" => "--with-cxxflags=-std=c++17")
   user mastodon_user
   cwd mastodon_path
   command "bundle install --without development,test --deployment"
 end
 
+# Node 24 ships corepack >= 0.30, for which `corepack prepare` without an
+# argument is no longer valid. Enable the shims as root and let yarn pick up
+# the version pinned in package.json instead.
+execute "corepack enable" do
+  command "corepack enable"
+end
+
 execute "yarn install" do
   environment deploy_env
   user mastodon_user
   cwd mastodon_path
-  command "corepack prepare && yarn install --immutable"
+  command "yarn install --immutable"
 end
 
 execute "rake assets:precompile" do
@@ -241,12 +247,37 @@ execute "rake assets:precompile" do
   command "bundle exec rake assets:precompile"
 end
 
-execute "rake db:migrate" do
+# Mastodon 4.4+ splits migrations into pre- and post-deployment phases.
+# Pre-deployment migrations must run before the services are (re)started.
+execute "rake db:migrate (pre-deployment)" do
+  environment deploy_env.merge("SKIP_POST_DEPLOYMENT_MIGRATIONS" => "true")
+  user mastodon_user
+  group mastodon_user
+  cwd mastodon_path
+  command "bundle exec rake db:migrate"
+  timeout 21_600
+end
+
+execute "rake db:migrate (post-deployment)" do
   environment deploy_env
   user mastodon_user
   group mastodon_user
   cwd mastodon_path
   command "bundle exec rake db:migrate"
+  timeout 21_600
+  action :nothing
+  notifies :run, "execute[tootctl search deploy accounts mapping]", :immediately
+end
+
+# Mastodon 4.4 changed the Elasticsearch `accounts` index mappings
+execute "tootctl search deploy accounts mapping" do
+  environment deploy_env
+  user mastodon_user
+  group mastodon_user
+  cwd mastodon_path
+  command "#{bundle_path} exec bin/tootctl search deploy --only-mapping --only=accounts"
+  timeout 3_600
+  action :nothing
 end
 
 service "mastodon-web" do

site-cookbooks/kosmos-mastodon/templates/default/env.erb

@@ -17,7 +17,6 @@ ACTIVE_RECORD_ENCRYPTION_KEY_DERIVATION_SALT=<%= @active_record_encryption_key_d
 ACTIVE_RECORD_ENCRYPTION_PRIMARY_KEY=<%= @active_record_encryption_primary_key %>
 PAPERCLIP_SECRET=<%= @paperclip_secret %>
 SECRET_KEY_BASE=<%= @secret_key_base %>
-OTP_SECRET=<%= @otp_secret %>
 
 # Registrations
 # Single user mode will disable registrations and redirect frontpage to the first profile
@@ -74,6 +73,10 @@ AWS_SECRET_ACCESS_KEY=<%= @aws_secret_access_key %>
 
 # locale
 DEFAULT_LOCALE=<%= @default_locale %>
+FORCE_DEFAULT_LOCALE=<%= @force_default_locale %>
+
+# Email subscriptions (Mastodon 4.6+)
+DISABLE_EMAIL_SUBSCRIPTIONS=<%= @disable_email_subscriptions %>
 
 <% if @libre_translate_endpoint %>
 # translate

site-cookbooks/kosmos-mastodon/templates/default/mastodon-sidekiq.systemd.service.erb

@@ -11,7 +11,7 @@ Environment="RAILS_ENV=production"
 Environment="DB_POOL=<%= @sidekiq_threads %>"
 Environment="MALLOC_ARENA_MAX=2"
 Environment="LD_PRELOAD=/usr/lib/x86_64-linux-gnu/libjemalloc.so.2"
-ExecStart=<%= @bundle_path %> exec sidekiq -c <%= @sidekiq_threads %> -q default -q mailers -q pull -q push -q ingress
+ExecStart=<%= @bundle_path %> exec sidekiq -c <%= @sidekiq_threads %> -q default -q mailers -q pull -q push -q ingress -q fasp
 TimeoutSec=15
 Restart=always