Refactor Mastodon nginx recipe for proxy usage

Works both as local deployment and proxy (via roles and environments)

* Use upstreams for proxy_pass
* Access static assets from proxy, configure caching for them
* Move Tor config to environment, install via role
* ...

roles/nginx_proxy.rb

@@ -1,17 +1,35 @@
 name "nginx_proxy"
 
+override_attributes(
+  'nginx' => {
+    'server_names_hash_bucket_size' => 128
+  },
+  'tor' => {
+    'HiddenServices' => {
+      'web' => {
+        'HiddenServicePorts' => ['80 127.0.0.1:80', '443 127.0.0.1:443']
+      }
+    }
+  }
+)
+
 default_run_list = %w(
+  tor-full
   kosmos_assets::nginx_site
   kosmos_discourse::nginx
   kosmos_drone::nginx
   kosmos_gitea::nginx
   kosmos_website::default
-  kosmos-ejabberd::nginx
   kosmos-akkounts::nginx_api
+  kosmos-ejabberd::nginx
   kosmos-hubot::nginx_botka_irc-libera-chat
   kosmos-hubot::nginx_hal8000_xmpp
   kosmos-ipfs::nginx_public_gateway
+  kosmos-mastodon::nginx
   remotestorage_discourse::nginx
+  kosmos_garage::default
+  kosmos_garage::firewall_rpc
+  kosmos_garage::nginx_web
 )
 
 env_run_lists(

site-cookbooks/kosmos-mastodon/attributes/default.rb

@@ -1,17 +1,14 @@
-node.default["kosmos-mastodon"]["repo"]            = "https://gitea.kosmos.org/kosmos/mastodon.git"
-node.default["kosmos-mastodon"]["revision"]        = "kosmos-production"
-node.default["kosmos-mastodon"]["directory"]       = "/opt/mastodon"
-node.default["kosmos-mastodon"]["server_name"]     = "kosmos.social"
-node.default["kosmos-mastodon"]["redis_url"]       = "redis://localhost:6379/0"
-node.default["kosmos-mastodon"]["sidekiq_threads"] = 25
+node.default["kosmos-mastodon"]["repo"]              = "https://gitea.kosmos.org/kosmos/mastodon.git"
+node.default["kosmos-mastodon"]["revision"]          = "kosmos-production"
+node.default["kosmos-mastodon"]["directory"]         = "/opt/mastodon"
 node.default["kosmos-mastodon"]["bind_ip"]           = "127.0.0.1"
 node.default["kosmos-mastodon"]["app_port"]          = 3000
 node.default["kosmos-mastodon"]["streaming_port"]    = 4000
+node.default["kosmos-mastodon"]["server_name"]       = "kosmos.social"
+node.default["kosmos-mastodon"]["redis_url"]         = "redis://localhost:6379/0"
+node.default["kosmos-mastodon"]["sidekiq_threads"]   = 25
+node.default["kosmos-mastodon"]["onion_address"]     = nil
 # Allocate this amount of RAM to the Java heap for Elasticsearch
 node.default["kosmos-mastodon"]["elasticsearch"]["allocated_memory"] = "1536m"
 
 node.override["redisio"]["version"] = "6.2.6"
-
-node.override["tor"]["HiddenServices"]["mastodon"] = {
-  "HiddenServicePorts" => ["80 127.0.0.1:80", "443 127.0.0.1:443"]
-}

site-cookbooks/kosmos-mastodon/recipes/nginx.rb

@@ -3,12 +3,23 @@
 # Recipe:: nginx
 #
 
-mastodon_path = node["kosmos-mastodon"]["directory"]
-server_name   = node["kosmos-mastodon"]["server_name"]
-
-node.override['nginx']['server_names_hash_bucket_size'] = 128
 include_recipe "kosmos-nginx"
 
+app_dir        = node["kosmos-mastodon"]["directory"]
+server_name    = node["kosmos-mastodon"]["server_name"]
+is_proxy       = node.roles.include?('nginx_proxy') rescue nil
+upstream_hosts = []
+
+if is_proxy
+  web_root_dir = "/var/www/#{server_name}/public"
+  search(:node, "role:mastodon").each do |node|
+    upstream_hosts << node["knife_zero"]["host"]
+  end
+else
+  web_root_dir = "#{app_dir}/public"
+  upstream_hosts << "localhost"
+end
+
 directory "#{node['nginx']['dir']}/snippets" do
   action :create
   owner 'www-data'
@@ -19,40 +30,30 @@ template "#{node['nginx']['dir']}/snippets/mastodon.conf" do
   source 'nginx_conf_shared.erb'
   owner 'www-data'
   mode 0640
-  variables streaming_port: node["kosmos-mastodon"]["streaming_port"],
-            puma_port:      node["kosmos-mastodon"]["puma_port"],
-            mastodon_path:  mastodon_path
+  variables web_root_dir: web_root_dir,
+            server_name:  server_name
   notifies :reload, 'service[nginx]', :delayed
 end
 
-onion_address = File.read("/var/lib/tor/mastodon/hostname").strip rescue nil
+nginx_certbot_site server_name
+
+onion_address = File.read("/var/lib/tor/web/hostname").strip rescue nil rescue nil
 
 template "#{node['nginx']['dir']}/sites-available/#{server_name}" do
   source 'nginx_conf_mastodon.erb'
   owner 'www-data'
   mode 0640
-  variables server_name:         server_name,
-            ssl_cert:            "/etc/letsencrypt/live/#{server_name}/fullchain.pem",
-            ssl_key:             "/etc/letsencrypt/live/#{server_name}/privkey.pem",
-            shared_config_path:  "#{node['nginx']['dir']}/snippets/mastodon.conf",
-            onion_address:       onion_address
+  variables server_name:        server_name,
+            ssl_cert:           "/etc/letsencrypt/live/#{server_name}/fullchain.pem",
+            ssl_key:            "/etc/letsencrypt/live/#{server_name}/privkey.pem",
+            shared_config_path: "#{node['nginx']['dir']}/snippets/mastodon.conf",
+            app_port:           node["kosmos-mastodon"]["app_port"],
+            streaming_port:     node["kosmos-mastodon"]["streaming_port"],
+            onion_address:      onion_address,
+            upstream_hosts:     upstream_hosts
   notifies :reload, 'service[nginx]', :delayed
 end
 
-# Legacy vhost
-nginx_site "mastodon" do
-  action :disable
-end
-
 nginx_site server_name do
   action :enable
 end
-
-nginx_certbot_site server_name
-
-#
-# Tor hidden service
-#
-# The attributes for the hidden service are set in attributes/default.rb, due
-# to the way the tor-full cookbook builds the path to the hidden service dir
-include_recipe "tor-full"

site-cookbooks/kosmos-mastodon/templates/default/env.production.erb

@@ -32,6 +32,9 @@ SMTP_FROM_ADDRESS=<%= @smtp_from_address %>
 # Optional asset host for multi-server setups
 # CDN_HOST=assets.example.com
 
+# Serve static files (to nginx proxy)
+RAILS_SERVE_STATIC_FILES=true
+
 # S3 (optional)
 S3_ENABLED=true
 S3_BUCKET=<%= @s3_bucket %>

site-cookbooks/kosmos-mastodon/templates/default/nginx_conf_mastodon.erb

@@ -1,17 +1,24 @@
-<% if @onion_address %>
-server {
-  listen 80;
-  server_name mastodon.<%= @onion_address %>;
-  include <%= @shared_config_path %>;
-}
+upstream mastodon_app {
+<% @upstream_hosts.each do |host| %>
+  server <%= host %>:<%= @app_port %>;
 <% end %>
+}
+
+upstream mastodon_streaming {
+<% @upstream_hosts.each do |host| %>
+  server <%= host %>:<%= @streaming_port %>;
+<% end %>
+}
 
 map $http_upgrade $connection_upgrade {
   default upgrade;
   ''      close;
 }
 
-<% if File.exist?(@ssl_cert) && File.exist?(@ssl_key) %>
+proxy_cache_path /var/cache/nginx/mastodon levels=1:2
+                 keys_zone=mastodon_cache:10m
+                 max_size=1g inactive=120m use_temp_path=off;
+
 server {
   listen 443 ssl http2;
   listen [::]:443 ssl http2;
@@ -22,11 +29,17 @@ server {
   ssl_certificate_key <%= @ssl_key %>;
 
   add_header Strict-Transport-Security "max-age=31536000";
+<% if @onion_address %>
   add_header Onion-Location https://mastodon.<%= @onion_address %>$request_uri;
-}
 <% end %>
+}
 
 <% if @onion_address %>
+server {
+  listen 80;
+  server_name mastodon.<%= @onion_address %>;
+  include <%= @shared_config_path %>;
+}
 server {
   listen 443 ssl http2;
   server_name mastodon.<%= @onion_address %>;

site-cookbooks/kosmos-mastodon/templates/default/nginx_conf_shared.erb

@@ -6,7 +6,7 @@ keepalive_timeout    70;
 sendfile             on;
 client_max_body_size 0;
 
-root <%= @mastodon_path %>/public;
+root <%= @web_root_dir %>;
 
 gzip on;
 gzip_disable "msie6";
@@ -19,20 +19,21 @@ gzip_types text/plain text/css application/json application/javascript text/xml
 
 location / {
 # If the maintenance file is present, show maintenance page
-  if (-f <%= @mastodon_path %>/public/maintenance.html) {
+  if (-f <%= @web_root_dir %>/maintenance.html) {
     return 503;
   }
-
   try_files $uri @proxy;
 }
 
 location /sw.js {
   add_header Cache-Control "max-age=0, no-cache, no-store, must-revalidate";
   add_header Pragma "no-cache";
+  try_files $uri @proxy;
 }
 
 location ~ ^/(emoji|packs|system/accounts/avatars|system/media_attachments/files) {
   add_header Cache-Control "public, max-age=31536000, immutable";
+  proxy_cache mastodon_cache;
   try_files $uri @proxy;
 }
 
@@ -44,7 +45,7 @@ location @proxy {
   proxy_set_header Proxy "";
   proxy_pass_header Server;
 
-  proxy_pass http://localhost:<%= @puma_port %>;
+  proxy_pass http://mastodon_app;
   proxy_buffering off;
   proxy_redirect off;
   proxy_http_version 1.1;
@@ -61,7 +62,7 @@ location /api/v1/streaming {
   proxy_set_header X-Forwarded-Proto https;
   proxy_set_header Proxy "";
 
-  proxy_pass http://localhost:<%= @streaming_port %>;
+  proxy_pass http://mastodon_streaming;
   proxy_buffering off;
   proxy_redirect off;
   proxy_http_version 1.1;
@@ -75,5 +76,5 @@ error_page 500 501 502 504 /500.html;
 error_page 503 /maintenance.html;
 
 location = /maintenance.html {
-  root <%= @mastodon_path %>/public;
+  root <%= @web_root_dir %>;
 }