Move Mastodon nginx to proxy/LB

Configure alternate_domains for Rails app to re-enable Tor access (was
throwing 403s without this config)

environments/production.json

@@ -16,6 +16,11 @@
           "use_ssl": "false"
         }
       }
+    },
+    "kosmos-mastodon": {
+      "alternate_domains": [
+        "mastodon.w7nooprauv6yrnhzh2ajpcnj3doinked2aaztlwfyt6u6pva2qdxqhid.onion"
+      ]
     }
   }
 }

nodes/fornax.kosmos.org.json

@@ -1,5 +1,6 @@
 {
   "name": "fornax.kosmos.org",
+  "chef_environment": "production",
   "normal": {
     "knife_zero": {
       "host": "10.1.1.147"
@@ -25,18 +26,25 @@
       "kosmos-base::default",
       "kosmos_kvm::host",
       "kosmos_kvm::backup",
+      "tor-full",
+      "tor-full::default",
       "kosmos_assets::nginx_site",
       "kosmos_discourse::nginx",
       "kosmos_drone::nginx",
       "kosmos_gitea::nginx",
       "kosmos_website",
       "kosmos_website::default",
-      "kosmos-ejabberd::nginx",
       "kosmos-akkounts::nginx_api",
+      "kosmos-ejabberd::nginx",
       "kosmos-hubot::nginx_botka_irc-libera-chat",
       "kosmos-hubot::nginx_hal8000_xmpp",
       "kosmos-ipfs::nginx_public_gateway",
+      "kosmos-mastodon::nginx",
       "remotestorage_discourse::nginx",
+      "kosmos_garage",
+      "kosmos_garage::default",
+      "kosmos_garage::firewall_rpc",
+      "kosmos_garage::nginx_web",
       "kosmos_zerotier::controller",
       "kosmos_zerotier::firewall",
       "kosmos_zerotier::zncui",

nodes/mastodon-3.json

@@ -24,7 +24,6 @@
       "kosmos_postgresql::hostsfile",
       "kosmos-mastodon",
       "kosmos-mastodon::default",
-      "kosmos-mastodon::nginx",
       "kosmos-mastodon::backup",
       "apt::default",
       "timezone_iii::default",
@@ -56,24 +55,14 @@
       "redisio::disable_os_default",
       "redisio::configure",
       "redisio::enable",
+      "firewall::default",
+      "chef-sugar::default",
       "nodejs::npm",
       "nodejs::install",
-      "kosmos-nginx::default",
-      "nginx::default",
-      "nginx::package",
-      "nginx::ohai_plugin",
-      "nginx::repo",
-      "nginx::commons",
-      "nginx::commons_dir",
-      "nginx::commons_script",
-      "nginx::commons_conf",
-      "kosmos-nginx::firewall",
-      "tor-full::default",
       "backup::default",
       "logrotate::default",
       "git::default",
-      "git::package",
+      "git::package"
-      "kosmos-base::letsencrypt"
     ],
     "platform": "ubuntu",
     "platform_version": "20.04",

roles/mastodon.rb

@@ -3,6 +3,5 @@ name "mastodon"
 run_list %w(
   role[postgresql_client]
   kosmos-mastodon
-  kosmos-mastodon::nginx
   kosmos-mastodon::backup
 )

roles/nginx_proxy.rb

@@ -1,17 +1,35 @@
 name "nginx_proxy"
 
+override_attributes(
+  'nginx' => {
+    'server_names_hash_bucket_size' => 128
+  },
+  'tor' => {
+    'HiddenServices' => {
+      'web' => {
+        'HiddenServicePorts' => ['80 127.0.0.1:80', '443 127.0.0.1:443']
+      }
+    }
+  }
+)
+
 default_run_list = %w(
+  tor-full
   kosmos_assets::nginx_site
   kosmos_discourse::nginx
   kosmos_drone::nginx
   kosmos_gitea::nginx
   kosmos_website::default
-  kosmos-ejabberd::nginx
   kosmos-akkounts::nginx_api
+  kosmos-ejabberd::nginx
   kosmos-hubot::nginx_botka_irc-libera-chat
   kosmos-hubot::nginx_hal8000_xmpp
   kosmos-ipfs::nginx_public_gateway
+  kosmos-mastodon::nginx
   remotestorage_discourse::nginx
+  kosmos_garage::default
+  kosmos_garage::firewall_rpc
+  kosmos_garage::nginx_web
 )
 
 env_run_lists(

site-cookbooks/kosmos-ipfs/templates/default/nginx_conf_ipfs.kosmos.org.erb

@@ -9,8 +9,6 @@ upstream _ipfs_api {
 <% end %>
 }
 
-limit_req_zone $binary_remote_addr zone=ipfsgateway:10m rate=10r/s;
-
 server {
   listen 443 ssl http2;
   listen [::]:443 ssl http2;
@@ -21,7 +19,6 @@ server {
   error_log  /var/log/nginx/<%= @server_name %>.error.log;
 
   location /ipfs {
-    limit_req zone=ipfsgateway burst=20 nodelay;
     proxy_pass http://_ipfs_gateway/ipfs;
   }
 

site-cookbooks/kosmos-mastodon/attributes/default.rb

@@ -1,17 +1,15 @@
 node.default["kosmos-mastodon"]["repo"]              = "https://gitea.kosmos.org/kosmos/mastodon.git"
 node.default["kosmos-mastodon"]["revision"]          = "kosmos-production"
 node.default["kosmos-mastodon"]["directory"]         = "/opt/mastodon"
-node.default["kosmos-mastodon"]["server_name"]     = "kosmos.social"
-node.default["kosmos-mastodon"]["redis_url"]       = "redis://localhost:6379/0"
-node.default["kosmos-mastodon"]["sidekiq_threads"] = 25
 node.default["kosmos-mastodon"]["bind_ip"]           = "127.0.0.1"
 node.default["kosmos-mastodon"]["app_port"]          = 3000
 node.default["kosmos-mastodon"]["streaming_port"]    = 4000
+node.default["kosmos-mastodon"]["server_name"]       = "kosmos.social"
+node.default["kosmos-mastodon"]["alternate_domains"] = []
+node.default["kosmos-mastodon"]["redis_url"]         = "redis://localhost:6379/0"
+node.default["kosmos-mastodon"]["sidekiq_threads"]   = 25
+node.default["kosmos-mastodon"]["onion_address"]     = nil
 # Allocate this amount of RAM to the Java heap for Elasticsearch
 node.default["kosmos-mastodon"]["elasticsearch"]["allocated_memory"] = "1536m"
 
 node.override["redisio"]["version"] = "6.2.6"
-
-node.override["tor"]["HiddenServices"]["mastodon"] = {
-  "HiddenServicePorts" => ["80 127.0.0.1:80", "443 127.0.0.1:443"]
-}

site-cookbooks/kosmos-mastodon/recipes/default.rb

@@ -42,7 +42,6 @@ elasticsearch_service 'elasticsearch'
 postgresql_data_bag_item = data_bag_item('credentials', 'postgresql')
 
 mastodon_path = node["kosmos-mastodon"]["directory"]
-
 mastodon_user = "mastodon"
 
 bind_ip = if node.chef_environment == "production"
@@ -160,6 +159,7 @@ application mastodon_path do
     group mastodon_user
     variables redis_url: node["kosmos-mastodon"]["redis_url"],
               domain: node["kosmos-mastodon"]["server_name"],
+              alternate_domains: node["kosmos-mastodon"]["alternate_domains"],
               paperclip_secret: mastodon_credentials['paperclip_secret'],
               secret_key_base: mastodon_credentials['secret_key_base'],
               otp_secret: mastodon_credentials['otp_secret'],

site-cookbooks/kosmos-mastodon/recipes/nginx.rb

@@ -3,12 +3,23 @@
 # Recipe:: nginx
 #
 
-mastodon_path = node["kosmos-mastodon"]["directory"]
-server_name   = node["kosmos-mastodon"]["server_name"]
-
-node.override['nginx']['server_names_hash_bucket_size'] = 128
 include_recipe "kosmos-nginx"
 
+app_dir        = node["kosmos-mastodon"]["directory"]
+server_name    = node["kosmos-mastodon"]["server_name"]
+is_proxy       = node.roles.include?('nginx_proxy') rescue nil
+upstream_hosts = []
+
+if is_proxy
+  web_root_dir = "/var/www/#{server_name}/public"
+  search(:node, "role:mastodon").each do |node|
+    upstream_hosts << node["knife_zero"]["host"]
+  end
+else
+  web_root_dir = "#{app_dir}/public"
+  upstream_hosts << "localhost"
+end
+
 directory "#{node['nginx']['dir']}/snippets" do
   action :create
   owner 'www-data'
@@ -19,13 +30,14 @@ template "#{node['nginx']['dir']}/snippets/mastodon.conf" do
   source 'nginx_conf_shared.erb'
   owner 'www-data'
   mode 0640
-  variables streaming_port: node["kosmos-mastodon"]["streaming_port"],
+  variables web_root_dir: web_root_dir,
-            puma_port:      node["kosmos-mastodon"]["puma_port"],
+            server_name:  server_name
-            mastodon_path:  mastodon_path
   notifies :reload, 'service[nginx]', :delayed
 end
 
-onion_address = File.read("/var/lib/tor/mastodon/hostname").strip rescue nil
+nginx_certbot_site server_name
+
+onion_address = File.read("/var/lib/tor/web/hostname").strip rescue nil rescue nil
 
 template "#{node['nginx']['dir']}/sites-available/#{server_name}" do
   source 'nginx_conf_mastodon.erb'
@@ -35,24 +47,13 @@ template "#{node['nginx']['dir']}/sites-available/#{server_name}" do
             ssl_cert:           "/etc/letsencrypt/live/#{server_name}/fullchain.pem",
             ssl_key:            "/etc/letsencrypt/live/#{server_name}/privkey.pem",
             shared_config_path: "#{node['nginx']['dir']}/snippets/mastodon.conf",
-            onion_address:       onion_address
+            app_port:           node["kosmos-mastodon"]["app_port"],
+            streaming_port:     node["kosmos-mastodon"]["streaming_port"],
+            onion_address:      onion_address,
+            upstream_hosts:     upstream_hosts
   notifies :reload, 'service[nginx]', :delayed
 end
 
-# Legacy vhost
-nginx_site "mastodon" do
-  action :disable
-end
-
 nginx_site server_name do
   action :enable
 end
-
-nginx_certbot_site server_name
-
-#
-# Tor hidden service
-#
-# The attributes for the hidden service are set in attributes/default.rb, due
-# to the way the tor-full cookbook builds the path to the hidden service dir
-include_recipe "tor-full"

site-cookbooks/kosmos-mastodon/templates/default/env.production.erb

@@ -32,6 +32,9 @@ SMTP_FROM_ADDRESS=<%= @smtp_from_address %>
 # Optional asset host for multi-server setups
 # CDN_HOST=assets.example.com
 
+# Serve static files (to nginx proxy)
+RAILS_SERVE_STATIC_FILES=true
+
 # S3 (optional)
 S3_ENABLED=true
 S3_BUCKET=<%= @s3_bucket %>
@@ -55,3 +58,4 @@ ES_HOST=localhost
 ES_PORT=9200
 
 ALLOW_ACCESS_TO_HIDDEN_SERVICE=true
+ALTERNATE_DOMAINS='<%= @alternate_domains.join(" ") %>'

site-cookbooks/kosmos-mastodon/templates/default/nginx_conf_mastodon.erb

@@ -1,17 +1,24 @@
-<% if @onion_address %>
+upstream mastodon_app {
-server {
+<% @upstream_hosts.each do |host| %>
-  listen 80;
+  server <%= host %>:<%= @app_port %>;
-  server_name mastodon.<%= @onion_address %>;
-  include <%= @shared_config_path %>;
-}
 <% end %>
+}
+
+upstream mastodon_streaming {
+<% @upstream_hosts.each do |host| %>
+  server <%= host %>:<%= @streaming_port %>;
+<% end %>
+}
 
 map $http_upgrade $connection_upgrade {
   default upgrade;
   ''      close;
 }
 
-<% if File.exist?(@ssl_cert) && File.exist?(@ssl_key) %>
+proxy_cache_path /var/cache/nginx/mastodon levels=1:2
+                 keys_zone=mastodon_cache:10m
+                 max_size=1g inactive=120m use_temp_path=off;
+
 server {
   listen 443 ssl http2;
   listen [::]:443 ssl http2;
@@ -22,11 +29,17 @@ server {
   ssl_certificate_key <%= @ssl_key %>;
 
   add_header Strict-Transport-Security "max-age=31536000";
+<% if @onion_address %>
   add_header Onion-Location https://mastodon.<%= @onion_address %>$request_uri;
-}
 <% end %>
+}
 
 <% if @onion_address %>
+server {
+  listen 80;
+  server_name mastodon.<%= @onion_address %>;
+  include <%= @shared_config_path %>;
+}
 server {
   listen 443 ssl http2;
   server_name mastodon.<%= @onion_address %>;

site-cookbooks/kosmos-mastodon/templates/default/nginx_conf_shared.erb

@@ -1,12 +1,12 @@
-access_log "/var/log/nginx/mastodon.access.log";
+access_log "/var/log/nginx/<%= @server_name %>.access.log";
-error_log "/var/log/nginx/mastodon.error.log";
+error_log "/var/log/nginx/<%= @server_name %>.error.log";
 
 keepalive_timeout    70;
 
 sendfile             on;
 client_max_body_size 0;
 
-root <%= @mastodon_path %>/public;
+root <%= @web_root_dir %>;
 
 gzip on;
 gzip_disable "msie6";
@@ -19,20 +19,21 @@ gzip_types text/plain text/css application/json application/javascript text/xml
 
 location / {
 # If the maintenance file is present, show maintenance page
-  if (-f <%= @mastodon_path %>/public/maintenance.html) {
+  if (-f <%= @web_root_dir %>/maintenance.html) {
     return 503;
   }
-
   try_files $uri @proxy;
 }
 
 location /sw.js {
   add_header Cache-Control "max-age=0, no-cache, no-store, must-revalidate";
   add_header Pragma "no-cache";
+  try_files $uri @proxy;
 }
 
 location ~ ^/(emoji|packs|system/accounts/avatars|system/media_attachments/files) {
   add_header Cache-Control "public, max-age=31536000, immutable";
+  proxy_cache mastodon_cache;
   try_files $uri @proxy;
 }
 
@@ -44,7 +45,7 @@ location @proxy {
   proxy_set_header Proxy "";
   proxy_pass_header Server;
 
-  proxy_pass http://localhost:<%= @puma_port %>;
+  proxy_pass http://mastodon_app;
   proxy_buffering off;
   proxy_redirect off;
   proxy_http_version 1.1;
@@ -61,7 +62,7 @@ location /api/v1/streaming {
   proxy_set_header X-Forwarded-Proto https;
   proxy_set_header Proxy "";
 
-  proxy_pass http://localhost:<%= @streaming_port %>;
+  proxy_pass http://mastodon_streaming;
   proxy_buffering off;
   proxy_redirect off;
   proxy_http_version 1.1;
@@ -75,5 +76,5 @@ error_page 500 501 502 504 /500.html;
 error_page 503 /maintenance.html;
 
 location = /maintenance.html {
-  root <%= @mastodon_path %>/public;
+  root <%= @web_root_dir %>;
 }