From 9efb9cd78c88c56e9aec25dfdd8f4d0d8d547a97 Mon Sep 17 00:00:00 2001 From: Sebastian Kippe Date: Mon, 23 Nov 2020 17:37:14 +0100 Subject: [PATCH 1/6] Configure/deploy HTTP upload service on uploads.kosmos.chat https://xmpp.org/extensions/xep-0363.html (Does not contain the config for ejabberd itself yet.) --- data_bags/credentials/ejabberd.json | 25 +++++--- .../kosmos-ejabberd/attributes/default.rb | 9 +++ site-cookbooks/kosmos-ejabberd/metadata.rb | 3 +- .../kosmos-ejabberd/recipes/upload_service.rb | 60 +++++++++++++++++++ .../templates/nginx_conf_upload_service.erb | 19 ++++++ .../kosmos-nginx/recipes/with_perl.rb | 33 ++++++++++ 6 files changed, 139 insertions(+), 10 deletions(-) create mode 100644 site-cookbooks/kosmos-ejabberd/recipes/upload_service.rb create mode 100644 site-cookbooks/kosmos-ejabberd/templates/nginx_conf_upload_service.erb create mode 100644 site-cookbooks/kosmos-nginx/recipes/with_perl.rb diff --git a/data_bags/credentials/ejabberd.json b/data_bags/credentials/ejabberd.json index bda5a71..fb14aca 100644 --- a/data_bags/credentials/ejabberd.json +++ b/data_bags/credentials/ejabberd.json @@ -1,23 +1,30 @@ { "id": "ejabberd", "5apps_ldap_password": { - "encrypted_data": "mfV9TyC4OM055JnyV73mq4qY840pH1tZC9LnIaA3A80CY2kVteC4\n", - "iv": "gpEC3IK9BN9RkaYz\n", - "auth_tag": "WXYWOjUCgEw5OR5VMh+Enw==\n", + "encrypted_data": "H7WrXu2iGreO5MSoaNKAAAQOxh92rij4j4UPffs7Rjq1mtd4dMed\n", + "iv": "uEOoET/OOSDjiELM\n", + "auth_tag": "ehYOXsKPHNXrYNy0xJ+BSw==\n", "version": 3, "cipher": "aes-256-gcm" }, "kosmos_ldap_password": { - "encrypted_data": "Q9znUOIIXU+XsPWet4rDCjHsPPxlA3EfNTkEER/EdfoCajd1Txuh\n", - "iv": "7SAOAwSU8rZGopB1\n", - "auth_tag": "X8yIyw2BFbQMAVTMYLA67g==\n", + "encrypted_data": "1u+tUrEj5JZ0F+j59f7VKztBTyn1vqT6V3H3K7uC9kHQCOUFmg3x\n", + "iv": "NjhasM5iVF6tBzps\n", + "auth_tag": "kSNqc3xEQavZifWcPeeFpA==\n", + "version": 3, + "cipher": "aes-256-gcm" + }, + "uploads_secret": { + "encrypted_data": "2IVxvsaGP1+D0zOT0g9+Zz4Eg42Y8FPe8GiwQDZq6I1f\n", + "iv": "+Ujln/JDnL/afzZ3\n", + "auth_tag": "v0QBCsEemxBaBvi6kazj+w==\n", "version": 3, "cipher": "aes-256-gcm" }, "admins": { - "encrypted_data": "xKtiBOgn4ysJt4byry31cVJUHEsatWDwHEzEve/N5NxTOh1f4QBD+Q68IYzv\nV0ulBjtW91yFcQqKNx/prAVcK3khbnsEzg8uoub9o6hSMwp16LL5x/u6T6u2\n5DwWBEy08yuaujkko57ir0Yv7mfRedT1i5SaH9pgg5VLm56G/PXrlPFfjwaU\n", - "iv": "fpL3EA1VbXxxi+yq\n", - "auth_tag": "iJMJAmw5gHWLFJM5kdzR9A==\n", + "encrypted_data": "3kH8Cbc4Wy1RMd8HLa7aOCZWCZEyjmXq7JC3T0875472F708JjuOXuEqmUeG\nI82OE7lfMVrOup+hiMk9aCTQqxArayWFRZeWnMN2Ji/dbl12wJ/zKWSOPDQ2\nBHzJ/U6NoHBzmAV/PyirmD8KBNkZxCN8vhCq5azZTnmNQmu8CBxM+qUDDhd1\n", + "iv": "dro4RKs1MDD+riaM\n", + "auth_tag": "7yiBW9jyMzcAPWw/XR8zNQ==\n", "version": 3, "cipher": "aes-256-gcm" } diff --git a/site-cookbooks/kosmos-ejabberd/attributes/default.rb b/site-cookbooks/kosmos-ejabberd/attributes/default.rb index f724394..9a91622 100644 --- a/site-cookbooks/kosmos-ejabberd/attributes/default.rb +++ b/site-cookbooks/kosmos-ejabberd/attributes/default.rb @@ -10,3 +10,12 @@ node.override["tor"]["HiddenServices"]["ejabberd"] = { "5269 127.0.0.1:5269" ] } + +node.default["kosmos-ejabberd"]["uploads"] = { + "domain" => "uploads.kosmos.chat", + "max_upload_size_mb" => "100", + "upload.pm" => { + "repo" => "https://gitea.kosmos.org/kosmos/ngx_http_upload.git", + "revision" => "0.2" + } +} diff --git a/site-cookbooks/kosmos-ejabberd/metadata.rb b/site-cookbooks/kosmos-ejabberd/metadata.rb index 1525cc0..4cf04db 100644 --- a/site-cookbooks/kosmos-ejabberd/metadata.rb +++ b/site-cookbooks/kosmos-ejabberd/metadata.rb @@ -19,8 +19,9 @@ chef_version '>= 12.14' if respond_to?(:chef_version) # # source_url 'https://github.com//kosmos-ejabberd' -depends "kosmos-postgresql" depends "kosmos-base" +depends "kosmos-postgresql" +depends "kosmos-nginx" depends "backup" depends "firewall" depends "tor-full" diff --git a/site-cookbooks/kosmos-ejabberd/recipes/upload_service.rb b/site-cookbooks/kosmos-ejabberd/recipes/upload_service.rb new file mode 100644 index 0000000..3fb4038 --- /dev/null +++ b/site-cookbooks/kosmos-ejabberd/recipes/upload_service.rb @@ -0,0 +1,60 @@ +# +# Cookbook:: kosmos-ejabberd +# Recipe:: upload_service +# + +include_recipe "kosmos-nginx::with_perl" + +ejabberd_credentials = data_bag_item("credentials", "ejabberd") +uploads_secret = ejabberd_credentials["uploads_secret"] + +upload_config = node["kosmos-ejabberd"]["uploads"] +domain = upload_config["domain"] + +git "/opt/upload.pm" do + repository upload_config["upload.pm"]["repo"] + revision upload_config["upload.pm"]["revision"] + action :sync +end + +directory "/var/www/upload" do + user node["nginx"]["user"] + group node["nginx"]["group"] + mode "0640" +end + +ruby_block "configure uploads secret" do + block do + file = Chef::Util::FileEdit.new("/opt/upload.pm/upload.pm") + file.search_file_replace(%r{it-is-secret}, uploads_secret) + file.write_file + end +end + +ruby_block "configure perl module in nginx" do + block do + file = Chef::Util::FileEdit.new("/etc/nginx/nginx.conf") + file.insert_line_after_match( + %r{types_hash_bucket_size}, + "\n\n perl_modules /opt/upload.pm;\n perl_require upload.pm;" + ) + file.write_file + end +end + +template "#{node["nginx"]["dir"]}/sites-available/#{domain}" do + source "nginx_conf_upload_service.erb" + owner node["nginx"]["user"] + mode 0640 + variables server_name: domain, + ssl_cert: "/etc/letsencrypt/live/#{domain}/fullchain.pem", + ssl_key: "/etc/letsencrypt/live/#{domain}/privkey.pem", + max_upload_size_mb: upload_config["max_upload_size_mb"] + notifies :reload, "service[nginx]", :delayed +end + +nginx_site domain do + action :enable +end + +nginx_certbot_site domain diff --git a/site-cookbooks/kosmos-ejabberd/templates/nginx_conf_upload_service.erb b/site-cookbooks/kosmos-ejabberd/templates/nginx_conf_upload_service.erb new file mode 100644 index 0000000..bbf25a5 --- /dev/null +++ b/site-cookbooks/kosmos-ejabberd/templates/nginx_conf_upload_service.erb @@ -0,0 +1,19 @@ +<% if File.exist?(@ssl_cert) && File.exist?(@ssl_key) -%> +# Generated by Chef + +server { + listen 443 ssl http2; + server_name <%= @server_name %>; + + ssl_certificate <%= @ssl_cert %>; + ssl_certificate_key <%= @ssl_key %>; + + root /var/www/upload; + + client_max_body_size <%= @max_upload_size_mb %>m; + + location / { + perl upload::handle; + } +} +<% end -%> diff --git a/site-cookbooks/kosmos-nginx/recipes/with_perl.rb b/site-cookbooks/kosmos-nginx/recipes/with_perl.rb new file mode 100644 index 0000000..bac0223 --- /dev/null +++ b/site-cookbooks/kosmos-nginx/recipes/with_perl.rb @@ -0,0 +1,33 @@ +node.override['nginx']['default_site_enabled'] = false +node.override['nginx']['server_tokens'] = 'off' + +node.override['nginx']['package_name'] = 'nginx-core' +include_recipe 'nginx' + +package 'libnginx-mod-http-perl' + +# Generate Strong Diffie-Hellman Group (increases security) +# https://weakdh.org/sysadmin.html +openssl_dhparam "/etc/ssl/private/dhparams.pem" do + key_length 2048 + mode 0600 + owner 'www-data' +end + +cookbook_file "#{node['nginx']['dir']}/conf.d/tls_config.conf" do + source 'nginx_tls_config.conf' + owner 'root' + group 'root' + mode '0644' + notifies :restart, 'service[nginx]' +end + +unless node.chef_environment == "development" + include_recipe 'kosmos-base::firewall' + + firewall_rule 'http/https' do + port [80, 443] + protocol :tcp + command :allow + end +end From 6a89bb9d2e075164d805b98d11030d9a1e35bfa7 Mon Sep 17 00:00:00 2001 From: Sebastian Kippe Date: Mon, 23 Nov 2020 17:39:47 +0100 Subject: [PATCH 2/6] Add node info for uploads-1 Runs uploads.kosmos.chat --- clients/uploads-1.json | 4 +++ nodes/uploads-1.json | 64 ++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 68 insertions(+) create mode 100644 clients/uploads-1.json create mode 100644 nodes/uploads-1.json diff --git a/clients/uploads-1.json b/clients/uploads-1.json new file mode 100644 index 0000000..fec1329 --- /dev/null +++ b/clients/uploads-1.json @@ -0,0 +1,4 @@ +{ + "name": "uploads-1", + "public_key": "-----BEGIN PUBLIC KEY-----\nMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAwJwWaz8TeGv3SFlKzLMx\nqN8GTL/c0N9ppBvv8xNSS/yF9Y40SbL418uxYzm9hIhOXgIygIgLT2EKIXX32t+R\neOJCdYycQFM3At2fhMkjhuUW0gmDRcYBcBJLC5hLh2EZ+A8V7k4qgrBpPLOjEv48\nhQY0vuAw2DGndWr4QLh5NLUmQiOrfuzcZSSNCBOTIgUZgNmRd9QcCHDq4WDH3poa\nosJo4a9JGEGUL1irOivvEdyJPwEd2f++nYAdWwj8pjCYgpRshQlLhxOlylMx7MxB\nQt2bgJC9sahfbfJCOqdlCU3DMJL0bRUiuxK77WeSsxWBJmrsiF3+Ljs2Ix+s7fnS\nywIDAQAB\n-----END PUBLIC KEY-----\n" +} \ No newline at end of file diff --git a/nodes/uploads-1.json b/nodes/uploads-1.json new file mode 100644 index 0000000..bb43002 --- /dev/null +++ b/nodes/uploads-1.json @@ -0,0 +1,64 @@ +{ + "name": "uploads-1", + "normal": { + "knife_zero": { + "host": "10.147.20.98" + } + }, + "automatic": { + "fqdn": "uploads-1", + "os": "linux", + "os_version": "5.4.0-54-generic", + "hostname": "uploads-1", + "ipaddress": "192.168.122.230", + "roles": [ + + ], + "recipes": [ + "kosmos-base", + "kosmos-base::default", + "kosmos-ejabberd::upload_service", + "apt::default", + "timezone_iii::default", + "timezone_iii::debian", + "ntp::default", + "ntp::apparmor", + "kosmos-base::systemd_emails", + "apt::unattended-upgrades", + "kosmos-base::firewall", + "kosmos-postfix::default", + "postfix::default", + "postfix::_common", + "postfix::_attributes", + "postfix::sasl_auth", + "hostname::default", + "kosmos-nginx::with_perl", + "nginx::default", + "nginx::package", + "nginx::ohai_plugin", + "nginx::repo", + "nginx::commons", + "nginx::commons_dir", + "nginx::commons_script", + "nginx::commons_conf", + "kosmos-base::letsencrypt" + ], + "platform": "ubuntu", + "platform_version": "20.04", + "cloud": null, + "chef_packages": { + "chef": { + "version": "15.14.0", + "chef_root": "/opt/chef/embedded/lib/ruby/gems/2.6.0/gems/chef-15.14.0/lib" + }, + "ohai": { + "version": "15.12.0", + "ohai_root": "/opt/chef/embedded/lib/ruby/gems/2.6.0/gems/ohai-15.12.0/lib/ohai" + } + } + }, + "run_list": [ + "recipe[kosmos-base]", + "recipe[kosmos-ejabberd::upload_service]" + ] +} \ No newline at end of file From 0bebdf7635fb3adbc5f0e79ea04fb166adf8bab0 Mon Sep 17 00:00:00 2001 From: Sebastian Kippe Date: Mon, 23 Nov 2020 17:40:14 +0100 Subject: [PATCH 3/6] Do not require nginx from certbot recipe In order to be able to use different nginx install recipes, we must not require a specific one from this recipe. --- site-cookbooks/kosmos-nginx/resources/nginx_certbot_site.rb | 2 -- 1 file changed, 2 deletions(-) diff --git a/site-cookbooks/kosmos-nginx/resources/nginx_certbot_site.rb b/site-cookbooks/kosmos-nginx/resources/nginx_certbot_site.rb index a3aecc2..9d9c0b6 100644 --- a/site-cookbooks/kosmos-nginx/resources/nginx_certbot_site.rb +++ b/site-cookbooks/kosmos-nginx/resources/nginx_certbot_site.rb @@ -8,8 +8,6 @@ property :site, String action :create do return if node.chef_environment == "development" - include_recipe "kosmos-nginx" - domain = new_resource.domain site = new_resource.site || domain root_directory = "/var/www/#{domain}" From 0aef830aa335ed95f7e085b3df9c55d87e3f2a9e Mon Sep 17 00:00:00 2001 From: Sebastian Kippe Date: Mon, 23 Nov 2020 20:50:01 +0100 Subject: [PATCH 4/6] Fix upload folder permissions Uploads are failing with the current mode. --- site-cookbooks/kosmos-ejabberd/recipes/upload_service.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/site-cookbooks/kosmos-ejabberd/recipes/upload_service.rb b/site-cookbooks/kosmos-ejabberd/recipes/upload_service.rb index 3fb4038..c772c0e 100644 --- a/site-cookbooks/kosmos-ejabberd/recipes/upload_service.rb +++ b/site-cookbooks/kosmos-ejabberd/recipes/upload_service.rb @@ -20,7 +20,7 @@ end directory "/var/www/upload" do user node["nginx"]["user"] group node["nginx"]["group"] - mode "0640" + mode "0755" end ruby_block "configure uploads secret" do From 0e29c930ed9b8fc2aa3e715cc5068326a83991fc Mon Sep 17 00:00:00 2001 From: Sebastian Kippe Date: Tue, 24 Nov 2020 15:33:34 +0100 Subject: [PATCH 5/6] Configure subdirectory level for upload.pm This allows to post to per-domain subdirectories from XMPP clients. --- data_bags/credentials/ejabberd.json | 24 +++++++++---------- .../kosmos-ejabberd/recipes/upload_service.rb | 6 ++++- 2 files changed, 17 insertions(+), 13 deletions(-) diff --git a/data_bags/credentials/ejabberd.json b/data_bags/credentials/ejabberd.json index fb14aca..5c0c89c 100644 --- a/data_bags/credentials/ejabberd.json +++ b/data_bags/credentials/ejabberd.json @@ -1,30 +1,30 @@ { "id": "ejabberd", "5apps_ldap_password": { - "encrypted_data": "H7WrXu2iGreO5MSoaNKAAAQOxh92rij4j4UPffs7Rjq1mtd4dMed\n", - "iv": "uEOoET/OOSDjiELM\n", - "auth_tag": "ehYOXsKPHNXrYNy0xJ+BSw==\n", + "encrypted_data": "RdzDZk2F4yBvgII84JGt8AF0LT4cyjRQFQvMJ5LhdB54T06Kjq3S\n", + "iv": "+3WlMHiNAFVE4iku\n", + "auth_tag": "mKheQu/KeHSyt8W783lrzA==\n", "version": 3, "cipher": "aes-256-gcm" }, "kosmos_ldap_password": { - "encrypted_data": "1u+tUrEj5JZ0F+j59f7VKztBTyn1vqT6V3H3K7uC9kHQCOUFmg3x\n", - "iv": "NjhasM5iVF6tBzps\n", - "auth_tag": "kSNqc3xEQavZifWcPeeFpA==\n", + "encrypted_data": "fABWhxMuLaF2qLFdIN//R6bgBkD60WRWiBZPErB1eBOxHqOp813o\n", + "iv": "uBPPYY/FM2hee05V\n", + "auth_tag": "cO+zP2QggWIzbuVxtkct2w==\n", "version": 3, "cipher": "aes-256-gcm" }, "uploads_secret": { - "encrypted_data": "2IVxvsaGP1+D0zOT0g9+Zz4Eg42Y8FPe8GiwQDZq6I1f\n", - "iv": "+Ujln/JDnL/afzZ3\n", - "auth_tag": "v0QBCsEemxBaBvi6kazj+w==\n", + "encrypted_data": "03Y8CNBstV7vYopx8X54hkRSlnwwbOg5Y0KwTPV4qys1\n", + "iv": "gLTP7Y2Y70jL+sxH\n", + "auth_tag": "HJoyOF4rYm9ayKfViuKBlA==\n", "version": 3, "cipher": "aes-256-gcm" }, "admins": { - "encrypted_data": "3kH8Cbc4Wy1RMd8HLa7aOCZWCZEyjmXq7JC3T0875472F708JjuOXuEqmUeG\nI82OE7lfMVrOup+hiMk9aCTQqxArayWFRZeWnMN2Ji/dbl12wJ/zKWSOPDQ2\nBHzJ/U6NoHBzmAV/PyirmD8KBNkZxCN8vhCq5azZTnmNQmu8CBxM+qUDDhd1\n", - "iv": "dro4RKs1MDD+riaM\n", - "auth_tag": "7yiBW9jyMzcAPWw/XR8zNQ==\n", + "encrypted_data": "mRX2Lxqxb//Gd76bk+G3V+eObaq+NILiMsHHjFvjBCvJrznvRzezqW1VHhwW\ndH/ZY2gM8CVCcmYNQ8Xtg/1loPYAUjROvDRirj5i9fP7zgJRc1anNmohDOle\n34aNPYverGm+IJ21sFrAv4Xe/KleJBO5ynuiInqqvljcu3LiuvSYBXW34yWB\n", + "iv": "QqJJM8gmox565JUd\n", + "auth_tag": "yWRLb22JwJjjoK6Wdr1ujg==\n", "version": 3, "cipher": "aes-256-gcm" } diff --git a/site-cookbooks/kosmos-ejabberd/recipes/upload_service.rb b/site-cookbooks/kosmos-ejabberd/recipes/upload_service.rb index c772c0e..6b5accd 100644 --- a/site-cookbooks/kosmos-ejabberd/recipes/upload_service.rb +++ b/site-cookbooks/kosmos-ejabberd/recipes/upload_service.rb @@ -23,10 +23,14 @@ directory "/var/www/upload" do mode "0755" end -ruby_block "configure uploads secret" do +ruby_block "configure uploads.pm" do block do file = Chef::Util::FileEdit.new("/opt/upload.pm/upload.pm") file.search_file_replace(%r{it-is-secret}, uploads_secret) + file.search_file_replace_line( + %r{my \$uri_prefix_components = 0;}, + 'my $uri_prefix_components = 1;' + ) file.write_file end end From f39f953b8a9c430c470aa8f6024c4ee3f15d73c9 Mon Sep 17 00:00:00 2001 From: Sebastian Kippe Date: Tue, 24 Nov 2020 15:44:59 +0100 Subject: [PATCH 6/6] Configure ejabberd nodes for HTTP upload service --- site-cookbooks/kosmos-ejabberd/recipes/default.rb | 10 ++++++++++ .../kosmos-ejabberd/templates/ejabberd.yml.erb | 4 ---- 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/site-cookbooks/kosmos-ejabberd/recipes/default.rb b/site-cookbooks/kosmos-ejabberd/recipes/default.rb index 2a73cdb..bc3a132 100644 --- a/site-cookbooks/kosmos-ejabberd/recipes/default.rb +++ b/site-cookbooks/kosmos-ejabberd/recipes/default.rb @@ -64,6 +64,11 @@ modules: max_user_conferences: 1000 default_room_options: mam: true + mod_http_upload: + put_url: "https://uploads.kosmos.chat/8af2c77" + external_secret: "#{ejabberd_credentials["uploads_secret"]}" + max_size: 104857600 + thumbnail: false # otherwise needs the identify command from ImageMagick installed EOF }, { @@ -89,6 +94,11 @@ modules: public_list: false persistent: true mam: true + mod_http_upload: + put_url: "https://uploads.kosmos.chat/2802cfe" + external_secret: "#{ejabberd_credentials["uploads_secret"]}" + max_size: 104857600 + thumbnail: false # otherwise needs the identify command from ImageMagick installed EOF } ] diff --git a/site-cookbooks/kosmos-ejabberd/templates/ejabberd.yml.erb b/site-cookbooks/kosmos-ejabberd/templates/ejabberd.yml.erb index 5ed892b..7889b20 100644 --- a/site-cookbooks/kosmos-ejabberd/templates/ejabberd.yml.erb +++ b/site-cookbooks/kosmos-ejabberd/templates/ejabberd.yml.erb @@ -191,10 +191,6 @@ modules: name: "abuse-addresses" urls: ["mailto:abuse@@HOST@"] mod_bosh: {} - mod_http_upload: - docroot: "/opt/ejabberd/uploads/xmpp.@HOST@/" - put_url: "https://xmpp.@HOST@:5443/upload" - thumbnail: false # otherwise needs the identify command from ImageMagick installed mod_last: {} mod_mam: default: always