Merge pull request 'Rate limit requests from bots on Gitea' (#656) from chore/rate_limit_bot_on_gitea into master

Reviewed-on: #656
Reviewed-by: Râu Cao <raucao@kosmos.org>

clients/redis-4.json

@@ -0,0 +1,4 @@
+{
+  "name": "redis-4",
+  "public_key": "-----BEGIN PUBLIC KEY-----\nMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAn3Zdzfao+n3PdtlJlViJ\nnpVDcL8ofw2rb2/VK7ivpWMKaL7mOq8HIQ+R+ghvllX/vPmN6co3+aUxOvvQ6KGC\nmAcnc7StqOpYUK0g6QZFROESO0vz1hy078AL9D2dEwOj7XS8leWCQyGgA1Z89T9n\naatXqEF1zGuvnVLA340prYkJk5S1X6KHq1WIi9H23hYehvGZlpkJ87wv8sKIo4Y3\nzIVA5hio0t0q+JyStOGr+xRl3mM43b25XcIh2ibrzKp3YoRWSCvMfhAdMKm1WEEI\njM/3fQRtkI1OQSb0o/wUoAmpUEEOTIvbWU2SuFVY7HYRuPwEilvbrGZKgrvbp8R+\nZQIDAQAB\n-----END PUBLIC KEY-----\n"
+}

environments/production.json

@@ -81,7 +81,7 @@
           "garage-",
           "lq-",
           "rsk-",
-          "postgres-6"
+          "postgres-11"
         ]
       }
     },

site-cookbooks/kosmos_gitea/attributes/default.rb

@@ -1,5 +1,5 @@
-node.default["gitea"]["version"] = "1.26.4"
+node.default["gitea"]["version"] = "1.27.1"
-node.default["gitea"]["checksum"] = "0faa36d151918f8f7d6e0f3ae67597d1c338583d695add146ac393109d0fc44a"
+node.default["gitea"]["checksum"] = "86a7ac26e7f9c9cca0f56c4fac07fff205d5fc3bca0e54af23a204f07b833bc9"
 node.default["gitea"]["repo"] = nil
 node.default["gitea"]["revision"] = nil
 node.default["gitea"]["working_directory"] = "/var/lib/gitea"

site-cookbooks/kosmos_gitea/files/rate_limits.conf

@@ -0,0 +1,25 @@
+# Increase if you have very long User-Agent strings
+map_hash_bucket_size 256;
+
+# Bot user agents
+map $http_user_agent $bot_name {
+    default              "";
+    ~*ClaudeBot          "claude";
+    ~*Sogou              "sogou";
+    ~*meta-externalagent "meta-externalagent";
+    # add more as needed
+}
+
+# Decide the rate-limit key (per-IP for each bot in this example)
+map $bot_name $bot_limit_key {
+    default              "";
+    "claude"             $binary_remote_addr;
+    "sogou"              $binary_remote_addr;
+    "meta-externalagent" $binary_remote_addr;
+}
+
+# Shared memory zone – only non-empty keys are counted
+limit_req_zone $bot_limit_key zone=bots:20m rate=240r/m; # 4r/s per IP+bot
+
+# Status code returned when the limit is exceeded
+limit_req_status 429;

site-cookbooks/kosmos_gitea/metadata.rb

@@ -4,7 +4,7 @@ maintainer_email 'ops@kosmos.org'
 license 'MIT'
 description 'Installs/configures Gitea'
 long_description 'Installs/configures Gitea'
-version '0.2.0'
+version '0.2.1'
 chef_version '>= 14.0'
 
 depends "firewall"

site-cookbooks/kosmos_gitea/recipes/nginx.rb

@@ -17,6 +17,14 @@ tls_cert_for domain do
   action :create
 end
 
+# Slow down requests from bots
+cookbook_file "#{node["openresty"]["dir"]}/conf.d/rate_limits.conf" do
+  source "rate_limits.conf"
+  owner "root"
+  group "root"
+  mode "0644"
+end
+
 openresty_site domain do
   template "nginx_conf_web.erb"
   variables server_name:   domain,

site-cookbooks/kosmos_gitea/templates/default/app.ini.erb

@@ -62,6 +62,9 @@ FROM = <%= @email %>
 INTERNAL_TOKEN = <%= @internal_token %>
 INSTALL_LOCK   = true
 SECRET_KEY     = <%= @secret_key %>
+<% if c = @config["webhook"] %>
+<% if c["allowed_host_list"] %>ALLOWED_HOST_LIST = <%= c["allowed_host_list"] %><% end %>
+<% end %>
 
 [service]
 REGISTER_EMAIL_CONFIRM            = false
@@ -101,11 +104,6 @@ RUN_AT_START = false
 NOTICE_ON_SUCCESS = false
 SCHEDULE = @every 15m
 
-<% if c = @config["webhook"] %>
-[webhook]
-<% if c["allowed_host_list"] %>ALLOWED_HOST_LIST = <%= c["allowed_host_list"] %><% end %>
-<% end %>
-
 <% if c = @config["storage"] %>
 [storage]
 <% if c["type"] == "minio" %>

site-cookbooks/kosmos_gitea/templates/default/nginx_conf_web.erb

@@ -20,6 +20,9 @@ server {
 
   proxy_intercept_errors on;
 
+  # Rate limit for bots. Defined in /etc/openresty/conf.d/rate_limits.conf
+  limit_req zone=bots burst=10; # allow a larger spike for asset bursts
+
   location ~ ^/(avatars|repo-avatars)/.*$ {
     proxy_buffers 1024 8k;
     proxy_pass http://_gitea_web;

site-cookbooks/kosmos_kvm/recipes/host.rb

@@ -5,6 +5,11 @@
 
 package %w(virtinst libvirt-daemon-system libvirt-clients)
 
+# Required on legacy hosts (Ubuntu 20.04) to build the NoCloud seed ISO
+# via cloud-localds. Modern virt-install (>= 4.0.0, Ubuntu >= 22.04)
+# handles cloud-init natively via --cloud-init and does not need it.
+package "cloud-image-utils" if node["platform_version"].to_f < 22.04
+
 directory "/var/lib/libvirt/images/base" do
   recursive true
   owner "libvirt-qemu"
@@ -24,7 +29,10 @@ end
 template "/usr/local/sbin/create_vm" do
   source "create_vm.erb"
   mode "0750"
-  variables base_image_path: node["kosmos_kvm"]["host"]["qemu_base_image"]["path"]
+  variables(
+    base_image_path: node["kosmos_kvm"]["host"]["qemu_base_image"]["path"],
+    modern_virt_install: node["platform_version"].to_f >= 22.04
+  )
 end
 
 firewall_rule 'ssh-alt-port' do

site-cookbooks/kosmos_kvm/templates/create_vm.erb

@@ -64,6 +64,7 @@ EOS
   popd
 fi
 
+<% if @modern_virt_install -%>
 virt-install \
   --name "$VMNAME" \
   --ram "$RAM" \
@@ -83,3 +84,30 @@ virt-install \
   --autostart \
   --import \
   --cloud-init root-password-generate=off,disable=on,meta-data=$CIDATA_PATH/meta-data,user-data=$CIDATA_PATH/user-data
+<% else -%>
+# Legacy path for hosts running virt-install < 4.0.0 (e.g. Ubuntu 20.04,
+# virt-install 2.2.x). These versions predate the --osinfo/--cloud-init
+# options, so we build a NoCloud seed ISO from the same user-data and
+# meta-data files and attach it as a cdrom. The guest receives the
+# exact same cloud-init configuration as on modern hosts.
+cloud-localds "$CIDATA_PATH/seed.iso" "$CIDATA_PATH/user-data" "$CIDATA_PATH/meta-data"
+
+virt-install \
+  --name "$VMNAME" \
+  --ram "$RAM" \
+  --vcpus "$CPUS" \
+  --cpu host \
+  --arch x86_64 \
+  --hvm \
+  --virt-type kvm \
+  --disk "$IMAGE_PATH" \
+  --disk "$CIDATA_PATH/seed.iso,device=cdrom" \
+  --boot hd \
+  --network=bridge=virbr0,model=virtio \
+  --graphics none \
+  --serial pty \
+  --console pty \
+  --channel unix,mode=bind,path=/var/lib/libvirt/qemu/$VMNAME.guest_agent.0,target_type=virtio,name=org.qemu.guest_agent.0 \
+  --autostart \
+  --import
+<% end -%>