Merge pull request 'Rate limit requests from bots on Gitea' (#656) from chore/rate_limit_bot_on_gitea into master

Reviewed-on: #656
Reviewed-by: Râu Cao <raucao@kosmos.org>

site-cookbooks/kosmos_gitea/files/rate_limits.conf

@@ -0,0 +1,25 @@
+# Increase if you have very long User-Agent strings
+map_hash_bucket_size 256;
+
+# Bot user agents
+map $http_user_agent $bot_name {
+    default              "";
+    ~*ClaudeBot          "claude";
+    ~*Sogou              "sogou";
+    ~*meta-externalagent "meta-externalagent";
+    # add more as needed
+}
+
+# Decide the rate-limit key (per-IP for each bot in this example)
+map $bot_name $bot_limit_key {
+    default              "";
+    "claude"             $binary_remote_addr;
+    "sogou"              $binary_remote_addr;
+    "meta-externalagent" $binary_remote_addr;
+}
+
+# Shared memory zone – only non-empty keys are counted
+limit_req_zone $bot_limit_key zone=bots:20m rate=240r/m; # 4r/s per IP+bot
+
+# Status code returned when the limit is exceeded
+limit_req_status 429;

site-cookbooks/kosmos_gitea/recipes/nginx.rb

@@ -17,6 +17,14 @@ tls_cert_for domain do
   action :create
 end
 
+# Slow down requests from bots
+cookbook_file "#{node["openresty"]["dir"]}/conf.d/rate_limits.conf" do
+  source "rate_limits.conf"
+  owner "root"
+  group "root"
+  mode "0644"
+end
+
 openresty_site domain do
   template "nginx_conf_web.erb"
   variables server_name:   domain,

site-cookbooks/kosmos_gitea/templates/default/nginx_conf_web.erb

@@ -20,6 +20,9 @@ server {
 
   proxy_intercept_errors on;
 
+  # Rate limit for bots. Defined in /etc/openresty/conf.d/rate_limits.conf
+  limit_req zone=bots burst=10; # allow a larger spike for asset bursts
+
   location ~ ^/(avatars|repo-avatars)/.*$ {
     proxy_buffers 1024 8k;
     proxy_pass http://_gitea_web;