Merge pull request 'Move Gitea bot controls into encrypted data bag' (#703) from gitea-bot-filter-data-bag into master

Reviewed-on: #703
Reviewed-by: Greg <greg@kosmos.org>

data_bags/credentials/gitea.json

@@ -1,58 +1,65 @@
 {
   "id": "gitea",
   "jwt_secret": {
-    "encrypted_data": "wMxs1Ec4vKRSzFtL2KuU1XfmR1t5KDx/7XBbI7V0QfgK+JwYbxU5w6feQCBE\nxOMepAXVUwU7RxPZ+hwQgPg=\n",
-    "iv": "F4vtuOL2B9e9LQnb\n",
-    "auth_tag": "NHATxHbr+3Y3Kxa68NwnjQ==\n",
+    "encrypted_data": "fZnJWLiiqzUpFtNirS33sclrlHmXPzKyMiIujp1hmwqWw6b3/ySeam4R/9F0\nqLr1jnW+fDmYmdUW6wPoFGk=\n",
+    "iv": "Wx88w/nBFI50Ej0b\n",
+    "auth_tag": "5JjuZuwouAvoMDk2LgAoLQ==\n",
     "version": 3,
     "cipher": "aes-256-gcm"
   },
   "internal_token": {
-    "encrypted_data": "mlvUtIjs6kcv7XcYCUOgOE/kDSE4Ts5G+CZuPrJapW9XwkebmyOnHJvXdihY\np/chUtar0pNB5Q16LeeZF9KrzOiDo/OXb40TPUzpsB0/607zV1z829STd4l7\nu5g4Zur13nxC9jT0zQL9QgDEobYdjgf/xu1BXxFT+Ue3lQ==\n",
-    "iv": "25+1a2OJYFNxdf1N\n",
-    "auth_tag": "aF8Gn6Mm7AwLjbR8cDnitg==\n",
+    "encrypted_data": "ASlbrrBLfMpuAHma3ApAs7I9Lm0xuUvLBz1IfNuaarjliQczOehbmZsqS883\nqCRV/rGwE8BO33VBTqmB49KUyee7B+KFbzyTVkXjrpHRofY4LOcYW7/OfMJP\n+hfAnWAlZeQud3Z8IfusXGDVHQ3U5ebFTwSaAMuMj4fPzg==\n",
+    "iv": "UHIYNKU0OUNjY3sN\n",
+    "auth_tag": "3laBspfgWoYdZqBD5pHVAA==\n",
     "version": 3,
     "cipher": "aes-256-gcm"
   },
   "secret_key": {
-    "encrypted_data": "xQuHuijNHoo2WicM2UvSGpwPHd0UilxlIl4BM2Rgyih5bdhjxB6UtUcY9uJQ\nYgxEd7y7R5+XhUAu87CEs4qAGtguDDxGtSGwgTSopvAYZewPFLw=\n",
-    "iv": "Kxwqagjps8kP7Dhz\n",
-    "auth_tag": "WGz5TzBzksf36hKPzBZTQQ==\n",
+    "encrypted_data": "4BVf7PGXARAaZbX7226LCF5xEpPhexxcLLfvGodGNuBjSOEliTXsQiHdFTIj\ng5nJzXx4D4sjhvY4FgcNEhVUreBWYke8dn0L3uz5xiogFSLtHlU=\n",
+    "iv": "JCJJRAS2iDRIo9U2\n",
+    "auth_tag": "i2mcB9IQWlXlavZ2zycoBw==\n",
     "version": 3,
     "cipher": "aes-256-gcm"
   },
   "postgresql_password": {
-    "encrypted_data": "ZziPtXhQM/TQBE+077smnjEPzfJOSo9Cj/CUnG/Be1AN0UAfielf68EhLg==\n",
-    "iv": "iBdSrY15vOc3eycF\n",
-    "auth_tag": "km2CkraKlpOygaz7Xy548Q==\n",
+    "encrypted_data": "5855l2MyYjqg/kUPyGPFcoqbObJZh/v79wFBAaBy0Vhe9VAo/G/1tqvvgA==\n",
+    "iv": "xwK5hEkkROs16hma\n",
+    "auth_tag": "sAPM/xsMMmoK71pj+5DqEQ==\n",
     "version": 3,
     "cipher": "aes-256-gcm"
   },
   "s3_key_id": {
-    "encrypted_data": "1j0znqBgNbHMyJIf51MmfkjkpU3SPv+EL8F30mrfQ44vsGziyeiWfp91hGUM\n",
-    "iv": "dzJM1EX/X8Qy5KbR\n",
-    "auth_tag": "2YUCCFG/oTph3svFYhhYzg==\n",
+    "encrypted_data": "K5fgxcHgxGcgDro96mgIowHJD6FFdea/+DtNyT8YMZ9OnrwBPWQTTtJqMpol\n",
+    "iv": "EZmdjcXjGUFKi5xP\n",
+    "auth_tag": "mRdO3XjYr2z+EZeEiorh/A==\n",
     "version": 3,
     "cipher": "aes-256-gcm"
   },
   "s3_secret_key": {
-    "encrypted_data": "7P3JUyl0LsGuGi8GhSYdXHm4bQhnkGfSrbEMGyfzjSYB5hqm17kYZwNbNA0O\nIUmJ6Kq9Nby7IFTd1qFo7aA+dXuvxJD5QXO8T5E+D0xIaWMHPco=\n",
-    "iv": "+ivHjYpQG/3gQWAi\n",
-    "auth_tag": "fftxN0Z/Kfrn+oFk07jKYQ==\n",
+    "encrypted_data": "S7xrrCDLuhOpprDNFTM/8ME2vDK3U2bhCKQalsFrxbsv1+ogHllF/G5zrL1p\n5H0pMkeDDvgN3oNNr2UbmYJJnmyyZyWDy5m1Dv9BiIv20RY1ucc=\n",
+    "iv": "nANI8bJR5UOd0YCP\n",
+    "auth_tag": "Wlkr2dEYJ2ytMO28VL8aKw==\n",
     "version": 3,
     "cipher": "aes-256-gcm"
   },
   "s3_bucket": {
-    "encrypted_data": "98UnmjwIlLjNFQojZlQRMZAWpI/7s9xJkgvh4sU5I2jWmYk=\n",
-    "iv": "zLck9Dp6OP+L3BwX\n",
-    "auth_tag": "Zc5G6bd7CbZfDCZ31YWxMg==\n",
+    "encrypted_data": "l9icd2M+a+P1japoRLGEYOXtkuMLnuotsE4nWyXER+i+XZE=\n",
+    "iv": "922rYakCutpFO5aP\n",
+    "auth_tag": "zDanzudY8PBBZBRGHvU6Hg==\n",
     "version": 3,
     "cipher": "aes-256-gcm"
   },
   "runners": {
-    "encrypted_data": "f0RRLCrGT7LDUEXcM6m2dJ7C95UPqVZz9dfNLsYa/3SZLDcm1p4FDIy0Su6R\nrXMoAI9IdLBN7/BDMMvqkULEq3Bx5vXn+oTUsUYuKxWmvKEUhC4virOApxh6\n5GbuqcOEPKaf9lHByL+2HKdAmJMzVRGD0t78ePS2pU4H6IFnS9V1p6opOEPr\nzTJ+0PM98eQ/voFKDHGNHUqgDs2qu9wUYNmcHe1eSimFdJiOCN0Mlszu3HL0\nXkHfrGbLrcW+8Ol7dTXdDJB7WAd3R3vddoZQ+mrwzGGDeSMm+ezeMzAX\n",
-    "iv": "NtZ9SbbscX47BXGH\n",
-    "auth_tag": "ZGBzxjNFB5WPnJCpdFwtAQ==\n",
+    "encrypted_data": "KR6YIzLKjcERHjwda5GHVN77qX6CDC/ihIRy8hZxoiRb+o+i3lJW3cX0EiM2\n36iK1apHRXZWCz7eM1SPRVU9AH/S4yaUBi/dhIvqta011KI2I8bgtkKvU9Af\nvemkIFfyDiq+fFU3irNoiK5tzUuU9FlwdCdohkZMINorcJq87cIvvzF5pDVv\niMLFqbRr7i4tUO9KYUGG6rnndz7crw/gjm6da64N9qSUKfxaaejjzT/ouruo\nxn8mJmHEfPyUf0A8mn/0p6VEnP/5AX5+YPRrOvuv2LxskpKp0hsIe7hU\n",
+    "iv": "eJVs/QvBuWfQO3k1\n",
+    "auth_tag": "crTUFAioi9e8k5I9TcTl3g==\n",
+    "version": 3,
+    "cipher": "aes-256-gcm"
+  },
+  "bot_filter": {
+    "encrypted_data": "lo3Nr87mIoMbjpQ2/Txc5MdWM3OPVEwZKCe2Uz+vkjjW4tVe7SzcJgQABR28\ng8nZfHRHPtwi5FJbEvOXDMAnQGduVoI/GMF+Be1OiS7efqZQ2qI4psGTrotc\nQmCbYr6XOXKz7v06y0i9pWZxBnZMr8j95ts5J5XWX9lc5pVY8q0ujm7j9Yb4\ntSUByBUCGJYkCg8uMqNMFEickBKsQodMlT/2VyYG/gFwEjUsB9f0COpTL1zN\n1hdWUtzk0eXVxmB50USYYrPQzgQrAQ0XwEkEq8mLeb7cAhGskQn7DNU+cZOZ\nCvtv2GzbSCIOAMgV5S4W23yZQUiaiB/B4t8jBrzoa4af91xJ228OUNGlCEUm\ncQB9f/0Zf9fRelOd/hpXqOakP4CfIVDBI/TOkFsQn2FurE2ndWZCzqqoFg3U\nR1fh1pwUbDc/n0Oi3L2g9fJlC3HzYmesgIpKYVp1Lak0w5093ewXQ3EeBD98\nRic+NNDAgeKkUQnLtNKTvjQFlYv+LYCaAyvMROfBLbr0iNtravB2QMchL104\nd7UYjw4VYThvcr7Wn/y7znmCyfXISDCrRd7x+4K0laSGSwnPrKqmmC+N+jxB\n2ydNf6zGZefcot2AIvxaKwV1kTsl5cxsH/cYrFsG2RvruPAqLAIDN06mT/Nz\nxJE=\n",
+    "iv": "If2EG9k+nEFssls0\n",
+    "auth_tag": "/2DlnFtWYLzJiJrS4Jm3Vg==\n",
     "version": 3,
     "cipher": "aes-256-gcm"
   }

site-cookbooks/kosmos_gitea/CHANGELOG.md

@@ -2,6 +2,30 @@
 
 This file is used to list changes made in each version of the kosmos_gitea cookbook.
 
+# 0.3.9
+
+- Move the bot-blocking configuration out of the cookbook and into the
+  encrypted `credentials/gitea` data bag (new `bot_filter` key), so the
+  rate-limited crawler names, the hard-banned user agents and the denied
+  networks are no longer published in the public repository and can be changed
+  by editing the data bag instead of the cookbook.
+  - `rate_limits.conf` and `robots.txt` are now rendered from the data bag
+    (`rate_limits.conf.erb`, `robots.txt.erb`). The rate values and the
+    expensive-route / robots route patterns stay static in the templates.
+  - `bot_filter` schema: `limit` (list of `{match, key}`), `deny_user_agents`
+    (list of user-agent patterns) and `deny_networks` (list of CIDRs).
+
+# 0.3.8
+
+- Broaden the Gitea edge bot controls on the openresty proxy:
+  - Rate-limit several additional AI crawlers and tighten the `bots` zone from
+    240r/m to 60r/m per IP. The heavy crawlers average under 1.5r/s, so the old
+    rate never engaged and the listed crawlers were effectively unthrottled.
+  - Hard-block a datacenter scraper network that spoofs browser User-Agents, so
+    it cannot be matched by UA, and enumerates historical commit pages.
+  - Disallow `/*/src/commit/` and `/*/commits/commit/` in robots.txt. These
+    combinatorial commit-browsing routes were ~56% of the traffic to Gitea.
+
 # 0.3.7
 
 - Upgrade the Gitea Actions runner from 5.2.4 to 5.4.0. No `config.yaml` changes

site-cookbooks/kosmos_gitea/files/custom/public/robots.txt

@@ -1,16 +0,0 @@
-# Hard-disallow the AI crawler that floods Gitea with combinatorial
-# filter-permutation requests.
-User-agent: PerplexityBot
-Disallow: /
-
-# Everyone else: keep normal pages crawlable, but never crawl filter
-# permutations on list pages or the expensive render routes.
-User-agent: *
-Disallow: /*?*
-Disallow: /*/blame/
-Disallow: /*/compare/
-Disallow: /*/raw/
-Disallow: /*/archive/
-Disallow: /*.patch$
-Disallow: /*.diff$
-Allow: /

site-cookbooks/kosmos_gitea/metadata.rb

@@ -4,7 +4,7 @@ maintainer_email 'ops@kosmos.org'
 license 'MIT'
 description 'Installs/configures Gitea'
 long_description 'Installs/configures Gitea'
-version '0.3.7'
+version '0.3.9'
 chef_version '>= 14.0'
 
 depends "firewall"

site-cookbooks/kosmos_gitea/recipes/default.rb

@@ -76,13 +76,17 @@ cookbook_file "#{working_directory}/custom/templates/custom/header.tmpl" do
   notifies :restart, "service[gitea]", :delayed
 end
 
-# robots.txt: disallow Perplexity and crawler-unfriendly filter permutations
-# and expensive render routes
-cookbook_file "#{working_directory}/custom/public/robots.txt" do
-  source "custom/public/robots.txt"
+# robots.txt: hard-ban the crawlers listed in the data bag, and disallow
+# crawler-unfriendly filter permutations, expensive render routes and
+# combinatorial commit-browsing routes.
+bot_deny_user_agents = (gitea_data_bag_item["bot_filter"] || {})["deny_user_agents"] || []
+
+template "#{working_directory}/custom/public/robots.txt" do
+  source "robots.txt.erb"
   owner "git"
   group "git"
   mode  "0644"
+  variables deny_user_agents: bot_deny_user_agents
 end
 
 directory config_directory do

site-cookbooks/kosmos_gitea/recipes/nginx.rb

@@ -17,12 +17,22 @@ tls_cert_for domain do
   action :create
 end
 
-# Slow down / block unwanted bot traffic and hard-limit expensive routes
-cookbook_file "#{node["openresty"]["dir"]}/conf.d/rate_limits.conf" do
-  source "rate_limits.conf"
+# Slow down / block unwanted bot traffic and hard-limit expensive routes.
+# The bot list and denied networks live in the `credentials/gitea` data bag so
+# they are not published in this repository.
+bot_filter       = data_bag_item("credentials", "gitea")["bot_filter"] || {}
+limit_bots       = bot_filter["limit"] || []
+deny_user_agents = bot_filter["deny_user_agents"] || []
+deny_networks    = bot_filter["deny_networks"] || []
+
+template "#{node["openresty"]["dir"]}/conf.d/rate_limits.conf" do
+  source "rate_limits.conf.erb"
   owner "root"
   group "root"
   mode "0644"
+  variables limit_bots:       limit_bots,
+            deny_user_agents: deny_user_agents,
+            deny_networks:    deny_networks
   notifies :reload, "service[openresty]", :delayed
 end
 

site-cookbooks/kosmos_gitea/templates/default/nginx_conf_web.erb

@@ -20,10 +20,13 @@ server {
 
   proxy_intercept_errors on;
 
-  # Hard ban Perplexity (does not honour robots.txt). Defined in
+  # Hard ban for user agents listed in the data bag. Defined in
   # /etc/openresty/conf.d/rate_limits.conf
   if ($deny_bot) { return 403; }
 
+  # Hard ban for networks listed in the data bag. Same file.
+  if ($deny_ip) { return 403; }
+
   # Rate limit for bots. Defined in /etc/openresty/conf.d/rate_limits.conf
   limit_req zone=bots burst=10; # allow a larger spike for asset bursts
 

site-cookbooks/kosmos_gitea/templates/default/rate_limits.conf.erb

@@ -1,33 +1,45 @@
-# Increase if you have very long User-Agent strings
+# Managed by Chef. The rate-limited bot list, the hard-banned user agents and
+# the denied networks come from the `credentials/gitea` data bag (`bot_filter`),
+# so they are not published in this repository. Edit them there.
 map_hash_bucket_size 256;
 
-# Bot user agents
+# Rate-limited bots: user-agent pattern -> label. First match wins.
 map $http_user_agent $bot_name {
     default              "";
-    ~*ClaudeBot          "claude";
-    ~*Sogou              "sogou";
-    ~*meta-externalagent "meta-externalagent";
-    # add more as needed
+<% @limit_bots.each do |bot| -%>
+    ~*<%= bot["match"] %> "<%= bot["key"] %>";
+<% end -%>
 }
 
-# Decide the rate-limit key (per-IP for each bot in this example)
+# Decide the rate-limit key (per-IP for each listed bot).
 map $bot_name $bot_limit_key {
     default              "";
-    "claude"             $binary_remote_addr;
-    "sogou"              $binary_remote_addr;
-    "meta-externalagent" $binary_remote_addr;
+<% @limit_bots.each do |bot| -%>
+    "<%= bot["key"] %>" $binary_remote_addr;
+<% end -%>
 }
 
-# Shared memory zone – only non-empty keys are counted
-limit_req_zone $bot_limit_key zone=bots:20m rate=240r/m; # 4r/s per IP+bot
+# Shared memory zone – only non-empty keys are counted.
+limit_req_zone $bot_limit_key zone=bots:20m rate=60r/m; # 1r/s per IP
 
-# Status code returned when the limit is exceeded
+# Status code returned when the limit is exceeded.
 limit_req_status 429;
 
-# Hard ban: Perplexity ignores robots.txt, so deny it at the edge.
+# Hard-banned user agents, denied at the edge.
 map $http_user_agent $deny_bot {
     default              0;
-    ~*PerplexityBot      1;
+<% @deny_user_agents.each do |ua| -%>
+    ~*<%= ua %> 1;
+<% end -%>
+}
+
+# Hard-banned networks: datacenter scrapers that spoof browser UAs, so they
+# cannot be caught by User-Agent.
+geo $deny_ip {
+    default          0;
+<% @deny_networks.each do |cidr| -%>
+    <%= cidr %> 1;
+<% end -%>
 }
 
 # Expensive routes and filter-permutation list pages get a hard per-IP limit.

site-cookbooks/kosmos_gitea/templates/default/robots.txt.erb

@@ -0,0 +1,20 @@
+# Managed by Chef. The hard-banned crawlers come from the `credentials/gitea`
+# data bag (`bot_filter`), so their names are not published in this repository.
+<% @deny_user_agents.each do |ua| -%>
+User-agent: <%= ua %>
+Disallow: /
+<% end -%>
+
+# Everyone else: keep normal pages crawlable, but never crawl filter
+# permutations on list pages, the expensive render routes or commit browsing.
+User-agent: *
+Disallow: /*?*
+Disallow: /*/blame/
+Disallow: /*/compare/
+Disallow: /*/raw/
+Disallow: /*/archive/
+Disallow: /*/src/commit/
+Disallow: /*/commits/commit/
+Disallow: /*.patch$
+Disallow: /*.diff$
+Allow: /