Merge pull request 'Slow down Gitea 404s to mess with scrapers/bots' (#626) from chore/gitea_scraping into master

Reviewed-on: #626
Reviewed-by: Greg <greg@kosmos.org>

AGENTS.md

@@ -0,0 +1,41 @@
+# AGENTS.md
+
+Welcome, AI Agent! This file contains essential context and rules for interacting with the Kosmos Chef repository. Read this carefully before planning or executing any changes.
+
+## 🏢 Project Overview
+This repository contains the infrastructure automation code used by Kosmos to provision and configure bare metal servers (KVM hosts) and Ubuntu virtual machines (KVM guests). 
+
+We use **Chef Infra**, managed locally via **Knife Zero** (agentless Chef), and **Berkshelf** for dependency management.
+
+## 📂 Directory Structure & Rules
+
+* **`site-cookbooks/`**: 🟢 **EDITABLE.** This directory contains all custom, internal cookbooks written specifically for Kosmos services (e.g., `kosmos-postgresql`, `kosmos_gitea`, `kosmos-mastodon`). *Active development happens here.*
+* **`cookbooks/`**: 🔴 **DO NOT EDIT.** This directory contains third-party/community cookbooks that are vendored. These are managed by Berkshelf. Modifying them directly will result in lost changes.
+* **`roles/`**: 🟢 **EDITABLE.** Contains Chef roles written in Ruby (e.g., `base.rb`, `kvm_guest.rb`, `postgresql_primary.rb`). These define run-lists and role-specific default attributes for servers.
+* **`environments/`**: Contains Chef environment definitions (like `production.rb`).
+* **`data_bags/`**: Contains data bag configurations, often encrypted. Be cautious and do not expose secrets. (Note: Agents should not manage data bag secrets directly unless provided the `.chef/encrypted_data_bag_secret`).
+* **`nodes/`**: Contains JSON state files for bootstrapped nodes. *Agents typically do not edit these directly unless cleaning up a deleted node.*
+* **`Berksfile`**: Defines community cookbook dependencies.
+* **`Vagrantfile` / `.kitchen/`**: Used for local virtualization and integration testing.
+
+## 🛠️ Tooling & Workflows
+
+1. **Dependency Management (Berkshelf)**
+   If a new community cookbook is required:
+   - Add it to the `Berksfile` at the root.
+   - Instruct the user to run `berks install` and `berks vendor cookbooks/ --delete` (or run it via the `bash` tool if permitted).
+
+2. **Provisioning (Knife Zero)**
+   - Bootstrapping and converging nodes is done using `knife zero`. 
+   - *Example:* `knife zero converge name:server-name.kosmos.org`
+
+3. **Code Style & Conventions**
+   - Chef recipes, resources, and roles are written in **Ruby**. 
+   - Follow standard Chef and Ruby (RuboCop) idioms. Look at neighboring files in `site-cookbooks/` or `roles/` to match formatting and naming conventions.
+
+## 🚨 Core Directives for AI Agents
+
+1. **Infrastructure as Code**: Manual server configurations are highly discouraged. All changes must be codified in a cookbook or role.
+2. **Test Safety Nets**: Look for `.kitchen.yml` within specific `site-cookbooks/<name>` to understand if local integration tests are available.
+3. **No Assumptions**: Do not assume standard test commands. Check `README.md` and repository config files first.
+4. **Secret Handling**: Avoid hardcoding passwords or API keys in recipes or roles. Assume sensitive information is managed via Chef `data_bags`.

clients/garage-14.json

@@ -1,4 +1,4 @@
 {
   "name": "garage-14",
-  "public_key": "-----BEGIN PUBLIC KEY-----\nMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAqNY8AuaM4byhaTZacfRJ\nv/qyHxcDJOMX/ElF1H908spdbB2ZiLXHOH1Ucw1d+NV6/QUtWk+ikKFPpasnatD7\nmjE57noH+H47Rll0nD7oT+in+fOBDHF9R0P6/qyRSdJbJkHOh0iC0MG4LcUfv0AY\nnVBW5iLZSe/PC3+PvhCv7yrx3ikSs0mg1ZWppw0ka5Ek3ZCZp5FB4L6++GYWpM+1\n6YI0CjMoRcXsaEQsJWhxHXT8/KDhW0BR8woZUGm0/Yn4teLYJzioxRfBep3lbygx\nOIsDN9IJzo2zVTGPDZQLXhVemIhzaepqTC77ibH7F0gN/1vsQBc/qf7UhbwaF4rR\ndQIDAQAB\n-----END PUBLIC KEY-----\n"
+  "public_key": "-----BEGIN PUBLIC KEY-----\nMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAypINv1zTZ7+pyT0iRhik\n0W70ASYADo7qK7QyE9/3nu2sUrP1IjoNFsv/ceKwicH7Fw2Ei1o+yKZlKn7zJzY7\n93YRZndF04VH2bmqy0uOWK0Bdat7gCld5bvS6FmRflg7g64LFb33/64QIVsVGHGL\nYF2TO//x79t9JKcQDa4h5MOWzJNTFuEcUGa0gJjMYpWGVHEJSgRuIgyhXmyIJJgY\nguj6ymTm5+3VS7NzoNy2fbTt1LRpHb5UWrCR15oiLZiDSMLMx0CcGOCmrhvODi4k\n0umw+2NPd1G50s9z7KVbTqybuQ65se2amRnkVcNfaBIU5qk9bVqcmhZlEozmBZCd\ndwIDAQAB\n-----END PUBLIC KEY-----\n"
 }

clients/garage-15.json

@@ -0,0 +1,4 @@
+{
+  "name": "garage-15",
+  "public_key": "-----BEGIN PUBLIC KEY-----\nMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAy14sTt5gxVZi9C3KIEBu\nDyUgbb6jc3/GR22fNPTqV6uDHhxzhE2UsYwY/7yuA1RasdwHEOBWZaoC0Om5/Zmi\n8gn6//v1ILyLNaAcw+SQcxZkCN8Sk/0atRS9HYk1agE8Mvh72Fe2z3l+92VMefy7\nJwJUNNBTbnV2WVCchChoWnfhI7bkSLSHp0M2MO2pI+lkpSdmfkJSa5z9zihgxKO8\nXfvhryDCZNvfRVHhwc+ffpap0gLF0H9riGKE4FwLy4YqbuW1Tgm6bObb9bpOIw6Q\nVfH3kC/KMK5FlnxGmYtDkhRJ/wjGInRBk9WK/QOmjyd2FVxipEQmA4RdjlznRC9I\nrwIDAQAB\n-----END PUBLIC KEY-----\n"
+}

clients/leo.json

@@ -0,0 +1,4 @@
+{
+  "name": "leo",
+  "public_key": "-----BEGIN PUBLIC KEY-----\nMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAnFfQsJnREjbXTtpT6BVt\naBaUzRmCQi8Du0TzeUG0ENrY0p5Exqleye2rC6bJlB3PER1xr5zdtuXLgbcVumIb\nzroU5JPtFbQk7r/pj0atT+UEYzl16iuEpprQ/bug+f0nE514USr6YG4G+tlZ/jBI\nSHsCQF1P8ufXFLW0ewC7rdvBkgA+DwK14naRxS4jO5MSl4wmNTjs/jymTg508mQq\nf5tG52t8qFdgn9pRdBXmyTpPtwK7I4rZ+1Qn+1E5m4oQUZsxh8Ba1bGbKotVO7Ua\nYL1yCGx7zRRUvLLIdSMvlRXTJBUSQtQ8P4QUDWTY1Na2w3t9sulKg2Lwsw8tktvC\nCwIDAQAB\n-----END PUBLIC KEY-----\n"
+}

clients/postgres-10.json

@@ -0,0 +1,4 @@
+{
+  "name": "postgres-10",
+  "public_key": "-----BEGIN PUBLIC KEY-----\nMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEA2oBb5omC7ZionWhudgFm\n2NGcRXsI0c7+g1+0duaxj5dziaRTltqzpRJTfiJD6R36FcvEqwGc+qQgYSMzc1Xd\nY4OTvJFIDiFAmROm/DZYgFtTDldVNJZO2bbU3COYf/Z2Poq56gC4zLLd/zf6shgb\n2Mty8PlQ82JJAY9EMI3aAifdnZ1k/g4weFC4LFg9lUcNNXOwlAjp//LJ3ku3aY1r\nwW74msSeWEjE44YZdWyMYgM7Fy1hz5giHFQtRdOLemRCWQ8h26wn/cmWld7lsLg+\nlYqxokxWXGv8r5zR8kDTBkd0dxY7ZMbo7oESY4Uhuf4UReMe2ZGHto1E7w3llSj+\n7wIDAQAB\n-----END PUBLIC KEY-----\n"
+}

data_bags/credentials/gandi_api.json

@@ -1,23 +1,16 @@
 {
   "id": "gandi_api",
-  "key": {
-    "encrypted_data": "lU7/xYTmP5Sb6SsK5TNNIyegWozzBtUzpg7oDdl6gcz9FEMmG2ft0Ljh5Q==\n",
-    "iv": "EZPQD3C+wsP/mBhF\n",
-    "auth_tag": "vF9E8Pj4Z8quJJdOMg/QTw==\n",
-    "version": 3,
-    "cipher": "aes-256-gcm"
-  },
   "access_token": {
-    "encrypted_data": "1Uw69JkNrmb8LU/qssuod1SlqxxrWR7TJQZeeivRrNzrMIVTEW/1uwJIYL6b\nM4GeeYl9lIRlMMmLBkc=\n",
+    "encrypted_data": "+skwxHnpAj/3d3e2u7s7B9EydbETj8b0flWahvb5gt/o4JYFWHrhIyX/0IVa\n4wgmu08eDgU51i0knGA=\n",
-    "iv": "cc1GJKu6Cf4DkIgX\n",
+    "iv": "ONKrFCt8Oj3GKIQ5\n",
-    "auth_tag": "ERem4S7ozG695kjvWIMghw==\n",
+    "auth_tag": "j9Hrk8ZZFMQub4NUO+2e4g==\n",
     "version": 3,
     "cipher": "aes-256-gcm"
   },
   "domains": {
-    "encrypted_data": "scZ5blsSjs54DlitR7KZ3enLbyceOR5q0wjHw1golQ==\n",
+    "encrypted_data": "lGfoPHdXEYYdJmoIA9M119wjVl1v4UzIv5gHADwx0A==\n",
-    "iv": "oDcHm7shAzW97b4t\n",
+    "iv": "q6XKbxhW7X9ONxNt\n",
-    "auth_tag": "62Zais9yf68SwmZRsmZ3hw==\n",
+    "auth_tag": "ns9WJH8Oe75siWu+sOZkRg==\n",
     "version": 3,
     "cipher": "aes-256-gcm"
   }

nodes/garage-14.json

@@ -3,15 +3,15 @@
   "chef_environment": "production",
   "normal": {
     "knife_zero": {
-      "host": "10.1.1.157"
+      "host": "10.1.1.151"
     }
   },
   "automatic": {
     "fqdn": "garage-14",
     "os": "linux",
-    "os_version": "5.15.0-1059-kvm",
+    "os_version": "5.15.0-1095-kvm",
     "hostname": "garage-14",
-    "ipaddress": "192.168.122.251",
+    "ipaddress": "192.168.122.36",
     "roles": [
       "base",
       "kvm_guest",
@@ -30,6 +30,7 @@
       "timezone_iii::debian",
       "ntp::default",
       "ntp::apparmor",
+      "kosmos-base::journald_conf",
       "kosmos-base::systemd_emails",
       "apt::unattended-upgrades",
       "kosmos-base::firewall",
@@ -46,13 +47,13 @@
     "cloud": null,
     "chef_packages": {
       "chef": {
-        "version": "18.8.54",
+        "version": "18.10.17",
-        "chef_root": "/opt/chef/embedded/lib/ruby/gems/3.1.0/gems/chef-18.8.54/lib",
+        "chef_root": "/opt/chef/embedded/lib/ruby/gems/3.1.0/gems/chef-18.10.17/lib",
         "chef_effortless": null
       },
       "ohai": {
-        "version": "18.2.8",
+        "version": "18.2.13",
-        "ohai_root": "/opt/chef/embedded/lib/ruby/gems/3.1.0/gems/ohai-18.2.8/lib/ohai"
+        "ohai_root": "/opt/chef/embedded/lib/ruby/gems/3.1.0/gems/ohai-18.2.13/lib/ohai"
       }
     }
   },

nodes/garage-15.json

@@ -0,0 +1,65 @@
+{
+  "name": "garage-15",
+  "chef_environment": "production",
+  "normal": {
+    "knife_zero": {
+      "host": "10.1.1.82"
+    }
+  },
+  "automatic": {
+    "fqdn": "garage-15",
+    "os": "linux",
+    "os_version": "5.15.0-1095-kvm",
+    "hostname": "garage-15",
+    "ipaddress": "192.168.122.57",
+    "roles": [
+      "base",
+      "kvm_guest",
+      "garage_node"
+    ],
+    "recipes": [
+      "kosmos-base",
+      "kosmos-base::default",
+      "kosmos_kvm::guest",
+      "kosmos_garage",
+      "kosmos_garage::default",
+      "kosmos_garage::firewall_rpc",
+      "kosmos_garage::firewall_apis",
+      "apt::default",
+      "timezone_iii::default",
+      "timezone_iii::debian",
+      "ntp::default",
+      "ntp::apparmor",
+      "kosmos-base::journald_conf",
+      "kosmos-base::systemd_emails",
+      "apt::unattended-upgrades",
+      "kosmos-base::firewall",
+      "kosmos-postfix::default",
+      "postfix::default",
+      "postfix::_common",
+      "postfix::_attributes",
+      "postfix::sasl_auth",
+      "hostname::default",
+      "firewall::default"
+    ],
+    "platform": "ubuntu",
+    "platform_version": "22.04",
+    "cloud": null,
+    "chef_packages": {
+      "chef": {
+        "version": "18.10.17",
+        "chef_root": "/opt/chef/embedded/lib/ruby/gems/3.1.0/gems/chef-18.10.17/lib",
+        "chef_effortless": null
+      },
+      "ohai": {
+        "version": "18.2.13",
+        "ohai_root": "/opt/chef/embedded/lib/ruby/gems/3.1.0/gems/ohai-18.2.13/lib/ohai"
+      }
+    }
+  },
+  "run_list": [
+    "role[base]",
+    "role[kvm_guest]",
+    "role[garage_node]"
+  ]
+}

nodes/leo.json

@@ -0,0 +1,56 @@
+{
+  "name": "leo",
+  "normal": {
+    "knife_zero": {
+      "host": "leo.kosmos.org"
+    }
+  },
+  "automatic": {
+    "fqdn": "leo",
+    "os": "linux",
+    "os_version": "5.15.0-164-generic",
+    "hostname": "leo",
+    "ipaddress": "5.9.81.116",
+    "roles": [
+      "base"
+    ],
+    "recipes": [
+      "kosmos-base",
+      "kosmos-base::default",
+      "kosmos_kvm::host",
+      "apt::default",
+      "timezone_iii::default",
+      "timezone_iii::debian",
+      "ntp::default",
+      "ntp::apparmor",
+      "kosmos-base::journald_conf",
+      "kosmos-base::systemd_emails",
+      "apt::unattended-upgrades",
+      "kosmos-base::firewall",
+      "kosmos-postfix::default",
+      "postfix::default",
+      "postfix::_common",
+      "postfix::_attributes",
+      "postfix::sasl_auth",
+      "hostname::default"
+    ],
+    "platform": "ubuntu",
+    "platform_version": "22.04",
+    "cloud": null,
+    "chef_packages": {
+      "chef": {
+        "version": "18.10.17",
+        "chef_root": "/opt/chef/embedded/lib/ruby/gems/3.1.0/gems/chef-18.10.17/lib",
+        "chef_effortless": null
+      },
+      "ohai": {
+        "version": "18.2.13",
+        "ohai_root": "/opt/chef/embedded/lib/ruby/gems/3.1.0/gems/ohai-18.2.13/lib/ohai"
+      }
+    }
+  },
+  "run_list": [
+    "role[base]",
+    "recipe[kosmos_kvm::host]"
+  ]
+}

nodes/postgres-10.json

@@ -0,0 +1,63 @@
+{
+  "name": "postgres-10",
+  "chef_environment": "production",
+  "normal": {
+    "knife_zero": {
+      "host": "10.1.1.176"
+    }
+  },
+  "automatic": {
+    "fqdn": "postgres-10",
+    "os": "linux",
+    "os_version": "5.15.0-1095-kvm",
+    "hostname": "postgres-10",
+    "ipaddress": "192.168.122.41",
+    "roles": [
+      "base",
+      "kvm_guest",
+      "postgresql_replica"
+    ],
+    "recipes": [
+      "kosmos-base",
+      "kosmos-base::default",
+      "kosmos_kvm::guest",
+      "kosmos_postgresql::hostsfile",
+      "kosmos_postgresql::replica",
+      "kosmos_postgresql::firewall",
+      "apt::default",
+      "timezone_iii::default",
+      "timezone_iii::debian",
+      "ntp::default",
+      "ntp::apparmor",
+      "kosmos-base::journald_conf",
+      "kosmos-base::systemd_emails",
+      "apt::unattended-upgrades",
+      "kosmos-base::firewall",
+      "kosmos-postfix::default",
+      "postfix::default",
+      "postfix::_common",
+      "postfix::_attributes",
+      "postfix::sasl_auth",
+      "hostname::default"
+    ],
+    "platform": "ubuntu",
+    "platform_version": "22.04",
+    "cloud": null,
+    "chef_packages": {
+      "chef": {
+        "version": "18.10.17",
+        "chef_root": "/opt/chef/embedded/lib/ruby/gems/3.1.0/gems/chef-18.10.17/lib",
+        "chef_effortless": null
+      },
+      "ohai": {
+        "version": "18.2.13",
+        "ohai_root": "/opt/chef/embedded/lib/ruby/gems/3.1.0/gems/ohai-18.2.13/lib/ohai"
+      }
+    }
+  },
+  "run_list": [
+    "role[base]",
+    "role[kvm_guest]",
+    "role[postgresql_replica]"
+  ]
+}

site-cookbooks/kosmos_garage/templates/nginx_conf_web.erb

@@ -18,6 +18,7 @@ server {
   }
 
   location / {
+    add_header 'Access-Control-Allow-Origin' '*' always;
     proxy_intercept_errors on;
     proxy_cache garage_cache;
     proxy_pass http://garage_web;

site-cookbooks/kosmos_gitea/templates/default/nginx_conf_web.erb

@@ -18,6 +18,8 @@ server {
 
   client_max_body_size 121M;
 
+  proxy_intercept_errors on;
+
   location ~ ^/(avatars|repo-avatars)/.*$ {
     proxy_buffers 1024 8k;
     proxy_pass http://_gitea_web;
@@ -52,5 +54,18 @@ server {
     proxy_set_header X-Real-IP $remote_addr;
     proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
     proxy_set_header X-Forwarded-Proto $scheme;
+    error_page 404 = @slow_404;
+  }
+
+  # Slow down 404 responses to make scraping random URLs less attractive
+  location @slow_404 {
+    internal;
+    default_type text/plain;
+    content_by_lua_block {
+      ngx.sleep(10)
+      ngx.status = 404
+      ngx.say("Not Found")
+      ngx.exit(ngx.HTTP_NOT_FOUND)
+    }
   }
 }