Merge pull request 'Improve BTC price tracking script' (#624) from feature/btc_price_tracker into master

Reviewed-on: #624

nodes/draco.kosmos.org.json

@@ -12,6 +12,7 @@
     },
     "openresty": {
       "listen_ip": "148.251.237.111",
+      "listen_ipv6": "2a01:4f8:202:804a::2",
       "log_formats": {
         "json": "{\"ip\":\"$remote_addr\",\"time\":\"$time_local\",\"host\":\"$host\",\"method\":\"$request_method\",\"uri\":\"$uri\",\"status\":$status,\"size\":$body_bytes_sent,\"referer\":\"$http_referer\",\"upstream_addr\":\"$upstream_addr\",\"upstream_response_time\":\"$upstream_response_time\",\"ua\":\"$http_user_agent\"}"
       }

nodes/gitea-2.json

@@ -50,13 +50,6 @@
       "postfix::sasl_auth",
       "hostname::default",
       "firewall::default",
-      "kosmos_gitea::compile_from_source",
-      "git::default",
-      "git::package",
-      "kosmos-nodejs::default",
-      "nodejs::nodejs_from_package",
-      "nodejs::repo",
-      "golang::default",
       "backup::default",
       "logrotate::default"
     ],

roles/gitea.rb

@@ -8,8 +8,8 @@ run_list %w(
 
 override_attributes(
   "gitea" => {
-    "repo" => "https://github.com/67P/gitea.git",
-    "revision" => "ldap_sync",
+    # "repo" => "https://github.com/67P/gitea.git",
+    # "revision" => "ldap_sync",
     "log" => { "level" => "Info" }
   },
 )

site-cookbooks/discourse/templates/nginx_conf.erb

@@ -8,8 +8,8 @@ upstream _<%= @upstream_name %> {
 <% if File.exist?(@ssl_cert) && File.exist?(@ssl_key) -%>
 server {
   server_name <%= @server_name %>;
-  listen 443 ssl http2;
-  listen [::]:443 ssl http2;
+  listen <%= "#{node['openresty']['listen_ip']}:" if node['openresty']['listen_ip'] %>443 ssl http2;
+  listen <%= "[#{node['openresty']['listen_ipv6']}]" %>:443 ssl http2;
 
   ssl_certificate     <%= @ssl_cert %>;
   ssl_certificate_key <%= @ssl_key %>;

site-cookbooks/kosmos-akkounts/templates/nginx_conf_akkounts.erb

@@ -11,7 +11,7 @@ proxy_cache_path <%= node[:openresty][:cache_dir] %>/akkounts levels=1:2
 
 server {
   listen <%= "#{node['openresty']['listen_ip']}:" if node['openresty']['listen_ip'] %>443 ssl http2;
-  listen [::]:443 ssl http2;
+  listen <%= "[#{node['openresty']['listen_ipv6']}]" %>:443 ssl http2;
   server_name <%= @domain %>;
 
   if ($host != $server_name) {

site-cookbooks/kosmos-akkounts/templates/nginx_conf_akkounts_api.erb

@@ -7,7 +7,7 @@ upstream _akkounts_api {
 
 server {
   listen <%= "#{node['openresty']['listen_ip']}:" if node['openresty']['listen_ip'] %>443 ssl http2;
-  listen [::]:443 ssl http2;
+  listen <%= "[#{node['openresty']['listen_ipv6']}]" %>:443 ssl http2;
   server_name <%= @domain %>;
 
   ssl_certificate     <%= @ssl_cert %>;

site-cookbooks/kosmos-bitcoin/templates/btc-price-tracker-daily.sh.erb

@@ -1,49 +1,86 @@
 #!/bin/bash
+set -e
+set -o pipefail
 
 # Calculate yesterday's date in YYYY-MM-DD format
 YESTERDAY=$(date -d "yesterday" +%Y-%m-%d)
 echo "Starting price tracking for $YESTERDAY" >&2
 
+# Helper function to perform HTTP requests with retries
+# Usage: make_request <retries> <method> <url> [data] [header1] [header2] ...
+make_request() {
+    local retries=$1
+    local method=$2
+    local url=$3
+    local data=$4
+    shift 4
+    local headers=("$@")
+
+    local count=0
+    local wait_time=3
+    local response
+
+    while [ "$count" -lt "$retries" ]; do
+        local curl_opts=(-s -S -f -X "$method")
+
+        if [ -n "$data" ]; then
+            curl_opts+=(-d "$data")
+        fi
+
+        for h in "${headers[@]}"; do
+            curl_opts+=(-H "$h")
+        done
+
+        if response=$(curl "${curl_opts[@]}" "$url"); then
+            echo "$response"
+            return 0
+        fi
+
+        echo "Request to $url failed (Attempt $((count+1))/$retries). Retrying in ${wait_time}s..." >&2
+        sleep "$wait_time"
+        count=$((count + 1))
+    done
+
+    echo "ERROR: Request to $url failed after $retries attempts" >&2
+    return 1
+}
+
 # Fetch and process rates for a fiat currency
 get_price_data() {
     local currency=$1
     local data avg open24 last
 
-    data=$(curl -s "https://www.bitstamp.net/api/v2/ticker/btc${currency,,}/")
-    if [ $? -eq 0 ] && [ ! -z "$data" ]; then
+    if data=$(make_request 3 "GET" "https://www.bitstamp.net/api/v2/ticker/btc${currency,,}/" ""); then
         echo "Successfully retrieved ${currency} price data" >&2
         open24=$(echo "$data" | jq -r '.open_24')
         last=$(echo "$data" | jq -r '.last')
-        avg=$(( (${open24%.*} + ${last%.*}) / 2 ))
+        avg=$(echo "$open24 $last" | awk '{printf "%.0f", ($1 + $2) / 2}')
         echo $avg
     else
         echo "ERROR: Failed to retrieve ${currency} price data" >&2
-        exit 1
+        return 1
     fi
 }
 
 # Get price data for each currency
-usd_avg=$(get_price_data "USD")
-eur_avg=$(get_price_data "EUR")
-gbp_avg=$(get_price_data "GBP")
+usd_avg=$(get_price_data "USD") || exit 1
+eur_avg=$(get_price_data "EUR") || exit 1
+gbp_avg=$(get_price_data "GBP") || exit 1
 
 # Create JSON
-json="{\"EUR\":$eur_avg,\"USD\":$usd_avg,\"GBP\":$gbp_avg}"
+json=$(jq -n \
+  --argjson eur "$eur_avg" \
+  --argjson usd "$usd_avg" \
+  --argjson gbp "$gbp_avg" \
+  '{"EUR": $eur, "USD": $usd, "GBP": $gbp}')
 echo "Rates: $json" >&2
 
 # PUT in remote storage
-response=$(curl -X PUT \
-    -H "Authorization: Bearer $RS_AUTH" \
-    -H "Content-Type: application/json" \
-    -d "$json" \
-    -w "%{http_code}" \
-    -s \
-    -o /dev/null \
-    "<%= @rs_base_url %>/$YESTERDAY")
-
-if [ "$response" -eq 200 ] || [ "$response" -eq 201 ]; then
+if make_request 3 "PUT" "<%= @rs_base_url %>/$YESTERDAY" "$json" \
+    "Authorization: Bearer $RS_AUTH" \
+    "Content-Type: application/json" > /dev/null; then
    echo "Successfully uploaded price data" >&2
 else
-   echo "ERROR: Failed to upload price data. HTTP status: $response" >&2
+   echo "ERROR: Failed to upload price data" >&2
    exit 1
 fi

site-cookbooks/kosmos-bitcoin/templates/nginx_conf_btcpayserver.erb

@@ -49,7 +49,7 @@ server {
   client_max_body_size 100M;
   server_name <%= @server_name %>;
   listen 443 ssl http2;
-  listen [::]:443 ssl http2;
+  listen <%= "[#{node['openresty']['listen_ipv6']}]" %>:443 ssl http2;
 
   access_log <%= node[:nginx][:log_dir] %>/btcpayserver.access.log json;
   error_log <%= node[:nginx][:log_dir] %>/btcpayserver.error.log warn;

site-cookbooks/kosmos-bitcoin/templates/nginx_conf_lndhub.erb

@@ -7,7 +7,7 @@ upstream _lndhub {
 
 server {
   listen <%= "#{node['openresty']['listen_ip']}:" if node['openresty']['listen_ip'] %>443 ssl http2;
-  listen [::]:443 ssl http2;
+  listen <%= "[#{node['openresty']['listen_ipv6']}]" %>:443 ssl http2;
   server_name <%= @server_name %>;
 
   add_header Strict-Transport-Security "max-age=15768000";

site-cookbooks/kosmos-btcpayserver/templates/nginx_conf_btcpayserver.erb

@@ -49,7 +49,7 @@ server {
   server_name <%= @server_name %>;
   <% if File.exist?(@ssl_cert) && File.exist?(@ssl_key) -%>
   listen 443 ssl http2;
-  listen [::]:443 ssl http2;
+  listen <%= "[#{node['openresty']['listen_ipv6']}]" %>:443 ssl http2;
   <% else -%>
   listen 80;
   <% end -%>

site-cookbooks/kosmos-ejabberd/templates/nginx_conf_upload_service.erb

@@ -3,7 +3,7 @@
 
 server {
   listen 443 ssl http2;
-  listen [::]:443 ssl http2;
+  listen <%= "[#{node['openresty']['listen_ipv6']}]" %>:443 ssl http2;
   server_name <%= @server_name %>;
 
   ssl_certificate     <%= @ssl_cert %>;

site-cookbooks/kosmos-hubot/templates/default/nginx_conf_hubot.erb

@@ -7,7 +7,7 @@ upstream _express_<%= @server_name.gsub(".", "_") %> {
 
 server {
   listen <%= "#{node['openresty']['listen_ip']}:" if node['openresty']['listen_ip'] %>443 ssl http2;
-  listen [::]:443 ssl http2;
+  listen <%= "[#{node['openresty']['listen_ipv6']}]" %>:443 ssl http2;
   server_name <%= @server_name %>;
 
   add_header Strict-Transport-Security "max-age=15768000";

site-cookbooks/kosmos-ipfs/templates/default/nginx_conf_ipfs.kosmos.org.erb

@@ -12,7 +12,7 @@ upstream _ipfs_api {
 server {
   server_name <%= @server_name %>;
   listen <%= "#{node['openresty']['listen_ip']}:" if node['openresty']['listen_ip'] %>443 ssl http2;
-  listen [::]:443 ssl http2;
+  listen <%= "[#{node['openresty']['listen_ipv6']}]" %>:443 ssl http2;
 
   access_log /var/log/nginx/<%= @server_name %>.access.log;
   error_log  /var/log/nginx/<%= @server_name %>.error.log;

site-cookbooks/kosmos-mastodon/templates/default/nginx_conf_mastodon.erb

@@ -21,7 +21,7 @@ proxy_cache_path /var/cache/nginx/mastodon levels=1:2
 
 server {
   listen <%= "#{node['openresty']['listen_ip']}:" if node['openresty']['listen_ip'] %>443 ssl http2;
-  listen [::]:443 ssl http2;
+  listen <%= "[#{node['openresty']['listen_ipv6']}]" %>:443 ssl http2;
   server_name <%= @server_name %>;
   include <%= @shared_config_path %>;
 

site-cookbooks/kosmos_assets/templates/nginx_conf_assets.erb

@@ -3,7 +3,7 @@
 
 server {
   listen <%= "#{node['openresty']['listen_ip']}:" if node['openresty']['listen_ip'] %>443 ssl http2;
-  listen [::]:443 ssl http2;
+  listen <%= "[#{node['openresty']['listen_ipv6']}]" %>:443 ssl http2;
   server_name <%= @domain %>;
 
   root /var/www/<%= @domain %>/site;

site-cookbooks/kosmos_discourse/templates/nginx_conf.erb

@@ -9,7 +9,7 @@ upstream _discourse {
 server {
   server_name <%= @server_name %>;
   listen <%= "#{node['openresty']['listen_ip']}:" if node['openresty']['listen_ip'] %>443 ssl http2;
-  listen [::]:443 ssl http2;
+  listen <%= "[#{node['openresty']['listen_ipv6']}]" %>:443 ssl http2;
 
   ssl_certificate     <%= @ssl_cert %>;
   ssl_certificate_key <%= @ssl_key %>;

site-cookbooks/kosmos_drone/templates/nginx_conf.erb

@@ -8,7 +8,7 @@ upstream _drone {
 server {
   server_name <%= @server_name %>;
   listen <%= "#{node['openresty']['listen_ip']}:" if node['openresty']['listen_ip'] %>443 ssl http2;
-  listen [::]:443 ssl http2;
+  listen <%= "[#{node['openresty']['listen_ipv6']}]" %>:443 ssl http2;
 
   ssl_certificate     <%= @ssl_cert %>;
   ssl_certificate_key <%= @ssl_key %>;

site-cookbooks/kosmos_garage/templates/nginx_conf_s3.erb

@@ -4,7 +4,7 @@ upstream garage_s3 {
 
 server {
   listen <%= "#{node[:openresty][:listen_ip]}:" if node[:openresty][:listen_ip] %>443 ssl http2;
-  listen [::]:443 http2 ssl;
+  listen <%= "[#{node['openresty']['listen_ipv6']}]" %>:443 ssl http2;
 
   ssl_certificate     <%= @ssl_cert %>;
   ssl_certificate_key <%= @ssl_key %>;

site-cookbooks/kosmos_garage/templates/nginx_conf_web.erb

@@ -1,6 +1,6 @@
 server {
   listen <%= "#{node[:openresty][:listen_ip]}:" if node[:openresty][:listen_ip] %>443 ssl http2;
-  listen [::]:443 http2 ssl;
+  listen <%= "[#{node['openresty']['listen_ipv6']}]" %>:443 ssl http2;
 
   server_name <%= @server_name %>;
 

site-cookbooks/kosmos_gitea/attributes/default.rb

@@ -1,5 +1,5 @@
-node.default["gitea"]["version"] = "1.23.8"
-node.default["gitea"]["checksum"] = "827037e7ca940866918abc62a7488736923396c467fcb4acd0dd9829bb6a6f4c"
+node.default["gitea"]["version"] = "1.25.4"
+node.default["gitea"]["checksum"] = "a3031853e67c53714728ef705642c9046a11fb0ea356aff592e23efe6114607d"
 node.default["gitea"]["repo"] = nil
 node.default["gitea"]["revision"] = nil
 node.default["gitea"]["working_directory"] = "/var/lib/gitea"
@@ -22,5 +22,5 @@ node.default["gitea"]["config"] = {
   }
 }
 
-node.default["gitea"]["act_runner"]["version"] = "0.2.6"
-node.default["gitea"]["act_runner"]["checksum"] = "234c2bdb871e7b0bfb84697f353395bfc7819faf9f0c0443845868b64a041057"
+node.default["gitea"]["act_runner"]["version"] = "0.2.13"
+node.default["gitea"]["act_runner"]["checksum"] = "3acac8b506ac8cadc88a55155b5d6378f0fab0b8f62d1e0c0450f4ccd69733e2"

site-cookbooks/kosmos_gitea/recipes/default.rb

@@ -129,7 +129,7 @@ template "/etc/systemd/system/gitea.service" do
             git_home_directory: git_home_directory,
             config_directory: config_directory,
             gitea_binary_path: gitea_binary_path
-  notifies :run, "execute[systemctl daemon-reload]", :delayed
+  notifies :run, "execute[systemctl daemon-reload]", :immediately
 end
 
 service "gitea" do

site-cookbooks/kosmos_gitea/templates/default/nginx_conf_ssh.erb

@@ -4,5 +4,6 @@ upstream _gitea_ssh {
 
 server {
   listen <%= "#{node['openresty']['listen_ip']}:" if node['openresty']['listen_ip'] %>22;
+  listen [::]:22;
   proxy_pass _gitea_ssh;
 }

site-cookbooks/kosmos_gitea/templates/default/nginx_conf_web.erb

@@ -6,7 +6,7 @@ upstream _gitea_web {
 server {
   server_name <%= @server_name %>;
   listen <%= "#{node['openresty']['listen_ip']}:" if node['openresty']['listen_ip'] %>443 ssl http2;
-  listen [::]:443 ssl http2;
+  listen <%= "[#{node['openresty']['listen_ipv6']}]" %>:443 ssl http2;
 
   ssl_certificate     <%= @ssl_cert %>;
   ssl_certificate_key <%= @ssl_key %>;

site-cookbooks/kosmos_liquor-cabinet/templates/nginx_conf_liquor-cabinet.erb

@@ -12,7 +12,7 @@ upstream _<%= @app_name %> {
 
 server {
   listen <%= "#{node['openresty']['listen_ip']}:" if node['openresty']['listen_ip'] %>443 ssl http2;
-  listen [::]:443 ssl http2;
+  listen <%= "[#{node['openresty']['listen_ipv6']}]" %>:443 ssl http2;
   server_name <%= @server_name %>;
 
   access_log <%= node[:nginx][:log_dir] %>/<%= @app_name %>.access.log; # TODO json_liquor_cabinet;

site-cookbooks/kosmos_openresty/attributes/default.rb

@@ -0,0 +1 @@
+node.default["openresty"]["listen_ipv6"] = "::"

site-cookbooks/kosmos_rsk/templates/nginx_conf_rskj.erb

@@ -6,7 +6,7 @@ upstream _<%= @upstream_name %> {
 
 server {
   listen <%= "#{node['openresty']['listen_ip']}:" if node['openresty']['listen_ip'] %>443 ssl http2;
-  listen [::]:443 ssl http2;
+  listen <%= "[#{node['openresty']['listen_ipv6']}]" %>:443 ssl http2;
 
   server_name <%= @domain %>;
 

site-cookbooks/kosmos_strfry/templates/nginx_conf_strfry.erb

@@ -13,7 +13,7 @@ upstream _substr {
 server {
   server_name <%= @domain %>;
   listen <%= "#{node['openresty']['listen_ip']}:" if node['openresty']['listen_ip'] %>443 ssl http2;
-  listen [::]:443 ssl http2;
+  listen <%= "[#{node['openresty']['listen_ipv6']}]" %>:443 ssl http2;
 
   access_log "/var/log/nginx/<%= @domain %>.access.log";
   error_log "/var/log/nginx/<%= @domain %>.error.log";

site-cookbooks/kosmos_website/templates/nginx_conf_redirect.erb

@@ -3,7 +3,7 @@
 server {
   server_name <%= @domain %>;
   listen <%= "#{node['openresty']['listen_ip']}:" if node['openresty']['listen_ip'] %>443 ssl http2;
-  listen [::]:443 ssl http2;
+  listen <%= "[#{node['openresty']['listen_ipv6']}]" %>:443 ssl http2;
 
   access_log <%= node[:openresty][:log_dir] %>/<%= @domain %>.access.log;
   error_log <%= node[:openresty][:log_dir] %>/<%= @domain %>.error.log warn;

site-cookbooks/kosmos_website/templates/nginx_conf_simple.erb

@@ -3,7 +3,7 @@
 server {
   server_name <%= @domain %>;
   listen <%= "#{node['openresty']['listen_ip']}:" if node['openresty']['listen_ip'] %>443 ssl http2;
-  listen [::]:443 ssl http2;
+  listen <%= "[#{node['openresty']['listen_ipv6']}]" %>:443 ssl http2;
 
   root /var/www/<%= @domain %>/public;
 

site-cookbooks/kosmos_website/templates/nginx_conf_website.erb

@@ -3,6 +3,7 @@
 server {
   server_name _;
   listen <%= "#{node['openresty']['listen_ip']}:" if node['openresty']['listen_ip'] %>80 default_server;
+  listen <%= "[#{node['openresty']['listen_ipv6']}]" %>:80 default_server;
 
   location / {
     return 301 https://<%= @domain %>;
@@ -12,7 +13,7 @@ server {
 server {
   server_name <%= @domain %>;
   listen <%= "#{node['openresty']['listen_ip']}:" if node['openresty']['listen_ip'] %>443 ssl http2 default_server;
-  listen [::]:443 ssl http2 default_server;
+  listen <%= "[#{node['openresty']['listen_ipv6']}]" %>:443 ssl http2 default_server;
 
   if ($host != $server_name) {
     return 307 $scheme://$server_name;

site-cookbooks/kredits-github/templates/default/nginx_conf.erb

@@ -5,8 +5,8 @@ upstream _<%= @app_name %> {
 
 <% if File.exist?(@ssl_cert) && File.exist?(@ssl_key) -%>
 server {
-  listen 443 ssl http2;
-  listen [::]:443 ssl http2;
+  listen <%= "#{node['openresty']['listen_ip']}:" if node['openresty']['listen_ip'] %>443 ssl http2;
+  listen <%= "[#{node['openresty']['listen_ipv6']}]" %>:443 ssl http2;
   server_name <%= @server_name %>;
 
   access_log <%= node[:nginx][:log_dir] %>/<%= @app_name %>.access.log json;

site-cookbooks/remotestorage_discourse/templates/nginx_conf.erb

@@ -8,7 +8,7 @@ upstream _rs_discourse {
 server {
   server_name <%= @server_name %>;
   listen <%= "#{node['openresty']['listen_ip']}:" if node['openresty']['listen_ip'] %>443 ssl http2;
-  listen [::]:443 ssl http2;
+  listen <%= "[#{node['openresty']['listen_ipv6']}]" %>:443 ssl http2;
 
   ssl_certificate     <%= @ssl_cert %>;
   ssl_certificate_key <%= @ssl_key %>;