From 0d9c580a0ac0f4439e752dbfa23982c56ba7d50f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A2u=20Cao?= Date: Tue, 6 Oct 2026 15:09:32 +0200 Subject: [PATCH 1/4] Automate first-run LDAP and database setup The documented flow started web (with Solid Queue in Puma) before the databases existed, and required manually creating the 389ds back-end and seeding the LDAP directory and databases. Automate it: - add an ldap-init one-shot service that creates the 389ds back-end if it does not exist - add a web entrypoint that seeds LDAP and the databases on first start, and runs db:prepare on later boots - update README and AGENTS accordingly, including the reset steps --- AGENTS.md | 9 ++++++--- README.md | 29 +++++++++++++++++------------ docker-compose.yml | 24 +++++++++++++++++++++--- docker/ldap-init.sh | 14 ++++++++++++++ docker/web-entrypoint.sh | 14 ++++++++++++++ 5 files changed, 72 insertions(+), 18 deletions(-) create mode 100644 docker/ldap-init.sh create mode 100644 docker/web-entrypoint.sh diff --git a/AGENTS.md b/AGENTS.md index 13e4939..6fe3e30 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -11,9 +11,12 @@ Start services: `docker compose up` (web, ldap, redis, minio, liquor-cabinet, st The `web` service runs `bin/dev` (foreman: Puma + Tailwind CSS watcher) and embeds Solid Queue workers (`SOLID_QUEUE_IN_PUMA=true`). -First-time LDAP setup (after creating the 389ds backend once): -`docker compose exec ldap dsconf localhost backend create --suffix="dc=kosmos,dc=org" --be_name="dev"` -then `docker compose run web bin/rails ldap:setup`. +First-time LDAP and database setup is automated: the `ldap-init` service +creates the 389ds back-end, then the `web` entrypoint seeds LDAP and the +databases on first start and runs `db:prepare` on every boot. Manual +equivalents: `docker compose exec ldap dsconf localhost backend create +--suffix="dc=kosmos,dc=org" --be-name="dev"` and `docker compose run --rm web +bin/rails ldap:setup`. ## Common commands (prefix with `docker compose exec web`) diff --git a/README.md b/README.md index e20a535..6739b35 100644 --- a/README.md +++ b/README.md @@ -14,13 +14,14 @@ so: 1. Make sure [Docker Compose is installed][1] and Docker is running (included in Docker Desktop) -3. Run `docker compose up --build` and wait until all services have started +2. Run `docker compose up --build` and wait until all services have started (389ds might take an extra minute to be ready). This will take a while when running for the first time, so you might want to do something else in the meantime. -4. `docker-compose exec ldap dsconf localhost backend create --suffix="dc=kosmos,dc=org" --be-name="dev"` -5. `docker compose run web rails ldap:setup` -6. `docker compose run web rails db:setup` + +On the first start, the `ldap-init` service creates the 389ds back-end, and the +`web` container then seeds the LDAP directory and the databases automatically. +On every start, `web` also applies any pending database migrations. After these steps, you should have a working Rails app with a handful of test users running on [http://localhost:3000](http://localhost:3000). @@ -71,15 +72,12 @@ containers you want to run to the `up` command, like so: #### LDAP server -After creating the Docker container for the first time (or after deleting it), -you need to run the following command once, in order to create the dirsrv -back-end: +On first start, the `ldap-init` service creates the dirsrv back-end +automatically, and the `web` container then seeds it with development entries. +To do either step manually (for example, after changing the setup), run: - docker-compose exec ldap dsconf localhost backend create --suffix="dc=kosmos,dc=org" --be-name="dev" - -Now you can seed the back-end with data using this Rails task: - - bundle exec rails ldap:setup + docker compose exec ldap dsconf localhost backend create --suffix="dc=kosmos,dc=org" --be-name="dev" + docker compose run --rm web bin/rails ldap:setup The setup task will first delete any existing entries in the directory tree ("dc=kosmos,dc=org"), and then create our development entries. @@ -88,6 +86,13 @@ Note that all 389ds data is stored in the `389ds-data` volume. So if you want to start over with a fresh installation, delete both that volume as well as the container. +To reset the development environment completely, remove all volumes plus the +generated database files and the first-run marker, then start over: + + docker compose down -v + rm -f db/*.sqlite3 tmp/.setup-complete + docker compose up --build + #### Minio / remoteStorage If you want to run remoteStorage accounts locally, you will have to create the diff --git a/docker-compose.yml b/docker-compose.yml index 61cd458..467cc45 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -12,6 +12,20 @@ services: DS_DM_PASSWORD: passthebutter SUFFIX_NAME: "dc=kosmos,dc=org" + ldap-init: + image: 4teamwork/389ds:latest + networks: + - internal_network + volumes: + - ./docker/ldap-init.sh:/ldap-init.sh:ro + environment: + LDAP_ADMIN_PASSWORD: passthebutter + LDAP_SUFFIX: "dc=kosmos,dc=org" + depends_on: + ldap: + condition: service_healthy + command: ["/bin/sh", "/ldap-init.sh"] + redis: restart: always image: redis:7-alpine @@ -26,7 +40,7 @@ services: web: build: . tty: true - command: bash -c "rm -f /akkounts/tmp/pids/server.pid; bin/dev" + command: ["bash", "docker/web-entrypoint.sh"] volumes: - .:/akkounts - /akkounts/node_modules @@ -57,8 +71,12 @@ services: NOSTR_PRIVATE_KEY: 7c3ef7e448505f0615137af38569d01807d3b05b5005d5ecf8aaafcd40323cea NOSTR_RELAY_URL: ws://strfry:7777 depends_on: - - ldap - - redis + ldap: + condition: service_started + ldap-init: + condition: service_completed_successfully + redis: + condition: service_started minio: image: quay.io/minio/minio:latest diff --git a/docker/ldap-init.sh b/docker/ldap-init.sh new file mode 100644 index 0000000..fc4ccc1 --- /dev/null +++ b/docker/ldap-init.sh @@ -0,0 +1,14 @@ +#!/bin/sh +set -e + +SUFFIX="${LDAP_SUFFIX:-dc=kosmos,dc=org}" +URI="ldap://ldap:3389" + +if dsconf -D "cn=Directory Manager" -w "$LDAP_ADMIN_PASSWORD" "$URI" \ + backend suffix list --suffix 2>/dev/null | grep -Fqx "$SUFFIX"; then + echo "LDAP backend for $SUFFIX already exists, skipping." +else + echo "Creating LDAP backend for $SUFFIX..." + dsconf -D "cn=Directory Manager" -w "$LDAP_ADMIN_PASSWORD" "$URI" \ + backend create --suffix "$SUFFIX" --be-name dev +fi diff --git a/docker/web-entrypoint.sh b/docker/web-entrypoint.sh new file mode 100644 index 0000000..2001ecb --- /dev/null +++ b/docker/web-entrypoint.sh @@ -0,0 +1,14 @@ +#!/usr/bin/env bash +set -e + +if [ ! -f tmp/.setup-complete ]; then + echo "First start: setting up LDAP entries and databases..." + bin/rails ldap:setup + bin/rails db:setup + touch tmp/.setup-complete +else + bin/rails db:prepare +fi + +rm -f tmp/pids/server.pid +exec bin/dev -- 2.50.1 From 6279699c11ddb539bf8980b5798380358a6254bb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A2u=20Cao?= Date: Tue, 6 Oct 2026 15:09:45 +0200 Subject: [PATCH 2/4] Switch dev object storage from MinIO to Garage MinIO no longer publishes prebuilt community images (quay.io returns 401, Docker Hub 404), so the dev setup cannot pull the image. Use a single-node Garage instance instead, which auto-creates the remotestorage bucket and a dev-key1 access key on first start, and point liquor-cabinet at it. --- AGENTS.md | 2 +- README.md | 26 ++++++++++++++------------ docker-compose.yml | 27 ++++++++++++++++----------- docker/garage/garage.toml | 14 ++++++++++++++ 4 files changed, 45 insertions(+), 24 deletions(-) create mode 100644 docker/garage/garage.toml diff --git a/AGENTS.md b/AGENTS.md index 6fe3e30..6bf026e 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -7,7 +7,7 @@ ejabberd, Discourse, Mastodon, remoteStorage, Nostr, LNDHub, and BTCPay. ## Development environment Development runs in Docker Compose — run all commands against the `web` container. -Start services: `docker compose up` (web, ldap, redis, minio, liquor-cabinet, strfry). +Start services: `docker compose up` (web, ldap, redis, garage, liquor-cabinet, strfry). The `web` service runs `bin/dev` (foreman: Puma + Tailwind CSS watcher) and embeds Solid Queue workers (`SOLID_QUEUE_IN_PUMA=true`). diff --git a/README.md b/README.md index 6739b35..5753f69 100644 --- a/README.md +++ b/README.md @@ -93,20 +93,22 @@ generated database files and the first-run marker, then start over: rm -f db/*.sqlite3 tmp/.setup-complete docker compose up --build -#### Minio / remoteStorage +#### Garage / remoteStorage -If you want to run remoteStorage accounts locally, you will have to create the -respective bucket first. With the `minio` container running (run by default -when using Docker Compose), follow these steps: +remoteStorage accounts use the `garage` S3-compatible object store. On first +start, Garage automatically configures a single-node cluster and creates the +`remotestorage` bucket together with a `dev-key1` access key (secret +`1234567890123456`), so no manual setup is required. -* `docker compose up web redis minio liquor-cabinet` -* Head to http://localhost:9001 and log in with user `minioadmin`, password - `minioadmin` -* Create a new bucket called `remotestorage` (or whatever you - change the `S3_BUCKET` config to) -* Create a new key with ID "dev-key" and secret "123456789" (or whatever you - change `S3_ACCESS_KEY` and `S3_SECRET_KEY` to). Leave the policy field empty, - as it will automatically allow access to the bucket you created. +If you want to run remoteStorage accounts locally, the `garage` container is +started by default when using Docker Compose. To run just the remoteStorage +stack: + +* `docker compose up web redis garage liquor-cabinet` + +The S3 API is available at http://localhost:3900 (region `garage`). If you want +to start over with a fresh storage, delete the `garage-data` volume as well as +the container. ### Adding npm modules to use with Stimulus controllers diff --git a/docker-compose.yml b/docker-compose.yml index 467cc45..b5e56c0 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -78,17 +78,21 @@ services: redis: condition: service_started - minio: - image: quay.io/minio/minio:latest - command: "server /data --console-address ':9001'" + garage: + image: dxflrs/garage:v2.4.1 + command: ["/garage", "server", "--single-node", "--default-bucket"] networks: - external_network - internal_network ports: - - "9000:9000" - - "9001:9001" + - "3900:3900" volumes: - - minio-data:/data + - ./docker/garage/garage.toml:/etc/garage.toml:ro + - garage-data:/var/lib/garage + environment: + GARAGE_DEFAULT_ACCESS_KEY: dev-key1 + GARAGE_DEFAULT_SECRET_KEY: "1234567890123456" + GARAGE_DEFAULT_BUCKET: remotestorage liquor-cabinet: image: gitea.kosmos.org/5apps/liquor-cabinet:2.0.0-rc.1 @@ -102,12 +106,13 @@ services: REDIS_HOST: redis REDIS_PORT: 6379 REDIS_DB: 1 - S3_ENDPOINT: http://minio:9000 - S3_ACCESS_KEY: dev-key - S3_SECRET_KEY: 123456789 + S3_ENDPOINT: http://garage:3900 + S3_REGION: garage + S3_ACCESS_KEY: dev-key1 + S3_SECRET_KEY: "1234567890123456" S3_BUCKET: remotestorage depends_on: - - minio + - garage - redis strfry: @@ -145,7 +150,7 @@ networks: volumes: 389ds-data: driver: local - minio-data: + garage-data: driver: local redis-data: driver: local diff --git a/docker/garage/garage.toml b/docker/garage/garage.toml new file mode 100644 index 0000000..9f6aadd --- /dev/null +++ b/docker/garage/garage.toml @@ -0,0 +1,14 @@ +metadata_dir = "/var/lib/garage/meta" +data_dir = "/var/lib/garage/data" +db_engine = "sqlite" + +replication_factor = 1 + +rpc_bind_addr = "[::]:3901" +rpc_public_addr = "127.0.0.1:3901" +rpc_secret = "1799bccfd7411eddcf9ebd316bc1f5287ad12a68094e1c6ac6abde7e6feae1ec" + +[s3_api] +s3_region = "garage" +api_bind_addr = "[::]:3900" +root_domain = ".s3.garage.localhost" -- 2.50.1 From bc37df52a079e76c2500b0dc3541742f49d57501 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A2u=20Cao?= Date: Tue, 6 Oct 2026 19:34:24 +0200 Subject: [PATCH 3/4] Allow remote access to dev services --- README.md | 13 +++++++++++++ .../services/rs_auths_controller.rb | 9 ++++++--- app/controllers/webfinger_controller.rb | 18 ++++++++++++++++-- 3 files changed, 35 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index 5753f69..0888d00 100644 --- a/README.md +++ b/README.md @@ -110,6 +110,19 @@ The S3 API is available at http://localhost:3900 (region `garage`). If you want to start over with a fresh storage, delete the `garage-data` volume as well as the container. +#### Accessing remoteStorage from another machine + +remoteStorage clients force HTTPS for any host except `localhost`, and browsers +block plain-HTTP requests to a LAN IP as mixed content. To connect to the dev +remoteStorage from a browser on another machine (including production apps such +as Inspektor), forward the ports over SSH and connect as `localhost`: + + ssh -N -L 3000:localhost:3000 -L 4567:localhost:4567 @ + +Then use `@localhost:3000` as the remoteStorage address in the client. +WeFinger discovery and storage requests are served through the forwarded ports, +so no TLS setup is needed. + ### Adding npm modules to use with Stimulus controllers The following command downloads the specified npm module to `vendor/javascript` diff --git a/app/controllers/services/rs_auths_controller.rb b/app/controllers/services/rs_auths_controller.rb index f9376d8..2fff161 100644 --- a/app/controllers/services/rs_auths_controller.rb +++ b/app/controllers/services/rs_auths_controller.rb @@ -22,9 +22,12 @@ class Services::RsAuthsController < Services::BaseController end def launch_app - user_address = Rails.env.development? ? - "#{current_user.cn}@localhost:3000" : - current_user.address + user_address = + if Rails.env.development? + "#{current_user.cn}@#{request.host_with_port}" + else + current_user.address + end launch_url = "#{@auth.launch_url}#remotestorage=#{user_address}" diff --git a/app/controllers/webfinger_controller.rb b/app/controllers/webfinger_controller.rb index a8e00f7..175fe5a 100644 --- a/app/controllers/webfinger_controller.rb +++ b/app/controllers/webfinger_controller.rb @@ -88,8 +88,14 @@ class WebfingerController < WellKnownController end def remotestorage_link - auth_url = new_rs_oauth_url(@username, host: Setting.rs_accounts_domain) - storage_url = "#{Setting.rs_storage_url}/#{@username}" + auth_url = + if Rails.env.development? + new_rs_oauth_url(@username) + else + new_rs_oauth_url(@username, host: Setting.rs_accounts_domain) + end + + storage_url = "#{remotestorage_storage_base_url}/#{@username}" { rel: "http://tools.ietf.org/id/draft-dejong-remotestorage", @@ -103,4 +109,12 @@ class WebfingerController < WellKnownController } } end + + def remotestorage_storage_base_url + return Setting.rs_storage_url unless Rails.env.development? + + uri = URI.parse(Setting.rs_storage_url) + uri.host = request.host + uri.to_s + end end -- 2.50.1 From 695977172ff95865f7b78f31bd28acbef38a8903 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A2u=20Cao?= Date: Tue, 6 Oct 2026 19:36:26 +0200 Subject: [PATCH 4/4] Fix Liquor Cabinet performance in dev Can't handle enough requests without ThreadPool, so once you hit the limit, requests take 5s+ (having to wait for the keepalive timeout) --- docker-compose.yml | 3 +++ docker/liquor-cabinet/rainbows.conf.rb | 5 +++++ 2 files changed, 8 insertions(+) create mode 100644 docker/liquor-cabinet/rainbows.conf.rb diff --git a/docker-compose.yml b/docker-compose.yml index b5e56c0..42a398a 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -101,6 +101,9 @@ services: - internal_network ports: - "4567:4567" + volumes: + - ./docker/liquor-cabinet/rainbows.conf.rb:/etc/liquor-cabinet/rainbows.conf.rb:ro + command: ["bundle", "exec", "rainbows", "-c", "/etc/liquor-cabinet/rainbows.conf.rb", "--listen", "0.0.0.0:4567"] environment: RACK_ENV: staging REDIS_HOST: redis diff --git a/docker/liquor-cabinet/rainbows.conf.rb b/docker/liquor-cabinet/rainbows.conf.rb new file mode 100644 index 0000000..5815c40 --- /dev/null +++ b/docker/liquor-cabinet/rainbows.conf.rb @@ -0,0 +1,5 @@ +Rainbows! do + use :ThreadPool + worker_connections 16 + client_max_body_size 100 * 1024 * 1024 +end -- 2.50.1