From 0cad4cdcfe1b296626b58937a740a0f133c71d47 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A2u=20Cao?= Date: Sun, 4 May 2025 17:40:33 +0400 Subject: [PATCH 1/7] WIP Switch from Sidekiq to Solid Queue --- Gemfile | 6 +- Gemfile.lock | 23 +++- README.md | 2 +- bin/jobs | 6 + config/application.rb | 3 +- config/database.yml | 4 + config/environments/development.rb | 3 + config/environments/production.rb | 5 +- config/initializers/devise_rails8_patch.rb | 11 ++ config/queue.yml | 21 ++++ config/recurring.yml | 10 ++ config/routes.rb | 2 +- db/queue_schema.rb | 129 +++++++++++++++++++++ docker-compose.yml | 27 +---- 14 files changed, 218 insertions(+), 34 deletions(-) create mode 100755 bin/jobs create mode 100644 config/initializers/devise_rails8_patch.rb create mode 100644 config/queue.yml create mode 100644 config/recurring.yml create mode 100644 db/queue_schema.rb diff --git a/Gemfile b/Gemfile index 3df7884..cd2e1b7 100644 --- a/Gemfile +++ b/Gemfile @@ -4,7 +4,7 @@ git_source(:github) { |repo| "https://github.com/#{repo}.git" } # Bundle edge Rails instead: gem 'rails', github: 'rails/rails' gem 'rails', '~> 8.0' # Use Puma as the app server -gem 'puma', '~> 4.1' +gem 'puma', '~> 6.6' # View components gem "view_component" # Asset bundler @@ -53,8 +53,8 @@ gem 'down' gem 'aws-sdk-s3', require: false # Background/scheduled jobs -gem 'sidekiq', '< 7' -gem 'sidekiq-scheduler' +gem 'solid_queue' +gem "mission_control-jobs" # Monitoring gem "sentry-ruby" diff --git a/Gemfile.lock b/Gemfile.lock index b829332..6689391 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -265,6 +265,16 @@ GEM mini_mime (1.1.5) mini_portile2 (2.8.8) minitest (5.25.5) + mission_control-jobs (1.0.2) + actioncable (>= 7.1) + actionpack (>= 7.1) + activejob (>= 7.1) + activerecord (>= 7.1) + importmap-rails (>= 1.2.1) + irb (~> 1.13) + railties (>= 7.1) + stimulus-rails + turbo-rails multipart-post (2.4.1) net-http (0.6.0) uri @@ -315,7 +325,7 @@ GEM date stringio public_suffix (6.0.1) - puma (4.3.12) + puma (6.6.0) nio4r (~> 2.0) raabro (1.4.0) racc (1.8.1) @@ -463,6 +473,13 @@ GEM tilt (~> 2.0) yard (~> 0.9, >= 0.9.24) yard-solargraph (~> 0.1) + solid_queue (1.1.5) + activejob (>= 7.1) + activerecord (>= 7.1) + concurrent-ruby (>= 1.3.1) + fugit (~> 1.11.0) + railties (>= 7.1) + thor (~> 1.3.1) sqlite3 (2.6.0) mini_portile2 (~> 2.8.0) sqlite3 (2.6.0-arm64-darwin) @@ -543,12 +560,13 @@ DEPENDENCIES lnurl lockbox manifique (~> 1.1.0) + mission_control-jobs net-ldap nostr (~> 0.6.0) pagy (~> 6.0, >= 6.0.2) pg (~> 1.5) propshaft - puma (~> 4.1) + puma (~> 6.6) rails (~> 8.0) rails-controller-testing rails-settings-cached (~> 2.8.3) @@ -559,6 +577,7 @@ DEPENDENCIES sidekiq (< 7) sidekiq-scheduler solargraph + solid_queue sqlite3 (>= 2.1) stimulus-rails turbo-rails diff --git a/README.md b/README.md index 13e20ac..530186b 100644 --- a/README.md +++ b/README.md @@ -57,7 +57,7 @@ Running the test suite: Running the test suite with Docker Compose requires overriding the Rails environment: - docker-compose run -e "RAILS_ENV=test" web rspec + docker-compose exec -e "RAILS_ENV=test" web rspec ### Docker Compose diff --git a/bin/jobs b/bin/jobs new file mode 100755 index 0000000..dcf59f3 --- /dev/null +++ b/bin/jobs @@ -0,0 +1,6 @@ +#!/usr/bin/env ruby + +require_relative "../config/environment" +require "solid_queue/cli" + +SolidQueue::Cli.start(ARGV) diff --git a/config/application.rb b/config/application.rb index 93f7f94..a59dd30 100644 --- a/config/application.rb +++ b/config/application.rb @@ -47,7 +47,8 @@ module Akkounts g.stylesheets false end - config.active_job.queue_adapter = :sidekiq + config.active_job.queue_adapter = :solid_queue + config.mission_control.jobs.http_basic_auth_enabled = false config.action_mailer.deliver_later_queue_name = nil # use "default" queue diff --git a/config/database.yml b/config/database.yml index 7f9bef7..339760c 100644 --- a/config/database.yml +++ b/config/database.yml @@ -7,6 +7,10 @@ development: primary: <<: *default database: db/development.sqlite3 + queue: + <<: *default + database: db/development_queue.sqlite3 + migrations_paths: db/queue_migrate lndhub: <<: *default adapter: postgresql diff --git a/config/environments/development.rb b/config/environments/development.rb index 1ec46a6..04a5849 100644 --- a/config/environments/development.rb +++ b/config/environments/development.rb @@ -54,6 +54,9 @@ Rails.application.configure do # Highlight code that enqueued background job in logs. config.active_job.verbose_enqueue_logs = true + # Solid Queue database + config.solid_queue.connects_to = { database: { writing: :queue } } + # Suppress logger output for asset requests. # config.assets.quiet = true diff --git a/config/environments/production.rb b/config/environments/production.rb index 439b155..727f90b 100644 --- a/config/environments/production.rb +++ b/config/environments/production.rb @@ -54,8 +54,9 @@ Rails.application.configure do # Replace the default in-process memory cache store with a durable alternative. # config.cache_store = :mem_cache_store - # Replace the default in-process and non-durable queuing backend for Active Job. - # config.active_job.queue_adapter = :resque + # Solid Queue database + config.solid_queue.connects_to = { database: { writing: :queue } } + # E-mail settings, adapted from https://github.com/mastodon/mastodon outgoing_email_address = ENV.fetch('SMTP_FROM_ADDRESS', 'accounts@localhost') diff --git a/config/initializers/devise_rails8_patch.rb b/config/initializers/devise_rails8_patch.rb new file mode 100644 index 0000000..b84a0ee --- /dev/null +++ b/config/initializers/devise_rails8_patch.rb @@ -0,0 +1,11 @@ +# See https://alvincrespo.hashnode.dev/rails-8s-lazy-route-loading-devise +# TODO remove when Devise is fixed +require 'devise' +Devise # make sure it's already loaded + +module Devise + def self.mappings + Rails.application.try(:reload_routes_unless_loaded) + @@mappings + end +end diff --git a/config/queue.yml b/config/queue.yml new file mode 100644 index 0000000..948271d --- /dev/null +++ b/config/queue.yml @@ -0,0 +1,21 @@ +default: &default + dispatchers: + - polling_interval: 1 + batch_size: 500 + workers: + - queues: "*" + threads: 3 + processes: <%= ENV.fetch("JOB_CONCURRENCY", 1) %> + polling_interval: 0.1 + +development: + <<: *default + workers: + - queues: "*" + threads: 1 + +test: + <<: *default + +production: + <<: *default diff --git a/config/recurring.yml b/config/recurring.yml new file mode 100644 index 0000000..d045b19 --- /dev/null +++ b/config/recurring.yml @@ -0,0 +1,10 @@ +# production: +# periodic_cleanup: +# class: CleanSoftDeletedRecordsJob +# queue: background +# args: [ 1000, { batch_size: 500 } ] +# schedule: every hour +# periodic_command: +# command: "SoftDeletedRecord.due.delete_all" +# priority: 2 +# schedule: at 5am every day diff --git a/config/routes.rb b/config/routes.rb index b9c771e..11a4d95 100644 --- a/config/routes.rb +++ b/config/routes.rb @@ -123,7 +123,7 @@ Rails.application.routes.draw do end authenticate :user, ->(user) { user.is_admin? } do - mount Sidekiq::Web, at: '/sidekiq' + mount MissionControl::Jobs::Engine, at: "/jobs" mount Flipper::UI.app(Flipper), at: '/flipper' end diff --git a/db/queue_schema.rb b/db/queue_schema.rb new file mode 100644 index 0000000..85194b6 --- /dev/null +++ b/db/queue_schema.rb @@ -0,0 +1,129 @@ +ActiveRecord::Schema[7.1].define(version: 1) do + create_table "solid_queue_blocked_executions", force: :cascade do |t| + t.bigint "job_id", null: false + t.string "queue_name", null: false + t.integer "priority", default: 0, null: false + t.string "concurrency_key", null: false + t.datetime "expires_at", null: false + t.datetime "created_at", null: false + t.index [ "concurrency_key", "priority", "job_id" ], name: "index_solid_queue_blocked_executions_for_release" + t.index [ "expires_at", "concurrency_key" ], name: "index_solid_queue_blocked_executions_for_maintenance" + t.index [ "job_id" ], name: "index_solid_queue_blocked_executions_on_job_id", unique: true + end + + create_table "solid_queue_claimed_executions", force: :cascade do |t| + t.bigint "job_id", null: false + t.bigint "process_id" + t.datetime "created_at", null: false + t.index [ "job_id" ], name: "index_solid_queue_claimed_executions_on_job_id", unique: true + t.index [ "process_id", "job_id" ], name: "index_solid_queue_claimed_executions_on_process_id_and_job_id" + end + + create_table "solid_queue_failed_executions", force: :cascade do |t| + t.bigint "job_id", null: false + t.text "error" + t.datetime "created_at", null: false + t.index [ "job_id" ], name: "index_solid_queue_failed_executions_on_job_id", unique: true + end + + create_table "solid_queue_jobs", force: :cascade do |t| + t.string "queue_name", null: false + t.string "class_name", null: false + t.text "arguments" + t.integer "priority", default: 0, null: false + t.string "active_job_id" + t.datetime "scheduled_at" + t.datetime "finished_at" + t.string "concurrency_key" + t.datetime "created_at", null: false + t.datetime "updated_at", null: false + t.index [ "active_job_id" ], name: "index_solid_queue_jobs_on_active_job_id" + t.index [ "class_name" ], name: "index_solid_queue_jobs_on_class_name" + t.index [ "finished_at" ], name: "index_solid_queue_jobs_on_finished_at" + t.index [ "queue_name", "finished_at" ], name: "index_solid_queue_jobs_for_filtering" + t.index [ "scheduled_at", "finished_at" ], name: "index_solid_queue_jobs_for_alerting" + end + + create_table "solid_queue_pauses", force: :cascade do |t| + t.string "queue_name", null: false + t.datetime "created_at", null: false + t.index [ "queue_name" ], name: "index_solid_queue_pauses_on_queue_name", unique: true + end + + create_table "solid_queue_processes", force: :cascade do |t| + t.string "kind", null: false + t.datetime "last_heartbeat_at", null: false + t.bigint "supervisor_id" + t.integer "pid", null: false + t.string "hostname" + t.text "metadata" + t.datetime "created_at", null: false + t.string "name", null: false + t.index [ "last_heartbeat_at" ], name: "index_solid_queue_processes_on_last_heartbeat_at" + t.index [ "name", "supervisor_id" ], name: "index_solid_queue_processes_on_name_and_supervisor_id", unique: true + t.index [ "supervisor_id" ], name: "index_solid_queue_processes_on_supervisor_id" + end + + create_table "solid_queue_ready_executions", force: :cascade do |t| + t.bigint "job_id", null: false + t.string "queue_name", null: false + t.integer "priority", default: 0, null: false + t.datetime "created_at", null: false + t.index [ "job_id" ], name: "index_solid_queue_ready_executions_on_job_id", unique: true + t.index [ "priority", "job_id" ], name: "index_solid_queue_poll_all" + t.index [ "queue_name", "priority", "job_id" ], name: "index_solid_queue_poll_by_queue" + end + + create_table "solid_queue_recurring_executions", force: :cascade do |t| + t.bigint "job_id", null: false + t.string "task_key", null: false + t.datetime "run_at", null: false + t.datetime "created_at", null: false + t.index [ "job_id" ], name: "index_solid_queue_recurring_executions_on_job_id", unique: true + t.index [ "task_key", "run_at" ], name: "index_solid_queue_recurring_executions_on_task_key_and_run_at", unique: true + end + + create_table "solid_queue_recurring_tasks", force: :cascade do |t| + t.string "key", null: false + t.string "schedule", null: false + t.string "command", limit: 2048 + t.string "class_name" + t.text "arguments" + t.string "queue_name" + t.integer "priority", default: 0 + t.boolean "static", default: true, null: false + t.text "description" + t.datetime "created_at", null: false + t.datetime "updated_at", null: false + t.index [ "key" ], name: "index_solid_queue_recurring_tasks_on_key", unique: true + t.index [ "static" ], name: "index_solid_queue_recurring_tasks_on_static" + end + + create_table "solid_queue_scheduled_executions", force: :cascade do |t| + t.bigint "job_id", null: false + t.string "queue_name", null: false + t.integer "priority", default: 0, null: false + t.datetime "scheduled_at", null: false + t.datetime "created_at", null: false + t.index [ "job_id" ], name: "index_solid_queue_scheduled_executions_on_job_id", unique: true + t.index [ "scheduled_at", "priority", "job_id" ], name: "index_solid_queue_dispatch_all" + end + + create_table "solid_queue_semaphores", force: :cascade do |t| + t.string "key", null: false + t.integer "value", default: 1, null: false + t.datetime "expires_at", null: false + t.datetime "created_at", null: false + t.datetime "updated_at", null: false + t.index [ "expires_at" ], name: "index_solid_queue_semaphores_on_expires_at" + t.index [ "key", "value" ], name: "index_solid_queue_semaphores_on_key_and_value" + t.index [ "key" ], name: "index_solid_queue_semaphores_on_key", unique: true + end + + add_foreign_key "solid_queue_blocked_executions", "solid_queue_jobs", column: "job_id", on_delete: :cascade + add_foreign_key "solid_queue_claimed_executions", "solid_queue_jobs", column: "job_id", on_delete: :cascade + add_foreign_key "solid_queue_failed_executions", "solid_queue_jobs", column: "job_id", on_delete: :cascade + add_foreign_key "solid_queue_ready_executions", "solid_queue_jobs", column: "job_id", on_delete: :cascade + add_foreign_key "solid_queue_recurring_executions", "solid_queue_jobs", column: "job_id", on_delete: :cascade + add_foreign_key "solid_queue_scheduled_executions", "solid_queue_jobs", column: "job_id", on_delete: :cascade +end diff --git a/docker-compose.yml b/docker-compose.yml index 0328ce0..f93cd3c 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -37,6 +37,9 @@ services: - "3000:3000" environment: RAILS_ENV: development + SOLID_QUEUE_IN_PUMA: true + LAUNCHY_DRY_RUN: true + BROWSER: /dev/null PRIMARY_DOMAIN: kosmos.org LDAP_HOST: ldap LDAP_PORT: 3389 @@ -54,30 +57,6 @@ services: - ldap - redis - sidekiq: - build: . - command: bash -c "bundle exec sidekiq -C config/sidekiq.yml" - volumes: - - .:/akkounts - networks: - - internal_network - environment: - RAILS_ENV: development - PRIMARY_DOMAIN: kosmos.org - LDAP_HOST: ldap - LDAP_PORT: 3389 - LDAP_ADMIN_PASSWORD: passthebutter - LDAP_USE_TLS: "false" - LAUNCHY_DRY_RUN: true - BROWSER: /dev/null - REDIS_URL: redis://redis:6379/0 - RS_REDIS_URL: redis://redis:6379/1 - RS_STORAGE_URL: "http://localhost:4567" - S3_ENABLED: false - depends_on: - - ldap - - redis - minio: image: quay.io/minio/minio:latest command: "server /data --console-address ':9001'" From b1dd5800b21b951cbecef60121e6621d7b2fea7e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A2u=20Cao?= Date: Sun, 4 May 2025 17:42:31 +0400 Subject: [PATCH 2/7] Update lockfile --- Gemfile.lock | 13 ------------- 1 file changed, 13 deletions(-) diff --git a/Gemfile.lock b/Gemfile.lock index 6689391..a590349 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -385,7 +385,6 @@ GEM logger rdoc (6.13.1) psych (>= 4.0.0) - redis (4.8.1) regexp_parser (2.10.0) reline (0.6.1) io-console (~> 0.5) @@ -434,8 +433,6 @@ GEM ruby-vips (2.2.3) ffi (~> 1.12) logger - rufus-scheduler (3.9.2) - fugit (~> 1.1, >= 1.11.1) sanitize (7.0.0) crass (~> 1.0.2) nokogiri (>= 1.16.8) @@ -446,14 +443,6 @@ GEM sentry-ruby (5.23.0) bigdecimal concurrent-ruby (~> 1.0, >= 1.0.2) - sidekiq (6.5.12) - connection_pool (>= 2.2.5, < 3) - rack (~> 2.0) - redis (>= 4.5.0, < 5) - sidekiq-scheduler (5.0.6) - rufus-scheduler (~> 3.2) - sidekiq (>= 6, < 8) - tilt (>= 1.4.0, < 3) solargraph (0.54.2) backport (~> 1.2) benchmark (~> 0.4) @@ -574,8 +563,6 @@ DEPENDENCIES rspec-rails sentry-rails sentry-ruby - sidekiq (< 7) - sidekiq-scheduler solargraph solid_queue sqlite3 (>= 2.1) From ecd814641a8e88fbda317993dec09ecb735d66a0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A2u=20Cao?= Date: Sun, 4 May 2025 17:44:37 +0400 Subject: [PATCH 3/7] Remove Sidekiq initializer --- config/initializers/sidekiq.rb | 5 ----- 1 file changed, 5 deletions(-) delete mode 100644 config/initializers/sidekiq.rb diff --git a/config/initializers/sidekiq.rb b/config/initializers/sidekiq.rb deleted file mode 100644 index 044cec0..0000000 --- a/config/initializers/sidekiq.rb +++ /dev/null @@ -1,5 +0,0 @@ -require_relative "../../app/models/setting" - -Sidekiq.configure_server do |config| - config.redis = { url: Setting.redis_url } -end From d1eea85b04033a6957dbfdf9d386a2421bbb32bb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A2u=20Cao?= Date: Sun, 4 May 2025 18:14:49 +0400 Subject: [PATCH 4/7] Add Redis gem explicitly, remove sidekiq require --- Gemfile | 3 +-- Gemfile.lock | 5 +++++ config/routes.rb | 2 -- 3 files changed, 6 insertions(+), 4 deletions(-) diff --git a/Gemfile b/Gemfile index cd2e1b7..78756d7 100644 --- a/Gemfile +++ b/Gemfile @@ -19,8 +19,6 @@ gem "turbo-rails" gem "stimulus-rails" # Build JSON APIs with ease. Read more: https://github.com/rails/jbuilder gem 'jbuilder', '~> 2.7' -# Use Redis adapter to run Action Cable in production -# gem 'redis', '~> 4.0' # Use Active Model has_secure_password gem 'bcrypt', '~> 3.1' @@ -65,6 +63,7 @@ gem 'discourse_api' gem "lnurl" gem 'manifique', '~> 1.1.0' gem 'nostr', '~> 0.6.0' +gem "redis", "~> 5.4" group :development, :test do # Use sqlite3 as the database for Active Record diff --git a/Gemfile.lock b/Gemfile.lock index a590349..c68cde3 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -385,6 +385,10 @@ GEM logger rdoc (6.13.1) psych (>= 4.0.0) + redis (5.4.0) + redis-client (>= 0.22.0) + redis-client (0.24.0) + connection_pool regexp_parser (2.10.0) reline (0.6.1) io-console (~> 0.5) @@ -559,6 +563,7 @@ DEPENDENCIES rails (~> 8.0) rails-controller-testing rails-settings-cached (~> 2.8.3) + redis (~> 5.4) rqrcode (~> 2.0) rspec-rails sentry-rails diff --git a/config/routes.rb b/config/routes.rb index 11a4d95..8344410 100644 --- a/config/routes.rb +++ b/config/routes.rb @@ -1,5 +1,3 @@ -require 'sidekiq/web' - Rails.application.routes.draw do devise_for :users, controllers: { confirmations: 'users/confirmations', From ee5c6d86d02e6ff661695004db605b4049fd71bb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A2u=20Cao?= Date: Mon, 5 May 2025 11:07:30 +0400 Subject: [PATCH 5/7] Port RS auth job removal to Solid Queue --- app/models/remote_storage_authorization.rb | 18 +++++++++++++----- spec/controllers/rs/oauth_controller_spec.rb | 1 + .../services/rs_auths_controller_spec.rb | 1 + ...te_storage_expire_authorization_job_spec.rb | 3 +++ .../remote_storage_authorization_spec.rb | 1 + 5 files changed, 19 insertions(+), 5 deletions(-) diff --git a/app/models/remote_storage_authorization.rb b/app/models/remote_storage_authorization.rb index 7eedd57..d2d3700 100644 --- a/app/models/remote_storage_authorization.rb +++ b/app/models/remote_storage_authorization.rb @@ -69,11 +69,19 @@ class RemoteStorageAuthorization < ApplicationRecord end def remove_token_expiry_job - queue = Sidekiq::Queue.new(RemoteStorageExpireAuthorizationJob.queue_name) - queue.each do |job| - next unless job.display_class == "RemoteStorageExpireAuthorizationJob" - job.delete if job.display_args == [id] - end + job_class = RemoteStorageExpireAuthorizationJob + job_args = [id] + + query = SolidQueue::Job.where(class_name: job_class.to_s) + + case ActiveRecord::Base.connection.adapter_name.downcase + when /sqlite/ + query.where("json_extract(arguments, '$.arguments') = ?", job_args.to_json) + when /postgres/ + query.where("arguments->>'arguments' = ?", job_args.to_json) + else + raise "Unsupported database adapter" + end.destroy_all end def find_or_create_web_app diff --git a/spec/controllers/rs/oauth_controller_spec.rb b/spec/controllers/rs/oauth_controller_spec.rb index 09b0750..03f4750 100644 --- a/spec/controllers/rs/oauth_controller_spec.rb +++ b/spec/controllers/rs/oauth_controller_spec.rb @@ -5,6 +5,7 @@ RSpec.describe Rs::OauthController, type: :controller do before do allow_any_instance_of(AppCatalog::WebApp).to receive(:update_metadata).and_return(true) + allow_any_instance_of(RemoteStorageAuthorization).to receive(:remove_token_expiry_job).and_return(nil) end describe "GET /rs/oauth/:username" do diff --git a/spec/controllers/services/rs_auths_controller_spec.rb b/spec/controllers/services/rs_auths_controller_spec.rb index 44bcdc0..61dc088 100644 --- a/spec/controllers/services/rs_auths_controller_spec.rb +++ b/spec/controllers/services/rs_auths_controller_spec.rb @@ -5,6 +5,7 @@ RSpec.describe Services::RsAuthsController, type: :controller do before do allow_any_instance_of(AppCatalog::WebApp).to receive(:update_metadata).and_return(true) + allow_any_instance_of(RemoteStorageAuthorization).to receive(:remove_token_expiry_job).and_return(nil) allow_any_instance_of(Flipper).to receive(:enabled?).and_return(true) end diff --git a/spec/jobs/remote_storage_expire_authorization_job_spec.rb b/spec/jobs/remote_storage_expire_authorization_job_spec.rb index 8e6cbb7..afe0f3e 100644 --- a/spec/jobs/remote_storage_expire_authorization_job_spec.rb +++ b/spec/jobs/remote_storage_expire_authorization_job_spec.rb @@ -5,6 +5,9 @@ RSpec.describe RemoteStorageExpireAuthorizationJob, type: :job do allow_any_instance_of(AppCatalog::WebApp).to( receive(:update_metadata).and_return(true) ) + allow_any_instance_of(RemoteStorageAuthorization).to( + receive(:remove_token_expiry_job).and_return(nil) + ) @user = create :user, cn: "ronald", ou: "kosmos.org" @rs_authorization = create :remote_storage_authorization, diff --git a/spec/models/remote_storage_authorization_spec.rb b/spec/models/remote_storage_authorization_spec.rb index 3673bde..2db3d7d 100644 --- a/spec/models/remote_storage_authorization_spec.rb +++ b/spec/models/remote_storage_authorization_spec.rb @@ -7,6 +7,7 @@ RSpec.describe RemoteStorageAuthorization, type: :model do before do allow_any_instance_of(AppCatalog::WebApp).to receive(:update_metadata).and_return(true) + allow_any_instance_of(RemoteStorageAuthorization).to receive(:remove_token_expiry_job).and_return(nil) end describe "#create" do From 906468d15632d108f27ce952906d0ee7c347d79a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A2u=20Cao?= Date: Mon, 5 May 2025 12:46:46 +0400 Subject: [PATCH 6/7] Allow to immediately expire auth via job When running the job before its schedule --- app/jobs/remote_storage_expire_authorization_job.rb | 2 -- 1 file changed, 2 deletions(-) diff --git a/app/jobs/remote_storage_expire_authorization_job.rb b/app/jobs/remote_storage_expire_authorization_job.rb index 62c240c..209c463 100644 --- a/app/jobs/remote_storage_expire_authorization_job.rb +++ b/app/jobs/remote_storage_expire_authorization_job.rb @@ -3,8 +3,6 @@ class RemoteStorageExpireAuthorizationJob < ApplicationJob def perform(rs_auth_id) rs_auth = RemoteStorageAuthorization.find rs_auth_id - return unless rs_auth.expire_at.nil? || rs_auth.expire_at <= DateTime.now - rs_auth.destroy! end end From 889c9ae824256aa3e69f5c10e9d0f9cd99a600da Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A2u=20Cao?= Date: Mon, 5 May 2025 15:17:32 +0400 Subject: [PATCH 7/7] Refactor database configs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Move postgres credentials to ENV vars * Allow postgres in development * Allow SQlite in production * Refactor optional lndhub db config Co-authored-by: Greg Karékinian --- .env.example | 9 ++++++ config/credentials.yml.enc | 2 +- config/database.yml | 56 +++++++++++++++++++++----------------- 3 files changed, 41 insertions(+), 26 deletions(-) diff --git a/.env.example b/.env.example index 438b759..92401ea 100644 --- a/.env.example +++ b/.env.example @@ -1,6 +1,15 @@ # PRIMARY_DOMAIN=kosmos.org # AKKOUNTS_DOMAIN=accounts.example.com +# The default backend is SQLite +# DB_ADAPTER=postgresql +# PG_HOST=localhost +# PG_PORT=5432 +# PG_DATABASE=akkounts +# PG_DATABASE_QUEUE=akkounts_queue +# PG_USERNAME=akkounts +# PG_PASSWORD= + # SMTP_SERVER=smtp.example.com # SMTP_PORT=587 # SMTP_LOGIN=accounts diff --git a/config/credentials.yml.enc b/config/credentials.yml.enc index de77871..49e6d9e 100644 --- a/config/credentials.yml.enc +++ b/config/credentials.yml.enc @@ -1 +1 @@ -tmI5vm7qZhaigr52jEBVWkRdj+EE+9OmPh3vWXC7kA/OHuuucpr7SodychuMkQDPLM0BLk88LFsqvRIR+mqnLWpRC+P9aeUFE6ohxSWzcAd7Y4sgxUD8zpCRPndrwTw0hxXXj1WZSYeWn4BoAB34aV+gYen2MajZF3a95hJGtS5yjgWxvLVkQQKqRDfykkfX6fCS0BPo5X7sT7m4xwCATD/D4219wajm5W3TIdkriHtwt28ZLspaRWA5e0UkzKf8+/Gaj2CrW7UWcvew8R93zQ5RA2/Sp3sDTVN+kLz9I9Q095lQC0ywCAEFYHeKmc2tjrzqRaAAWu06xmWLqGIg21G+A/UU9lUJOkIpxQACWoOfS2IoXR1nXhgXMopkz3aCBXDxKw554v4H2QyOceOsuRf2C685ibMqzQkKMmJ4tcbiOJL77DUc08JTjB8Dq4Ohr8sMzXbV/hATevjYoRP0XarLekqhLv90ZLuIVY16DwB0CzACeNBKeKbeLqJF51upRRWgi+gTbYpV04yUwnXdyssF8mydWocgihrTryBi8F6PsuhBGcaYdP+0yibnGxDCC4x2rupbBfMj2OIX7pYzgtIHB3Eo954Y+bCoggqbE/Qrb9VVXNMgtKgLt8EGWU2tg6wl9QicitIq87uLDAade93zTn6rmcKPywjMDo6jbVIs653ZdUhiKdHGdpnJccbgQ/iLSPB1umNnCeaEX5jM+K9zBvl7ZMCdSk1YIQ==--ekKumqLiSlVJNwMe--K/ecXmmMT1x+WnIXMbHBDw== \ No newline at end of file +zaTF2ZJaU/M9CYmXEJoDmS1oniSV/1YT0UnM1jysEhMknn3bYOzRBZM3eGJ5Mr6bJYz6cv5hSDL5pT0/6hqgpV04dc/fVDplWO4eEpD1kBFM3LjIPCe9REbRlRUwODpoV/y6wWOFme8nFMS9uOSFkL6RtMuQli0os5Rp+2Jal2lJwAujFjFwuuj+1iRzqc3pzeIIy0clPmR9ANxXS+rPL3jmxty6QzVMr9Q658roVD38yRg0CNgs09eZ/FvqeqXlQkwDfg2/zX9tg5ocGwvHzmicZJ/yU6kl6liqNAJvEDrolm0gSzemY8NfhCySd5wjEvpP+uvKxbd5M3rhagC8S9MmpmxewuOKbDFEyTSRO6Kp2yakdcLSn12ZPB+X0nlMRno+UEzh0EvcX2mxRXppIKrsRUGVeYmj4GMI0vyLW1eCuyPLueN7sOnSjhtM84URNWkSnb0LTxlYxOJGbnwzn85QVpFdXtDktbaXvMWcVdH9XdMDbaBs1G7BIA6Z8i+mxLVVEbQWQM6VBrUhpkpWfphLmN5b16LYbGTzdKnR0iPcsNr8Tsl8vYfeWH8S0ujD105lS39v37YimN6E5l4X2CrqG+DNBtKfbqC0E3lhZMqDRWetxzaxE47oe4g=--0EKrvwe2YTfsbssz--SbmUH0sMiy5uUhpxFImgMA== \ No newline at end of file diff --git a/config/database.yml b/config/database.yml index 339760c..782a22d 100644 --- a/config/database.yml +++ b/config/database.yml @@ -1,25 +1,37 @@ default: &default - adapter: sqlite3 + adapter: <%= ENV["DB_ADAPTER"] || "sqlite3" %> pool: <%= ENV["DB_POOL"] || ENV['MAX_THREADS'] || 5 %> timeout: 5000 +<% if ENV["DB_ADAPTER"] == "postgresql" %> + host: <%= ENV["PG_HOST"] || 'localhost' %> + port: <%= ENV["PG_PORT"] || 5432 %> + username: <%= ENV["PG_USERNAME"] || 'akkounts' %> + password: <%= ENV["PG_PASSWORD"] %> +<% end %> + +<% if ENV["LNDHUB_PG_HOST"].present? %> +lndhub: &lndhub + adapter: postgresql + database_tasks: false + host: <%= ENV["LNDHUB_PG_HOST"] %> + port: <%= ENV["LNDHUB_PG_PORT"] || 5432 %> + database: <%= ENV["LNDHUB_PG_DATABASE"] || 'lndhub' %> + username: <%= ENV["LNDHUB_PG_USERNAME"] || 'lndhub' %> + password: <%= ENV["LNDHUB_PG_PASSWORD"] %> +<% end %> development: primary: <<: *default - database: db/development.sqlite3 + database: <%= ENV["DB_ADAPTER"] == "postgresql" ? ENV["PG_DATABASE"] : "db/development.sqlite3" %> queue: <<: *default - database: db/development_queue.sqlite3 + database: <%= ENV["DB_ADAPTER"] == "postgresql" ? ENV["PG_DATABASE_QUEUE"] : "db/development_queue.sqlite3" %> migrations_paths: db/queue_migrate +<% if ENV["LNDHUB_PG_HOST"].present? %> lndhub: - <<: *default - adapter: postgresql - database_tasks: false - host: <%= ENV["LNDHUB_PG_HOST"] || 'localhost' %> - port: <%= ENV["LNDHUB_PG_PORT"] || 5432 %> - database: <%= ENV["LNDHUB_PG_DATABASE"] || 'lndhub' %> - username: <%= ENV["LNDHUB_PG_USERNAME"] || 'lndhub' %> - password: <%= ENV["LNDHUB_PG_PASSWORD"] %> + <<: *lndhub +<% end %> # Warning: The database defined as "test" will be erased and # re-generated from your development database when you run "rake". @@ -36,18 +48,12 @@ test: production: primary: <<: *default - adapter: postgresql - database: akkounts - port: 5432 - host: <%= Rails.application.credentials.postgres[:host] rescue nil %> - username: <%= Rails.application.credentials.postgres[:username] rescue nil %> - password: <%= Rails.application.credentials.postgres[:password] rescue nil %> - lndhub: + database: <%= ENV["DB_ADAPTER"] == "postgresql" ? ENV["PG_DATABASE"] : "db/production.sqlite3" %> + queue: <<: *default - adapter: postgresql - database_tasks: false - host: <%= ENV["LNDHUB_PG_HOST"] || 'localhost' %> - port: <%= ENV["LNDHUB_PG_PORT"] || 5432 %> - database: <%= ENV["LNDHUB_PG_DATABASE"] || 'lndhub' %> - username: <%= ENV["LNDHUB_PG_USERNAME"] || 'lndhub' %> - password: <%= ENV["LNDHUB_PG_PASSWORD"] %> + database: <%= ENV["DB_ADAPTER"] == "postgresql" ? ENV["PG_DATABASE_QUEUE"] : "db/production_queue.sqlite3" %> + migrations_paths: db/queue_migrate +<% if ENV["LNDHUB_PG_HOST"].present? %> + lndhub: + <<: *lndhub +<% end %>