diff --git a/.gitignore b/.gitignore index e02bc74..aaab8ab 100644 --- a/.gitignore +++ b/.gitignore @@ -39,3 +39,6 @@ yarn-debug.log* # Ignore local dotenv config file .env + +# Ignore redis dumps from sidekiq +dump.rdb diff --git a/Gemfile b/Gemfile index 8a5b728..6d0b53d 100644 --- a/Gemfile +++ b/Gemfile @@ -21,15 +21,22 @@ gem 'jbuilder', '~> 2.7' # Reduces boot times through caching; required in config/boot.rb gem 'bootsnap', '>= 1.4.2', require: false +# Configuration gem 'dotenv-rails' +# Authentication gem 'warden' gem 'devise' gem 'devise_ldap_authenticatable' gem 'net-ldap' +# HTTP requests gem 'faraday' +# Background/scheduled jobs +gem 'sidekiq' +gem 'sidekiq-scheduler' + group :development, :test do # Use sqlite3 as the database for Active Record gem 'sqlite3', '~> 1.4' diff --git a/Gemfile.lock b/Gemfile.lock index 922b81a..6a4c8c2 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -73,6 +73,7 @@ GEM regexp_parser (~> 1.5) xpath (~> 3.2) concurrent-ruby (1.1.7) + connection_pool (2.2.3) crack (0.4.3) safe_yaml (~> 1.0.0) crass (1.0.6) @@ -91,7 +92,10 @@ GEM dotenv-rails (2.7.2) dotenv (= 2.7.2) railties (>= 3.2, < 6.1) + e2mmap (0.1.0) erubi (1.9.0) + et-orbi (1.2.4) + tzinfo factory_bot (6.1.0) activesupport (>= 5.0.0) factory_bot_rails (6.1.0) @@ -100,6 +104,9 @@ GEM faraday (0.17.0) multipart-post (>= 1.2, < 3) ffi (1.13.1) + fugit (1.4.2) + et-orbi (~> 1.1, >= 1.1.8) + raabro (~> 1.4) globalid (0.4.2) activesupport (>= 4.2.0) hashdiff (0.4.0) @@ -141,6 +148,7 @@ GEM public_suffix (4.0.6) puma (4.3.6) nio4r (~> 2.0) + raabro (1.4.0) rack (2.2.3) rack-proxy (0.6.5) rack @@ -176,6 +184,7 @@ GEM rb-fsevent (0.10.4) rb-inotify (0.10.1) ffi (~> 1.0) + redis (4.2.5) regexp_parser (1.8.2) responders (3.0.1) actionpack (>= 5.0) @@ -197,6 +206,8 @@ GEM rspec-mocks (~> 3.9) rspec-support (~> 3.9) rspec-support (3.10.0) + rufus-scheduler (3.7.0) + fugit (~> 1.1, >= 1.1.6) safe_yaml (1.0.5) sass-rails (6.0.0) sassc-rails (~> 2.1, >= 2.1.1) @@ -208,6 +219,17 @@ GEM sprockets (> 3.0) sprockets-rails tilt + sidekiq (6.1.3) + connection_pool (>= 2.2.2) + rack (~> 2.0) + redis (>= 4.2.0) + sidekiq-scheduler (3.0.1) + e2mmap + redis (>= 3, < 5) + rufus-scheduler (~> 3.2) + sidekiq (>= 3) + thwait + tilt (>= 1.4.0) spring (2.1.1) spring-watcher-listen (2.0.1) listen (>= 2.7, < 4.0) @@ -222,6 +244,8 @@ GEM sqlite3 (1.4.2) thor (1.0.1) thread_safe (0.3.6) + thwait (0.2.0) + e2mmap tilt (2.0.10) turbolinks (5.2.1) turbolinks-source (~> 5.2) @@ -274,6 +298,8 @@ DEPENDENCIES rails (~> 6.0.3, >= 6.0.3.4) rspec-rails sass-rails (>= 6) + sidekiq + sidekiq-scheduler spring spring-watcher-listen (~> 2.0.0) sqlite3 (~> 1.4) diff --git a/README.md b/README.md index 82c130b..61287a9 100644 --- a/README.md +++ b/README.md @@ -38,6 +38,10 @@ Running the dev server: bundle exec rails server +Running the background workers (requires Redis): + + bundle exec sidekiq -C config/sidekiq.yml + Running all specs: bundle exec rspec @@ -62,6 +66,11 @@ manual LDIF imports etc. (or provide a staging instance) * [devise_ldap_authenticatable](https://github.com/cschiewek/devise_ldap_authenticatable) * [net/ldap](https://www.rubydoc.info/gems/net-ldap/Net/LDAP) +### Asynchronous jobs/workers + +* [Sidekiq](https://github.com/mperham/sidekiq/wiki/) +* [ActiveJob](https://github.com/mperham/sidekiq/wiki/Active-Job) + ## License [GNU Affero General Public License v3.0](https://choosealicense.com/licenses/agpl-3.0/) diff --git a/app/jobs/create_ldap_user_job.rb b/app/jobs/create_ldap_user_job.rb new file mode 100644 index 0000000..da5e533 --- /dev/null +++ b/app/jobs/create_ldap_user_job.rb @@ -0,0 +1,32 @@ +class CreateLdapUserJob < ApplicationJob + queue_as :default + + def perform(username, domain, email, hashed_pw) + dn = "cn=#{username},ou=#{domain},cn=users,dc=kosmos,dc=org" + attr = { + objectclass: ["top", "account", "person", "extensibleObject"], + cn: username, + sn: username, + uid: username, + mail: email, + userPassword: hashed_pw + } + + ldap_client.add(dn: dn, attributes: attr) + end + + def ldap_client + ldap_client ||= Net::LDAP.new host: ldap_config['host'], + port: ldap_config['port'], + encryption: ldap_config['ssl'], + auth: { + method: :simple, + username: ldap_config['admin_user'], + password: ldap_config['admin_password'] + } + end + + def ldap_config + ldap_config ||= YAML.load(ERB.new(File.read("#{Rails.root}/config/ldap.yml")).result)[Rails.env] + end +end diff --git a/app/jobs/exchange_xmpp_contacts_job.rb b/app/jobs/exchange_xmpp_contacts_job.rb new file mode 100644 index 0000000..5e5caa2 --- /dev/null +++ b/app/jobs/exchange_xmpp_contacts_job.rb @@ -0,0 +1,18 @@ +class ExchangeXmppContactsJob < ApplicationJob + queue_as :default + + def perform(inviter, username, domain) + ejabberd = EjabberdApiClient.new + + ejabberd.add_rosteritem({ + "localuser": username, "localhost": domain, + "user": inviter.cn, "host": inviter.ou, + "nick": inviter.cn, "group": "Friends", "subs": "both" + }) + ejabberd.add_rosteritem({ + "localuser": inviter.cn, "localhost": inviter.ou, + "user": username, "host": domain, + "nick": username, "group": "Friends", "subs": "both" + }) + end +end diff --git a/app/services/create_account.rb b/app/services/create_account.rb index eb60824..64901d4 100644 --- a/app/services/create_account.rb +++ b/app/services/create_account.rb @@ -33,51 +33,15 @@ class CreateAccount < ApplicationService @invitation.update! invited_user_id: user_id, used_at: DateTime.now end + # TODO move to confirmation def add_ldap_document - dn = "cn=#{@username},ou=kosmos.org,cn=users,dc=kosmos,dc=org" - attr = { - objectclass: ["top", "account", "person", "extensibleObject"], - cn: @username, - sn: @username, - uid: @username, - mail: @email, - userPassword: Devise.ldap_auth_password_builder.call(@password) - } - - ldap_client.add(dn: dn, attributes: attr) - end - - def ldap_client - ldap_client ||= Net::LDAP.new host: ldap_config['host'], - port: ldap_config['port'], - encryption: ldap_config['ssl'], - auth: { - method: :simple, - username: ldap_config['admin_user'], - password: ldap_config['admin_password'] - } - end - - def ldap_config - ldap_config ||= YAML.load(ERB.new(File.read("#{Rails.root}/config/ldap.yml")).result)[Rails.env] + hashed_pw = Devise.ldap_auth_password_builder.call(@password) + CreateLdapUserJob.perform_later(@username, @domain, @email, hashed_pw) end def exchange_xmpp_contacts #TODO enable in development when we have easy setup of ejabberd etc. return if Rails.env.development? - - ejabberd = EjabberdApiClient.new - inviter = @invitation.user - - ejabberd.add_rosteritem({ - "localuser": @username, "localhost": @domain, - "user": inviter.cn, "host": inviter.ou, - "nick": inviter.cn, "group": "Friends", "subs": "both" - }) - ejabberd.add_rosteritem({ - "localuser": inviter.cn, "localhost": inviter.ou, - "user": @username, "host": @domain, - "nick": @username, "group": "Friends", "subs": "both" - }) + ExchangeXmppContactsJob.perform_later(@invitation.user, @username, @domain) end end diff --git a/config/application.rb b/config/application.rb index 127cf2f..98cc27c 100644 --- a/config/application.rb +++ b/config/application.rb @@ -39,5 +39,8 @@ module Akkounts g.fixture_replacement :factory_bot, suffix_factory: 'factory', dir: 'spec/factories' g.stylesheets false end + + config.active_job.queue_adapter = :sidekiq + config.action_mailer.deliver_later_queue_name = nil # use "default" queue end end diff --git a/config/environments/test.rb b/config/environments/test.rb index ac775c3..f3c0b8f 100644 --- a/config/environments/test.rb +++ b/config/environments/test.rb @@ -49,4 +49,6 @@ Rails.application.configure do protocol: "https", from: "accounts@kosmos.org" } + + config.active_job.queue_adapter = :test end diff --git a/config/routes.rb b/config/routes.rb index ae5bdc1..2a3592d 100644 --- a/config/routes.rb +++ b/config/routes.rb @@ -1,3 +1,5 @@ +require 'sidekiq/web' + Rails.application.routes.draw do resources :donations devise_for :users @@ -23,6 +25,10 @@ Rails.application.routes.draw do resources :donations end + authenticate :user, ->(user) { user.is_admin? } do + mount Sidekiq::Web => '/sidekiq' + end + # Letter Opener (open "sent" emails in dev and staging) if Rails.env.match(/staging|development/) mount LetterOpenerWeb::Engine, at: "letter_opener" diff --git a/config/sidekiq.yml b/config/sidekiq.yml new file mode 100644 index 0000000..615bb16 --- /dev/null +++ b/config/sidekiq.yml @@ -0,0 +1,3 @@ +:concurrency: 2 +:queues: + - default diff --git a/spec/jobs/create_ldap_user_job_spec.rb b/spec/jobs/create_ldap_user_job_spec.rb new file mode 100644 index 0000000..78aae67 --- /dev/null +++ b/spec/jobs/create_ldap_user_job_spec.rb @@ -0,0 +1,34 @@ +require 'rails_helper' + +RSpec.describe CreateLdapUserJob, type: :job do + let(:ldap_client_mock) { instance_double(Net::LDAP) } + + subject(:job) { + described_class.any_instance.stub(:ldap_client).and_return(ldap_client_mock) + described_class.perform_later( + 'halfinney', 'kosmos.org', 'halfinney@example.com', + 'remember-remember-the-5th-of-november' + ) + } + + it "creates a new document with the correct attributes" do + ldap_client_mock.should_receive(:add).with( + dn: "cn=halfinney,ou=kosmos.org,cn=users,dc=kosmos,dc=org", + attributes: { + objectclass: ["top", "account", "person", "extensibleObject"], + cn: "halfinney", + sn: "halfinney", + uid: "halfinney", + mail: "halfinney@example.com", + userPassword: "remember-remember-the-5th-of-november" + } + ) + + perform_enqueued_jobs { job } + end + + after do + clear_enqueued_jobs + clear_performed_jobs + end +end diff --git a/spec/jobs/exchange_xmpp_contacts_job_spec.rb b/spec/jobs/exchange_xmpp_contacts_job_spec.rb new file mode 100644 index 0000000..177e03c --- /dev/null +++ b/spec/jobs/exchange_xmpp_contacts_job_spec.rb @@ -0,0 +1,29 @@ +require 'rails_helper' +require 'webmock/rspec' + +RSpec.describe ExchangeXmppContactsJob, type: :job do + let(:user) { create :user, cn: "willherschel", ou: "kosmos.org" } + + subject(:job) { + described_class.perform_later(user, 'isaacnewton', 'kosmos.org') + } + + before do + stub_request(:post, "http://xmpp.example.com/api/add_rosteritem") + .to_return(status: 200, body: "", headers: {}) + end + + it "posts add_rosteritem commands to the ejabberd API" do + perform_enqueued_jobs { job } + + expect(WebMock).to have_requested(:post, "http://xmpp.example.com/api/add_rosteritem") + .with { |req| req.body == '{"localuser":"isaacnewton","localhost":"kosmos.org","user":"willherschel","host":"kosmos.org","nick":"willherschel","group":"Friends","subs":"both"}' } + expect(WebMock).to have_requested(:post, "http://xmpp.example.com/api/add_rosteritem") + .with { |req| req.body == '{"localuser":"willherschel","localhost":"kosmos.org","user":"isaacnewton","host":"kosmos.org","nick":"isaacnewton","group":"Friends","subs":"both"}' } + end + + after do + clear_enqueued_jobs + clear_performed_jobs + end +end diff --git a/spec/rails_helper.rb b/spec/rails_helper.rb index d11311e..c4a74a4 100644 --- a/spec/rails_helper.rb +++ b/spec/rails_helper.rb @@ -69,5 +69,6 @@ RSpec.configure do |config| config.include Devise::Test::ControllerHelpers, :type => :controller config.include Warden::Test::Helpers config.include FactoryBot::Syntax::Methods + config.include ActiveJob::TestHelper, type: :job config.extend ControllerMacros, :type => :controller end diff --git a/spec/services/create_account_spec.rb b/spec/services/create_account_spec.rb index c6e74d5..e010735 100644 --- a/spec/services/create_account_spec.rb +++ b/spec/services/create_account_spec.rb @@ -1,14 +1,6 @@ require 'rails_helper' -require 'webmock/rspec' -require 'json' RSpec.describe CreateAccount, type: :model do - let(:ldap_client_mock) { instance_double(Net::LDAP) } - - before do - allow(service).to receive(:ldap_client).and_return(ldap_client_mock) - end - describe "#create_user_in_database" do let(:service) { CreateAccount.new( username: 'isaacnewton', @@ -48,30 +40,34 @@ RSpec.describe CreateAccount, type: :model do end describe "#add_ldap_document" do + include ActiveJob::TestHelper + let(:service) { CreateAccount.new( username: 'halfinney', email: 'halfinney@example.com', password: 'remember-remember-the-5th-of-november' )} - it "creates a new document with the correct attributes" do - expect(ldap_client_mock).to receive(:add).with( - dn: "cn=halfinney,ou=kosmos.org,cn=users,dc=kosmos,dc=org", - attributes: { - objectclass: ["top", "account", "person", "extensibleObject"], - cn: "halfinney", - sn: "halfinney", - uid: "halfinney", - mail: "halfinney@example.com", - userPassword: /^{SSHA512}.{171}=/ - } - ) - + it "enqueues a job to create the LDAP user document" do service.send(:add_ldap_document) + + expect(enqueued_jobs.size).to eq(1) + + args = enqueued_jobs.first['arguments'] + expect(args[0]).to eq('halfinney') + expect(args[1]).to eq('kosmos.org') + expect(args[2]).to eq('halfinney@example.com') + expect(args[3]).to match(/^{SSHA512}.{171}=/) + end + + after do + clear_enqueued_jobs end end describe "#exchange_xmpp_contacts" do + include ActiveJob::TestHelper + let(:inviter) { create :user, cn: "willherschel", ou: "kosmos.org" } let(:invitation) { create :invitation, user: inviter } let(:service) { CreateAccount.new( @@ -81,18 +77,19 @@ RSpec.describe CreateAccount, type: :model do invitation: invitation )} - before do - stub_request(:post, "http://xmpp.example.com/api/add_rosteritem") - .to_return(status: 200, body: "", headers: {}) - end - - it "posts add_rosteritem commands to the ejabberd API" do + it "enqueues a job to exchange XMPP contacts between inviter and invitee" do service.send(:exchange_xmpp_contacts) - expect(WebMock).to have_requested(:post, "http://xmpp.example.com/api/add_rosteritem") - .with { |req| req.body == '{"localuser":"isaacnewton","localhost":"kosmos.org","user":"willherschel","host":"kosmos.org","nick":"willherschel","group":"Friends","subs":"both"}' } - expect(WebMock).to have_requested(:post, "http://xmpp.example.com/api/add_rosteritem") - .with { |req| req.body == '{"localuser":"willherschel","localhost":"kosmos.org","user":"isaacnewton","host":"kosmos.org","nick":"isaacnewton","group":"Friends","subs":"both"}' } + expect(enqueued_jobs.size).to eq(1) + + args = enqueued_jobs.first['arguments'] + expect(args[0]['_aj_globalid']).to match('gid://akkounts/User') + expect(args[1]).to eq('isaacnewton') + expect(args[2]).to eq('kosmos.org') + end + + after do + clear_enqueued_jobs end end end