Merge pull request 'Set up async workers/jobs via Sidekiq' (#26) from feature/sidekiq into master

Reviewed-on: #26

.gitignore

@@ -39,3 +39,6 @@ yarn-debug.log*
 
 # Ignore local dotenv config file
 .env
+
+# Ignore redis dumps from sidekiq
+dump.rdb

Gemfile

@@ -21,15 +21,22 @@ gem 'jbuilder', '~> 2.7'
 # Reduces boot times through caching; required in config/boot.rb
 gem 'bootsnap', '>= 1.4.2', require: false
 
+# Configuration
 gem 'dotenv-rails'
 
+# Authentication
 gem 'warden'
 gem 'devise'
 gem 'devise_ldap_authenticatable'
 gem 'net-ldap'
 
+# HTTP requests
 gem 'faraday'
 
+# Background/scheduled jobs
+gem 'sidekiq'
+gem 'sidekiq-scheduler'
+
 group :development, :test do
   # Use sqlite3 as the database for Active Record
   gem 'sqlite3', '~> 1.4'

Gemfile.lock

@@ -73,6 +73,7 @@ GEM
       regexp_parser (~> 1.5)
       xpath (~> 3.2)
     concurrent-ruby (1.1.7)
+    connection_pool (2.2.3)
     crack (0.4.3)
       safe_yaml (~> 1.0.0)
     crass (1.0.6)
@@ -91,7 +92,10 @@ GEM
     dotenv-rails (2.7.2)
       dotenv (= 2.7.2)
       railties (>= 3.2, < 6.1)
+    e2mmap (0.1.0)
     erubi (1.9.0)
+    et-orbi (1.2.4)
+      tzinfo
     factory_bot (6.1.0)
       activesupport (>= 5.0.0)
     factory_bot_rails (6.1.0)
@@ -100,6 +104,9 @@ GEM
     faraday (0.17.0)
       multipart-post (>= 1.2, < 3)
     ffi (1.13.1)
+    fugit (1.4.2)
+      et-orbi (~> 1.1, >= 1.1.8)
+      raabro (~> 1.4)
     globalid (0.4.2)
       activesupport (>= 4.2.0)
     hashdiff (0.4.0)
@@ -141,6 +148,7 @@ GEM
     public_suffix (4.0.6)
     puma (4.3.6)
       nio4r (~> 2.0)
+    raabro (1.4.0)
     rack (2.2.3)
     rack-proxy (0.6.5)
       rack
@@ -176,6 +184,7 @@ GEM
     rb-fsevent (0.10.4)
     rb-inotify (0.10.1)
       ffi (~> 1.0)
+    redis (4.2.5)
     regexp_parser (1.8.2)
     responders (3.0.1)
       actionpack (>= 5.0)
@@ -197,6 +206,8 @@ GEM
       rspec-mocks (~> 3.9)
       rspec-support (~> 3.9)
     rspec-support (3.10.0)
+    rufus-scheduler (3.7.0)
+      fugit (~> 1.1, >= 1.1.6)
     safe_yaml (1.0.5)
     sass-rails (6.0.0)
       sassc-rails (~> 2.1, >= 2.1.1)
@@ -208,6 +219,17 @@ GEM
       sprockets (> 3.0)
       sprockets-rails
       tilt
+    sidekiq (6.1.3)
+      connection_pool (>= 2.2.2)
+      rack (~> 2.0)
+      redis (>= 4.2.0)
+    sidekiq-scheduler (3.0.1)
+      e2mmap
+      redis (>= 3, < 5)
+      rufus-scheduler (~> 3.2)
+      sidekiq (>= 3)
+      thwait
+      tilt (>= 1.4.0)
     spring (2.1.1)
     spring-watcher-listen (2.0.1)
       listen (>= 2.7, < 4.0)
@@ -222,6 +244,8 @@ GEM
     sqlite3 (1.4.2)
     thor (1.0.1)
     thread_safe (0.3.6)
+    thwait (0.2.0)
+      e2mmap
     tilt (2.0.10)
     turbolinks (5.2.1)
       turbolinks-source (~> 5.2)
@@ -274,6 +298,8 @@ DEPENDENCIES
   rails (~> 6.0.3, >= 6.0.3.4)
   rspec-rails
   sass-rails (>= 6)
+  sidekiq
+  sidekiq-scheduler
   spring
   spring-watcher-listen (~> 2.0.0)
   sqlite3 (~> 1.4)

README.md

@@ -38,6 +38,10 @@ Running the dev server:
 
     bundle exec rails server
 
+Running the background workers (requires Redis):
+
+    bundle exec sidekiq -C config/sidekiq.yml
+
 Running all specs:
 
     bundle exec rspec
@@ -62,6 +66,11 @@ manual LDIF imports etc. (or provide a staging instance)
 * [devise_ldap_authenticatable](https://github.com/cschiewek/devise_ldap_authenticatable)
 * [net/ldap](https://www.rubydoc.info/gems/net-ldap/Net/LDAP)
 
+### Asynchronous jobs/workers
+
+* [Sidekiq](https://github.com/mperham/sidekiq/wiki/)
+* [ActiveJob](https://github.com/mperham/sidekiq/wiki/Active-Job)
+
 ## License
 
 [GNU Affero General Public License v3.0](https://choosealicense.com/licenses/agpl-3.0/)

app/jobs/create_ldap_user_job.rb

@@ -0,0 +1,32 @@
+class CreateLdapUserJob < ApplicationJob
+  queue_as :default
+
+  def perform(username, domain, email, hashed_pw)
+    dn = "cn=#{username},ou=#{domain},cn=users,dc=kosmos,dc=org"
+    attr = {
+      objectclass: ["top", "account", "person", "extensibleObject"],
+      cn: username,
+      sn: username,
+      uid: username,
+      mail: email,
+      userPassword: hashed_pw
+    }
+
+    ldap_client.add(dn: dn, attributes: attr)
+  end
+
+  def ldap_client
+    ldap_client ||= Net::LDAP.new host: ldap_config['host'],
+      port: ldap_config['port'],
+      encryption: ldap_config['ssl'],
+      auth: {
+        method: :simple,
+        username: ldap_config['admin_user'],
+        password: ldap_config['admin_password']
+      }
+  end
+
+  def ldap_config
+    ldap_config ||= YAML.load(ERB.new(File.read("#{Rails.root}/config/ldap.yml")).result)[Rails.env]
+  end
+end

app/jobs/exchange_xmpp_contacts_job.rb

@@ -0,0 +1,18 @@
+class ExchangeXmppContactsJob < ApplicationJob
+  queue_as :default
+
+  def perform(inviter, username, domain)
+    ejabberd = EjabberdApiClient.new
+
+    ejabberd.add_rosteritem({
+      "localuser": username, "localhost": domain,
+      "user": inviter.cn, "host": inviter.ou,
+      "nick": inviter.cn, "group": "Friends", "subs": "both"
+    })
+    ejabberd.add_rosteritem({
+      "localuser": inviter.cn, "localhost": inviter.ou,
+      "user": username, "host": domain,
+      "nick": username, "group": "Friends", "subs": "both"
+    })
+  end
+end

app/services/create_account.rb

@@ -33,51 +33,15 @@ class CreateAccount < ApplicationService
     @invitation.update! invited_user_id: user_id, used_at: DateTime.now
   end
 
+  # TODO move to confirmation
   def add_ldap_document
-    dn = "cn=#{@username},ou=kosmos.org,cn=users,dc=kosmos,dc=org"
-    attr = {
-      objectclass: ["top", "account", "person", "extensibleObject"],
-      cn: @username,
-      sn: @username,
-      uid: @username,
-      mail: @email,
-      userPassword: Devise.ldap_auth_password_builder.call(@password)
-    }
-
-    ldap_client.add(dn: dn, attributes: attr)
-  end
-
-  def ldap_client
-    ldap_client ||= Net::LDAP.new host: ldap_config['host'],
-      port: ldap_config['port'],
-      encryption: ldap_config['ssl'],
-      auth: {
-        method: :simple,
-        username: ldap_config['admin_user'],
-        password: ldap_config['admin_password']
-      }
-  end
-
-  def ldap_config
-    ldap_config ||= YAML.load(ERB.new(File.read("#{Rails.root}/config/ldap.yml")).result)[Rails.env]
+    hashed_pw = Devise.ldap_auth_password_builder.call(@password)
+    CreateLdapUserJob.perform_later(@username, @domain, @email, hashed_pw)
   end
 
   def exchange_xmpp_contacts
     #TODO enable in development when we have easy setup of ejabberd etc.
     return if Rails.env.development?
-
-    ejabberd = EjabberdApiClient.new
-    inviter = @invitation.user
-
-    ejabberd.add_rosteritem({
-      "localuser": @username, "localhost": @domain,
-      "user": inviter.cn, "host": inviter.ou,
-      "nick": inviter.cn, "group": "Friends", "subs": "both"
-    })
-    ejabberd.add_rosteritem({
-      "localuser": inviter.cn, "localhost": inviter.ou,
-      "user": @username, "host": @domain,
-      "nick": @username, "group": "Friends", "subs": "both"
-    })
+    ExchangeXmppContactsJob.perform_later(@invitation.user, @username, @domain)
   end
 end

config/application.rb

@@ -39,5 +39,8 @@ module Akkounts
       g.fixture_replacement :factory_bot, suffix_factory: 'factory', dir: 'spec/factories'
       g.stylesheets         false
     end
+
+    config.active_job.queue_adapter = :sidekiq
+    config.action_mailer.deliver_later_queue_name = nil # use "default" queue
   end
 end

config/environments/test.rb

@@ -49,4 +49,6 @@ Rails.application.configure do
     protocol: "https",
     from: "accounts@kosmos.org"
   }
+
+  config.active_job.queue_adapter = :test
 end

config/routes.rb

@@ -1,3 +1,5 @@
+require 'sidekiq/web'
+
 Rails.application.routes.draw do
   resources :donations
   devise_for :users
@@ -23,6 +25,10 @@ Rails.application.routes.draw do
     resources :donations
   end
 
+  authenticate :user, ->(user) { user.is_admin? } do
+    mount Sidekiq::Web => '/sidekiq'
+  end
+
   # Letter Opener (open "sent" emails in dev and staging)
   if Rails.env.match(/staging|development/)
     mount LetterOpenerWeb::Engine, at: "letter_opener"

config/sidekiq.yml

@@ -0,0 +1,3 @@
+:concurrency: 2
+:queues:
+  - default

spec/jobs/create_ldap_user_job_spec.rb

@@ -0,0 +1,34 @@
+require 'rails_helper'
+
+RSpec.describe CreateLdapUserJob, type: :job do
+  let(:ldap_client_mock) { instance_double(Net::LDAP) }
+
+  subject(:job) {
+    described_class.any_instance.stub(:ldap_client).and_return(ldap_client_mock)
+    described_class.perform_later(
+      'halfinney', 'kosmos.org', 'halfinney@example.com',
+      'remember-remember-the-5th-of-november'
+    )
+  }
+
+  it "creates a new document with the correct attributes" do
+    ldap_client_mock.should_receive(:add).with(
+      dn: "cn=halfinney,ou=kosmos.org,cn=users,dc=kosmos,dc=org",
+      attributes: {
+        objectclass: ["top", "account", "person", "extensibleObject"],
+        cn: "halfinney",
+        sn: "halfinney",
+        uid: "halfinney",
+        mail: "halfinney@example.com",
+        userPassword: "remember-remember-the-5th-of-november"
+      }
+    )
+
+    perform_enqueued_jobs { job }
+  end
+
+  after do
+    clear_enqueued_jobs
+    clear_performed_jobs
+  end
+end

spec/jobs/exchange_xmpp_contacts_job_spec.rb

@@ -0,0 +1,29 @@
+require 'rails_helper'
+require 'webmock/rspec'
+
+RSpec.describe ExchangeXmppContactsJob, type: :job do
+  let(:user) { create :user, cn: "willherschel", ou: "kosmos.org" }
+
+  subject(:job) {
+    described_class.perform_later(user, 'isaacnewton', 'kosmos.org')
+  }
+
+  before do
+    stub_request(:post, "http://xmpp.example.com/api/add_rosteritem")
+      .to_return(status: 200, body: "", headers: {})
+  end
+
+  it "posts add_rosteritem commands to the ejabberd API" do
+    perform_enqueued_jobs { job }
+
+    expect(WebMock).to have_requested(:post, "http://xmpp.example.com/api/add_rosteritem")
+      .with { |req| req.body == '{"localuser":"isaacnewton","localhost":"kosmos.org","user":"willherschel","host":"kosmos.org","nick":"willherschel","group":"Friends","subs":"both"}' }
+    expect(WebMock).to have_requested(:post, "http://xmpp.example.com/api/add_rosteritem")
+      .with { |req| req.body == '{"localuser":"willherschel","localhost":"kosmos.org","user":"isaacnewton","host":"kosmos.org","nick":"isaacnewton","group":"Friends","subs":"both"}' }
+  end
+
+  after do
+    clear_enqueued_jobs
+    clear_performed_jobs
+  end
+end

spec/rails_helper.rb

@@ -69,5 +69,6 @@ RSpec.configure do |config|
   config.include Devise::Test::ControllerHelpers, :type => :controller
   config.include Warden::Test::Helpers
   config.include FactoryBot::Syntax::Methods
+  config.include ActiveJob::TestHelper, type: :job
   config.extend ControllerMacros, :type => :controller
 end

spec/services/create_account_spec.rb

@@ -1,14 +1,6 @@
 require 'rails_helper'
-require 'webmock/rspec'
-require 'json'
 
 RSpec.describe CreateAccount, type: :model do
-  let(:ldap_client_mock) { instance_double(Net::LDAP) }
-
-  before do
-    allow(service).to receive(:ldap_client).and_return(ldap_client_mock)
-  end
-
   describe "#create_user_in_database" do
     let(:service) { CreateAccount.new(
       username: 'isaacnewton',
@@ -48,30 +40,34 @@ RSpec.describe CreateAccount, type: :model do
   end
 
   describe "#add_ldap_document" do
+    include ActiveJob::TestHelper
+
     let(:service) { CreateAccount.new(
       username: 'halfinney',
       email: 'halfinney@example.com',
       password: 'remember-remember-the-5th-of-november'
     )}
 
-    it "creates a new document with the correct attributes" do
-      expect(ldap_client_mock).to receive(:add).with(
-        dn: "cn=halfinney,ou=kosmos.org,cn=users,dc=kosmos,dc=org",
-        attributes: {
-          objectclass: ["top", "account", "person", "extensibleObject"],
-          cn: "halfinney",
-          sn: "halfinney",
-          uid: "halfinney",
-          mail: "halfinney@example.com",
-          userPassword: /^{SSHA512}.{171}=/
-        }
-      )
-
+    it "enqueues a job to create the LDAP user document" do
       service.send(:add_ldap_document)
+
+      expect(enqueued_jobs.size).to eq(1)
+
+      args = enqueued_jobs.first['arguments']
+      expect(args[0]).to eq('halfinney')
+      expect(args[1]).to eq('kosmos.org')
+      expect(args[2]).to eq('halfinney@example.com')
+      expect(args[3]).to match(/^{SSHA512}.{171}=/)
+    end
+
+    after do
+      clear_enqueued_jobs
     end
   end
 
   describe "#exchange_xmpp_contacts" do
+    include ActiveJob::TestHelper
+
     let(:inviter) { create :user, cn: "willherschel", ou: "kosmos.org" }
     let(:invitation) { create :invitation, user: inviter }
     let(:service) { CreateAccount.new(
@@ -81,18 +77,19 @@ RSpec.describe CreateAccount, type: :model do
       invitation: invitation
     )}
 
-    before do
-      stub_request(:post, "http://xmpp.example.com/api/add_rosteritem")
-        .to_return(status: 200, body: "", headers: {})
-    end
-
-    it "posts add_rosteritem commands to the ejabberd API" do
+    it "enqueues a job to exchange XMPP contacts between inviter and invitee" do
       service.send(:exchange_xmpp_contacts)
 
-      expect(WebMock).to have_requested(:post, "http://xmpp.example.com/api/add_rosteritem")
-        .with { |req| req.body == '{"localuser":"isaacnewton","localhost":"kosmos.org","user":"willherschel","host":"kosmos.org","nick":"willherschel","group":"Friends","subs":"both"}' }
-      expect(WebMock).to have_requested(:post, "http://xmpp.example.com/api/add_rosteritem")
-        .with { |req| req.body == '{"localuser":"willherschel","localhost":"kosmos.org","user":"isaacnewton","host":"kosmos.org","nick":"isaacnewton","group":"Friends","subs":"both"}' }
+      expect(enqueued_jobs.size).to eq(1)
+
+      args = enqueued_jobs.first['arguments']
+      expect(args[0]['_aj_globalid']).to match('gid://akkounts/User')
+      expect(args[1]).to eq('isaacnewton')
+      expect(args[2]).to eq('kosmos.org')
+    end
+
+    after do
+      clear_enqueued_jobs
     end
   end
 end