Merge pull request #59 from 5apps/swift

Add Swift provider

Gemfile

@@ -5,6 +5,8 @@ gem "sinatra-contrib"
 gem "activesupport", '~> 3.2'
 gem "riak-client", :github => "5apps/riak-ruby-client", :branch => "invalid_uri_error"
 gem "fog"
+gem "rest-client"
+gem "redis"
 
 group :test do
   gem 'rake'
@@ -14,4 +16,5 @@ end
 
 group :staging, :production do
   gem "rainbows"
+  gem "sentry-raven"
 end

Gemfile.lock

@@ -21,6 +21,8 @@ GEM
     builder (3.2.2)
     eventmachine (1.0.3)
     excon (0.16.10)
+    faraday (0.9.1)
+      multipart-post (>= 1.2, < 3)
     fog (1.7.0)
       builder
       excon (~> 0.14)
@@ -42,6 +44,7 @@ GEM
     mime-types (1.23)
     minitest (2.10.0)
     multi_json (1.10.0)
+    multipart-post (2.0.0)
     net-scp (1.0.4)
       net-ssh (>= 1.99.1)
     net-ssh (2.6.7)
@@ -59,7 +62,12 @@ GEM
       unicorn (~> 4.8)
     raindrops (0.13.0)
     rake (0.9.2.2)
+    redis (3.0.7)
+    rest-client (1.6.7)
+      mime-types (>= 1.16)
     ruby-hmac (0.4.0)
+    sentry-raven (0.13.1)
+      faraday (>= 0.7.6)
     sinatra (1.4.5)
       rack (~> 1.4)
       rack-protection (~> 1.4)
@@ -87,6 +95,9 @@ DEPENDENCIES
   purdytest
   rainbows
   rake
+  redis
+  rest-client
   riak-client!
+  sentry-raven
   sinatra (~> 1.4)
   sinatra-contrib

config.yml.example

@@ -13,6 +13,13 @@ development: &defaults
       cs_binaries: rs.binaries
       authorizations: rs_authorizations
       opslog: rs_opslog
+  # # uncomment this section and comment the riak one
+  # swift: &swift_defaults
+  #   host: "https://swift.example.com"
+  # # Redis is needed for the swift backend
+  # redis:
+  #   host: localhost
+  #   port: 6379
 
 test:
   <<: *defaults

lib/remote_storage/riak.rb

@@ -15,7 +15,7 @@ module RemoteStorage
       self.settings = settings
       self.server = server
 
-      credentials = File.read(settings['riak_cs']['credentials_file'])
+      credentials = File.read(settings.riak['riak_cs']['credentials_file'])
       self.cs_credentials = JSON.parse(credentials)
     end
 
@@ -229,8 +229,7 @@ module RemoteStorage
 
     # A URI object that can be used with HTTP backend methods
     def riak_uri(bucket, key)
-      rc = settings.symbolize_keys
+      URI.parse "http://#{settings.riak["host"]}:#{settings.riak["http_port"]}/riak/#{bucket}/#{key}"
-      URI.parse "http://#{rc[:host]}:#{rc[:http_port]}/riak/#{bucket}/#{key}"
     end
 
     def serializer_for(content_type)
@@ -471,13 +470,13 @@ module RemoteStorage
     end
 
     def client
-      @client ||= ::Riak::Client.new(:host      => settings['host'],
+      @client ||= ::Riak::Client.new(:host      => settings.riak['host'],
-                                     :http_port => settings['http_port'])
+                                     :http_port => settings.riak['http_port'])
     end
 
     def data_bucket
       @data_bucket ||= begin
-                         bucket = client.bucket(settings['buckets']['data'])
+                         bucket = client.bucket(settings.riak['buckets']['data'])
                          bucket.allow_mult = false
                          bucket
                        end
@@ -485,7 +484,7 @@ module RemoteStorage
 
     def directory_bucket
       @directory_bucket ||= begin
-                              bucket = client.bucket(settings['buckets']['directories'])
+                              bucket = client.bucket(settings.riak['buckets']['directories'])
                               bucket.allow_mult = false
                               bucket
                             end
@@ -493,7 +492,7 @@ module RemoteStorage
 
     def auth_bucket
       @auth_bucket ||= begin
-                         bucket = client.bucket(settings['buckets']['authorizations'])
+                         bucket = client.bucket(settings.riak['buckets']['authorizations'])
                          bucket.allow_mult = false
                          bucket
                        end
@@ -501,7 +500,7 @@ module RemoteStorage
 
     def binary_bucket
       @binary_bucket ||= begin
-                           bucket = client.bucket(settings['buckets']['binaries'])
+                           bucket = client.bucket(settings.riak['buckets']['binaries'])
                            bucket.allow_mult = false
                            bucket
                          end
@@ -509,7 +508,7 @@ module RemoteStorage
 
     def opslog_bucket
       @opslog_bucket ||= begin
-                           bucket = client.bucket(settings['buckets']['opslog'])
+                           bucket = client.bucket(settings.riak['buckets']['opslog'])
                            bucket.allow_mult = false
                            bucket
                          end
@@ -520,12 +519,12 @@ module RemoteStorage
         :provider                 => 'AWS',
         :aws_access_key_id        => cs_credentials['key_id'],
         :aws_secret_access_key    => cs_credentials['key_secret'],
-        :endpoint                 => settings['riak_cs']['endpoint']
+        :endpoint                 => settings.riak['riak_cs']['endpoint']
       })
     end
 
     def cs_binary_bucket
-      @cs_binary_bucket ||= cs_client.directories.create(:key => settings['buckets']['cs_binaries'])
+      @cs_binary_bucket ||= cs_client.directories.create(:key => settings.riak['buckets']['cs_binaries'])
     end
 
   end

lib/remote_storage/swift.rb

@@ -0,0 +1,311 @@
+require "rest_client"
+require "json"
+require "cgi"
+require "active_support/core_ext/time/conversions"
+require "active_support/core_ext/numeric/time"
+require "redis"
+
+module RemoteStorage
+  class Swift
+
+    attr_accessor :settings, :server
+
+    def initialize(settings, server)
+      self.settings = settings
+      self.server = server
+    end
+
+    def authorize_request(user, directory, token, listing=false)
+      request_method = server.env["REQUEST_METHOD"]
+
+      if directory.split("/").first == "public"
+        return true if ["GET", "HEAD"].include?(request_method) && !listing
+      end
+
+      authorizations = redis.smembers("authorizations:#{user}:#{token}")
+      permission = directory_permission(authorizations, directory)
+
+      server.halt 401 unless permission
+      if ["PUT", "DELETE"].include? request_method
+        server.halt 401 unless permission == "rw"
+      end
+    end
+
+    def get_head(user, directory, key)
+      url = url_for(user, directory, key)
+
+      res = do_head_request(url)
+
+      set_response_headers(res)
+    rescue RestClient::ResourceNotFound
+      server.halt 404
+    end
+
+    def get_data(user, directory, key)
+      url = url_for(user, directory, key)
+
+      res = do_get_request(url)
+
+      set_response_headers(res)
+
+      none_match = (server.env["HTTP_IF_NONE_MATCH"] || "").split(",").map(&:strip)
+      server.halt 304 if none_match.include? %Q("#{res.headers[:etag]}")
+
+      return res.body
+    rescue RestClient::ResourceNotFound
+      server.halt 404
+    end
+
+    def get_head_directory_listing(user, directory)
+      res = do_head_request("#{container_url_for(user)}/#{directory}/")
+
+      server.headers["Content-Type"] = "application/json"
+      server.headers["ETag"]         = %Q("#{res.headers[:etag]}")
+    rescue RestClient::ResourceNotFound
+      server.halt 404
+    end
+
+    def get_directory_listing(user, directory)
+      server.headers["Content-Type"] = "application/json"
+
+      do_head_request("#{container_url_for(user)}/#{directory}/") do |response|
+        if response.code == 404
+          return directory_listing([]).to_json
+        else
+          server.headers["ETag"] = %Q("#{response.headers[:etag]}")
+          none_match = (server.env["HTTP_IF_NONE_MATCH"] || "").split(",").map(&:strip)
+          server.halt 304 if none_match.include? %Q("#{response.headers[:etag]}")
+        end
+      end
+
+      res = do_get_request("#{container_url_for(user)}/?format=json&path=#{directory}/")
+
+      if body = JSON.parse(res.body)
+        listing = directory_listing(body)
+      else
+        puts "listing not JSON"
+      end
+
+      listing.to_json
+    end
+
+    def put_data(user, directory, key, data, content_type)
+      server.halt 409 if has_name_collision?(user, directory, key)
+
+      url = url_for(user, directory, key)
+
+      if required_match = server.env["HTTP_IF_MATCH"]
+        do_head_request(url) do |response|
+          server.halt 412 unless required_match == %Q("#{response.headers[:etag]}")
+        end
+      end
+      if server.env["HTTP_IF_NONE_MATCH"] == "*"
+        do_head_request(url) do |response|
+          server.halt 412 unless response.code == 404
+        end
+      end
+
+      res = do_put_request(url, data, content_type)
+
+      if update_dir_objects(user, directory)
+        server.headers["ETag"] = %Q("#{res.headers[:etag]}")
+        server.halt 200
+      else
+        server.halt 500
+      end
+    end
+
+    def delete_data(user, directory, key)
+      url = url_for(user, directory, key)
+
+      if required_match = server.env["HTTP_IF_MATCH"]
+        do_head_request(url) do |response|
+          server.halt 412 unless required_match == %Q("#{response.headers[:etag]}")
+        end
+      end
+
+      do_delete_request(url)
+      delete_dir_objects(user, directory)
+
+      server.halt 200
+    rescue RestClient::ResourceNotFound
+      server.halt 404
+    end
+
+    private
+
+    def set_response_headers(response)
+      server.headers["ETag"]           = %Q("#{response.headers[:etag]}")
+      server.headers["Content-Type"]   = response.headers[:content_type]
+      server.headers["Content-Length"] = response.headers[:content_length]
+      server.headers["Last-Modified"]  = response.headers[:last_modified]
+    end
+
+    def extract_category(directory)
+      if directory.match(/^public\//)
+        "public/#{directory.split('/')[1]}"
+      else
+        directory.split('/').first
+      end
+    end
+
+    def directory_permission(authorizations, directory)
+      authorizations = authorizations.map do |auth|
+        auth.index(":") ? auth.split(":") : [auth, "rw"]
+      end
+      authorizations = Hash[*authorizations.flatten]
+
+      permission = authorizations[""]
+
+      authorizations.each do |key, value|
+        if directory.match(/^(public\/)?#{key}(\/|$)/)
+          if permission.nil? || permission == "r"
+            permission = value
+          end
+          return permission if permission == "rw"
+        end
+      end
+
+      permission
+    end
+
+    def directory_listing(res_body)
+      listing = {
+        "@context" => "http://remotestorage.io/spec/folder-description",
+        "items"    => {}
+      }
+
+      res_body.each do |entry|
+        name = entry["name"].gsub("#{File.dirname(entry["name"])}/", '')
+        if name[-1] == "/"
+          listing["items"].merge!({
+            name => {
+              "ETag"           => entry["hash"],
+            }
+          })
+        else
+          listing["items"].merge!({
+            name => {
+              "ETag"           => entry["hash"],
+              "Content-Type"   => entry["content_type"],
+              "Content-Length" => entry["bytes"]
+            }
+          })
+        end
+      end
+
+      listing
+    end
+
+    def has_name_collision?(user, directory, key)
+      # check for existing directory with the same name as the document
+      do_head_request("#{container_url_for(user)}/#{directory}/#{key}/") do |res|
+        return true if res.code == 200
+      end
+
+      # check for existing documents with the same name as one of the parent directories
+      parent_directories_for(directory).each do |dir|
+        do_head_request("#{container_url_for(user)}/#{dir}/") do |res_dir|
+          if res_dir.code == 200
+            return false
+          else
+            do_head_request("#{container_url_for(user)}/#{dir}") do |res_key|
+              if res_key.code == 200
+                return true
+              else
+                next
+              end
+            end
+          end
+        end
+      end
+
+      false
+    end
+
+    def parent_directories_for(directory)
+      directories = directory.split("/")
+      parent_directories = []
+
+      while directories.any?
+        parent_directories << directories.join("/")
+        directories.pop
+      end
+
+      parent_directories
+    end
+
+    def update_dir_objects(user, directory)
+      timestamp = (Time.now.to_f * 1000).to_i
+
+      parent_directories_for(directory).each do |dir|
+        do_put_request("#{container_url_for(user)}/#{dir}/", timestamp.to_s, "text/plain")
+      end
+
+      true
+    rescue
+      parent_directories_for(directory).each do |dir|
+        do_delete_request("#{container_url_for(user)}/#{dir}/") rescue false
+      end
+
+      false
+    end
+
+    def delete_dir_objects(user, directory)
+      parent_directories_for(directory).each do |dir|
+        if dir_empty?(user, dir)
+          do_delete_request("#{container_url_for(user)}/#{dir}/")
+        else
+          timestamp = (Time.now.to_f * 1000).to_i
+          do_put_request("#{container_url_for(user)}/#{dir}/", timestamp.to_s, "text/plain")
+        end
+      end
+    end
+
+    def dir_empty?(user, dir)
+      do_get_request("#{container_url_for(user)}/?format=plain&limit=1&path=#{dir}/") do |res|
+        return res.headers[:content_length] == "0"
+      end
+    end
+
+    def container_url_for(user)
+      "#{base_url}/#{container_for(user)}"
+    end
+
+    def url_for(user, directory, key)
+      "#{container_url_for(user)}/#{directory}/#{key}"
+    end
+
+    def base_url
+      @base_url ||= settings.swift["host"]
+    end
+
+    def container_for(user)
+      "rs:#{settings.environment.to_s.chars.first}:#{user}"
+    end
+
+    def default_headers
+      @default_headers ||= {"x-auth-token" => settings.swift["token"]}
+    end
+
+    def do_put_request(url, data, content_type)
+      RestClient.put(url, data, default_headers.merge({content_type: content_type}))
+    end
+
+    def do_get_request(url, &block)
+      RestClient.get(url, default_headers, &block)
+    end
+
+    def do_head_request(url, &block)
+      RestClient.head(url, default_headers, &block)
+    end
+
+    def do_delete_request(url)
+      RestClient.delete(url, default_headers)
+    end
+
+    def redis
+      @redis ||= Redis.new(host: settings.redis["host"], port: settings.redis["port"])
+    end
+  end
+end

liquor-cabinet.rb

@@ -5,6 +5,7 @@ require "sinatra/base"
 require 'sinatra/config_file'
 require "sinatra/reloader"
 require "remote_storage/riak"
+require "remote_storage/swift"
 
 class LiquorCabinet < Sinatra::Base
 
@@ -23,11 +24,22 @@ class LiquorCabinet < Sinatra::Base
 
   configure :development do
     register Sinatra::Reloader
+    also_reload "lib/remote_storage/*.rb"
     enable :logging
   end
 
   configure :production, :staging do
     require "rack/common_logger"
+    if ENV['SENTRY_DSN']
+      require "raven"
+
+      Raven.configure do |config|
+        config.dsn = ENV['SENTRY_DSN']
+        config.tags = { environment: settings.environment.to_s }
+      end
+
+      use Raven::Rack
+    end
   end
 
   #
@@ -62,17 +74,19 @@ class LiquorCabinet < Sinatra::Base
   end
 
   ["/:user/*/:key", "/:user/:key"].each do |path|
-    get path do
-      storage.get_data(@user, @directory, @key)
-    end
-
     head path do
       storage.get_head(@user, @directory, @key)
     end
 
+    get path do
+      storage.get_data(@user, @directory, @key)
+    end
+
     put path do
       data = request.body.read
 
+      halt 422 unless env['CONTENT_TYPE']
+
       if env['CONTENT_TYPE'] == "application/x-www-form-urlencoded"
         content_type = "text/plain; charset=utf-8"
       else
@@ -88,23 +102,29 @@ class LiquorCabinet < Sinatra::Base
   end
 
   ["/:user/*/", "/:user/"].each do |path|
-    get path do
-      storage.get_directory_listing(@user, @directory)
-    end
-
     head path do
       storage.get_head_directory_listing(@user, @directory)
     end
+
+    get path do
+      storage.get_directory_listing(@user, @directory)
+    end
   end
 
   private
 
   def storage
     @storage ||= begin
-      if settings.riak
+      if settings.respond_to? :riak
-        RemoteStorage::Riak.new(settings.riak, self)
+        RemoteStorage::Riak.new(settings, self)
-      # elsif settings.redis
+      elsif settings.respond_to? :swift
-      #  include RemoteStorage::Redis
+        settings.swift["token"] = File.read("tmp/swift_token.txt")
+        RemoteStorage::Swift.new(settings, self)
+      else
+        puts <<-EOF
+You need to set one storage backend in your config.yml file.
+Riak and Swift are currently supported. See config.yml.example.
+        EOF
       end
     end
   end

spec/riak_spec.rb

@@ -17,19 +17,11 @@ describe "App with Riak backend" do
       head "/jimmy/public/foo"
     end
 
-    it "returns an empty body" do
+    it "works" do
       last_response.status.must_equal 200
       last_response.body.must_equal ""
-    end
-
-    it "has an ETag header set" do
-      last_response.status.must_equal 200
       last_response.headers["ETag"].wont_be_nil
-    end
+      last_response.headers["Content-Length"].must_equal "14"
-
-    it "has a Content-Length header set" do
-      last_response.status.must_equal 200
-      last_response.headers["Content-Length"].must_equal '14'
     end
   end
 
@@ -44,26 +36,13 @@ describe "App with Riak backend" do
         get "/jimmy/public/foo"
       end
 
-      it "returns the value on all get requests" do
+      it "works" do
         last_response.status.must_equal 200
         last_response.body.must_equal "some text data"
-      end
-
-      it "has an ETag header set" do
-        last_response.status.must_equal 200
         last_response.headers["ETag"].wont_be_nil
-      end
-
-      it "has a Content-Length header set" do
-        last_response.status.must_equal 200
         last_response.headers["Content-Length"].must_equal "14"
-      end
-
-      it "has caching headers set" do
-        last_response.status.must_equal 200
         last_response.headers["Expires"].must_equal "0"
       end
-    end
 
       describe "empty file" do
         before do
@@ -124,19 +103,11 @@ describe "App with Riak backend" do
           head "/jimmy/documents/foo"
         end
 
-      it "returns an empty body" do
+        it "works" do
           last_response.status.must_equal 200
           last_response.body.must_equal ""
-      end
-
-      it "has an ETag header set" do
-        last_response.status.must_equal 200
           last_response.headers["ETag"].wont_be_nil
-      end
+          last_response.headers["Content-Length"].must_equal "22"
-
-      it "has a Content-Length header set" do
-        last_response.status.must_equal 200
-        last_response.headers["Content-Length"].must_equal '22'
         end
       end
 
@@ -325,7 +296,7 @@ describe "App with Riak backend" do
           end
         end
 
-      describe "naming collissions between documents and directories" do
+        describe "naming collisions between documents and directories" do
           before do
             put "/jimmy/documents/archive/document", "lorem ipsum"
           end
@@ -801,3 +772,4 @@ describe "App with Riak backend" do
       end
     end
   end
+end

spec/spec_helper.rb

@@ -28,8 +28,7 @@ def write_last_response_to_file(filename = "last_response.html")
 end
 
 alias context describe
-
+if app.settings.respond_to? :riak
-if app.settings.riak
   ::Riak.disable_list_keys_warnings = true
 
   def client