Merge branch 'master' into stable

.travis.yml

@@ -4,13 +4,10 @@ rvm:
   - 2.4.1
 services:
   - redis-server
-before_install:
-  - sh .travis/install_riakcs.sh
-  - gem install bundler
 before_script:
-  - cp config.yml.example config.yml
+  - cp config.yml.example.$BACKEND config.yml
   - mkdir -p tmp && echo "swifttoken" > tmp/swift_token.txt
-script: ruby spec/swift/*
+script: ruby spec/$BACKEND/*
 branches:
   only:
     - master
@@ -22,6 +19,8 @@ notifications:
       - http://hook-juggler.herokuapp.com/hooks/travis
     on_success: always
     on_failure: always
-# Force legacy Blue Box build for now
+env:
-sudo: required
+  - BACKEND=s3
-group: legacy
+  - BACKEND=swift
+# Run on Docker infrastructure
+sudo: false

Gemfile

@@ -3,8 +3,6 @@ source "https://rubygems.org"
 gem "sinatra"
 gem "sinatra-contrib"
 gem "activesupport"
-gem "riak-client", git: "https://github.com/5apps/riak-ruby-client", branch: "invalid_uri_error"
-gem "fog-aws"
 gem "rest-client", "~> 2.1.0.rc1" # Fixes a memory leak in Ruby 2.4
 gem "redis"
 # Remove require when we can update to 3.0, which sets the new storage
@@ -17,6 +15,7 @@ group :test do
   gem 'purdytest', :require => false
   gem 'm'
   gem 'minitest-stub_any_instance'
+  gem 'webmock'
 end
 
 group :staging, :production do

Gemfile.lock

@@ -1,121 +1,96 @@
-GIT
-  remote: https://github.com/5apps/riak-ruby-client
-  revision: 5f21df86b14339aeb252374851d29ad813cca1dd
-  branch: invalid_uri_error
-  specs:
-    riak-client (1.4.0)
-      beefcake (~> 0.3.7)
-      builder (>= 2.1.2)
-      i18n (>= 0.4.0)
-      innertube (~> 1.0.2)
-      multi_json (~> 1.0)
-
 GEM
   remote: https://rubygems.org/
   specs:
-    activesupport (5.1.2)
+    activesupport (5.2.0)
       concurrent-ruby (~> 1.0, >= 1.0.2)
-      i18n (~> 0.7)
+      i18n (>= 0.7, < 2)
       minitest (~> 5.1)
       tzinfo (~> 1.1)
-    backports (3.8.0)
+    addressable (2.5.2)
-    beefcake (0.3.7)
+      public_suffix (>= 2.0.2, < 4.0)
-    builder (3.2.3)
+    backports (3.11.2)
     concurrent-ruby (1.0.5)
+    crack (0.4.3)
+      safe_yaml (~> 1.0.0)
     domain_name (0.5.20170404)
       unf (>= 0.0.5, < 1.0.0)
-    excon (0.57.1)
+    faraday (0.14.0)
-    faraday (0.12.1)
       multipart-post (>= 1.2, < 3)
-    fog-aws (1.4.0)
+    hashdiff (0.3.7)
-      fog-core (~> 1.38)
-      fog-json (~> 1.0)
-      fog-xml (~> 0.1)
-      ipaddress (~> 0.8)
-    fog-core (1.44.3)
-      builder
-      excon (~> 0.49)
-      formatador (~> 0.2)
-    fog-json (1.0.2)
-      fog-core (~> 1.0)
-      multi_json (~> 1.10)
-    fog-xml (0.1.3)
-      fog-core
-      nokogiri (>= 1.5.11, < 2.0.0)
-    formatador (0.2.5)
     http-accept (1.7.0)
     http-cookie (1.0.3)
       domain_name (~> 0.5)
-    i18n (0.8.4)
+    i18n (1.0.0)
-    innertube (1.0.2)
+      concurrent-ruby (~> 1.0)
-    ipaddress (0.8.3)
+    kgio (2.11.2)
-    kgio (2.11.0)
     m (1.5.1)
       method_source (>= 0.6.7)
       rake (>= 0.9.2.2)
-    method_source (0.8.2)
+    method_source (0.9.0)
     mime-types (3.1)
       mime-types-data (~> 3.2015)
     mime-types-data (3.2016.0521)
-    mini_portile2 (2.2.0)
+    minitest (5.11.3)
-    minitest (5.10.2)
+    minitest-stub_any_instance (1.0.2)
-    minitest-stub_any_instance (1.0.1)
+    multi_json (1.13.1)
-    multi_json (1.12.1)
     multipart-post (2.0.0)
-    mustermann (1.0.0)
+    mustermann (1.0.2)
     netrc (0.11.0)
-    nokogiri (1.8.0)
+    public_suffix (3.0.2)
-      mini_portile2 (~> 2.2.0)
     purdytest (2.0.0)
       minitest (~> 5.5)
-    rack (2.0.3)
+    rack (2.0.4)
-    rack-protection (2.0.0)
+    rack-protection (2.0.1)
       rack
-    rack-test (0.6.3)
+    rack-test (1.0.0)
-      rack (>= 1.0)
+      rack (>= 1.0, < 3)
     rainbows (5.1.1)
       kgio (~> 2.5)
       rack (>= 1.1, < 3.0)
       unicorn (~> 5.1)
-    raindrops (0.18.0)
+    raindrops (0.19.0)
-    rake (12.0.0)
+    rake (12.3.1)
-    redis (3.3.3)
+    redis (4.0.1)
     rest-client (2.1.0.rc1)
       http-accept (>= 1.7.0, < 2.0)
       http-cookie (>= 1.0.2, < 2.0)
       mime-types (>= 1.16, < 4.0)
       netrc (~> 0.8)
-    sentry-raven (2.5.3)
+    safe_yaml (1.0.4)
+    sentry-raven (2.7.2)
       faraday (>= 0.7.6, < 1.0)
-    sinatra (2.0.0)
+    sinatra (2.0.1)
       mustermann (~> 1.0)
       rack (~> 2.0)
-      rack-protection (= 2.0.0)
+      rack-protection (= 2.0.1)
       tilt (~> 2.0)
-    sinatra-contrib (2.0.0)
+    sinatra-contrib (2.0.1)
       backports (>= 2.0)
       multi_json
       mustermann (~> 1.0)
-      rack-protection (= 2.0.0)
+      rack-protection (= 2.0.1)
-      sinatra (= 2.0.0)
+      sinatra (= 2.0.1)
       tilt (>= 1.3, < 3)
     thread_safe (0.3.6)
-    tilt (2.0.7)
+    tilt (2.0.8)
-    tzinfo (1.2.3)
+    tzinfo (1.2.5)
       thread_safe (~> 0.1)
     unf (0.1.4)
       unf_ext
-    unf_ext (0.0.7.4)
+    unf_ext (0.0.7.5)
-    unicorn (5.3.0)
+    unicorn (5.4.0)
       kgio (~> 2.6)
       raindrops (~> 0.7)
+    webmock (3.3.0)
+      addressable (>= 2.3.6)
+      crack (>= 0.3.2)
+      hashdiff
 
 PLATFORMS
   ruby
 
 DEPENDENCIES
   activesupport
-  fog-aws
   m
   mime-types
   minitest-stub_any_instance
@@ -125,10 +100,10 @@ DEPENDENCIES
   rake
   redis
   rest-client (~> 2.1.0.rc1)
-  riak-client!
   sentry-raven
   sinatra
   sinatra-contrib
+  webmock
 
 BUNDLED WITH
-   1.15.1
+   1.16.0

README.md

@@ -3,16 +3,19 @@
 # Liquor Cabinet
 
 Liquor Cabinet is where Frank stores all his stuff. It's a
-remoteStorage-compatible storage provider API, based on Sinatra and currently
+[remoteStorage](https://remotestorage.io) HTTP API, based on Sinatra. The
-using Riak as backend. You can use it on its own, or e.g. mount it from a Rails
+metadata and OAuth tokens are stored in Redis, and documents can be stored in
-application.
+anything that supports the storage API of either Openstack Swift or Amazon S3.
 
-It's merely implementing the storage API, not including the Webfinger and OAuth
+Liquor Cabinet only implements the storage API part of the remoteStorage
-parts of remoteStorage. You have to set the authorization keys/values in the
+protocol, but does not include the Webfinger and OAuth parts. It is meant to be
-database yourself.
+added to existing systems and user accounts, so you will have to add your own
+OAuth dialog for remoteStorage authorizations and persist the tokens in Redis.
 
-If you have any questions about this thing, drop by #remotestorage on Freenode, and
+If you have any questions about this program, drop by #remotestorage on
-we'll happily answer them.
+Freenode, or [post to the RS
+forums](https://community.remotestorage.io/c/server-development), and we'll
+happily answer them.
 
 ## Contributing
 

config.yml.example

@@ -1,45 +0,0 @@
-development: &defaults
-  maintenance: false
-  # riak: &riak_defaults
-  #   host: localhost
-  #   http_port: 8098
-  #   riak_cs:
-  #     credentials_file: "cs_credentials.json"
-  #     endpoint: "http://cs.example.com:8080"
-  #   buckets:
-  #     data: rs_data
-  #     directories: rs_directories
-  #     binaries: rs_binaries
-  #     cs_binaries: rs.binaries
-  #     authorizations: rs_authorizations
-  #     opslog: rs_opslog
-  # # uncomment this section and comment the riak one
-  # swift: &swift_defaults
-  #   host: "https://swift.example.com"
-  # # Redis is needed for the swift backend
-  # redis:
-  #   host: localhost
-  #   port: 6379
-
-test:
-  <<: *defaults
-  # riak:
-  #   <<: *riak_defaults
-  #   buckets:
-  #     data: rs_data_test
-  #     directories: rs_directories_test
-  #     binaries: rs_binaries_test
-  #     cs_binaries: rs.binaries.test
-  #     authorizations: rs_authorizations_test
-  #     opslog: rs_opslog_test
-  swift:
-    host: "https://swift.example.com"
-  redis:
-    host: localhost
-    port: 6379
-
-staging:
-  <<: *defaults
-
-production:
-  <<: *defaults

config.yml.example.s3

@@ -0,0 +1,26 @@
+development: &defaults
+  maintenance: false
+  s3:
+    endpoint: "https://some-endpoint"
+    region: "region"
+    access_key_id: ""
+    secret_key_id: ""
+    bucket: "test-bucket"
+  redis:
+    host: localhost
+    port: 6379
+
+test:
+  <<: *defaults
+  s3:
+    endpoint: "https://some-endpoint"
+    region: "region"
+    access_key_id: ""
+    secret_key_id: ""
+    bucket: "test-bucket"
+
+staging:
+  <<: *defaults
+
+production:
+  <<: *defaults

config.yml.example.swift

@@ -0,0 +1,18 @@
+development: &defaults
+  maintenance: false
+  swift: &swift_defaults
+    host: "https://swift.example.com"
+  redis:
+    host: localhost
+    port: 6379
+
+test:
+  <<: *defaults
+  swift:
+    host: "https://swift.example.com"
+
+staging:
+  <<: *defaults
+
+production:
+  <<: *defaults

lib/remote_storage/rest_provider.rb

@@ -0,0 +1,503 @@
+require "rest_client"
+require "json"
+require "cgi"
+require "active_support/core_ext/time/conversions"
+require "active_support/core_ext/numeric/time"
+require "active_support/core_ext/hash"
+require "redis"
+require "digest/md5"
+
+module RemoteStorage
+  module RestProvider
+
+    attr_accessor :settings, :server
+
+    def initialize(settings, server)
+      @settings = settings
+      @server   = server
+    end
+
+    def authorize_request(user, directory, token, listing=false)
+      request_method = server.env["REQUEST_METHOD"]
+
+      if directory.split("/").first == "public"
+        return true if ["GET", "HEAD"].include?(request_method) && !listing
+      end
+
+      server.halt 401, "Unauthorized" if token.nil? || token.empty?
+
+      authorizations = redis.smembers("authorizations:#{user}:#{token}")
+      permission = directory_permission(authorizations, directory)
+
+      server.halt 401, "Unauthorized" unless permission
+      if ["PUT", "DELETE"].include? request_method
+        server.halt 401, "Unauthorized" unless permission == "rw"
+      end
+    end
+
+    def get_head(user, directory, key)
+      none_match = (server.env["HTTP_IF_NONE_MATCH"] || "").split(",")
+                                                           .map(&:strip)
+                                                           .map { |s| s.gsub(/^"?W\//, "") }
+      metadata = redis.hgetall redis_metadata_object_key(user, directory, key)
+
+      server.halt 404 if metadata.empty?
+
+      # Set the response headers for a 304 or 200 response
+      server.headers["ETag"]          = %Q("#{metadata["e"]}")
+      server.headers["Last-Modified"] = Time.at(metadata["m"].to_i / 1000).httpdate
+      server.headers["Content-Type"]  = metadata["t"]
+      server.headers["Content-Length"]  = metadata["s"]
+
+      if none_match.include? %Q("#{metadata["e"]}")
+        server.halt 304
+      end
+    end
+
+    def get_data(user, directory, key)
+      none_match = (server.env["HTTP_IF_NONE_MATCH"] || "").split(",")
+                                                           .map(&:strip)
+                                                           .map { |s| s.gsub(/^"?W\//, "") }
+      metadata = redis.hgetall redis_metadata_object_key(user, directory, key)
+      if none_match.include? %Q("#{metadata["e"]}")
+        server.headers["ETag"]           = %Q("#{metadata["e"]}")
+        server.headers["Last-Modified"]  = Time.at(metadata["m"].to_i / 1000).httpdate
+        server.halt 304
+      end
+
+      url = url_for_key(user, directory, key)
+
+      res = do_get_request(url)
+
+      set_response_headers(res.headers)
+
+      return res.body
+    rescue RestClient::ResourceNotFound
+      server.halt 404, "Not Found"
+    end
+
+    def get_head_directory_listing(user, directory)
+      get_directory_listing(user, directory)
+
+      "" # just return empty body, headers are set by get_directory_listing
+    end
+
+    def get_directory_listing(user, directory)
+      etag = redis.hget "rs:m:#{user}:#{directory}/", "e"
+
+      server.headers["Content-Type"] = "application/ld+json"
+
+      none_match = (server.env["HTTP_IF_NONE_MATCH"] || "").split(",")
+                                                           .map(&:strip)
+                                                           .map { |s| s.gsub(/^"?W\//, "") }
+
+      if etag
+        server.halt 304 if none_match.include? %Q("#{etag}")
+
+        items = get_directory_listing_from_redis_via_lua(user, directory)
+      else
+        etag = etag_for(user, directory)
+        items = {}
+
+        server.halt 304 if none_match.include? %Q("#{etag}")
+      end
+
+      server.headers["ETag"] = %Q("#{etag}")
+
+      listing = {
+        "@context" => "http://remotestorage.io/spec/folder-description",
+        "items"    => items
+      }
+
+      listing.to_json
+    end
+
+    def put_data(user, directory, key, data, content_type)
+      server.halt 400 if server.env["HTTP_CONTENT_RANGE"]
+      server.halt 409, "Conflict" if has_name_collision?(user, directory, key)
+
+      existing_metadata = redis.hgetall redis_metadata_object_key(user, directory, key)
+      url = url_for_key(user, directory, key)
+
+      if required_match = server.env["HTTP_IF_MATCH"]
+        required_match = required_match.gsub(/^"?W\//, "")
+        unless required_match == %Q("#{existing_metadata["e"]}")
+
+          # get actual metadata and compare in case redis metadata became out of sync
+          begin
+            head_res = do_head_request(url)
+          # The file doesn't exist, return 412
+          rescue RestClient::ResourceNotFound
+            server.halt 412, "Precondition Failed"
+          end
+
+          if required_match == format_etag(head_res.headers[:etag])
+            # log previous size difference that was missed ealier because of redis failure
+            log_size_difference(user, existing_metadata["s"], head_res.headers[:content_length])
+          else
+            server.halt 412, "Precondition Failed"
+          end
+        end
+      end
+      if server.env["HTTP_IF_NONE_MATCH"] == "*"
+        server.halt 412, "Precondition Failed" unless existing_metadata.empty?
+      end
+
+      etag, timestamp = do_put_request(url, data, content_type)
+
+      metadata = {
+        e: etag,
+        s: data.size,
+        t: content_type,
+        m: timestamp
+      }
+
+      if update_metadata_object(user, directory, key, metadata)
+        if metadata_changed?(existing_metadata, metadata)
+          update_dir_objects(user, directory, timestamp, checksum_for(data))
+          log_size_difference(user, existing_metadata["s"], metadata[:s])
+        end
+
+        server.headers["ETag"] = %Q("#{etag}")
+        server.halt existing_metadata.empty? ? 201 : 200
+      else
+        server.halt 500
+      end
+    end
+
+    def delete_data(user, directory, key)
+      url = url_for_key(user, directory, key)
+
+      existing_metadata = redis.hgetall "rs:m:#{user}:#{directory}/#{key}"
+
+      if required_match = server.env["HTTP_IF_MATCH"]
+        unless required_match.gsub(/^"?W\//, "") == %Q("#{existing_metadata["e"]}")
+          server.halt 412, "Precondition Failed"
+        end
+      end
+
+      found = try_to_delete(url)
+
+      log_size_difference(user, existing_metadata["s"], 0)
+      delete_metadata_objects(user, directory, key)
+      delete_dir_objects(user, directory)
+
+      if found
+        server.headers["Etag"] = %Q("#{existing_metadata["e"]}")
+        server.halt 200
+      else
+        server.halt 404, "Not Found"
+      end
+    end
+
+    private
+
+    # Implement this method in your class that includes this module. For example
+    # %Q("#{etag}") if the ETag does not already have quotes around it
+    def format_etag(etag)
+      NotImplementedError
+    end
+
+    def base_url
+      NotImplementedError
+    end
+
+    def container_url_for(user)
+      NotImplementedError
+    end
+
+    def default_headers
+      raise NotImplementedError
+    end
+
+    def set_response_headers(headers)
+      server.headers["ETag"]           = format_etag(headers[:etag])
+      server.headers["Content-Type"]   = headers[:content_type]
+      server.headers["Content-Length"] = headers[:content_length]
+      server.headers["Last-Modified"]  = headers[:last_modified]
+    end
+
+    def extract_category(directory)
+      if directory.match(/^public\//)
+        "public/#{directory.split('/')[1]}"
+      else
+        directory.split('/').first
+      end
+    end
+
+    def directory_permission(authorizations, directory)
+      authorizations = authorizations.map do |auth|
+        auth.index(":") ? auth.split(":") : [auth, "rw"]
+      end
+      authorizations = Hash[*authorizations.flatten]
+
+      permission = authorizations[""]
+
+      authorizations.each do |key, value|
+        if directory.match(/^(public\/)?#{key}(\/|$)/)
+          if permission.nil? || permission == "r"
+            permission = value
+          end
+          return permission if permission == "rw"
+        end
+      end
+
+      permission
+    end
+
+    def has_name_collision?(user, directory, key)
+      lua_script = <<-EOF
+        local user = ARGV[1]
+        local directory = ARGV[2]
+        local key = ARGV[3]
+
+        -- build table with parent directories from remaining arguments
+        local parent_dir_count = #ARGV - 3
+        local parent_directories = {}
+        for i = 4, 4 + parent_dir_count do
+          table.insert(parent_directories, ARGV[i])
+        end
+
+        -- check for existing directory with the same name as the document
+        local redis_key = "rs:m:"..user..":"
+        if directory == "" then
+          redis_key = redis_key..key.."/"
+        else
+          redis_key = redis_key..directory.."/"..key.."/"
+        end
+        if redis.call("hget", redis_key, "e") then
+          return true
+        end
+
+        for index, dir in pairs(parent_directories) do
+          if redis.call("hget", "rs:m:"..user..":"..dir.."/", "e") then
+            -- the directory already exists, no need to do further checks
+            return false
+          else
+            -- check for existing document with same name as directory
+            if redis.call("hget", "rs:m:"..user..":"..dir, "e") then
+              return true
+            end
+          end
+        end
+
+        return false
+      EOF
+
+      parent_directories = parent_directories_for(directory)
+
+      redis.eval(lua_script, nil, [user, directory, key, *parent_directories])
+    end
+
+    def metadata_changed?(old_metadata, new_metadata)
+      # check metadata relevant to the directory listing
+      return old_metadata["e"] != new_metadata[:e]      ||
+             old_metadata["s"] != new_metadata[:s].to_s ||
+             old_metadata["m"] != new_metadata[:m]      ||
+             old_metadata["t"] != new_metadata[:t]
+    end
+
+    def timestamp_for(date)
+      return DateTime.parse(date).strftime("%Q").to_i
+    end
+
+    def log_size_difference(user, old_size, new_size)
+      delta = new_size.to_i - old_size.to_i
+      redis.incrby "rs:s:#{user}", delta
+    end
+
+    def checksum_for(data)
+      Digest::MD5.hexdigest(data)
+    end
+
+    def parent_directories_for(directory)
+      directories = directory.split("/")
+      parent_directories = []
+
+      while directories.any?
+        parent_directories << directories.join("/")
+        directories.pop
+      end
+
+      parent_directories << "" # add empty string for the root directory
+
+      parent_directories
+    end
+
+    def top_directory(directory)
+      if directory.match(/\//)
+        directory.split("/").last
+      elsif directory != ""
+        return directory
+      end
+    end
+
+    def parent_directory_for(directory)
+      if directory.match(/\//)
+        return directory[0..directory.rindex("/")]
+      elsif directory != ""
+        return "/"
+      end
+    end
+
+    def update_metadata_object(user, directory, key, metadata)
+      redis_key = redis_metadata_object_key(user, directory, key)
+      redis.hmset(redis_key, *metadata)
+      redis.sadd "rs:m:#{user}:#{directory}/:items", key
+
+      true
+    end
+
+    def update_dir_objects(user, directory, timestamp, checksum)
+      parent_directories_for(directory).each do |dir|
+        etag = etag_for(dir, timestamp, checksum)
+
+        key = "rs:m:#{user}:#{dir}/"
+        metadata = {e: etag, m: timestamp}
+        redis.hmset(key, *metadata)
+        redis.sadd "rs:m:#{user}:#{parent_directory_for(dir)}:items", "#{top_directory(dir)}/"
+      end
+    end
+
+    def delete_metadata_objects(user, directory, key)
+      redis.del redis_metadata_object_key(user, directory, key)
+      redis.srem "rs:m:#{user}:#{directory}/:items", key
+    end
+
+    def delete_dir_objects(user, directory)
+      timestamp = (Time.now.to_f * 1000).to_i
+
+      parent_directories_for(directory).each do |dir|
+        if dir_empty?(user, dir)
+          redis.del "rs:m:#{user}:#{dir}/"
+          redis.srem "rs:m:#{user}:#{parent_directory_for(dir)}:items", "#{top_directory(dir)}/"
+        else
+          etag = etag_for(dir, timestamp)
+
+          metadata = {e: etag, m: timestamp}
+          redis.hmset("rs:m:#{user}:#{dir}/", *metadata)
+        end
+      end
+    end
+
+    def dir_empty?(user, dir)
+      redis.smembers("rs:m:#{user}:#{dir}/:items").empty?
+    end
+
+    def redis_metadata_object_key(user, directory, key)
+      "rs:m:#{user}:#{[directory, key].delete_if(&:empty?).join("/")}"
+    end
+
+    def url_for_key(user, directory, key)
+      File.join [container_url_for(user), escape(directory), escape(key)].compact
+    end
+
+    def do_put_request(url, data, content_type)
+      deal_with_unauthorized_requests do
+        res = RestClient.put(url, data, default_headers.merge({content_type: content_type}))
+
+        return [
+          res.headers[:etag],
+          timestamp_for(res.headers[:last_modified])
+        ]
+      end
+    end
+
+    def do_get_request(url, &block)
+      deal_with_unauthorized_requests do
+        RestClient.get(url, default_headers, &block)
+      end
+    end
+
+    def do_head_request(url, &block)
+      deal_with_unauthorized_requests do
+        RestClient.head(url, default_headers, &block)
+      end
+    end
+
+    def do_delete_request(url)
+      deal_with_unauthorized_requests do
+        RestClient.delete(url, default_headers)
+      end
+    end
+
+    def escape(url)
+      # We want spaces to turn into %20 and slashes to stay slashes
+      CGI::escape(url).gsub('+', '%20').gsub('%2F', '/')
+    end
+
+    def redis
+      @redis ||= Redis.new(settings.redis.symbolize_keys)
+    end
+
+    def etag_for(*args)
+      Digest::MD5.hexdigest args.join(":")
+    end
+
+    def deal_with_unauthorized_requests(&block)
+      begin
+        block.call
+      rescue RestClient::Unauthorized => ex
+        Raven.capture_exception(ex)
+        server.halt 500
+      end
+    end
+
+    def try_to_delete(url)
+      found = true
+
+      begin
+        do_delete_request(url)
+      rescue RestClient::ResourceNotFound
+        found = false
+      end
+
+      found
+    end
+
+    def get_directory_listing_from_redis_via_lua(user, directory)
+      lua_script = <<-EOF
+        local user = ARGV[1]
+        local directory = ARGV[2]
+        local items = redis.call("smembers", "rs:m:"..user..":"..directory.."/:items")
+        local listing = {}
+
+        for index, name in pairs(items) do
+          local redis_key = "rs:m:"..user..":"
+          if directory == "" then
+            redis_key = redis_key..name
+          else
+            redis_key = redis_key..directory.."/"..name
+          end
+
+          local metadata_values = redis.call("hgetall", redis_key)
+          local metadata = {}
+
+          -- redis returns hashes as a single list of alternating keys and values
+          -- this collates it into a table
+          for idx = 1, #metadata_values, 2 do
+            metadata[metadata_values[idx]] = metadata_values[idx + 1]
+          end
+
+          listing[name] = {["ETag"] = metadata["e"]}
+          if string.sub(name, -1) ~= "/" then
+            listing[name]["Content-Type"]   = metadata["t"]
+            listing[name]["Content-Length"] = tonumber(metadata["s"])
+            listing[name]["Last-Modified"]  = tonumber(metadata["m"])
+          end
+        end
+
+        return cjson.encode(listing)
+      EOF
+
+      items = JSON.parse(redis.eval(lua_script, nil, [user, directory]))
+
+      items.reject{|k, _| k.end_with? "/"}.each do |_, v|
+        v["Last-Modified"] = Time.at(v["Last-Modified"]/1000).httpdate
+      end
+
+      items
+    end
+
+  end
+end

lib/remote_storage/riak.rb

@@ -1,531 +0,0 @@
-require "riak"
-require "json"
-require "cgi"
-require "active_support/core_ext/time/conversions"
-require "active_support/core_ext/numeric/time"
-
-module RemoteStorage
-  class Riak
-
-    ::Riak.url_decoding = true
-
-    attr_accessor :settings, :server, :cs_credentials
-
-    def initialize(settings, server)
-      self.settings = settings
-      self.server = server
-
-      credentials = File.read(settings.riak['riak_cs']['credentials_file'])
-      self.cs_credentials = JSON.parse(credentials)
-    end
-
-    def authorize_request(user, directory, token, listing=false)
-      request_method = server.env["REQUEST_METHOD"]
-
-      if directory.split("/").first == "public"
-        return true if ["GET", "HEAD"].include?(request_method) && !listing
-      end
-
-      authorizations = auth_bucket.get("#{user}:#{token}").data
-      permission = directory_permission(authorizations, directory)
-
-      server.halt 401 unless permission
-      if ["PUT", "DELETE"].include? request_method
-        server.halt 401 unless permission == "rw"
-      end
-    rescue ::Riak::HTTPFailedRequest
-      server.halt 401
-    end
-
-    def get_head(user, directory, key)
-      object = data_bucket.get("#{user}:#{directory}:#{key}")
-      set_object_response_headers(object)
-      server.halt 200
-    rescue ::Riak::HTTPFailedRequest
-      server.halt 404
-    end
-
-    def get_data(user, directory, key)
-      object = data_bucket.get("#{user}:#{directory}:#{key}")
-
-      set_object_response_headers(object)
-
-      none_match = (server.env["HTTP_IF_NONE_MATCH"] || "").split(",").map(&:strip)
-      server.halt 304 if none_match.include? object.etag
-
-      if binary_key = object.meta["binary_key"]
-        object = cs_binary_bucket.files.get(binary_key[0])
-
-        case object.content_type[/^[^;\s]+/]
-        when "application/json"
-          return object.body.to_json
-        else
-          return object.body
-        end
-      end
-
-      case object.content_type[/^[^;\s]+/]
-      when "application/json"
-        return object.data.to_json
-      else
-        data = serializer_for(object.content_type) ? object.data : object.raw_data
-
-        # Never return nil, always turn data into a string
-        return data.nil? ? '' : data
-      end
-    rescue ::Riak::HTTPFailedRequest
-      server.halt 404
-    end
-
-    def get_head_directory_listing(user, directory)
-      directory_object = directory_bucket.get("#{user}:#{directory}")
-      set_directory_response_headers(directory_object)
-      server.halt 200
-    rescue ::Riak::HTTPFailedRequest
-      server.halt 404
-    end
-
-    def get_directory_listing(user, directory)
-      directory_object = directory_bucket.get("#{user}:#{directory}")
-
-      set_directory_response_headers(directory_object)
-
-      none_match = (server.env["HTTP_IF_NONE_MATCH"] || "").split(",").map(&:strip)
-      server.halt 304 if none_match.include? directory_object.etag
-
-      listing = directory_listing(user, directory)
-
-      return listing.to_json
-    rescue ::Riak::HTTPFailedRequest
-      server.halt 404
-    end
-
-    def put_data(user, directory, key, data, content_type=nil)
-      server.halt 409 if has_name_collision?(user, directory, key)
-
-      object = build_data_object(user, directory, key, data, content_type)
-
-      if required_match = server.env["HTTP_IF_MATCH"]
-        server.halt 412 unless required_match == object.etag
-      end
-
-      object_exists = !object.raw_data.nil? || !object.meta["binary_key"].nil?
-      existing_object_size = object_size(object)
-
-      server.halt 412 if object_exists && server.env["HTTP_IF_NONE_MATCH"] == "*"
-
-      timestamp = (Time.now.to_f * 1000).to_i
-      object.meta["timestamp"] = timestamp
-
-      if binary_data?(object.content_type, data)
-        save_binary_data(object, data) or server.halt 422
-        new_object_size = data.size
-      else
-        set_object_data(object, data) or server.halt 422
-        new_object_size = object.raw_data.size
-      end
-
-      object.store
-
-      log_count = object_exists ? 0 : 1
-      log_operation(user, directory, log_count, new_object_size, existing_object_size)
-
-      update_all_directory_objects(user, directory, timestamp)
-
-      server.headers["ETag"] = object.etag
-      server.halt object_exists ? 200 : 201
-    rescue ::Riak::HTTPFailedRequest
-      server.halt 422
-    end
-
-    def delete_data(user, directory, key)
-      object = data_bucket.get("#{user}:#{directory}:#{key}")
-      existing_object_size = object_size(object)
-      etag = object.etag
-
-      if required_match = server.env["HTTP_IF_MATCH"]
-        server.halt 412 unless required_match == etag
-      end
-
-      if binary_key = object.meta["binary_key"]
-        object = cs_binary_bucket.files.get(binary_key[0])
-        object.destroy
-      end
-
-      riak_response = data_bucket.delete("#{user}:#{directory}:#{key}")
-
-      if riak_response[:code] != 404
-        log_operation(user, directory, -1, 0, existing_object_size)
-      end
-
-      timestamp = (Time.now.to_f * 1000).to_i
-      delete_or_update_directory_objects(user, directory, timestamp)
-
-      server.halt 200
-    rescue ::Riak::HTTPFailedRequest
-      server.halt 404
-    end
-
-    private
-
-    def set_object_response_headers(object)
-      server.headers["Content-Type"]   = object.content_type
-      server.headers["ETag"]           = object.etag
-      server.headers["Content-Length"] = object_size(object).to_s
-    end
-
-    def set_directory_response_headers(directory_object)
-      server.headers["Content-Type"] = "application/json"
-      server.headers["ETag"]         = directory_object.etag
-    end
-
-    def extract_category(directory)
-      if directory.match(/^public\//)
-        "public/#{directory.split('/')[1]}"
-      else
-        directory.split('/').first
-      end
-    end
-
-    def build_data_object(user, directory, key, data, content_type=nil)
-      object = data_bucket.get_or_new("#{user}:#{directory}:#{key}")
-
-      object.content_type = content_type || "text/plain; charset=utf-8"
-
-      directory_index = directory == "" ? "/" : directory
-      object.indexes.merge!({:user_id_bin   => [user],
-                             :directory_bin => [directory_index]})
-
-      object
-    end
-
-    def log_operation(user, directory, count, new_size=0, old_size=0)
-      size = (-old_size + new_size)
-      return if count == 0 && size == 0
-
-      log_entry = opslog_bucket.new
-      log_entry.content_type = "application/json"
-      log_entry.data = {
-        "count"    => count,
-        "size"     => size,
-        "category" => extract_category(directory)
-      }
-      log_entry.indexes.merge!({:user_id_bin => [user]})
-      log_entry.store
-    end
-
-    def object_size(object)
-      if binary_key = object.meta["binary_key"]
-        response = cs_client.head_object cs_binary_bucket.key, binary_key[0]
-        response.headers["Content-Length"].to_i
-      else
-        object.raw_data.nil? ? 0 : object.raw_data.size
-      end
-    end
-
-    def escape(string)
-      ::Riak.escaper.escape(string).gsub("+", "%20").gsub('/', "%2F")
-    end
-
-    # A URI object that can be used with HTTP backend methods
-    def riak_uri(bucket, key)
-      URI.parse "http://#{settings.riak["host"]}:#{settings.riak["http_port"]}/riak/#{bucket}/#{key}"
-    end
-
-    def serializer_for(content_type)
-      ::Riak::Serializers[content_type[/^[^;\s]+/]]
-    end
-
-    def directory_permission(authorizations, directory)
-      authorizations = authorizations.map do |auth|
-        auth.index(":") ? auth.split(":") : [auth, "rw"]
-      end
-      authorizations = Hash[*authorizations.flatten]
-
-      permission = authorizations[""]
-
-      authorizations.each do |key, value|
-        if directory.match(/^(public\/)?#{key}(\/|$)/)
-          if permission.nil? || permission == "r"
-            permission = value
-          end
-          return permission if permission == "rw"
-        end
-      end
-
-      permission
-    end
-
-    def directory_listing(user, directory)
-      listing = {
-        "@context" => "http://remotestorage.io/spec/folder-description",
-        "items"    => {}
-      }
-
-      sub_directories(user, directory).each do |entry|
-        directory_name = entry["name"].split("/").last
-        etag           = entry["etag"]
-
-        listing["items"].merge!({ "#{directory_name}/" => { "ETag" => etag }})
-      end
-
-      directory_entries(user, directory).each do |entry|
-        entry_name     = entry["name"]
-        etag           = entry["etag"]
-        content_type   = entry["contentType"]
-        content_length = entry["contentLength"].to_i
-
-        listing["items"].merge!({
-          entry_name => {
-            "ETag"           => etag,
-            "Content-Type"   => content_type,
-            "Content-Length" => content_length
-          }
-        })
-      end
-
-      listing
-    end
-
-    def directory_entries(user, directory)
-      all_keys = user_directory_keys(user, directory, data_bucket)
-      return [] if all_keys.empty?
-
-      map_query = <<-EOH
-        function(v){
-          var metadata = v.values[0]['metadata'];
-          var dir_name = metadata['index']['directory_bin'];
-          if (dir_name === '/') {
-            dir_name = '';
-          }
-          var name = v.key.match(/^[^:]*:(.*)/)[1]; // strip username from key
-          name = name.replace(dir_name + ':', ''); // strip directory from key
-          var etag = metadata['X-Riak-VTag'];
-          var contentType = metadata['content-type'];
-          var contentLength = metadata['X-Riak-Meta']['X-Riak-Meta-Content_length'] || 0;
-
-          return [{
-            name:          name,
-            etag:          etag,
-            contentType:   contentType,
-            contentLength: contentLength
-          }];
-        }
-      EOH
-
-      run_map_reduce(data_bucket, all_keys, map_query)
-    end
-
-    def sub_directories(user, directory)
-      all_keys = user_directory_keys(user, directory, directory_bucket)
-      return [] if all_keys.empty?
-
-      map_query = <<-EOH
-        function(v){
-          var name = v.key.match(/^[^:]*:(.*)/)[1]; // strip username from key
-          var etag = v.values[0]['metadata']['X-Riak-VTag'];
-
-          return [{
-            name: name,
-            etag: etag
-          }];
-        }
-      EOH
-
-      run_map_reduce(directory_bucket, all_keys, map_query)
-    end
-
-    def user_directory_keys(user, directory, bucket)
-      directory = "/" if directory == ""
-
-      user_keys = bucket.get_index("user_id_bin", user)
-      directory_keys = bucket.get_index("directory_bin", directory)
-
-      user_keys & directory_keys
-    end
-
-    def run_map_reduce(bucket, keys, map_query)
-      map_reduce = ::Riak::MapReduce.new(client)
-      keys.each do |key|
-        map_reduce.add(bucket.name, key)
-      end
-
-      map_reduce.
-        map(map_query, :keep => true).
-        run
-    end
-
-    def update_all_directory_objects(user, directory, timestamp)
-      parent_directories_for(directory).each do |parent_directory|
-        update_directory_object(user, parent_directory, timestamp)
-      end
-    end
-
-    def update_directory_object(user, directory, timestamp)
-      if directory.match(/\//)
-        parent_directory = directory[0..directory.rindex("/")-1]
-      elsif directory != ""
-        parent_directory = "/"
-      end
-
-      directory_object = directory_bucket.new("#{user}:#{directory}")
-      directory_object.content_type = "text/plain; charset=utf-8"
-      directory_object.data = timestamp.to_s
-      directory_object.indexes.merge!({:user_id_bin => [user]})
-      if parent_directory
-        directory_object.indexes.merge!({:directory_bin => [parent_directory]})
-      end
-      directory_object.store
-    end
-
-    def delete_or_update_directory_objects(user, directory, timestamp)
-      parent_directories_for(directory).each do |parent_directory|
-        existing_files = directory_entries(user, parent_directory)
-        existing_subdirectories = sub_directories(user, parent_directory)
-
-        if existing_files.empty? && existing_subdirectories.empty?
-          directory_bucket.delete "#{user}:#{parent_directory}"
-        else
-          update_directory_object(user, parent_directory, timestamp)
-        end
-      end
-    end
-
-    def set_object_data(object, data)
-      if object.content_type[/^[^;\s]+/] == "application/json"
-        data = "{}" if data.blank?
-        data = JSON.parse(data)
-      end
-
-      object.meta["content_length"] = data.size
-
-      if serializer_for(object.content_type)
-        object.data = data
-      else
-        object.raw_data = data
-      end
-    rescue JSON::ParserError
-      return false
-    end
-
-    def save_binary_data(object, data)
-      cs_binary_object = cs_binary_bucket.files.create(
-        :key          => object.key,
-        :body         => data,
-        :content_type => object.content_type
-      )
-
-      object.meta["binary_key"]     = cs_binary_object.key
-      object.meta["content_length"] = cs_binary_object.content_length
-      object.raw_data = ""
-    end
-
-    def binary_data?(content_type, data)
-      return true if content_type[/[^;\s]+$/] == "charset=binary"
-
-      original_encoding = data.encoding
-      data.force_encoding("UTF-8")
-      is_binary = !data.valid_encoding?
-      data.force_encoding(original_encoding)
-
-      is_binary
-    end
-
-    def parent_directories_for(directory)
-      directories = directory.split("/")
-      parent_directories = []
-
-      while directories.any?
-        parent_directories << directories.join("/")
-        directories.pop
-      end
-
-      parent_directories << ""
-    end
-
-    def has_name_collision?(user, directory, key)
-      parent_directories = parent_directories_for(directory).reverse
-      parent_directories.shift # remove root dir entry
-
-      # check for existing documents with the same name as one of the parent directories
-      parent_directories.each do |dir|
-        begin
-          parts = dir.split("/")
-          document_key = parts.pop
-          directory_name = parts.join("/")
-          data_bucket.get("#{user}:#{directory_name}:#{document_key}")
-          return true
-        rescue ::Riak::HTTPFailedRequest
-        end
-      end
-
-      # check for an existing directory with same name as document
-      begin
-        directory_bucket.get("#{user}:#{directory}/#{key}")
-        return true
-      rescue ::Riak::HTTPFailedRequest
-      end
-
-      false
-    end
-
-    def client
-      @client ||= ::Riak::Client.new(:host      => settings.riak['host'],
-                                     :http_port => settings.riak['http_port'])
-    end
-
-    def data_bucket
-      @data_bucket ||= begin
-                         bucket = client.bucket(settings.riak['buckets']['data'])
-                         bucket.allow_mult = false
-                         bucket
-                       end
-    end
-
-    def directory_bucket
-      @directory_bucket ||= begin
-                              bucket = client.bucket(settings.riak['buckets']['directories'])
-                              bucket.allow_mult = false
-                              bucket
-                            end
-    end
-
-    def auth_bucket
-      @auth_bucket ||= begin
-                         bucket = client.bucket(settings.riak['buckets']['authorizations'])
-                         bucket.allow_mult = false
-                         bucket
-                       end
-    end
-
-    def binary_bucket
-      @binary_bucket ||= begin
-                           bucket = client.bucket(settings.riak['buckets']['binaries'])
-                           bucket.allow_mult = false
-                           bucket
-                         end
-    end
-
-    def opslog_bucket
-      @opslog_bucket ||= begin
-                           bucket = client.bucket(settings.riak['buckets']['opslog'])
-                           bucket.allow_mult = false
-                           bucket
-                         end
-    end
-
-    def cs_client
-      @cs_client ||= Fog::Storage.new({
-        :provider                 => 'AWS',
-        :aws_access_key_id        => cs_credentials['key_id'],
-        :aws_secret_access_key    => cs_credentials['key_secret'],
-        :endpoint                 => settings.riak['riak_cs']['endpoint']
-      })
-    end
-
-    def cs_binary_bucket
-      @cs_binary_bucket ||= cs_client.directories.create(:key => settings.riak['buckets']['cs_binaries'])
-    end
-
-  end
-end

lib/remote_storage/s3.rb

@@ -0,0 +1,113 @@
+require "remote_storage/rest_provider"
+require "digest"
+require "base64"
+require "openssl"
+require "webrick/httputils"
+
+module RemoteStorage
+  class S3
+    include RestProvider
+
+    private
+
+    # S3 already wraps the ETag around quotes
+    def format_etag(etag)
+      etag
+    end
+
+    def do_put_request(url, data, content_type)
+      deal_with_unauthorized_requests do
+        md5 = Digest::MD5.base64digest(data)
+        authorization_headers = authorization_headers_for(
+          "PUT", url, md5, content_type
+        ).merge({ "Content-Type" => content_type, "Content-Md5" => md5 })
+        res = RestClient.put(url, data, authorization_headers)
+        # S3 does not return a Last-Modified response header on PUTs
+        head_res = do_head_request(url)
+
+        return [
+          res.headers[:etag].delete('"'),
+          timestamp_for(head_res.headers[:last_modified])
+        ]
+      end
+    end
+
+    def do_get_request(url, &block)
+      deal_with_unauthorized_requests do
+        authorization_headers = authorization_headers_for("GET", url)
+        RestClient.get(url, authorization_headers, &block)
+      end
+    end
+
+    def do_head_request(url, &block)
+      deal_with_unauthorized_requests do
+        authorization_headers = authorization_headers_for("HEAD", url)
+        RestClient.head(url, authorization_headers, &block)
+      end
+    end
+
+    def do_delete_request(url)
+      deal_with_unauthorized_requests do
+        authorization_headers = authorization_headers_for("DELETE", url)
+        RestClient.delete(url, authorization_headers)
+      end
+    end
+
+    def try_to_delete(url)
+      found = true
+
+      begin
+        do_head_request(url)
+      rescue RestClient::ResourceNotFound
+        found = false
+      end
+
+      do_delete_request(url) if found
+
+      return found
+    end
+
+    # This is using the S3 authorizations, not the newer AW V4 Signatures
+    # (https://s3.amazonaws.com/doc/s3-developer-guide/RESTAuthentication.html)
+    def authorization_headers_for(http_verb, url, md5 = nil, content_type = nil)
+      url = File.join("/", url.gsub(base_url, ""))
+      date = Time.now.httpdate
+      signed_data = generate_s3_signature(http_verb, md5, content_type, date, url)
+      {
+        "Authorization" => "AWS #{credentials[:access_key_id]}:#{signed_data}",
+        "Date" => date
+      }
+    end
+
+    def credentials
+      @credentials ||= { access_key_id: settings.s3["access_key_id"], secret_key_id: settings.s3["secret_key_id"] }
+    end
+
+    def digest(secret, string_to_sign)
+      Base64.encode64(hmac(secret, string_to_sign)).strip
+    end
+
+    def hmac(key, value)
+      OpenSSL::HMAC.digest(OpenSSL::Digest.new('sha1'), key, value)
+    end
+
+    def uri_escape(s)
+      WEBrick::HTTPUtils.escape(s).gsub('%5B', '[').gsub('%5D', ']')
+    end
+
+    def generate_s3_signature(http_verb, md5, content_type, date, url)
+      string_to_sign = [http_verb, md5, content_type, date, url].join "\n"
+      signature = digest(credentials[:secret_key_id], string_to_sign)
+      uri_escape(signature)
+    end
+
+    def base_url
+      @base_url ||= settings.s3["endpoint"]
+    end
+
+    def container_url_for(user)
+      "#{base_url}#{settings.s3["bucket"]}/#{user}"
+    end
+  end
+
+end

lib/remote_storage/swift.rb

@@ -1,459 +1,32 @@
 require "rest_client"
-require "json"
-require "cgi"
 require "active_support/core_ext/time/conversions"
 require "active_support/core_ext/numeric/time"
 require "active_support/core_ext/hash"
-require "redis"
+require "remote_storage/rest_provider"
-require "digest/md5"
 
 module RemoteStorage
   class Swift
-
+    include RestProvider
-    attr_accessor :settings, :server
-
-    def initialize(settings, server)
-      @settings = settings
-      @server   = server
-    end
-
-    def authorize_request(user, directory, token, listing=false)
-      request_method = server.env["REQUEST_METHOD"]
-
-      if directory.split("/").first == "public"
-        return true if ["GET", "HEAD"].include?(request_method) && !listing
-      end
-
-      server.halt 401, "Unauthorized" if token.nil? || token.empty?
-
-      authorizations = redis.smembers("authorizations:#{user}:#{token}")
-      permission = directory_permission(authorizations, directory)
-
-      server.halt 401, "Unauthorized" unless permission
-      if ["PUT", "DELETE"].include? request_method
-        server.halt 401, "Unauthorized" unless permission == "rw"
-      end
-    end
-
-    def get_head(user, directory, key)
-      url = url_for_key(user, directory, key)
-
-      res = do_head_request(url)
-
-      set_response_headers(res)
-    rescue RestClient::ResourceNotFound
-      server.halt 404
-    end
-
-    def get_data(user, directory, key)
-      url = url_for_key(user, directory, key)
-
-      res = do_get_request(url)
-
-      set_response_headers(res)
-
-      none_match = (server.env["HTTP_IF_NONE_MATCH"] || "").split(",")
-                                                           .map(&:strip)
-                                                           .map { |s| s.gsub(/^"?W\//, "") }
-      server.halt 304 if none_match.include? %Q("#{res.headers[:etag]}")
-
-      return res.body
-    rescue RestClient::ResourceNotFound
-      server.halt 404, "Not Found"
-    end
-
-    def get_head_directory_listing(user, directory)
-      get_directory_listing(user, directory)
-
-      "" # just return empty body, headers are set by get_directory_listing
-    end
-
-    def get_directory_listing(user, directory)
-      etag = redis.hget "rs:m:#{user}:#{directory}/", "e"
-
-      server.headers["Content-Type"] = "application/ld+json"
-
-      none_match = (server.env["HTTP_IF_NONE_MATCH"] || "").split(",")
-                                                           .map(&:strip)
-                                                           .map { |s| s.gsub(/^"?W\//, "") }
-
-      if etag
-        server.halt 304 if none_match.include? %Q("#{etag}")
-
-        items = get_directory_listing_from_redis_via_lua(user, directory)
-      else
-        etag = etag_for(user, directory)
-        items = {}
-
-        server.halt 304 if none_match.include? %Q("#{etag}")
-      end
-
-      server.headers["ETag"] = %Q("#{etag}")
-
-      listing = {
-        "@context" => "http://remotestorage.io/spec/folder-description",
-        "items"    => items
-      }
-
-      listing.to_json
-    end
-
-    def get_directory_listing_from_redis_via_lua(user, directory)
-      lua_script = <<-EOF
-        local user = ARGV[1]
-        local directory = ARGV[2]
-        local items = redis.call("smembers", "rs:m:"..user..":"..directory.."/:items")
-        local listing = {}
-
-        for index, name in pairs(items) do
-          local redis_key = "rs:m:"..user..":"
-          if directory == "" then
-            redis_key = redis_key..name
-          else
-            redis_key = redis_key..directory.."/"..name
-          end
-
-          local metadata_values = redis.call("hgetall", redis_key)
-          local metadata = {}
-
-          -- redis returns hashes as a single list of alternating keys and values
-          -- this collates it into a table
-          for idx = 1, #metadata_values, 2 do
-            metadata[metadata_values[idx]] = metadata_values[idx + 1]
-          end
-
-          listing[name] = {["ETag"] = metadata["e"]}
-          if string.sub(name, -1) ~= "/" then
-            listing[name]["Content-Type"]   = metadata["t"]
-            listing[name]["Content-Length"] = tonumber(metadata["s"])
-          end
-        end
-
-        return cjson.encode(listing)
-      EOF
-
-      JSON.parse(redis.eval(lua_script, nil, [user, directory]))
-    end
-
-    def put_data(user, directory, key, data, content_type)
-      server.halt 400 if server.env["HTTP_CONTENT_RANGE"]
-      server.halt 409, "Conflict" if has_name_collision?(user, directory, key)
-
-      existing_metadata = redis.hgetall redis_metadata_object_key(user, directory, key)
-      url = url_for_key(user, directory, key)
-
-      if required_match = server.env["HTTP_IF_MATCH"]
-        required_match = required_match.gsub(/^"?W\//, "")
-        unless required_match == %Q("#{existing_metadata["e"]}")
-
-          # get actual metadata and compare in case redis metadata became out of sync
-          begin
-            head_res = do_head_request(url)
-          # The file doesn't exist in Orbit, return 412
-          rescue RestClient::ResourceNotFound
-            server.halt 412, "Precondition Failed"
-          end
-
-          if required_match == %Q("#{head_res.headers[:etag]}")
-            # log previous size difference that was missed ealier because of redis failure
-            log_size_difference(user, existing_metadata["s"], head_res.headers[:content_length])
-          else
-            server.halt 412, "Precondition Failed"
-          end
-        end
-      end
-      if server.env["HTTP_IF_NONE_MATCH"] == "*"
-        server.halt 412, "Precondition Failed" unless existing_metadata.empty?
-      end
-
-      res = do_put_request(url, data, content_type)
-
-      timestamp = timestamp_for(res.headers[:last_modified])
-
-      metadata = {
-        e: res.headers[:etag],
-        s: data.size,
-        t: content_type,
-        m: timestamp
-      }
-
-      if update_metadata_object(user, directory, key, metadata)
-        if metadata_changed?(existing_metadata, metadata)
-          update_dir_objects(user, directory, timestamp, checksum_for(data))
-          log_size_difference(user, existing_metadata["s"], metadata[:s])
-        end
-
-        server.headers["ETag"] = %Q("#{res.headers[:etag]}")
-        server.halt existing_metadata.empty? ? 201 : 200
-      else
-        server.halt 500
-      end
-    end
-
-    def log_size_difference(user, old_size, new_size)
-      delta = new_size.to_i - old_size.to_i
-      redis.incrby "rs:s:#{user}", delta
-    end
-
-    def checksum_for(data)
-      Digest::MD5.hexdigest(data)
-    end
-
-    def delete_data(user, directory, key)
-      url = url_for_key(user, directory, key)
-      not_found = false
-
-      existing_metadata = redis.hgetall "rs:m:#{user}:#{directory}/#{key}"
-
-      if required_match = server.env["HTTP_IF_MATCH"]
-        unless required_match.gsub(/^"?W\//, "") == %Q("#{existing_metadata["e"]}")
-          server.halt 412, "Precondition Failed"
-        end
-      end
-
-      begin
-        do_delete_request(url)
-      rescue RestClient::ResourceNotFound
-        not_found = true
-      end
-
-      log_size_difference(user, existing_metadata["s"], 0)
-      delete_metadata_objects(user, directory, key)
-      delete_dir_objects(user, directory)
-
-      if not_found
-        server.halt 404, "Not Found"
-      else
-        server.headers["Etag"] = %Q("#{existing_metadata["e"]}")
-        server.halt 200
-      end
-    end
 
     private
 
-    def set_response_headers(response)
+    # Add quotes around the ETag
-      server.headers["ETag"]           = %Q("#{response.headers[:etag]}")
+    def format_etag(etag)
-      server.headers["Content-Type"]   = response.headers[:content_type]
+      %Q("#{etag}")
-      server.headers["Content-Length"] = response.headers[:content_length]
-      server.headers["Last-Modified"]  = response.headers[:last_modified]
-    end
-
-    def extract_category(directory)
-      if directory.match(/^public\//)
-        "public/#{directory.split('/')[1]}"
-      else
-        directory.split('/').first
-      end
-    end
-
-    def directory_permission(authorizations, directory)
-      authorizations = authorizations.map do |auth|
-        auth.index(":") ? auth.split(":") : [auth, "rw"]
-      end
-      authorizations = Hash[*authorizations.flatten]
-
-      permission = authorizations[""]
-
-      authorizations.each do |key, value|
-        if directory.match(/^(public\/)?#{key}(\/|$)/)
-          if permission.nil? || permission == "r"
-            permission = value
-          end
-          return permission if permission == "rw"
-        end
-      end
-
-      permission
-    end
-
-    def has_name_collision?(user, directory, key)
-      lua_script = <<-EOF
-        local user = ARGV[1]
-        local directory = ARGV[2]
-        local key = ARGV[3]
-
-        -- build table with parent directories from remaining arguments
-        local parent_dir_count = #ARGV - 3
-        local parent_directories = {}
-        for i = 4, 4 + parent_dir_count do
-          table.insert(parent_directories, ARGV[i])
-        end
-
-        -- check for existing directory with the same name as the document
-        local redis_key = "rs:m:"..user..":"
-        if directory == "" then
-          redis_key = redis_key..key.."/"
-        else
-          redis_key = redis_key..directory.."/"..key.."/"
-        end
-        if redis.call("hget", redis_key, "e") then
-          return true
-        end
-
-        for index, dir in pairs(parent_directories) do
-          if redis.call("hget", "rs:m:"..user..":"..dir.."/", "e") then
-            -- the directory already exists, no need to do further checks
-            return false
-          else
-            -- check for existing document with same name as directory
-            if redis.call("hget", "rs:m:"..user..":"..dir, "e") then
-              return true
-            end
-          end
-        end
-
-        return false
-      EOF
-
-      parent_directories = parent_directories_for(directory)
-
-      redis.eval(lua_script, nil, [user, directory, key, *parent_directories])
-    end
-
-    def metadata_changed?(old_metadata, new_metadata)
-      # check metadata relevant to the directory listing
-      # ie. the timestamp (m) is not relevant, because it's not used in
-      # the listing
-      return old_metadata["e"] != new_metadata[:e]      ||
-             old_metadata["s"] != new_metadata[:s].to_s ||
-             old_metadata["t"] != new_metadata[:t]
-    end
-
-    def timestamp_for(date)
-      return DateTime.parse(date).strftime("%Q").to_i
-    end
-
-    def parent_directories_for(directory)
-      directories = directory.split("/")
-      parent_directories = []
-
-      while directories.any?
-        parent_directories << directories.join("/")
-        directories.pop
-      end
-
-      parent_directories << "" # add empty string for the root directory
-
-      parent_directories
-    end
-
-    def top_directory(directory)
-      if directory.match(/\//)
-        directory.split("/").last
-      elsif directory != ""
-        return directory
-      end
-    end
-
-    def parent_directory_for(directory)
-      if directory.match(/\//)
-        return directory[0..directory.rindex("/")]
-      elsif directory != ""
-        return "/"
-      end
-    end
-
-    def update_metadata_object(user, directory, key, metadata)
-      redis_key = redis_metadata_object_key(user, directory, key)
-      redis.hmset(redis_key, *metadata)
-      redis.sadd "rs:m:#{user}:#{directory}/:items", key
-
-      true
-    end
-
-    def update_dir_objects(user, directory, timestamp, checksum)
-      parent_directories_for(directory).each do |dir|
-        etag = etag_for(dir, timestamp, checksum)
-
-        key = "rs:m:#{user}:#{dir}/"
-        metadata = {e: etag, m: timestamp}
-        redis.hmset(key, *metadata)
-        redis.sadd "rs:m:#{user}:#{parent_directory_for(dir)}:items", "#{top_directory(dir)}/"
-      end
-    end
-
-    def delete_metadata_objects(user, directory, key)
-      redis.del redis_metadata_object_key(user, directory, key)
-      redis.srem "rs:m:#{user}:#{directory}/:items", key
-    end
-
-    def delete_dir_objects(user, directory)
-      timestamp = (Time.now.to_f * 1000).to_i
-
-      parent_directories_for(directory).each do |dir|
-        if dir_empty?(user, dir)
-          redis.del "rs:m:#{user}:#{dir}/"
-          redis.srem "rs:m:#{user}:#{parent_directory_for(dir)}:items", "#{top_directory(dir)}/"
-        else
-          etag = etag_for(dir, timestamp)
-
-          metadata = {e: etag, m: timestamp}
-          redis.hmset("rs:m:#{user}:#{dir}/", *metadata)
-        end
-      end
-    end
-
-    def dir_empty?(user, dir)
-      redis.smembers("rs:m:#{user}:#{dir}/:items").empty?
-    end
-
-    def redis_metadata_object_key(user, directory, key)
-      "rs:m:#{user}:#{[directory, key].delete_if(&:empty?).join("/")}"
-    end
-
-    def container_url_for(user)
-      "#{base_url}/rs:documents:#{settings.environment.to_s}/#{user}"
-    end
-
-    def url_for_key(user, directory, key)
-      File.join [container_url_for(user), escape(directory), escape(key)].compact
     end
 
     def base_url
       @base_url ||= settings.swift["host"]
     end
 
+    def container_url_for(user)
+      "#{base_url}/rs:documents:#{settings.environment.to_s}/#{user}"
+    end
+
     def default_headers
       {"x-auth-token" => swift_token}
     end
 
-    def do_put_request(url, data, content_type)
-      deal_with_unauthorized_requests do
-        RestClient.put(url, data, default_headers.merge({content_type: content_type}))
-      end
-    end
-
-    def do_get_request(url, &block)
-      deal_with_unauthorized_requests do
-        RestClient.get(url, default_headers, &block)
-      end
-    end
-
-    def do_head_request(url, &block)
-      deal_with_unauthorized_requests do
-        RestClient.head(url, default_headers, &block)
-      end
-    end
-
-    def do_delete_request(url)
-      deal_with_unauthorized_requests do
-        RestClient.delete(url, default_headers)
-      end
-    end
-
-    def escape(url)
-      # We want spaces to turn into %20 and slashes to stay slashes
-      CGI::escape(url).gsub('+', '%20').gsub('%2F', '/')
-    end
-
-    def redis
-      @redis ||= Redis.new(settings.redis.symbolize_keys)
-    end
-
-    def etag_for(*args)
-      Digest::MD5.hexdigest args.join(":")
-    end
-
     def reload_swift_token
       server.logger.debug "Reloading swift token. Old token: #{settings.swift_token}"
       # Remove the line break from the token file. The line break that the

liquor-cabinet.rb

@@ -4,8 +4,8 @@ require "json"
 require "sinatra/base"
 require 'sinatra/config_file'
 require "sinatra/reloader"
-require "remote_storage/riak"
 require "remote_storage/swift"
+require "remote_storage/s3"
 
 class LiquorCabinet < Sinatra::Base
 
@@ -129,10 +129,10 @@ class LiquorCabinet < Sinatra::Base
 
   def storage
     @storage ||= begin
-      if settings.respond_to? :riak
+      if settings.respond_to? :swift
-        RemoteStorage::Riak.new(settings, self)
-      elsif settings.respond_to? :swift
         RemoteStorage::Swift.new(settings, self)
+      elsif settings.respond_to? :s3
+        RemoteStorage::S3.new(settings, self)
       else
         puts <<-EOF
 You need to set one storage backend in your config.yml file.

spec/riak/app_spec.rb

@@ -1,15 +0,0 @@
-require_relative "../spec_helper"
-
-describe "App" do
-  include Rack::Test::Methods
-
-  def app
-    LiquorCabinet
-  end
-
-  it "returns 404 on non-existing routes" do
-    get "/virginmargarita"
-    last_response.status.must_equal 404
-  end
-
-end

spec/riak/directories_spec.rb

@@ -1,640 +0,0 @@
-require_relative "../spec_helper"
-
-describe "Directories" do
-  include Rack::Test::Methods
-
-  before do
-    purge_all_buckets
-
-    auth = auth_bucket.new("jimmy:123")
-    auth.data = [":r", "documents:r", "tasks:rw"]
-    auth.store
-
-    header "Authorization", "Bearer 123"
-  end
-
-  describe "HEAD listing" do
-    before do
-      put "/jimmy/tasks/foo", "do the laundry"
-      put "/jimmy/tasks/http%3A%2F%2F5apps.com", "prettify design"
-
-      head "/jimmy/tasks/"
-    end
-
-    it "has an empty body" do
-      last_response.status.must_equal 200
-      last_response.body.must_equal ""
-    end
-
-    it "has an ETag header set" do
-      last_response.status.must_equal 200
-      last_response.headers["ETag"].wont_be_nil
-
-      # check that ETag stays the same
-      etag = last_response.headers["ETag"]
-      get "/jimmy/tasks/"
-      last_response.headers["ETag"].must_equal etag
-    end
-
-    it "has CORS headers set" do
-      last_response.status.must_equal 200
-      last_response.headers["Access-Control-Allow-Origin"].must_equal "*"
-      last_response.headers["Access-Control-Allow-Methods"].must_equal "GET, PUT, DELETE"
-      last_response.headers["Access-Control-Allow-Headers"].must_equal "Authorization, Content-Type, Origin, If-Match, If-None-Match"
-      last_response.headers["Access-Control-Expose-Headers"].must_equal "ETag, Content-Length"
-    end
-
-    context "for an empty or absent directory" do
-      it "responds with 404" do
-        head "/jimmy/documents/"
-
-        last_response.status.must_equal 404
-      end
-    end
-  end
-
-  describe "GET listing" do
-    before do
-      put "/jimmy/tasks/foo", "do the laundry"
-      put "/jimmy/tasks/http%3A%2F%2F5apps.com", "prettify design"
-      put "/jimmy/tasks/%3A/foo%3Abar%40foo.org", "hello world"
-    end
-
-    it "lists the objects with version, length and content-type" do
-      get "/jimmy/tasks/"
-
-      last_response.status.must_equal 200
-      last_response.content_type.must_equal "application/json"
-
-      foo = data_bucket.get("jimmy:tasks:foo")
-
-      content = JSON.parse(last_response.body)
-      content["items"]["http://5apps.com"].wont_be_nil
-      content["items"][":/"].wont_be_nil
-      content["items"]["foo"].wont_be_nil
-      content["items"]["foo"]["ETag"].must_equal foo.etag.gsub(/"/, "")
-      content["items"]["foo"]["Content-Type"].must_equal "text/plain"
-      content["items"]["foo"]["Content-Length"].must_equal 14
-    end
-
-    it "has an ETag header set" do
-      get "/jimmy/tasks/"
-
-      last_response.status.must_equal 200
-      last_response.headers["ETag"].wont_be_nil
-
-      # check that ETag stays the same
-      etag = last_response.headers["ETag"]
-      get "/jimmy/tasks/"
-      last_response.headers["ETag"].must_equal etag
-    end
-
-    it "has CORS headers set" do
-      get "/jimmy/tasks/"
-
-      last_response.status.must_equal 200
-      last_response.headers["Access-Control-Allow-Origin"].must_equal "*"
-      last_response.headers["Access-Control-Allow-Methods"].must_equal "GET, PUT, DELETE"
-      last_response.headers["Access-Control-Allow-Headers"].must_equal "Authorization, Content-Type, Origin, If-Match, If-None-Match"
-      last_response.headers["Access-Control-Expose-Headers"].must_equal "ETag, Content-Length"
-    end
-
-    it "has caching headers set" do
-      get "/jimmy/tasks/"
-
-      last_response.status.must_equal 200
-      last_response.headers["Expires"].must_equal "0"
-    end
-
-    it "doesn't choke on colons in the directory name" do
-      get "/jimmy/tasks/%3A/"
-
-      last_response.status.must_equal 200
-      last_response.content_type.must_equal "application/json"
-
-      content = JSON.parse(last_response.body)
-      content["items"]["foo:bar@foo.org"].wont_be_nil
-    end
-
-    context "when If-None-Match header is set" do
-      before do
-        get "/jimmy/tasks/"
-
-        @etag = last_response.headers["ETag"]
-      end
-
-      it "responds with 'not modified' when it matches the current ETag" do
-        header "If-None-Match", @etag
-        get "/jimmy/tasks/"
-
-        last_response.status.must_equal 304
-        last_response.body.must_be_empty
-        last_response.headers["ETag"].must_equal @etag
-      end
-
-      it "responds normally when it does not match the current ETag" do
-        header "If-None-Match", "FOO"
-        get "/jimmy/tasks/"
-
-        last_response.status.must_equal 200
-        last_response.body.wont_be_empty
-      end
-    end
-
-    describe "when If-None-Match header is set with multiple revisions" do
-      before do
-        get "/jimmy/tasks/"
-
-        @etag = last_response.headers["ETag"]
-      end
-
-      it "responds with 'not modified' when it contains the current ETag" do
-        header "If-None-Match", "DEADBEEF,#{@etag} ,F00BA4"
-        get "/jimmy/tasks/"
-
-        last_response.status.must_equal 304
-        last_response.body.must_be_empty
-        last_response.headers["ETag"].must_equal @etag
-      end
-
-      it "responds normally when it does not contain the current ETag" do
-        header "If-None-Match", "FOO,BAR"
-        get "/jimmy/tasks/"
-
-        last_response.status.must_equal 200
-        last_response.body.wont_be_empty
-      end
-    end
-
-    context "with sub-directories" do
-      before do
-        get "/jimmy/tasks/"
-        @old_etag = last_response.headers["ETag"]
-
-        put "/jimmy/tasks/home/laundry", "do the laundry"
-      end
-
-      it "lists the containing objects as well as the direct sub-directories" do
-        get "/jimmy/tasks/"
-
-        last_response.status.must_equal 200
-
-        home = directory_bucket.get("jimmy:tasks/home")
-
-        content = JSON.parse(last_response.body)
-        content["items"]["foo"].wont_be_nil
-        content["items"]["http://5apps.com"].wont_be_nil
-        content["items"]["home/"].wont_be_nil
-        content["items"]["home/"]["ETag"].must_equal home.etag.gsub(/"/, "")
-      end
-
-      it "updates the ETag of the parent directory" do
-        get "/jimmy/tasks/"
-
-        last_response.headers["ETag"].wont_be_nil
-        last_response.headers["ETag"].wont_equal @old_etag
-      end
-
-      context "for a different user" do
-        before do
-          auth = auth_bucket.new("alice:321")
-          auth.data = [":r", "documents:r", "tasks:rw"]
-          auth.store
-
-          header "Authorization", "Bearer 321"
-
-          put "/alice/tasks/homework", "write an essay"
-        end
-
-        it "does not list the directories of jimmy" do
-          get "/alice/tasks/"
-
-          last_response.status.must_equal 200
-
-          content = JSON.parse(last_response.body)
-          content["items"]["/"].must_be_nil
-          content["items"]["tasks/"].must_be_nil
-          content["items"]["home/"].must_be_nil
-          content["items"]["homework"].wont_be_nil
-        end
-      end
-
-      context "sub-directories without objects" do
-        it "lists the direct sub-directories" do
-          put "/jimmy/tasks/private/projects/world-domination/start", "write a manifesto"
-          get "/jimmy/tasks/private/"
-
-          last_response.status.must_equal 200
-
-          projects = directory_bucket.get("jimmy:tasks/private/projects")
-
-          content = JSON.parse(last_response.body)
-          content["items"]["projects/"]["ETag"].must_equal projects.etag.gsub(/"/, "")
-        end
-
-        it "updates the timestamps of the existing directory objects" do
-          directory = directory_bucket.new("jimmy:tasks")
-          directory.content_type = "text/plain"
-          directory.data = (2.seconds.ago.to_f * 1000).to_i
-          directory.store
-
-          put "/jimmy/tasks/private/projects/world-domination/start", "write a manifesto"
-
-          object = data_bucket.get("jimmy:tasks/private/projects/world-domination:start")
-          directory = directory_bucket.get("jimmy:tasks")
-
-          directory.data.to_i.must_equal object.meta['timestamp'][0].to_i
-        end
-      end
-
-      context "with binary data" do
-        context "charset given in content-type header" do
-          before do
-            header "Content-Type", "image/jpeg; charset=binary"
-            filename = File.join(File.expand_path(File.dirname(__FILE__)), "..", "fixtures", "rockrule.jpeg")
-            @image = File.open(filename, "r").read
-            put "/jimmy/tasks/jaypeg.jpg", @image
-          end
-
-          it "lists the binary files" do
-            get "/jimmy/tasks/"
-
-            last_response.status.must_equal 200
-
-            jaypeg = data_bucket.get("jimmy:tasks:jaypeg.jpg")
-
-            content = JSON.parse(last_response.body)
-            content["items"]["jaypeg.jpg"]["ETag"].must_equal jaypeg.etag.gsub(/"/, "")
-            content["items"]["jaypeg.jpg"]["Content-Type"].must_equal "image/jpeg"
-            content["items"]["jaypeg.jpg"]["Content-Length"].must_equal 16044
-          end
-        end
-
-        context "no charset in content-type header" do
-          before do
-            header "Content-Type", "image/jpeg"
-            filename = File.join(File.expand_path(File.dirname(__FILE__)), "..", "fixtures", "rockrule.jpeg")
-            @image = File.open(filename, "r").read
-            put "/jimmy/tasks/jaypeg.jpg", @image
-          end
-
-          it "lists the binary files" do
-            get "/jimmy/tasks/"
-
-            last_response.status.must_equal 200
-
-            jaypeg = data_bucket.get("jimmy:tasks:jaypeg.jpg")
-
-            content = JSON.parse(last_response.body)
-            content["items"]["jaypeg.jpg"]["ETag"].must_equal jaypeg.etag.gsub(/"/, "")
-            content["items"]["jaypeg.jpg"]["Content-Type"].must_equal "image/jpeg"
-            content["items"]["jaypeg.jpg"]["Content-Length"].must_equal 16044
-          end
-        end
-      end
-    end
-
-    context "for a sub-directory" do
-      before do
-        put "/jimmy/tasks/home/laundry", "do the laundry"
-      end
-
-      it "lists the objects with timestamp" do
-        get "/jimmy/tasks/home/"
-
-        last_response.status.must_equal 200
-
-        laundry = data_bucket.get("jimmy:tasks/home:laundry")
-
-        content = JSON.parse(last_response.body)
-        content["items"]["laundry"]["ETag"].must_equal laundry.etag.gsub(/"/, "")
-      end
-    end
-
-    context "for an empty or absent directory" do
-      it "returns an empty listing" do
-        get "/jimmy/documents/notfound/"
-
-        last_response.status.must_equal 404
-      end
-    end
-
-    context "special characters in directory name" do
-      before do
-        put "/jimmy/tasks/foo~bar/task1", "some task"
-      end
-
-      it "lists the directory in the parent directory" do
-        get "/jimmy/tasks/"
-
-        last_response.status.must_equal 200
-
-        content = JSON.parse(last_response.body)
-        content["items"]["foo~bar/"].wont_be_nil
-      end
-
-      it "lists the containing objects" do
-        get "/jimmy/tasks/foo~bar/"
-
-        last_response.status.must_equal 200
-
-        content = JSON.parse(last_response.body)
-        content["items"]["task1"].wont_be_nil
-      end
-
-      it "returns the requested object" do
-        get "/jimmy/tasks/foo~bar/task1"
-
-        last_response.status.must_equal 200
-
-        last_response.body.must_equal "some task"
-      end
-    end
-
-    context "special characters in object name" do
-      before do
-        put "/jimmy/tasks/bla~blub", "some task"
-      end
-
-      it "lists the containing object" do
-        get "/jimmy/tasks/"
-
-        last_response.status.must_equal 200
-
-        content = JSON.parse(last_response.body)
-        content["items"]["bla~blub"].wont_be_nil
-      end
-    end
-
-    context "for the root directory" do
-      before do
-        auth = auth_bucket.new("jimmy:123")
-        auth.data = [":rw"]
-        auth.store
-
-        put "/jimmy/root-1", "Put my root down"
-        put "/jimmy/root-2", "Back to the roots"
-      end
-
-      it "lists the containing objects and direct sub-directories" do
-        get "/jimmy/"
-
-        last_response.status.must_equal 200
-
-        tasks = directory_bucket.get("jimmy:tasks")
-
-        content = JSON.parse(last_response.body)
-        content["items"]["root-1"].wont_be_nil
-        content["items"]["root-2"].wont_be_nil
-        content["items"]["tasks/"].wont_be_nil
-        content["items"]["tasks/"]["ETag"].must_equal tasks.etag.gsub(/"/, "")
-      end
-
-      it "has an ETag header set" do
-        get "/jimmy/"
-
-        last_response.status.must_equal 200
-        last_response.headers["ETag"].wont_be_nil
-      end
-    end
-
-    context "for the public directory" do
-      before do
-        auth = auth_bucket.new("jimmy:123")
-        auth.data = ["documents:r", "bookmarks:rw"]
-        auth.store
-
-        put "/jimmy/public/bookmarks/5apps", "http://5apps.com"
-      end
-
-      context "when authorized for the category" do
-        it "lists the files" do
-          get "/jimmy/public/bookmarks/"
-
-          last_response.status.must_equal 200
-
-          content = JSON.parse(last_response.body)
-          content["items"]["5apps"].wont_be_nil
-        end
-
-        it "has an ETag header set" do
-          get "/jimmy/public/bookmarks/"
-
-          last_response.status.must_equal 200
-          last_response.headers["ETag"].wont_be_nil
-        end
-      end
-
-      context "when directly authorized for the public directory" do
-        before do
-          auth = auth_bucket.new("jimmy:123")
-          auth.data = ["documents:r", "public/bookmarks:rw"]
-          auth.store
-        end
-
-        it "lists the files" do
-          get "/jimmy/public/bookmarks/"
-
-          last_response.status.must_equal 200
-
-          content = JSON.parse(last_response.body)
-          content["items"]["5apps"].wont_be_nil
-        end
-      end
-
-      context "when not authorized" do
-        before do
-          auth_bucket.delete("jimmy:123")
-        end
-
-        it "does not allow a directory listing of the public root" do
-          get "/jimmy/public/"
-
-          last_response.status.must_equal 401
-        end
-
-        it "does not allow a directory listing of a sub-directory" do
-          get "/jimmy/public/bookmarks/"
-
-          last_response.status.must_equal 401
-        end
-      end
-    end
-  end
-
-  describe "directory object" do
-    describe "PUT file" do
-      context "no existing directory object" do
-        before do
-          put "/jimmy/tasks/home/trash", "take out the trash"
-        end
-
-        it "creates a new directory object" do
-          object = data_bucket.get("jimmy:tasks/home:trash")
-          directory = directory_bucket.get("jimmy:tasks/home")
-
-          directory.data.wont_be_nil
-          directory.data.to_i.must_equal object.meta['timestamp'][0].to_i
-        end
-
-        it "sets the correct index for the directory object" do
-          object = directory_bucket.get("jimmy:tasks/home")
-          object.indexes["directory_bin"].must_include "tasks"
-        end
-
-        it "creates directory objects for the parent directories" do
-          object = directory_bucket.get("jimmy:tasks")
-          object.indexes["directory_bin"].must_include "/"
-          object.data.wont_be_nil
-
-          object = directory_bucket.get("jimmy:")
-          object.indexes["directory_bin"].must_be_empty
-          object.data.wont_be_nil
-        end
-      end
-
-      context "existing directory object" do
-        before do
-          put "/jimmy/tasks/home/trash", "collect some trash"
-        end
-
-        it "updates the timestamp of the directory" do
-          put "/jimmy/tasks/home/trash", "take out the trash"
-
-          last_response.status.must_equal 200
-
-          object = data_bucket.get("jimmy:tasks/home:trash")
-          directory = directory_bucket.get("jimmy:tasks/home")
-
-          directory.data.to_i.must_equal object.meta['timestamp'][0].to_i
-        end
-      end
-    end
-  end
-
-  describe "OPTIONS listing" do
-    it "has CORS headers set" do
-      options "/jimmy/tasks/"
-
-      last_response.status.must_equal 200
-
-      last_response.headers["Access-Control-Allow-Origin"].must_equal "*"
-      last_response.headers["Access-Control-Allow-Methods"].must_equal "GET, PUT, DELETE"
-      last_response.headers["Access-Control-Allow-Headers"].must_equal "Authorization, Content-Type, Origin, If-Match, If-None-Match"
-      last_response.headers["Access-Control-Expose-Headers"].must_equal "ETag, Content-Length"
-    end
-
-    context "sub-directories" do
-      it "has CORS headers set" do
-        options "/jimmy/tasks/foo/bar/"
-
-        last_response.status.must_equal 200
-
-        last_response.headers["Access-Control-Allow-Origin"].must_equal "*"
-        last_response.headers["Access-Control-Allow-Methods"].must_equal "GET, PUT, DELETE"
-        last_response.headers["Access-Control-Allow-Headers"].must_equal "Authorization, Content-Type, Origin, If-Match, If-None-Match"
-        last_response.headers["Access-Control-Expose-Headers"].must_equal "ETag, Content-Length"
-      end
-    end
-
-    context "root directory" do
-      it "has CORS headers set" do
-        options "/jimmy/"
-
-        last_response.status.must_equal 200
-
-        last_response.headers["Access-Control-Allow-Origin"].must_equal "*"
-        last_response.headers["Access-Control-Allow-Methods"].must_equal "GET, PUT, DELETE"
-        last_response.headers["Access-Control-Allow-Headers"].must_equal "Authorization, Content-Type, Origin, If-Match, If-None-Match"
-        last_response.headers["Access-Control-Expose-Headers"].must_equal "ETag, Content-Length"
-      end
-    end
-  end
-
-  describe "DELETE file" do
-    context "last file in directory" do
-      before do
-        put "/jimmy/tasks/home/trash", "take out the trash"
-      end
-
-      it "deletes the directory objects for all empty parent directories" do
-        delete "/jimmy/tasks/home/trash"
-
-        last_response.status.must_equal 200
-
-        lambda {
-          directory_bucket.get("jimmy:tasks/home")
-        }.must_raise Riak::HTTPFailedRequest
-
-        lambda {
-          directory_bucket.get("jimmy:tasks")
-        }.must_raise Riak::HTTPFailedRequest
-
-        lambda {
-          directory_bucket.get("jimmy:")
-        }.must_raise Riak::HTTPFailedRequest
-      end
-    end
-
-    context "with additional files in directory" do
-      before do
-        put "/jimmy/tasks/home/trash", "take out the trash"
-        put "/jimmy/tasks/home/laundry/washing", "wash the clothes"
-      end
-
-      it "does not delete the directory objects for the parent directories" do
-        delete "/jimmy/tasks/home/trash"
-
-        directory_bucket.get("jimmy:tasks/home").wont_be_nil
-        directory_bucket.get("jimmy:tasks").wont_be_nil
-        directory_bucket.get("jimmy:").wont_be_nil
-      end
-
-      it "updates the ETag headers of all parent directories" do
-        get "/jimmy/tasks/home/"
-        home_etag = last_response.headers["ETag"]
-
-        get "/jimmy/tasks/"
-        tasks_etag = last_response.headers["ETag"]
-
-        get "/jimmy/"
-        root_etag = last_response.headers["ETag"]
-
-        delete "/jimmy/tasks/home/trash"
-
-        get "/jimmy/tasks/home/"
-        last_response.headers["ETag"].wont_be_nil
-        last_response.headers["ETag"].wont_equal home_etag
-
-        get "/jimmy/tasks/"
-        last_response.headers["ETag"].wont_be_nil
-        last_response.headers["ETag"].wont_equal tasks_etag
-
-        get "/jimmy/"
-        last_response.headers["ETag"].wont_be_nil
-        last_response.headers["ETag"].wont_equal root_etag
-      end
-
-      describe "timestamps" do
-        before do
-          @old_timestamp = (2.seconds.ago.to_f * 1000).to_i
-
-          ["tasks/home", "tasks", ""].each do |dir|
-            directory = directory_bucket.get("jimmy:#{dir}")
-            directory.data = @old_timestamp.to_s
-            directory.store
-          end
-        end
-
-        it "updates the timestamp for the parent directories" do
-          delete "/jimmy/tasks/home/trash"
-
-          directory_bucket.get("jimmy:tasks/home").data.to_i.must_be :>, @old_timestamp
-          directory_bucket.get("jimmy:tasks").data.to_i.must_be :>, @old_timestamp
-          directory_bucket.get("jimmy:").data.to_i.must_be :>, @old_timestamp
-        end
-      end
-    end
-  end
-
-end

spec/riak/permissions_spec.rb

@@ -1,424 +0,0 @@
-require_relative "../spec_helper"
-
-describe "Permissions" do
-  include Rack::Test::Methods
-
-  before do
-    purge_all_buckets
-  end
-
-  describe "GET" do
-    context "public data" do
-      before do
-        object = data_bucket.new("jimmy:public:foo")
-        object.content_type = "text/plain"
-        object.data = "some text data"
-        object.store
-
-        object = data_bucket.new("jimmy:public/documents:foo")
-        object.content_type = "text/plain"
-        object.data = "some text data"
-        object.store
-      end
-
-      it "returns the value on all get requests" do
-        get "/jimmy/public/foo"
-
-        last_response.status.must_equal 200
-        last_response.body.must_equal "some text data"
-      end
-
-      it "returns the value from a sub-directory" do
-        get "/jimmy/public/documents/foo"
-
-        last_response.status.must_equal 200
-        last_response.body.must_equal "some text data"
-      end
-    end
-
-    context "private data" do
-      before do
-        object = data_bucket.new("jimmy:documents:foo")
-        object.content_type = "text/plain"
-        object.data = "some private, authorized text data"
-        object.store
-
-        object = data_bucket.new("jimmy:documents/very/interesting:text")
-        object.content_type = "text/plain"
-        object.data = "some very interesting writing"
-        object.store
-
-        object = data_bucket.new("jimmy:confidential:bar")
-        object.content_type = "text/plain"
-        object.data = "some private, non-authorized text data"
-        object.store
-
-        auth = auth_bucket.new("jimmy:123")
-        auth.data = ["documents:r", "tasks:rw"]
-        auth.store
-
-        header "Authorization", "Bearer 123"
-      end
-
-      context "when authorized" do
-        it "returns the value for a key in a top-level directory" do
-          get "/jimmy/documents/foo"
-
-          last_response.status.must_equal 200
-          last_response.body.must_equal "some private, authorized text data"
-        end
-
-        it "returns the value for a key in a sub-directory" do
-          get "/jimmy/documents/very/interesting/text"
-
-          last_response.status.must_equal 200
-          last_response.body.must_equal "some very interesting writing"
-        end
-      end
-
-      context "when not authorized" do
-        it "returns a 401 for a key in a top-level directory" do
-          get "/jimmy/confidential/bar"
-
-          last_response.status.must_equal 401
-        end
-      end
-    end
-  end
-
-  describe "PUT" do
-    before do
-      auth = auth_bucket.new("jimmy:123")
-      auth.data = ["documents:r", "contacts:rw", "tasks:r", "tasks/home:rw"]
-      auth.store
-
-      header "Authorization", "Bearer 123"
-    end
-
-    context "to a top-level directory" do
-      it "saves the value when there are write permissions" do
-        put "/jimmy/contacts/1", "John Doe"
-
-        last_response.status.must_equal 201
-        data_bucket.get("jimmy:contacts:1").data.must_equal "John Doe"
-      end
-
-      it "returns a 401 when there are read permissions only" do
-        put "/jimmy/documents/foo", "some text"
-
-        last_response.status.must_equal 401
-      end
-    end
-
-    context "to a sub-directory" do
-      it "saves the value when there are direct write permissions" do
-        put "/jimmy/tasks/home/1", "take out the trash"
-
-        last_response.status.must_equal 201
-        data_bucket.get("jimmy:tasks/home:1").data.must_equal "take out the trash"
-      end
-
-      it "saves the value when there are write permissions for a parent directory" do
-        put "/jimmy/contacts/family/1", "Bobby Brother"
-
-        last_response.status.must_equal 201
-        data_bucket.get("jimmy:contacts/family:1").data.must_equal "Bobby Brother"
-      end
-
-      it "returns a 401 when there are read permissions only" do
-        put "/jimmy/documents/business/1", "some text"
-
-        last_response.status.must_equal 401
-      end
-    end
-
-    context "to the public directory" do
-      context "when authorized for the corresponding category" do
-        it "saves the value" do
-          put "/jimmy/public/contacts/foo", "Foo Bar"
-
-          last_response.status.must_equal 201
-          data_bucket.get("jimmy:public/contacts:foo").data.must_equal "Foo Bar"
-        end
-
-        it "saves the value to a sub-directory" do
-          put "/jimmy/public/contacts/family/foo", "Foo Bar"
-
-          last_response.status.must_equal 201
-          data_bucket.get("jimmy:public/contacts/family:foo").data.must_equal "Foo Bar"
-        end
-      end
-
-      context "when not authorized for the corresponding category" do
-        it "returns a 401" do
-          put "/jimmy/public/documents/foo", "Foo Bar"
-
-          last_response.status.must_equal 401
-        end
-      end
-    end
-  end
-
-  describe "DELETE" do
-    before do
-      auth = auth_bucket.new("jimmy:123")
-      auth.data = ["documents:r", "tasks:rw"]
-      auth.store
-
-      header "Authorization", "Bearer 123"
-    end
-
-    context "when authorized" do
-      before do
-        object = data_bucket.new("jimmy:tasks:1")
-        object.content_type = "text/plain"
-        object.data = "do the laundry"
-        object.store
-
-        object = data_bucket.new("jimmy:tasks/home:1")
-        object.content_type = "text/plain"
-        object.data = "take out the trash"
-        object.store
-      end
-
-      it "removes the key from a top-level directory" do
-        delete "/jimmy/tasks/1"
-
-        last_response.status.must_equal 200
-        lambda {
-          data_bucket.get("jimmy:tasks:1")
-        }.must_raise Riak::HTTPFailedRequest
-      end
-
-      it "removes the key from a top-level directory" do
-        delete "/jimmy/tasks/home/1"
-
-        last_response.status.must_equal 200
-        lambda {
-          data_bucket.get("jimmy:tasks/home:1")
-        }.must_raise Riak::HTTPFailedRequest
-      end
-
-      context "public directory" do
-        before do
-          object = data_bucket.new("jimmy:public/tasks:open")
-          object.content_type = "text/plain"
-          object.data = "hello world"
-          object.store
-        end
-
-        it "removes the key" do
-          delete "/jimmy/public/tasks/open"
-
-          last_response.status.must_equal 200
-          lambda {
-            data_bucket.get("jimmy:public/tasks:open")
-          }.must_raise Riak::HTTPFailedRequest
-        end
-      end
-    end
-
-    context "when not authorized" do
-      before do
-        object = data_bucket.new("jimmy:documents:private")
-        object.content_type = "text/plain"
-        object.data = "some private, authorized text data"
-        object.store
-
-        object = data_bucket.new("jimmy:documents/business:foo")
-        object.content_type = "text/plain"
-        object.data = "some private, authorized text data"
-        object.store
-      end
-
-      it "returns a 401 for a key in a top-level directory" do
-        delete "/jimmy/documents/private"
-
-        last_response.status.must_equal 401
-      end
-
-      it "returns a 401 for a key in a sub-directory" do
-        delete "/jimmy/documents/business/foo"
-
-        last_response.status.must_equal 401
-      end
-
-      context "public directory" do
-        before do
-          object = data_bucket.new("jimmy:public/documents:foo")
-          object.content_type = "text/plain"
-          object.data = "some private, authorized text data"
-          object.store
-        end
-
-        it "returns a 401" do
-          delete "/jimmy/public/documents/foo"
-
-          last_response.status.must_equal 401
-        end
-      end
-    end
-  end
-
-  describe "global permissions" do
-    before do
-      object = data_bucket.new("jimmy:documents/very/interesting:text")
-      object.content_type = "text/plain"
-      object.data = "some very interesting writing"
-      object.store
-    end
-
-    context "write all" do
-      before do
-        auth = auth_bucket.new("jimmy:123")
-        auth.data = [":rw", "documents:r"]
-        auth.store
-
-        header "Authorization", "Bearer 123"
-      end
-
-      it "allows GET requests" do
-        get "/jimmy/documents/very/interesting/text"
-
-        last_response.status.must_equal 200
-        last_response.body.must_equal "some very interesting writing"
-      end
-
-      it "allows PUT requests" do
-        put "/jimmy/contacts/1", "John Doe"
-
-        last_response.status.must_equal 201
-        data_bucket.get("jimmy:contacts:1").data.must_equal "John Doe"
-      end
-
-      it "allows DELETE requests" do
-        delete "/jimmy/documents/very/interesting/text"
-
-        last_response.status.must_equal 200
-        lambda {
-          data_bucket.get("jimmy:documents/very/interesting:text")
-        }.must_raise Riak::HTTPFailedRequest
-      end
-
-      context "root directory" do
-        before do
-          object = data_bucket.new("jimmy::root")
-          object.content_type = "text/plain"
-          object.data = "Back to the roots"
-          object.store
-        end
-
-        it "allows GET requests" do
-          get "/jimmy/root"
-
-          last_response.status.must_equal 200
-          last_response.body.must_equal "Back to the roots"
-        end
-
-        it "allows PUT requests" do
-          put "/jimmy/1", "Gonna kick it root down"
-
-          last_response.status.must_equal 201
-          data_bucket.get("jimmy::1").data.must_equal "Gonna kick it root down"
-        end
-
-        it "allows DELETE requests" do
-          delete "/jimmy/root"
-
-          last_response.status.must_equal 200
-          lambda {
-            data_bucket.get("jimmy::root")
-          }.must_raise Riak::HTTPFailedRequest
-        end
-      end
-
-      context "public directory" do
-        before do
-          object = data_bucket.new("jimmy:public/tasks:hello")
-          object.content_type = "text/plain"
-          object.data = "Hello World"
-          object.store
-        end
-
-        it "allows GET requests" do
-          get "/jimmy/public/tasks/"
-
-          last_response.status.must_equal 404
-        end
-
-        it "allows PUT requests" do
-          put "/jimmy/public/1", "Hello World"
-
-          last_response.status.must_equal 201
-          data_bucket.get("jimmy:public:1").data.must_equal "Hello World"
-        end
-
-        it "allows DELETE requests" do
-          delete "/jimmy/public/tasks/hello"
-
-          last_response.status.must_equal 200
-          lambda {
-            data_bucket.get("jimmy:public/tasks:hello")
-          }.must_raise Riak::HTTPFailedRequest
-        end
-      end
-    end
-
-    context "read all" do
-      before do
-        auth = auth_bucket.new("jimmy:123")
-        auth.data = [":r", "contacts:rw"]
-        auth.store
-
-        header "Authorization", "Bearer 123"
-      end
-
-      it "allows GET requests" do
-        get "/jimmy/documents/very/interesting/text"
-
-        last_response.status.must_equal 200
-        last_response.body.must_equal "some very interesting writing"
-      end
-
-      it "disallows PUT requests" do
-        put "/jimmy/documents/foo", "some text"
-
-        last_response.status.must_equal 401
-      end
-
-      it "disallows DELETE requests" do
-        delete "/jimmy/documents/very/interesting/text"
-
-        last_response.status.must_equal 401
-      end
-
-      context "public directory" do
-        before do
-          object = data_bucket.new("jimmy:public/tasks:hello")
-          object.content_type = "text/plain"
-          object.data = "Hello World"
-          object.store
-        end
-
-        it "allows GET requests" do
-          get "/jimmy/public/tasks/"
-
-          last_response.status.must_equal 404
-        end
-
-        it "disallows PUT requests" do
-          put "/jimmy/public/tasks/foo", "some text"
-
-          last_response.status.must_equal 401
-        end
-
-        it "disallows DELETE requests" do
-          delete "/jimmy/public/tasks/hello"
-
-          last_response.status.must_equal 401
-        end
-      end
-    end
-  end
-
-end

spec/riak/riak_spec.rb

@@ -1,775 +0,0 @@
-require_relative "../spec_helper"
-
-describe "App with Riak backend" do
-  include Rack::Test::Methods
-
-  before do
-    purge_all_buckets
-  end
-
-  describe "HEAD public data" do
-    before do
-      object = data_bucket.new("jimmy:public:foo")
-      object.content_type = "text/plain"
-      object.data = "some text data"
-      object.store
-
-      head "/jimmy/public/foo"
-    end
-
-    it "works" do
-      last_response.status.must_equal 200
-      last_response.body.must_equal ""
-      last_response.headers["ETag"].wont_be_nil
-      last_response.headers["Content-Length"].must_equal "14"
-    end
-  end
-
-  describe "GET public data" do
-    describe "file with content" do
-      before do
-        object = data_bucket.new("jimmy:public:foo")
-        object.content_type = "text/plain"
-        object.data = "some text data"
-        object.store
-
-        get "/jimmy/public/foo"
-      end
-
-      it "works" do
-        last_response.status.must_equal 200
-        last_response.body.must_equal "some text data"
-        last_response.headers["ETag"].wont_be_nil
-        last_response.headers["Content-Length"].must_equal "14"
-        last_response.headers["Expires"].must_equal "0"
-      end
-
-      describe "empty file" do
-        before do
-          object = data_bucket.new("jimmy:public:empty")
-          object.content_type = "text/plain"
-          object.data = ""
-          object.store
-
-          get "/jimmy/public/empty"
-        end
-
-        it "returns an empty body" do
-          last_response.status.must_equal 200
-          # Rack::MockRequest turns the body into a string. We can't use
-          # `last_response.body` to check for nil, because:
-          # >> [nil].join
-          # => ""
-          last_response.body.must_equal ''
-          last_response.headers["Content-Length"].must_equal '0'
-        end
-      end
-    end
-
-    describe "GET data with custom content type" do
-      before do
-        object = data_bucket.new("jimmy:public:magic")
-        object.content_type = "text/magic"
-        object.raw_data = "some text data"
-        object.store
-      end
-
-      it "returns the value with the correct content type" do
-        get "/jimmy/public/magic"
-
-        last_response.status.must_equal 200
-        last_response.content_type.must_equal "text/magic"
-        last_response.body.must_equal "some text data"
-      end
-    end
-
-    describe "private data" do
-      before do
-        object = data_bucket.new("jimmy:documents:foo")
-        object.content_type = "text/plain"
-        object.data = "some private text data"
-        object.store
-
-        @etag = object.etag
-
-        auth = auth_bucket.new("jimmy:123")
-        auth.data = ["documents", "public"]
-        auth.store
-      end
-
-      describe "HEAD" do
-        before do
-          header "Authorization", "Bearer 123"
-          head "/jimmy/documents/foo"
-        end
-
-        it "works" do
-          last_response.status.must_equal 200
-          last_response.body.must_equal ""
-          last_response.headers["ETag"].wont_be_nil
-          last_response.headers["Content-Length"].must_equal "22"
-        end
-      end
-
-      describe "HEAD nonexisting key" do
-        it "returns a 404" do
-          header "Authorization", "Bearer 123"
-          head "/jimmy/documents/somestupidkey"
-
-          last_response.status.must_equal 404
-        end
-      end
-
-      describe "GET" do
-        before do
-          header "Authorization", "Bearer 123"
-        end
-
-        it "returns the value" do
-          get "/jimmy/documents/foo"
-
-          last_response.status.must_equal 200
-          last_response.body.must_equal "some private text data"
-        end
-
-        describe "when If-None-Match header is set" do
-          it "responds with 'not modified' when it matches the current ETag" do
-            header "If-None-Match", @etag
-            get "/jimmy/documents/foo"
-
-            last_response.status.must_equal 304
-            last_response.body.must_be_empty
-            last_response.headers["ETag"].must_equal @etag
-          end
-
-          it "responds normally when it does not match the current ETag" do
-            header "If-None-Match", "FOO"
-            get "/jimmy/documents/foo"
-
-            last_response.status.must_equal 200
-            last_response.body.must_equal "some private text data"
-          end
-        end
-
-        describe "when If-None-Match header is set with multiple revisions" do
-          it "responds with 'not modified' when it contains the current ETag" do
-            header "If-None-Match", "DEADBEEF,#{@etag},F00BA4"
-            get "/jimmy/documents/foo"
-
-            last_response.status.must_equal 304
-            last_response.body.must_be_empty
-            last_response.headers["ETag"].must_equal @etag
-          end
-
-          it "responds normally when it does not contain the current ETag" do
-            header "If-None-Match", "FOO,BAR"
-            get "/jimmy/documents/foo"
-
-            last_response.status.must_equal 200
-            last_response.body.must_equal "some private text data"
-          end
-        end
-      end
-
-      describe "GET nonexisting key" do
-        it "returns a 404" do
-          header "Authorization", "Bearer 123"
-          get "/jimmy/documents/somestupidkey"
-
-          last_response.status.must_equal 404
-        end
-      end
-
-      describe "PUT" do
-        before do
-          header "Authorization", "Bearer 123"
-        end
-
-        describe "with implicit content type" do
-          before do
-            put "/jimmy/documents/bar", "another text"
-          end
-
-          it "saves the value" do
-            last_response.status.must_equal 201
-            last_response.body.must_equal ""
-            data_bucket.get("jimmy:documents:bar").data.must_equal "another text"
-          end
-
-          it "stores the data as plain text with utf-8 encoding" do
-            data_bucket.get("jimmy:documents:bar").content_type.must_equal "text/plain; charset=utf-8"
-          end
-
-          it "sets the ETag header" do
-            last_response.headers["ETag"].wont_be_nil
-          end
-
-          it "indexes the data set" do
-            indexes = data_bucket.get("jimmy:documents:bar").indexes
-            indexes["user_id_bin"].must_be_kind_of Set
-            indexes["user_id_bin"].must_include "jimmy"
-
-            indexes["directory_bin"].must_include "documents"
-          end
-
-          it "logs the operation" do
-            objects = []
-            opslog_bucket.keys.each { |k| objects << opslog_bucket.get(k) rescue nil }
-
-            log_entry = objects.select{|o| o.data["count"] == 1}.first
-            log_entry.data["size"].must_equal 12
-            log_entry.data["category"].must_equal "documents"
-            log_entry.indexes["user_id_bin"].must_include "jimmy"
-          end
-        end
-
-        describe "with explicit content type" do
-          before do
-            header "Content-Type", "application/json"
-            put "/jimmy/documents/jason", '{"foo": "bar", "unhosted": 1}'
-          end
-
-          it "saves the value (as JSON)" do
-            last_response.status.must_equal 201
-            data_bucket.get("jimmy:documents:jason").data.must_be_kind_of Hash
-            data_bucket.get("jimmy:documents:jason").data.must_equal({"foo" => "bar", "unhosted" => 1})
-          end
-
-          it "uses the requested content type" do
-            data_bucket.get("jimmy:documents:jason").content_type.must_equal "application/json"
-          end
-
-          it "delivers the data correctly" do
-            header "Authorization", "Bearer 123"
-            get "/jimmy/documents/jason"
-
-            last_response.body.must_equal '{"foo":"bar","unhosted":1}'
-            last_response.content_type.must_equal "application/json"
-          end
-        end
-
-        describe "with arbitrary content type" do
-          before do
-            header "Content-Type", "text/magic"
-            put "/jimmy/documents/magic", "pure magic"
-          end
-
-          it "saves the value" do
-            last_response.status.must_equal 201
-            data_bucket.get("jimmy:documents:magic").raw_data.must_equal "pure magic"
-          end
-
-          it "uses the requested content type" do
-            data_bucket.get("jimmy:documents:magic").content_type.must_equal "text/magic"
-          end
-
-          it "delivers the data correctly" do
-            header "Authorization", "Bearer 123"
-            get "/jimmy/documents/magic"
-
-            last_response.body.must_equal "pure magic"
-            last_response.content_type.must_equal "text/magic"
-          end
-        end
-
-        describe "with content type containing the encoding" do
-          before do
-            header "Content-Type", "application/json; charset=UTF-8"
-            put "/jimmy/documents/jason", '{"foo": "bar", "unhosted": 1}'
-          end
-
-          it "saves the value (as JSON)" do
-            last_response.status.must_equal 201
-            data_bucket.get("jimmy:documents:jason").data.must_be_kind_of Hash
-            data_bucket.get("jimmy:documents:jason").data.must_equal({"foo" => "bar", "unhosted" => 1})
-          end
-
-          it "uses the requested content type" do
-            data_bucket.get("jimmy:documents:jason").content_type.must_equal "application/json; charset=UTF-8"
-          end
-
-          it "delivers the data correctly" do
-            get "/jimmy/documents/jason"
-
-            last_response.body.must_equal '{"foo":"bar","unhosted":1}'
-            last_response.content_type.must_equal "application/json; charset=UTF-8"
-          end
-        end
-
-        describe "naming collisions between documents and directories" do
-          before do
-            put "/jimmy/documents/archive/document", "lorem ipsum"
-          end
-
-          it "responds with 409 when directory with same name already exists" do
-            put "/jimmy/documents/archive", "some awesome content"
-
-            last_response.status.must_equal 409
-
-            lambda {
-              data_bucket.get("jimmy:documents/archive")
-            }.must_raise Riak::HTTPFailedRequest
-          end
-
-          it "responds with 409 when there is an existing document with same name as one of the directories" do
-            put "/jimmy/documents/archive/document/subdir/doc", "some awesome content"
-
-            last_response.status.must_equal 409
-
-            lambda {
-              data_bucket.get("jimmy:documents/archive/document/subdir/doc")
-            }.must_raise Riak::HTTPFailedRequest
-          end
-        end
-
-        describe "with existing content" do
-          before do
-            put "/jimmy/documents/archive/foo", "lorem ipsum"
-          end
-
-          it "saves the value" do
-            put "/jimmy/documents/archive/foo", "some awesome content"
-
-            last_response.status.must_equal 200
-            data_bucket.get("jimmy:documents/archive:foo").data.must_equal "some awesome content"
-          end
-
-          it "logs the operations" do
-            put "/jimmy/documents/archive/foo", "some awesome content"
-
-            objects = []
-            opslog_bucket.keys.each { |k| objects << opslog_bucket.get(k) rescue nil }
-
-            create_entry = objects.select{|o| o.data["count"] == 1}.first
-            create_entry.data["size"].must_equal 11
-            create_entry.data["category"].must_equal "documents"
-            create_entry.indexes["user_id_bin"].must_include "jimmy"
-
-            update_entry = objects.select{|o| o.data["count"] == 0}.first
-            update_entry.data["size"].must_equal 9
-            update_entry.data["category"].must_equal "documents"
-            update_entry.indexes["user_id_bin"].must_include "jimmy"
-          end
-
-          it "changes the ETag header" do
-            old_etag = last_response.headers["ETag"]
-            put "/jimmy/documents/archive/foo", "some awesome content"
-
-            last_response.headers["ETag"].wont_be_nil
-            last_response.headers["ETag"].wont_equal old_etag
-          end
-
-          describe "when If-Match header is set" do
-            it "allows the request if the header matches the current ETag" do
-              old_etag = last_response.headers["ETag"]
-              header "If-Match", old_etag
-
-              put "/jimmy/documents/archive/foo", "some awesome content"
-              last_response.status.must_equal 200
-
-              get "/jimmy/documents/archive/foo"
-              last_response.body.must_equal "some awesome content"
-            end
-
-            it "fails the request if the header does not match the current ETag" do
-              header "If-Match", "WONTMATCH"
-
-              put "/jimmy/documents/archive/foo", "some awesome content"
-              last_response.status.must_equal 412
-
-              get "/jimmy/documents/archive/foo"
-              last_response.body.must_equal "lorem ipsum"
-            end
-          end
-
-          describe "when If-None-Match header is set" do
-            before do
-              header "If-None-Match", "*"
-            end
-
-            it "fails when the document already exists" do
-              put "/jimmy/documents/archive/foo", "some awesome content"
-
-              last_response.status.must_equal 412
-
-              get "/jimmy/documents/archive/foo"
-              last_response.body.must_equal "lorem ipsum"
-            end
-
-            it "succeeds when the document does not exist" do
-              put "/jimmy/documents/archive/bar", "my little content"
-
-              last_response.status.must_equal 201
-            end
-          end
-        end
-
-        describe "exsting content without serializer registered for the given content-type" do
-          before do
-            header "Content-Type", "text/html; charset=UTF-8"
-            put "/jimmy/documents/html", '<html></html>'
-            put "/jimmy/documents/html", '<html><body></body></html>'
-          end
-
-          it "saves the value" do
-            last_response.status.must_equal 200
-            data_bucket.get("jimmy:documents:html").raw_data.must_equal "<html><body></body></html>"
-          end
-
-          it "uses the requested content type" do
-            data_bucket.get("jimmy:documents:html").content_type.must_equal "text/html; charset=UTF-8"
-          end
-        end
-
-        describe "public data" do
-          before do
-            put "/jimmy/public/documents/notes/foo", "note to self"
-          end
-
-          it "saves the value" do
-            last_response.status.must_equal 201
-            data_bucket.get("jimmy:public/documents/notes:foo").data.must_equal "note to self"
-          end
-
-          it "logs the operation" do
-            objects = []
-            opslog_bucket.keys.each { |k| objects << opslog_bucket.get(k) rescue nil }
-
-            log_entry = objects.select{|o| o.data["count"] == 1}.first
-            log_entry.data["size"].must_equal 12
-            log_entry.data["category"].must_equal "public/documents"
-            log_entry.indexes["user_id_bin"].must_include "jimmy"
-          end
-        end
-
-        context "with binary data" do
-          context "binary charset in content-type header" do
-            before do
-              header "Content-Type", "image/jpeg; charset=binary"
-              filename = File.join(File.expand_path(File.dirname(__FILE__)), "..", "fixtures", "rockrule.jpeg")
-              @image = File.open(filename, "r").read
-              put "/jimmy/documents/jaypeg", @image
-            end
-
-            it "uses the requested content type" do
-              get "/jimmy/documents/jaypeg"
-
-              last_response.status.must_equal 200
-              last_response.content_type.must_equal "image/jpeg; charset=binary"
-            end
-
-            it "delivers the data correctly" do
-              get "/jimmy/documents/jaypeg"
-
-              last_response.status.must_equal 200
-              last_response.body.must_equal @image
-            end
-
-            it "responds with an ETag header" do
-              last_response.headers["ETag"].wont_be_nil
-              etag = last_response.headers["ETag"]
-
-              get "/jimmy/documents/jaypeg"
-
-              last_response.headers["ETag"].wont_be_nil
-              last_response.headers["ETag"].must_equal etag
-            end
-
-            it "responds with a Content-Length header" do
-              get "/jimmy/documents/jaypeg"
-
-              last_response.headers["Content-Length"].must_equal "16044"
-            end
-
-            it "changes the ETag when updating the file" do
-              old_etag = last_response.headers["ETag"]
-              put "/jimmy/documents/jaypeg", @image
-
-              last_response.headers["ETag"].wont_be_nil
-              last_response.headers["ETag"].wont_equal old_etag
-            end
-
-            it "logs the operation" do
-              objects = []
-              opslog_bucket.keys.each { |k| objects << opslog_bucket.get(k) rescue nil }
-
-              log_entry = objects.select{|o| o.data["count"] == 1}.first
-              log_entry.data["size"].must_equal 16044
-              log_entry.data["category"].must_equal "documents"
-              log_entry.indexes["user_id_bin"].must_include "jimmy"
-            end
-
-            context "overwriting existing file with same file" do
-              before do
-                header "Content-Type", "image/jpeg; charset=binary"
-                filename = File.join(File.expand_path(File.dirname(__FILE__)), "..", "fixtures", "rockrule.jpeg")
-                @image = File.open(filename, "r").read
-                put "/jimmy/documents/jaypeg", @image
-              end
-
-              it "doesn't log the operation" do
-                objects = []
-                opslog_bucket.keys.each { |k| objects << opslog_bucket.get(k) rescue nil }
-
-                objects.size.must_equal 1
-              end
-            end
-
-            context "overwriting existing file with different file" do
-              before do
-                header "Content-Type", "image/jpeg; charset=binary"
-                filename = File.join(File.expand_path(File.dirname(__FILE__)), "..", "fixtures", "rockrule.jpeg")
-                @image = File.open(filename, "r").read
-                put "/jimmy/documents/jaypeg", @image+"foo"
-              end
-
-              it "logs the operation changing only the size" do
-                objects = []
-                opslog_bucket.keys.each { |k| objects << opslog_bucket.get(k) rescue nil }
-
-                objects.size.must_equal 2
-
-                log_entry = objects.select{|o| o.data["count"] == 0}.first
-                log_entry.data["size"].must_equal 3
-                log_entry.data["category"].must_equal "documents"
-                log_entry.indexes["user_id_bin"].must_include "jimmy"
-              end
-            end
-          end
-
-          context "no binary charset in content-type header" do
-            before do
-              header "Content-Type", "image/jpeg"
-              filename = File.join(File.expand_path(File.dirname(__FILE__)), "..", "fixtures", "rockrule.jpeg")
-              @image = File.open(filename, "r").read
-              put "/jimmy/documents/jaypeg", @image
-            end
-
-            it "uses the requested content type" do
-              get "/jimmy/documents/jaypeg"
-
-              last_response.status.must_equal 200
-              last_response.content_type.must_equal "image/jpeg"
-            end
-
-            it "delivers the data correctly" do
-              get "/jimmy/documents/jaypeg"
-
-              last_response.status.must_equal 200
-              last_response.body.must_equal @image
-            end
-          end
-        end
-
-        context "with escaped key" do
-          before do
-            put "/jimmy/documents/bar%3Abaz/http%3A%2F%2F5apps.com", "super website"
-          end
-
-          it "delivers the data correctly" do
-            header "Authorization", "Bearer 123"
-            get "/jimmy/documents/bar%3Abaz/http%3A%2F%2F5apps.com"
-
-            last_response.body.must_equal 'super website'
-          end
-        end
-
-        context "with unescaped key" do
-          before do
-            put "/jimmy/documents/bar:baz/john@doe.com", "John Doe"
-          end
-
-          it "lists the document in the directory" do
-            get "/jimmy/documents/bar:baz/"
-
-            content = JSON.parse(last_response.body)
-            content["items"]["john@doe.com"].wont_be_nil
-          end
-
-          it "delivers the data correctly" do
-            get "/jimmy/documents/bar:baz/john@doe.com"
-
-            last_response.body.must_equal "John Doe"
-          end
-        end
-
-        context "escaped square brackets in key" do
-          before do
-            put "/jimmy/documents/gracehopper%5B1%5D.jpg", "super image"
-          end
-
-          it "delivers the data correctly" do
-            header "Authorization", "Bearer 123"
-            get "/jimmy/documents/gracehopper%5B1%5D.jpg"
-
-            last_response.body.must_equal "super image"
-          end
-        end
-
-        context "invalid JSON" do
-          context "empty body" do
-            before do
-              header "Content-Type", "application/json"
-              put "/jimmy/documents/jason", ""
-            end
-
-            it "saves an empty JSON object" do
-              last_response.status.must_equal 201
-              data_bucket.get("jimmy:documents:jason").data.must_be_kind_of Hash
-              data_bucket.get("jimmy:documents:jason").data.must_equal({})
-            end
-          end
-
-          context "unparsable JSON" do
-            before do
-              header "Content-Type", "application/json"
-              put "/jimmy/documents/jason", "foo"
-            end
-
-            it "returns a 422" do
-              last_response.status.must_equal 422
-            end
-          end
-        end
-      end
-
-      describe "DELETE" do
-        before do
-          header "Authorization", "Bearer 123"
-        end
-
-        describe "basics" do
-          before do
-            delete "/jimmy/documents/foo"
-          end
-
-          it "removes the key" do
-            last_response.status.must_equal 200
-            lambda {
-              data_bucket.get("jimmy:documents:foo")
-            }.must_raise Riak::HTTPFailedRequest
-          end
-
-          it "logs the operation" do
-            objects = []
-            opslog_bucket.keys.each { |k| objects << opslog_bucket.get(k) rescue nil }
-
-            log_entry = objects.select{|o| o.data["count"] == -1}.first
-            log_entry.data["size"].must_equal(-22)
-            log_entry.data["category"].must_equal "documents"
-            log_entry.indexes["user_id_bin"].must_include "jimmy"
-          end
-        end
-
-        context "non-existing object" do
-          before do
-            delete "/jimmy/documents/foozius"
-          end
-
-          it "responds with 404" do
-            last_response.status.must_equal 404
-          end
-
-          it "doesn't log the operation" do
-            objects = []
-            opslog_bucket.keys.each { |k| objects << opslog_bucket.get(k) rescue nil }
-            objects.select{|o| o.data["count"] == -1}.size.must_equal 0
-          end
-        end
-
-        context "when an If-Match header is given" do
-          it "allows the request if it matches the current ETag" do
-            get "/jimmy/documents/foo"
-            old_etag = last_response.headers["ETag"]
-            header "If-Match", old_etag
-
-            delete "/jimmy/documents/foo"
-            last_response.status.must_equal 200
-
-            get "/jimmy/documents/foo"
-            last_response.status.must_equal 404
-          end
-
-          it "fails the request if it does not match the current ETag" do
-            header "If-Match", "WONTMATCH"
-
-            delete "/jimmy/documents/foo"
-            last_response.status.must_equal 412
-
-            get "/jimmy/documents/foo"
-            last_response.status.must_equal 200
-            last_response.body.must_equal "some private text data"
-          end
-        end
-
-        context "binary data" do
-          before do
-            header "Content-Type", "image/jpeg; charset=binary"
-            filename = File.join(File.expand_path(File.dirname(__FILE__)), "..", "fixtures", "rockrule.jpeg")
-            @image = File.open(filename, "r").read
-            put "/jimmy/documents/jaypeg", @image
-
-            delete "/jimmy/documents/jaypeg"
-          end
-
-          it "removes the main object" do
-            last_response.status.must_equal 200
-            lambda {
-              data_bucket.get("jimmy:documents:jaypeg")
-            }.must_raise Riak::HTTPFailedRequest
-          end
-
-          it "removes the binary object" do
-            last_response.status.must_equal 200
-
-            binary = cs_binary_bucket.files.get("jimmy:documents:jaypeg")
-            binary.must_be_nil
-          end
-
-          it "logs the operation" do
-            objects = []
-            opslog_bucket.keys.each { |k| objects << opslog_bucket.get(k) rescue nil }
-
-            log_entry = objects.select{|o| o.data["count"] == -1 && o.data["size"] == -16044}.first
-            log_entry.data["category"].must_equal "documents"
-            log_entry.indexes["user_id_bin"].must_include "jimmy"
-          end
-        end
-      end
-    end
-
-    describe "unauthorized access" do
-      before do
-        auth = auth_bucket.new("jimmy:123")
-        auth.data = ["documents", "public"]
-        auth.store
-
-        header "Authorization", "Bearer 321"
-      end
-
-      describe "GET" do
-        it "returns a 401" do
-          get "/jimmy/documents/foo"
-
-          last_response.status.must_equal 401
-        end
-      end
-
-      describe "PUT" do
-        it "returns a 401" do
-          put "/jimmy/documents/foo", "some text"
-
-          last_response.status.must_equal 401
-        end
-      end
-
-      describe "DELETE" do
-        it "returns a 401" do
-          delete "/jimmy/documents/foo"
-
-          last_response.status.must_equal 401
-        end
-      end
-    end
-  end
-end

spec/s3/app_spec.rb

@@ -0,0 +1,64 @@
+require_relative "../spec_helper"
+
+describe "S3 provider" do
+  def container_url_for(user)
+    "#{app.settings.s3["endpoint"]}#{app.settings.s3["bucket"]}/#{user}"
+  end
+
+  def storage_class
+    RemoteStorage::S3
+  end
+
+  before do
+    stub_request(:put, "#{container_url_for("phil")}/food/aguacate").
+      to_return(status: 200, headers: { etag: '"0815etag"' })
+    # Write new content with an If-Match header (a new Etag is returned)
+    stub_request(:put, "#{container_url_for("phil")}/food/aguacate").
+      with(body: "aye").
+      to_return(status: 200, headers: { etag: '"0915etag"' })
+    stub_request(:head, "#{container_url_for("phil")}/food/aguacate").
+      to_return(status: 200, headers: { last_modified: "Fri, 04 Mar 2016 12:20:18 GMT" })
+    stub_request(:get, "#{container_url_for("phil")}/food/aguacate").
+      to_return(status: 200, body: "rootbody", headers: { etag: '"0817etag"', content_type: "text/plain; charset=utf-8" })
+    stub_request(:delete, "#{container_url_for("phil")}/food/aguacate").
+      to_return(status: 200, headers: { etag: '"0815etag"' })
+
+    # Write new content to check the metadata in Redis
+    stub_request(:put, "#{container_url_for("phil")}/food/banano").
+      with(body: "si").
+      to_return(status: 200, headers: { etag: '"0815etag"' })
+    stub_request(:put, "#{container_url_for("phil")}/food/banano").
+      with(body: "oh, no").
+      to_return(status: 200, headers: { etag: '"0817etag"' })
+    stub_request(:head, "#{container_url_for("phil")}/food/banano").
+      to_return(status: 200, headers: { last_modified: "Fri, 04 Mar 2016 12:20:20 GMT" })
+
+    stub_request(:put, "#{container_url_for("phil")}/food/camaron").
+      to_return(status: 200, headers: { etag: '"0816etag"' })
+    stub_request(:head, "#{container_url_for("phil")}/food/camaron").
+      to_return(status: 200, headers: { last_modified: "Fri, 04 Mar 2016 12:20:18 GMT" })
+    stub_request(:delete, "#{container_url_for("phil")}/food/camaron").
+      to_return(status: 200, headers: { etag: '"0816etag"' })
+
+    stub_request(:put, "#{container_url_for("phil")}/food/desayunos/bolon").
+      to_return(status: 200, headers: { etag: '"0817etag"' })
+    stub_request(:head, "#{container_url_for("phil")}/food/desayunos/bolon").
+      to_return(status: 200, headers: { last_modified: "Fri, 04 Mar 2016 12:20:18 GMT" })
+    stub_request(:delete, "#{container_url_for("phil")}/food/desayunos/bolon").
+      to_return(status: 200, headers: { etag: '"0817etag"' })
+
+    # objects in root dir
+    stub_request(:put, "#{container_url_for("phil")}/bamboo.txt").
+      to_return(status: 200, headers: { etag: '"0818etag"' })
+    stub_request(:head, "#{container_url_for("phil")}/bamboo.txt").
+      to_return(status: 200, headers: { last_modified: "Fri, 04 Mar 2016 12:20:18 GMT" })
+
+    # 404
+    stub_request(:head, "#{container_url_for("phil")}/food/steak").
+      to_return(status: 404)
+    stub_request(:get, "#{container_url_for("phil")}/food/steak").
+      to_return(status: 404)
+  end
+
+  it_behaves_like 'a REST adapter'
+end

spec/shared_examples.rb

@@ -0,0 +1,645 @@
+require_relative "./spec_helper"
+
+shared_examples_for 'a REST adapter' do
+  include Rack::Test::Methods
+
+  def container_url_for(user)
+    raise NotImplementedError
+  end
+
+  def storage_class
+    raise NotImplementedError
+  end
+
+  it "returns 404 on non-existing routes" do
+    get "/virginmargarita"
+    last_response.status.must_equal 404
+  end
+
+  describe "PUT requests" do
+
+    before do
+      purge_redis
+    end
+
+    context "authorized" do
+      before do
+        redis.sadd "authorizations:phil:amarillo", [":rw"]
+        header "Authorization", "Bearer amarillo"
+      end
+
+      it "creates the metadata object in redis" do
+        put "/phil/food/aguacate", "si"
+
+        metadata = redis.hgetall "rs:m:phil:food/aguacate"
+        metadata["s"].must_equal "2"
+        metadata["t"].must_equal "text/plain; charset=utf-8"
+        metadata["e"].must_equal "0815etag"
+        metadata["m"].length.must_equal 13
+      end
+
+      it "updates the metadata object in redis when it changes" do
+        put "/phil/food/banano", "si"
+        put "/phil/food/banano", "oh, no"
+
+        metadata = redis.hgetall "rs:m:phil:food/banano"
+        metadata["s"].must_equal "6"
+        metadata["t"].must_equal "text/plain; charset=utf-8"
+        metadata["e"].must_equal "0817etag"
+        metadata["m"].must_equal "1457094020000"
+      end
+
+      it "creates the directory objects metadata in redis" do
+        put "/phil/food/aguacate", "si"
+        put "/phil/food/camaron", "yummi"
+
+        metadata = redis.hgetall "rs:m:phil:/"
+        metadata["e"].must_equal "fe2976909daaf074660981ab563fe65d"
+        metadata["m"].length.must_equal 13
+
+        metadata = redis.hgetall "rs:m:phil:food/"
+        metadata["e"].must_equal "926f98ff820f2f9764fd3c60a22865ad"
+        metadata["m"].length.must_equal 13
+
+        food_items = redis.smembers "rs:m:phil:food/:items"
+        food_items.each do |food_item|
+          ["camaron", "aguacate"].must_include food_item
+        end
+
+        root_items = redis.smembers "rs:m:phil:/:items"
+        root_items.must_equal ["food/"]
+      end
+
+      context "response code" do
+        it "is 201 for newly created objects" do
+          put "/phil/food/aguacate", "ci"
+
+          last_response.status.must_equal 201
+        end
+
+        it "is 200 for updated objects" do
+          put "/phil/food/aguacate", "deliciosa"
+          put "/phil/food/aguacate", "muy deliciosa"
+
+          last_response.status.must_equal 200
+        end
+      end
+
+      context "logging usage size" do
+        it "logs the complete size when creating new objects" do
+          put "/phil/food/aguacate", "1234567890"
+
+          size_log = redis.get "rs:s:phil"
+          size_log.must_equal "10"
+        end
+
+        it "logs the size difference when updating existing objects" do
+          put "/phil/food/camaron",  "1234567890"
+          put "/phil/food/aguacate", "1234567890"
+          put "/phil/food/aguacate", "123"
+
+          size_log = redis.get "rs:s:phil"
+          size_log.must_equal "13"
+        end
+      end
+
+      describe "objects in root dir" do
+        before do
+          put "/phil/bamboo.txt", "shir kan"
+        end
+
+        it "are listed in the directory listing with all metadata" do
+          get "phil/"
+
+          last_response.status.must_equal 200
+          last_response.content_type.must_equal "application/ld+json"
+
+          content = JSON.parse(last_response.body)
+          content["items"]["bamboo.txt"].wont_be_nil
+          content["items"]["bamboo.txt"]["ETag"].must_equal "0818etag"
+          content["items"]["bamboo.txt"]["Content-Type"].must_equal "text/plain; charset=utf-8"
+          content["items"]["bamboo.txt"]["Content-Length"].must_equal 8
+          content["items"]["bamboo.txt"]["Last-Modified"].must_equal "Fri, 04 Mar 2016 12:20:18 GMT"
+        end
+      end
+
+      describe "name collision checks" do
+        it "is successful when there is no name collision" do
+          put "/phil/food/aguacate", "si"
+
+          last_response.status.must_equal 201
+
+          metadata = redis.hgetall "rs:m:phil:food/aguacate"
+          metadata["s"].must_equal "2"
+        end
+
+        it "conflicts when there is a directory with same name as document" do
+          put "/phil/food/aguacate", "si"
+          put "/phil/food", "wontwork"
+
+          last_response.status.must_equal 409
+          last_response.body.must_equal "Conflict"
+
+          metadata = redis.hgetall "rs:m:phil:food"
+          metadata.must_be_empty
+        end
+
+        it "conflicts when there is a document with same name as directory" do
+          put "/phil/food/aguacate", "si"
+          put "/phil/food/aguacate/empanado", "wontwork"
+
+          last_response.status.must_equal 409
+
+          metadata = redis.hgetall "rs:m:phil:food/aguacate/empanado"
+          metadata.must_be_empty
+        end
+
+        it "returns 400 when a Content-Range header is sent" do
+          header "Content-Range", "bytes 0-3/3"
+
+          put "/phil/food/aguacate", "si"
+
+          last_response.status.must_equal 400
+        end
+      end
+
+      describe "If-Match header" do
+        before do
+          put "/phil/food/aguacate", "si"
+        end
+
+        it "allows the request if the header matches the current ETag" do
+          header "If-Match", "\"0815etag\""
+
+          put "/phil/food/aguacate", "aye"
+
+          last_response.status.must_equal 200
+          last_response.headers["Etag"].must_equal "\"0915etag\""
+        end
+
+        it "allows the request if the header contains a weak ETAG matching the current ETag" do
+          header "If-Match", "W/\"0815etag\""
+
+          put "/phil/food/aguacate", "aye"
+
+          last_response.status.must_equal 200
+          last_response.headers["Etag"].must_equal "\"0915etag\""
+        end
+
+        it "allows the request if the header contains a weak ETAG with leading quote matching the current ETag" do
+          header "If-Match", "\"W/\"0815etag\""
+
+          put "/phil/food/aguacate", "aye"
+
+          last_response.status.must_equal 200
+          last_response.headers["Etag"].must_equal "\"0915etag\""
+        end
+
+        it "fails the request if the header does not match the current ETag" do
+          header "If-Match", "someotheretag"
+
+          put "/phil/food/aguacate", "aye"
+
+          last_response.status.must_equal 412
+          last_response.body.must_equal "Precondition Failed"
+        end
+
+        it "allows the request if redis metadata became out of sync" do
+          header "If-Match", "\"0815etag\""
+
+          put "/phil/food/aguacate", "aye"
+
+          last_response.status.must_equal 200
+        end
+      end
+
+      describe "If-None-Match header set to '*'" do
+        it "succeeds when the document doesn't exist yet" do
+          header "If-None-Match", "*"
+
+          put "/phil/food/aguacate", "si"
+
+          last_response.status.must_equal 201
+        end
+
+        it "fails the request if the document already exists" do
+          put "/phil/food/aguacate", "si"
+
+          header "If-None-Match", "*"
+          put "/phil/food/aguacate", "si"
+
+          last_response.status.must_equal 412
+          last_response.body.must_equal "Precondition Failed"
+        end
+      end
+    end
+
+  end
+
+  describe "DELETE requests" do
+
+    before do
+      purge_redis
+    end
+
+    context "not authorized" do
+      describe "with no token" do
+        it "says it's not authorized" do
+          delete "/phil/food/aguacate"
+
+          last_response.status.must_equal 401
+          last_response.body.must_equal "Unauthorized"
+        end
+      end
+
+      describe "with empty token" do
+        it "says it's not authorized" do
+          header "Authorization", "Bearer "
+          delete "/phil/food/aguacate"
+
+          last_response.status.must_equal 401
+          last_response.body.must_equal "Unauthorized"
+        end
+      end
+
+      describe "with wrong token" do
+        it "says it's not authorized" do
+          header "Authorization", "Bearer wrongtoken"
+          delete "/phil/food/aguacate"
+
+          last_response.status.must_equal 401
+          last_response.body.must_equal "Unauthorized"
+        end
+      end
+
+    end
+
+    context "authorized" do
+      before do
+        redis.sadd "authorizations:phil:amarillo", [":rw"]
+        header "Authorization", "Bearer amarillo"
+
+        put "/phil/food/aguacate", "si"
+        put "/phil/food/camaron", "yummi"
+        put "/phil/food/desayunos/bolon", "wow"
+      end
+
+      it "decreases the size log by size of deleted object" do
+        delete "/phil/food/aguacate"
+
+        size_log = redis.get "rs:s:phil"
+        size_log.must_equal "8"
+      end
+
+      it "deletes the metadata object in redis" do
+        delete "/phil/food/aguacate"
+
+        metadata = redis.hgetall "rs:m:phil:food/aguacate"
+        metadata.must_be_empty
+      end
+
+      it "deletes the directory objects metadata in redis" do
+        old_metadata = redis.hgetall "rs:m:phil:food/"
+
+        storage_class.stub_any_instance :etag_for, "newetag" do
+          delete "/phil/food/aguacate"
+        end
+
+        metadata = redis.hgetall "rs:m:phil:food/"
+        metadata["e"].must_equal "newetag"
+        metadata["m"].length.must_equal 13
+        metadata["m"].wont_equal old_metadata["m"]
+
+        food_items = redis.smembers "rs:m:phil:food/:items"
+        food_items.sort.must_equal ["camaron", "desayunos/"]
+
+        root_items = redis.smembers "rs:m:phil:/:items"
+        root_items.must_equal ["food/"]
+      end
+
+      it "deletes the parent directory objects metadata when deleting all items" do
+        delete "/phil/food/aguacate"
+        delete "/phil/food/camaron"
+        delete "/phil/food/desayunos/bolon"
+
+        redis.smembers("rs:m:phil:food/desayunos:items").must_be_empty
+        redis.hgetall("rs:m:phil:food/desayunos/").must_be_empty
+
+        redis.smembers("rs:m:phil:food/:items").must_be_empty
+        redis.hgetall("rs:m:phil:food/").must_be_empty
+
+        redis.smembers("rs:m:phil:/:items").must_be_empty
+      end
+
+      it "responds with the ETag of the deleted item in the header" do
+        delete "/phil/food/aguacate"
+
+        last_response.headers["ETag"].must_equal "\"0815etag\""
+      end
+
+      context "when item doesn't exist" do
+        before do
+          purge_redis
+
+          delete "/phil/food/steak"
+        end
+
+        it "returns a 404" do
+          last_response.status.must_equal 404
+          last_response.body.must_equal "Not Found"
+        end
+
+        it "deletes any metadata that might still exist" do
+          delete "/phil/food/steak"
+
+          metadata = redis.hgetall "rs:m:phil:food/steak"
+          metadata.must_be_empty
+
+          redis.smembers("rs:m:phil:food/:items").must_be_empty
+          redis.hgetall("rs:m:phil:food/").must_be_empty
+
+          redis.smembers("rs:m:phil:/:items").must_be_empty
+        end
+      end
+
+      describe "If-Match header" do
+        it "succeeds when the header matches the current ETag" do
+          header "If-Match", "\"0815etag\""
+
+          delete "/phil/food/aguacate"
+
+          last_response.status.must_equal 200
+        end
+
+        it "succeeds when the header contains a weak ETAG matching the current ETag" do
+          header "If-Match", "W/\"0815etag\""
+
+          delete "/phil/food/aguacate"
+
+          last_response.status.must_equal 200
+        end
+
+        it "fails the request if it does not match the current ETag" do
+          header "If-Match", "someotheretag"
+
+          delete "/phil/food/aguacate"
+
+          last_response.status.must_equal 412
+          last_response.body.must_equal "Precondition Failed"
+        end
+      end
+    end
+  end
+
+  describe "GET requests" do
+
+    before do
+      purge_redis
+    end
+
+    context "not authorized" do
+
+      describe "without token" do
+        it "says it's not authorized" do
+          get "/phil/food/"
+
+          last_response.status.must_equal 401
+          last_response.body.must_equal "Unauthorized"
+        end
+      end
+
+      describe "with wrong token" do
+        it "says it's not authorized" do
+          header "Authorization", "Bearer wrongtoken"
+          get "/phil/food/"
+
+          last_response.status.must_equal 401
+          last_response.body.must_equal "Unauthorized"
+        end
+      end
+
+    end
+
+    context "authorized" do
+
+      before do
+        redis.sadd "authorizations:phil:amarillo", [":rw"]
+        header "Authorization", "Bearer amarillo"
+
+        put "/phil/food/aguacate", "si"
+        put "/phil/food/camaron", "yummi"
+        put "/phil/food/desayunos/bolon", "wow"
+      end
+
+      describe "documents" do
+
+        it "returns the required response headers" do
+          get "/phil/food/aguacate"
+
+          last_response.status.must_equal 200
+          last_response.headers["ETag"].must_equal "\"0817etag\""
+          last_response.headers["Cache-Control"].must_equal "no-cache"
+          last_response.headers["Content-Type"].must_equal "text/plain; charset=utf-8"
+        end
+
+        it "returns a 404 when data doesn't exist" do
+          get "/phil/food/steak"
+
+          last_response.status.must_equal 404
+          last_response.body.must_equal "Not Found"
+        end
+
+        it "responds with 304 when IF_NONE_MATCH header contains the ETag" do
+          header "If-None-Match", "\"0815etag\""
+
+          get "/phil/food/aguacate"
+
+          last_response.status.must_equal 304
+          last_response.headers["ETag"].must_equal "\"0815etag\""
+          last_response.headers["Last-Modified"].must_equal "Fri, 04 Mar 2016 12:20:18 GMT"
+        end
+
+        it "responds with 304 when IF_NONE_MATCH header contains weak ETAG matching the current ETag" do
+          header "If-None-Match", "W/\"0815etag\""
+
+          get "/phil/food/aguacate"
+
+          last_response.status.must_equal 304
+          last_response.headers["ETag"].must_equal "\"0815etag\""
+          last_response.headers["Last-Modified"].must_equal "Fri, 04 Mar 2016 12:20:18 GMT"
+        end
+
+      end
+
+      describe "directory listings" do
+
+        it "returns the correct ETag header" do
+          get "/phil/food/"
+
+          last_response.status.must_equal 200
+          last_response.headers["ETag"].must_equal "\"f9f85fbf5aa1fa378fd79ac8aa0a457d\""
+        end
+
+        it "returns a Cache-Control header with value 'no-cache'" do
+          get "/phil/food/"
+
+          last_response.status.must_equal 200
+          last_response.headers["Cache-Control"].must_equal "no-cache"
+        end
+
+        it "responds with 304 when IF_NONE_MATCH header contains the ETag" do
+          header "If-None-Match", "\"f9f85fbf5aa1fa378fd79ac8aa0a457d\""
+          get "/phil/food/"
+
+          last_response.status.must_equal 304
+        end
+
+        it "responds with 304 when IF_NONE_MATCH header contains weak ETAG matching the ETag" do
+          header "If-None-Match", "W/\"f9f85fbf5aa1fa378fd79ac8aa0a457d\""
+          get "/phil/food/"
+
+          last_response.status.must_equal 304
+        end
+
+        it "contains all items in the directory" do
+          get "/phil/food/"
+
+          last_response.status.must_equal 200
+          last_response.content_type.must_equal "application/ld+json"
+
+          content = JSON.parse(last_response.body)
+          content["@context"].must_equal "http://remotestorage.io/spec/folder-description"
+          content["items"]["aguacate"].wont_be_nil
+          content["items"]["aguacate"]["Content-Type"].must_equal "text/plain; charset=utf-8"
+          content["items"]["aguacate"]["Content-Length"].must_equal 2
+          content["items"]["aguacate"]["ETag"].must_equal "0815etag"
+          content["items"]["camaron"].wont_be_nil
+          content["items"]["camaron"]["Content-Type"].must_equal "text/plain; charset=utf-8"
+          content["items"]["camaron"]["Content-Length"].must_equal 5
+          content["items"]["camaron"]["ETag"].must_equal "0816etag"
+          content["items"]["desayunos/"].wont_be_nil
+          content["items"]["desayunos/"]["ETag"].must_equal "dd36e3cfe52b5f33421150b289a7d48d"
+        end
+
+        it "contains all items in the root directory" do
+          get "phil/"
+
+          last_response.status.must_equal 200
+          last_response.content_type.must_equal "application/ld+json"
+
+          content = JSON.parse(last_response.body)
+          content["items"]["food/"].wont_be_nil
+          content["items"]["food/"]["ETag"].must_equal "f9f85fbf5aa1fa378fd79ac8aa0a457d"
+        end
+
+        it "responds with an empty directory liting when directory doesn't exist" do
+          get "phil/some-non-existing-dir/"
+
+          last_response.status.must_equal 200
+          last_response.content_type.must_equal "application/ld+json"
+
+          content = JSON.parse(last_response.body)
+          content["items"].must_equal({})
+        end
+
+      end
+    end
+
+  end
+
+  describe "HEAD requests" do
+
+    before do
+      purge_redis
+    end
+
+    context "not authorized" do
+
+      describe "without token" do
+        it "says it's not authorized" do
+          head "/phil/food/camarones"
+
+          last_response.status.must_equal 401
+          last_response.body.must_be_empty
+        end
+      end
+
+      describe "with wrong token" do
+        it "says it's not authorized" do
+          header "Authorization", "Bearer wrongtoken"
+          head "/phil/food/camarones"
+
+          last_response.status.must_equal 401
+          last_response.body.must_be_empty
+        end
+      end
+
+    end
+
+    context "authorized" do
+
+      before do
+        redis.sadd "authorizations:phil:amarillo", [":rw"]
+        header "Authorization", "Bearer amarillo"
+
+        put "/phil/food/aguacate", "si"
+        put "/phil/food/camaron", "yummi"
+        put "/phil/food/desayunos/bolon", "wow"
+      end
+
+      describe "directory listings" do
+        it "returns the correct header information" do
+          get "/phil/food/"
+
+          last_response.status.must_equal 200
+          last_response.content_type.must_equal "application/ld+json"
+          last_response.headers["ETag"].must_equal "\"f9f85fbf5aa1fa378fd79ac8aa0a457d\""
+        end
+      end
+
+      describe "documents" do
+        context "when the document doesn't exist" do
+          it "returns a 404" do
+            head "/phil/food/steak"
+
+            last_response.status.must_equal 404
+            last_response.body.must_be_empty
+          end
+        end
+
+        context "when the document exists" do
+          it "returns the required response headers" do
+            head "/phil/food/aguacate"
+
+            last_response.status.must_equal 200
+            last_response.headers["ETag"].must_equal "\"0815etag\""
+            last_response.headers["Cache-Control"].must_equal "no-cache"
+            last_response.headers["Last-Modified"].must_equal "Fri, 04 Mar 2016 12:20:18 GMT"
+            last_response.headers["Content-Type"].must_equal "text/plain; charset=utf-8"
+            last_response.headers["Content-Length"].must_equal "2"
+          end
+
+          it "responds with 304 when IF_NONE_MATCH header contains the ETag" do
+            header "If-None-Match", "\"0815etag\""
+
+            head "/phil/food/aguacate"
+
+            last_response.status.must_equal 304
+            last_response.headers["ETag"].must_equal "\"0815etag\""
+            last_response.headers["Last-Modified"].must_equal "Fri, 04 Mar 2016 12:20:18 GMT"
+          end
+
+          it "responds with 304 when IF_NONE_MATCH header contains weak ETAG matching the current ETag" do
+            header "If-None-Match", "W/\"0815etag\""
+
+            head "/phil/food/aguacate"
+
+            last_response.status.must_equal 304
+            last_response.headers["ETag"].must_equal "\"0815etag\""
+            last_response.headers["Last-Modified"].must_equal "Fri, 04 Mar 2016 12:20:18 GMT"
+          end
+        end
+      end
+    end
+  end
+end

spec/spec_helper.rb

@@ -6,13 +6,15 @@ Bundler.require
 
 require_relative '../liquor-cabinet'
 require 'minitest/autorun'
+require "minitest/stub_any_instance"
 require 'rack/test'
 require 'purdytest'
-require 'riak'
 require "redis"
 require "rest_client"
-require "minitest/stub_any_instance"
 require "ostruct"
+require 'webmock/minitest'
+
+WebMock.disable_net_connect!
 
 def app
   LiquorCabinet
@@ -20,22 +22,11 @@ end
 
 app.set :environment, :test
 
-def wait_a_second
-  now = Time.now.to_i
-  while Time.now.to_i == now; end
-end
-
-def write_last_response_to_file(filename = "last_response.html")
-  File.open(filename, "w") do |f|
-    f.write last_response.body
-  end
-end
-
 alias context describe
 
 if app.settings.respond_to? :redis
   def redis
-    @redis ||= Redis.new(host: app.settings.redis["host"], port: app.settings.redis["port"])
+    @redis ||= Redis.new(app.settings.redis.symbolize_keys)
   end
 
   def purge_redis
@@ -45,73 +36,22 @@ if app.settings.respond_to? :redis
   end
 end
 
-if app.settings.respond_to? :riak
+MiniTest::Spec.class_eval do
-  ::Riak.disable_list_keys_warnings = true
+  def self.shared_examples
-
+    @shared_examples ||= {}
-  def client
-    @client ||= ::Riak::Client.new(:host => app.settings.riak['host'],
-                                   :http_port => app.settings.riak['http_port'])
-  end
-
-  def data_bucket
-    @data_bucket ||= begin
-                       bucket = client.bucket(app.settings.riak['buckets']['data'])
-                       bucket.allow_mult = false
-                       bucket
-                     end
-  end
-
-  def directory_bucket
-    @directory_bucket ||= begin
-                            bucket = client.bucket(app.settings.riak['buckets']['directories'])
-                            bucket.allow_mult = false
-                            bucket
-                          end
-  end
-
-  def auth_bucket
-    @auth_bucket ||= begin
-                       bucket = client.bucket(app.settings.riak['buckets']['authorizations'])
-                       bucket.allow_mult = false
-                       bucket
-                     end
-  end
-
-  def opslog_bucket
-    @opslog_bucket ||= begin
-                         bucket = client.bucket(app.settings.riak['buckets']['opslog'])
-                         bucket.allow_mult = false
-                         bucket
-                       end
-  end
-
-  def cs_credentials
-    @cs_credentials ||= begin
-                          credentials = File.read(app.settings.riak['riak_cs']['credentials_file'])
-                          JSON.parse(credentials)
-                        end
-  end
-
-  def cs_client
-    @cs_client ||= Fog::Storage.new({
-      :provider                 => 'AWS',
-      :aws_access_key_id        => cs_credentials['key_id'],
-      :aws_secret_access_key    => cs_credentials['key_secret'],
-      :endpoint                 => app.settings.riak['riak_cs']['endpoint']
-    })
-  end
-
-  def cs_binary_bucket
-    @cs_binary_bucket ||= cs_client.directories.create(:key => app.settings.riak['buckets']['cs_binaries'])
-  end
-
-  def purge_all_buckets
-    [data_bucket, directory_bucket, auth_bucket, opslog_bucket].each do |bucket|
-      bucket.keys.each {|key| bucket.delete key}
-    end
-
-    cs_binary_bucket.files.each do |file|
-      file.destroy
-    end
   end
 end
+
+module MiniTest::Spec::SharedExamples
+  def shared_examples_for(desc, &block)
+    MiniTest::Spec.shared_examples[desc] = block
+  end
+
+  def it_behaves_like(desc)
+    self.instance_eval(&MiniTest::Spec.shared_examples[desc])
+  end
+end
+
+Object.class_eval { include(MiniTest::Spec::SharedExamples) }
+
+require_relative 'shared_examples'

spec/swift/app_spec.rb

@@ -1,826 +1,58 @@
 require_relative "../spec_helper"
 
-describe "App" do
+describe "Swift provider" do
-  include Rack::Test::Methods
+  def container_url_for(user)
-
+    "#{app.settings.swift["host"]}/rs:documents:test/#{user}"
-  def app
-    LiquorCabinet
   end
 
-  it "returns 404 on non-existing routes" do
+  def storage_class
-    get "/virginmargarita"
+    RemoteStorage::Swift
-    last_response.status.must_equal 404
   end
 
-  describe "PUT requests" do
+  before do
-
+    stub_request(:put, "#{container_url_for("phil")}/food/aguacate").
-    before do
+      to_return(status: 200, headers: { etag: "0815etag", last_modified: "Fri, 04 Mar 2016 12:20:18 GMT" })
-      purge_redis
+    # Write new content with an If-Match header (a new Etag is returned)
-    end
+    stub_request(:put, "#{container_url_for("phil")}/food/aguacate").
-
+      with(body: "aye").
-    context "authorized" do
+      to_return(status: 200, headers: { etag: "0915etag", last_modified: "Fri, 04 Mar 2016 12:20:18 GMT" })
-      before do
+    stub_request(:head, "#{container_url_for("phil")}/food/aguacate").
-        redis.sadd "authorizations:phil:amarillo", [":rw"]
+      to_return(status: 200, headers: { last_modified: "Fri, 04 Mar 2016 12:20:18 GMT" })
-        header "Authorization", "Bearer amarillo"
+    stub_request(:get, "#{container_url_for("phil")}/food/aguacate").
-      end
+      to_return(status: 200, body: "rootbody", headers: { etag: "0817etag", content_type: "text/plain; charset=utf-8" })
-
+    stub_request(:delete, "#{container_url_for("phil")}/food/aguacate").
-      it "creates the metadata object in redis" do
+      to_return(status: 200, headers: { etag: "0815etag" })
-        put_stub = OpenStruct.new(headers: {
+
-          etag: "bla",
+    # Write new content to check the metadata in Redis
-          last_modified: "Fri, 04 Mar 2016 12:20:18 GMT"
+    stub_request(:put, "#{container_url_for("phil")}/food/banano").
-        })
+      with(body: "si").
-
+      to_return(status: 200, headers: { etag: "0815etag", last_modified: "Fri, 04 Mar 2016 12:20:18 GMT" })
-        RestClient.stub :put, put_stub do
+    stub_request(:put, "#{container_url_for("phil")}/food/banano").
-          put "/phil/food/aguacate", "si"
+      with(body: "oh, no").
-        end
+      to_return(status: 200, headers: { etag: "0817etag", last_modified: "Fri, 04 Mar 2016 12:20:20 GMT" })
-
+
-        metadata = redis.hgetall "rs:m:phil:food/aguacate"
+    stub_request(:put, "#{container_url_for("phil")}/food/camaron").
-        metadata["s"].must_equal "2"
+      to_return(status: 200, headers: { etag: "0816etag", last_modified: "Fri, 04 Mar 2016 12:20:18 GMT" })
-        metadata["t"].must_equal "text/plain; charset=utf-8"
+    stub_request(:delete, "#{container_url_for("phil")}/food/camaron").
-        metadata["e"].must_equal "bla"
+      to_return(status: 200, headers: { etag: "0816etag" })
-        metadata["m"].length.must_equal 13
+
-      end
+    stub_request(:put, "#{container_url_for("phil")}/food/desayunos/bolon").
-
+      to_return(status: 200, headers: { etag: "0817etag", last_modified: "Fri, 04 Mar 2016 12:20:18 GMT" })
-      it "creates the directory objects metadata in redis" do
+    stub_request(:delete, "#{container_url_for("phil")}/food/desayunos/bolon").
-        put_stub = OpenStruct.new(headers: {
+      to_return(status: 200, headers: { etag: "0817etag" })
-          etag: "bla",
+
-          last_modified: "Fri, 04 Mar 2016 12:20:18 GMT"
+    # objects in root dir
-        })
+    stub_request(:put, "#{container_url_for("phil")}/bamboo.txt").
-        get_stub = OpenStruct.new(body: "rootbody")
+      to_return(status: 200, headers: { etag: "0818etag", last_modified: "Fri, 04 Mar 2016 12:20:18 GMT" })
-
+
-        RestClient.stub :put, put_stub do
+    # 404
-          RestClient.stub :get, get_stub do
+    stub_request(:head, "#{container_url_for("phil")}/food/steak").
-            RemoteStorage::Swift.stub_any_instance :etag_for, "newetag" do
+      to_return(status: 404)
-              put "/phil/food/aguacate", "si"
+    stub_request(:get, "#{container_url_for("phil")}/food/steak").
-              put "/phil/food/camaron", "yummi"
+      to_return(status: 404)
-            end
+    stub_request(:delete, "#{container_url_for("phil")}/food/steak").
-          end
+      to_return(status: 404)
-        end
-
-        metadata = redis.hgetall "rs:m:phil:/"
-        metadata["e"].must_equal "newetag"
-        metadata["m"].length.must_equal 13
-
-        metadata = redis.hgetall "rs:m:phil:food/"
-        metadata["e"].must_equal "newetag"
-        metadata["m"].length.must_equal 13
-
-        food_items = redis.smembers "rs:m:phil:food/:items"
-        food_items.each do |food_item|
-          ["camaron", "aguacate"].must_include food_item
-        end
-
-        root_items = redis.smembers "rs:m:phil:/:items"
-        root_items.must_equal ["food/"]
-      end
-
-      context "response code" do
-        before do
-          @put_stub = OpenStruct.new(headers: {
-            etag: "bla",
-            last_modified: "Fri, 04 Mar 2016 12:20:18 GMT"
-          })
-        end
-
-        it "is 201 for newly created objects" do
-          RestClient.stub :put, @put_stub do
-            put "/phil/food/aguacate", "muy deliciosa"
-          end
-
-          last_response.status.must_equal 201
-        end
-
-        it "is 200 for updated objects" do
-          RestClient.stub :put, @put_stub do
-            put "/phil/food/aguacate", "deliciosa"
-            put "/phil/food/aguacate", "muy deliciosa"
-          end
-
-          last_response.status.must_equal 200
-        end
-      end
-
-      context "logging usage size" do
-        before do
-          @put_stub = OpenStruct.new(headers: {
-            etag: "bla",
-            last_modified: "Fri, 04 Mar 2016 12:20:18 GMT"
-          })
-        end
-
-        it "logs the complete size when creating new objects" do
-          RestClient.stub :put, @put_stub do
-            put "/phil/food/aguacate", "1234567890"
-          end
-
-          size_log = redis.get "rs:s:phil"
-          size_log.must_equal "10"
-        end
-
-        it "logs the size difference when updating existing objects" do
-          RestClient.stub :put, @put_stub do
-            put "/phil/food/camaron",  "1234567890"
-            put "/phil/food/aguacate", "1234567890"
-            put "/phil/food/aguacate", "123"
-          end
-
-          size_log = redis.get "rs:s:phil"
-          size_log.must_equal "13"
-        end
-      end
-
-      describe "objects in root dir" do
-        before do
-          put_stub = OpenStruct.new(headers: {
-            etag: "bla",
-            last_modified: "Fri, 04 Mar 2016 12:20:18 GMT"
-          })
-
-          RestClient.stub :put, put_stub do
-            put "/phil/bamboo.txt", "shir kan"
-          end
-        end
-
-        it "are listed in the directory listing with all metadata" do
-          get "phil/"
-
-          last_response.status.must_equal 200
-          last_response.content_type.must_equal "application/ld+json"
-
-          content = JSON.parse(last_response.body)
-          content["items"]["bamboo.txt"].wont_be_nil
-          content["items"]["bamboo.txt"]["ETag"].must_equal "bla"
-          content["items"]["bamboo.txt"]["Content-Type"].must_equal "text/plain; charset=utf-8"
-          content["items"]["bamboo.txt"]["Content-Length"].must_equal 8
-        end
-      end
-
-      describe "name collision checks" do
-        it "is successful when there is no name collision" do
-          put_stub = OpenStruct.new(headers: {
-            etag: "bla",
-            last_modified: "Fri, 04 Mar 2016 12:20:18 GMT"
-          })
-          get_stub = OpenStruct.new(body: "rootbody")
-
-          RestClient.stub :put, put_stub do
-            RestClient.stub :get, get_stub do
-              RemoteStorage::Swift.stub_any_instance :etag_for, "rootetag" do
-                put "/phil/food/aguacate", "si"
-              end
-            end
-          end
-
-          last_response.status.must_equal 201
-
-          metadata = redis.hgetall "rs:m:phil:food/aguacate"
-          metadata["s"].must_equal "2"
-        end
-
-        it "conflicts when there is a directory with same name as document" do
-          put_stub = OpenStruct.new(headers: {
-            etag: "bla",
-            last_modified: "Fri, 04 Mar 2016 12:20:18 GMT"
-          })
-
-          RestClient.stub :put, put_stub do
-            put "/phil/food/aguacate", "si"
-            put "/phil/food", "wontwork"
-          end
-
-          last_response.status.must_equal 409
-          last_response.body.must_equal "Conflict"
-
-          metadata = redis.hgetall "rs:m:phil:food"
-          metadata.must_be_empty
-        end
-
-        it "conflicts when there is a document with same name as directory" do
-          put_stub = OpenStruct.new(headers: {
-            etag: "bla",
-            last_modified: "Fri, 04 Mar 2016 12:20:18 GMT"
-          })
-
-          RestClient.stub :put, put_stub do
-            put "/phil/food/aguacate", "si"
-            put "/phil/food/aguacate/empanado", "wontwork"
-          end
-
-          last_response.status.must_equal 409
-
-          metadata = redis.hgetall "rs:m:phil:food/aguacate/empanado"
-          metadata.must_be_empty
-        end
-
-        it "returns 400 when a Content-Range header is sent" do
-          header "Content-Range", "bytes 0-3/3"
-
-          put "/phil/food/aguacate", "si"
-
-          last_response.status.must_equal 400
-        end
-      end
-
-      describe "If-Match header" do
-        before do
-          put_stub = OpenStruct.new(headers: {
-            etag: "oldetag",
-            last_modified: "Fri, 04 Mar 2016 12:20:18 GMT"
-          })
-
-          RestClient.stub :put, put_stub do
-            put "/phil/food/aguacate", "si"
-          end
-        end
-
-        it "allows the request if the header matches the current ETag" do
-          header "If-Match", "\"oldetag\""
-
-          put_stub = OpenStruct.new(headers: {
-            etag: "newetag",
-            last_modified: "Fri, 04 Mar 2016 12:20:18 GMT"
-          })
-
-          RestClient.stub :put, put_stub do
-            put "/phil/food/aguacate", "aye"
-          end
-
-          last_response.status.must_equal 200
-          last_response.headers["Etag"].must_equal "\"newetag\""
-        end
-
-        it "allows the request if the header contains a weak ETAG matching the current ETag" do
-          header "If-Match", "W/\"oldetag\""
-
-          put_stub = OpenStruct.new(headers: {
-            etag: "newetag",
-            last_modified: "Fri, 04 Mar 2016 12:20:18 GMT"
-          })
-
-          RestClient.stub :put, put_stub do
-            put "/phil/food/aguacate", "aye"
-          end
-
-          last_response.status.must_equal 200
-          last_response.headers["Etag"].must_equal "\"newetag\""
-        end
-
-        it "allows the request if the header contains a weak ETAG with leading quote matching the current ETag" do
-          header "If-Match", "\"W/\"oldetag\""
-
-          put_stub = OpenStruct.new(headers: {
-            etag: "newetag",
-            last_modified: "Fri, 04 Mar 2016 12:20:18 GMT"
-          })
-
-          RestClient.stub :put, put_stub do
-            put "/phil/food/aguacate", "aye"
-          end
-
-          last_response.status.must_equal 200
-          last_response.headers["Etag"].must_equal "\"newetag\""
-        end
-
-        it "fails the request if the header does not match the current ETag" do
-          header "If-Match", "someotheretag"
-
-          head_stub = OpenStruct.new(headers: {
-            etag: "oldetag",
-            last_modified: "Fri, 04 Mar 2016 12:20:18 GMT",
-            content_type: "text/plain",
-            content_length: 23
-          })
-
-          RestClient.stub :head, head_stub do
-            put "/phil/food/aguacate", "aye"
-          end
-
-          last_response.status.must_equal 412
-          last_response.body.must_equal "Precondition Failed"
-        end
-
-        it "allows the request if redis metadata became out of sync" do
-          header "If-Match", "\"existingetag\""
-
-          head_stub = OpenStruct.new(headers: {
-            etag: "existingetag",
-            last_modified: "Fri, 04 Mar 2016 12:20:18 GMT",
-            content_type: "text/plain",
-            content_length: 23
-          })
-
-          put_stub = OpenStruct.new(headers: {
-            etag: "newetag",
-            last_modified: "Fri, 04 Mar 2016 12:20:18 GMT"
-          })
-
-          RestClient.stub :head, head_stub do
-            RestClient.stub :put, put_stub do
-              put "/phil/food/aguacate", "aye"
-            end
-          end
-
-          last_response.status.must_equal 200
-        end
-      end
-
-      describe "If-None-Match header set to '*'" do
-        it "succeeds when the document doesn't exist yet" do
-          put_stub = OpenStruct.new(headers: {
-            etag: "someetag",
-            last_modified: "Fri, 04 Mar 2016 12:20:18 GMT"
-          })
-
-          header "If-None-Match", "*"
-
-          RestClient.stub :put, put_stub do
-            put "/phil/food/aguacate", "si"
-          end
-
-          last_response.status.must_equal 201
-        end
-
-        it "fails the request if the document already exists" do
-          put_stub = OpenStruct.new(headers: {
-            etag: "someetag",
-            last_modified: "Fri, 04 Mar 2016 12:20:18 GMT"
-          })
-
-          RestClient.stub :put, put_stub do
-            put "/phil/food/aguacate", "si"
-          end
-
-          header "If-None-Match", "*"
-          RestClient.stub :put, put_stub do
-            put "/phil/food/aguacate", "si"
-          end
-
-          last_response.status.must_equal 412
-          last_response.body.must_equal "Precondition Failed"
-        end
-      end
-    end
-
-  end
-
-  describe "DELETE requests" do
-
-    before do
-      purge_redis
-    end
-
-    context "not authorized" do
-      describe "with no token" do
-        it "says it's not authorized" do
-          delete "/phil/food/aguacate"
-
-          last_response.status.must_equal 401
-          last_response.body.must_equal "Unauthorized"
-        end
-      end
-
-      describe "with empty token" do
-        it "says it's not authorized" do
-          header "Authorization", "Bearer "
-          delete "/phil/food/aguacate"
-
-          last_response.status.must_equal 401
-          last_response.body.must_equal "Unauthorized"
-        end
-      end
-
-      describe "with wrong token" do
-        it "says it's not authorized" do
-          header "Authorization", "Bearer wrongtoken"
-          delete "/phil/food/aguacate"
-
-          last_response.status.must_equal 401
-          last_response.body.must_equal "Unauthorized"
-        end
-      end
-
-    end
-
-    context "authorized" do
-      before do
-        redis.sadd "authorizations:phil:amarillo", [":rw"]
-        header "Authorization", "Bearer amarillo"
-
-        put_stub = OpenStruct.new(headers: {
-          etag: "bla",
-          last_modified: "Fri, 04 Mar 2016 12:20:18 GMT"
-        })
-
-        RestClient.stub :put, put_stub do
-          put "/phil/food/aguacate", "si"
-          put "/phil/food/camaron", "yummi"
-          put "/phil/food/desayunos/bolon", "wow"
-        end
-      end
-
-      it "decreases the size log by size of deleted object" do
-        RestClient.stub :delete, "" do
-          RemoteStorage::Swift.stub_any_instance :etag_for, "rootetag" do
-            delete "/phil/food/aguacate"
-          end
-        end
-
-        size_log = redis.get "rs:s:phil"
-        size_log.must_equal "8"
-      end
-
-      it "deletes the metadata object in redis" do
-        RestClient.stub :delete, "" do
-          RemoteStorage::Swift.stub_any_instance :etag_for, "rootetag" do
-            delete "/phil/food/aguacate"
-          end
-        end
-
-        metadata = redis.hgetall "rs:m:phil:food/aguacate"
-        metadata.must_be_empty
-      end
-
-      it "deletes the directory objects metadata in redis" do
-        old_metadata = redis.hgetall "rs:m:phil:food/"
-
-        RestClient.stub :delete, "" do
-          RemoteStorage::Swift.stub_any_instance :etag_for, "newetag" do
-            delete "/phil/food/aguacate"
-          end
-        end
-
-        metadata = redis.hgetall "rs:m:phil:food/"
-        metadata["e"].must_equal "newetag"
-        metadata["m"].length.must_equal 13
-        metadata["m"].wont_equal old_metadata["m"]
-
-        food_items = redis.smembers "rs:m:phil:food/:items"
-        food_items.sort.must_equal ["camaron", "desayunos/"]
-
-        root_items = redis.smembers "rs:m:phil:/:items"
-        root_items.must_equal ["food/"]
-      end
-
-      it "deletes the parent directory objects metadata when deleting all items" do
-        RestClient.stub :delete, "" do
-          RemoteStorage::Swift.stub_any_instance :etag_for, "rootetag" do
-            delete "/phil/food/aguacate"
-            delete "/phil/food/camaron"
-            delete "/phil/food/desayunos/bolon"
-          end
-        end
-
-        redis.smembers("rs:m:phil:food/desayunos:items").must_be_empty
-        redis.hgetall("rs:m:phil:food/desayunos/").must_be_empty
-
-        redis.smembers("rs:m:phil:food/:items").must_be_empty
-        redis.hgetall("rs:m:phil:food/").must_be_empty
-
-        redis.smembers("rs:m:phil:/:items").must_be_empty
-      end
-
-      it "responds with the ETag of the deleted item in the header" do
-        RestClient.stub :delete, "" do
-          delete "/phil/food/aguacate"
-        end
-
-        last_response.headers["ETag"].must_equal "\"bla\""
-      end
-
-      context "when item doesn't exist" do
-        before do
-          purge_redis
-
-          put_stub = OpenStruct.new(headers: {
-            etag: "bla",
-            last_modified: "Fri, 04 Mar 2016 12:20:18 GMT"
-          })
-
-          RestClient.stub :put, put_stub do
-            put "/phil/food/steak", "si"
-          end
-
-          raises_exception = ->(url, headers) { raise RestClient::ResourceNotFound.new }
-          RestClient.stub :delete, raises_exception do
-            delete "/phil/food/steak"
-          end
-        end
-
-        it "returns a 404" do
-          last_response.status.must_equal 404
-          last_response.body.must_equal "Not Found"
-        end
-
-        it "deletes any metadata that might still exist" do
-          raises_exception = ->(url, headers) { raise RestClient::ResourceNotFound.new }
-          RestClient.stub :delete, raises_exception do
-            delete "/phil/food/steak"
-          end
-
-          metadata = redis.hgetall "rs:m:phil:food/steak"
-          metadata.must_be_empty
-
-          redis.smembers("rs:m:phil:food/:items").must_be_empty
-          redis.hgetall("rs:m:phil:food/").must_be_empty
-
-          redis.smembers("rs:m:phil:/:items").must_be_empty
-        end
-      end
-
-      describe "If-Match header" do
-        it "succeeds when the header matches the current ETag" do
-          header "If-Match", "\"bla\""
-
-          RestClient.stub :delete, "" do
-            delete "/phil/food/aguacate"
-          end
-
-          last_response.status.must_equal 200
-        end
-
-        it "succeeds when the header contains a weak ETAG matching the current ETag" do
-          header "If-Match", "W/\"bla\""
-
-          RestClient.stub :delete, "" do
-            delete "/phil/food/aguacate"
-          end
-
-          last_response.status.must_equal 200
-        end
-
-        it "fails the request if it does not match the current ETag" do
-          header "If-Match", "someotheretag"
-
-          delete "/phil/food/aguacate"
-
-          last_response.status.must_equal 412
-          last_response.body.must_equal "Precondition Failed"
-        end
-      end
-    end
-  end
-
-  describe "GET requests" do
-
-    before do
-      purge_redis
-    end
-
-    context "not authorized" do
-
-      describe "without token" do
-        it "says it's not authorized" do
-          get "/phil/food/"
-
-          last_response.status.must_equal 401
-          last_response.body.must_equal "Unauthorized"
-        end
-      end
-
-      describe "with wrong token" do
-        it "says it's not authorized" do
-          header "Authorization", "Bearer wrongtoken"
-          get "/phil/food/"
-
-          last_response.status.must_equal 401
-          last_response.body.must_equal "Unauthorized"
-        end
-      end
-
-    end
-
-    context "authorized" do
-
-      before do
-        redis.sadd "authorizations:phil:amarillo", [":rw"]
-        header "Authorization", "Bearer amarillo"
-
-        put_stub = OpenStruct.new(headers: {
-          etag: "bla",
-          last_modified: "Fri, 04 Mar 2016 12:20:18 GMT"
-        })
-
-        RestClient.stub :put, put_stub do
-          put "/phil/food/aguacate", "si"
-          put "/phil/food/camaron", "yummi"
-          put "/phil/food/desayunos/bolon", "wow"
-        end
-      end
-
-      describe "documents" do
-
-        it "returns the required response headers" do
-          get_stub = OpenStruct.new(body: "si", headers: {
-            etag: "0815etag",
-            last_modified: "Fri, 04 Mar 2016 12:20:18 GMT",
-            content_type: "text/plain; charset=utf-8",
-            content_length: 2
-          })
-
-          RestClient.stub :get, get_stub do
-            get "/phil/food/aguacate"
-          end
-
-          last_response.status.must_equal 200
-          last_response.headers["ETag"].must_equal "\"0815etag\""
-          last_response.headers["Cache-Control"].must_equal "no-cache"
-          last_response.headers["Content-Type"].must_equal "text/plain; charset=utf-8"
-        end
-
-        it "returns a 404 when data doesn't exist" do
-          raises_exception = ->(url, headers) { raise RestClient::ResourceNotFound.new }
-          RestClient.stub :get, raises_exception do
-            get "/phil/food/steak"
-          end
-
-          last_response.status.must_equal 404
-          last_response.body.must_equal "Not Found"
-        end
-
-        it "responds with 304 when IF_NONE_MATCH header contains the ETag" do
-          header "If-None-Match", "\"0815etag\""
-
-          get_stub = OpenStruct.new(body: "si", headers: {
-            etag: "0815etag",
-            last_modified: "Fri, 04 Mar 2016 12:20:18 GMT",
-            content_type: "text/plain; charset=utf-8",
-            content_length: 2
-          })
-
-          RestClient.stub :get, get_stub do
-            get "/phil/food/aguacate"
-          end
-
-          last_response.status.must_equal 304
-        end
-
-        it "responds with 304 when IF_NONE_MATCH header contains weak ETAG matching the current ETag" do
-          header "If-None-Match", "W/\"0815etag\""
-
-          get_stub = OpenStruct.new(body: "si", headers: {
-            etag: "0815etag",
-            last_modified: "Fri, 04 Mar 2016 12:20:18 GMT",
-            content_type: "text/plain; charset=utf-8",
-            content_length: 2
-          })
-
-          RestClient.stub :get, get_stub do
-            get "/phil/food/aguacate"
-          end
-
-          last_response.status.must_equal 304
-        end
-
-      end
-
-      describe "directory listings" do
-
-        it "returns the correct ETag header" do
-          get "/phil/food/"
-
-          last_response.status.must_equal 200
-          last_response.headers["ETag"].must_equal "\"f9f85fbf5aa1fa378fd79ac8aa0a457d\""
-        end
-
-        it "returns a Cache-Control header with value 'no-cache'" do
-          get "/phil/food/"
-
-          last_response.status.must_equal 200
-          last_response.headers["Cache-Control"].must_equal "no-cache"
-        end
-
-        it "responds with 304 when IF_NONE_MATCH header contains the ETag" do
-          header "If-None-Match", "\"f9f85fbf5aa1fa378fd79ac8aa0a457d\""
-          get "/phil/food/"
-
-          last_response.status.must_equal 304
-        end
-
-        it "responds with 304 when IF_NONE_MATCH header contains weak ETAG matching the ETag" do
-          header "If-None-Match", "W/\"f9f85fbf5aa1fa378fd79ac8aa0a457d\""
-          get "/phil/food/"
-
-          last_response.status.must_equal 304
-        end
-
-        it "contains all items in the directory" do
-          get "/phil/food/"
-
-          last_response.status.must_equal 200
-          last_response.content_type.must_equal "application/ld+json"
-
-          content = JSON.parse(last_response.body)
-          content["@context"].must_equal "http://remotestorage.io/spec/folder-description"
-          content["items"]["aguacate"].wont_be_nil
-          content["items"]["aguacate"]["Content-Type"].must_equal "text/plain; charset=utf-8"
-          content["items"]["aguacate"]["Content-Length"].must_equal 2
-          content["items"]["aguacate"]["ETag"].must_equal "bla"
-          content["items"]["camaron"].wont_be_nil
-          content["items"]["camaron"]["Content-Type"].must_equal "text/plain; charset=utf-8"
-          content["items"]["camaron"]["Content-Length"].must_equal 5
-          content["items"]["camaron"]["ETag"].must_equal "bla"
-          content["items"]["desayunos/"].wont_be_nil
-          content["items"]["desayunos/"]["ETag"].must_equal "dd36e3cfe52b5f33421150b289a7d48d"
-        end
-
-        it "contains all items in the root directory" do
-          get "phil/"
-
-          last_response.status.must_equal 200
-          last_response.content_type.must_equal "application/ld+json"
-
-          content = JSON.parse(last_response.body)
-          content["items"]["food/"].wont_be_nil
-          content["items"]["food/"]["ETag"].must_equal "f9f85fbf5aa1fa378fd79ac8aa0a457d"
-        end
-
-        it "responds with an empty directory liting when directory doesn't exist" do
-          get "phil/some-non-existing-dir/"
-
-          last_response.status.must_equal 200
-          last_response.content_type.must_equal "application/ld+json"
-
-          content = JSON.parse(last_response.body)
-          content["items"].must_equal({})
-        end
-
-      end
-    end
-
-  end
-
-  describe "HEAD requests" do
-
-    before do
-      purge_redis
-    end
-
-    context "not authorized" do
-
-      describe "without token" do
-        it "says it's not authorized" do
-          head "/phil/food/camarones"
-
-          last_response.status.must_equal 401
-          last_response.body.must_be_empty
-        end
-      end
-
-      describe "with wrong token" do
-        it "says it's not authorized" do
-          header "Authorization", "Bearer wrongtoken"
-          head "/phil/food/camarones"
-
-          last_response.status.must_equal 401
-          last_response.body.must_be_empty
-        end
-      end
-
-    end
-
-    context "authorized" do
-
-      before do
-        redis.sadd "authorizations:phil:amarillo", [":rw"]
-        header "Authorization", "Bearer amarillo"
-
-        put_stub = OpenStruct.new(headers: {
-          etag: "bla",
-          last_modified: "Fri, 04 Mar 2016 12:20:18 GMT"
-        })
-
-        RestClient.stub :put, put_stub do
-          put "/phil/food/aguacate", "si"
-          put "/phil/food/camaron", "yummi"
-          put "/phil/food/desayunos/bolon", "wow"
-        end
-      end
-
-      describe "directory listings" do
-        it "returns the correct header information" do
-          get "/phil/food/"
-
-          last_response.status.must_equal 200
-          last_response.content_type.must_equal "application/ld+json"
-          last_response.headers["ETag"].must_equal "\"f9f85fbf5aa1fa378fd79ac8aa0a457d\""
-        end
-      end
-
-      describe "documents" do
-        it "returns a 404 when the document doesn't exist" do
-          raises_exception = ->(url, headers) { raise RestClient::ResourceNotFound.new }
-          RestClient.stub :head, raises_exception do
-            head "/phil/food/steak"
-          end
-
-          last_response.status.must_equal 404
-          last_response.body.must_be_empty
-        end
-      end
-
-    end
-
   end
 
+  it_behaves_like 'a REST adapter'
 end
-