From 77bd54b0097ce97bc00373ecb586769ae8e59262 Mon Sep 17 00:00:00 2001
From: Garret Alfert <garret@5apps.com>
Date: Wed, 20 Jan 2016 15:32:44 -0500
Subject: [PATCH 01/28] Move old specs into riak subdir

---
 spec/{ => riak}/app_spec.rb         | 1 +
 spec/{ => riak}/directories_spec.rb | 0
 spec/{ => riak}/permissions_spec.rb | 0
 spec/{ => riak}/riak_spec.rb        | 0
 4 files changed, 1 insertion(+)
 rename spec/{ => riak}/app_spec.rb (99%)
 rename spec/{ => riak}/directories_spec.rb (100%)
 rename spec/{ => riak}/permissions_spec.rb (100%)
 rename spec/{ => riak}/riak_spec.rb (100%)

diff --git a/spec/app_spec.rb b/spec/riak/app_spec.rb
similarity index 99%
rename from spec/app_spec.rb
rename to spec/riak/app_spec.rb
index ccdb8a1..1eb0ef1 100644
--- a/spec/app_spec.rb
+++ b/spec/riak/app_spec.rb
@@ -11,4 +11,5 @@ describe "App" do
     get "/virginmargarita"
     last_response.status.must_equal 404
   end
+
 end
diff --git a/spec/directories_spec.rb b/spec/riak/directories_spec.rb
similarity index 100%
rename from spec/directories_spec.rb
rename to spec/riak/directories_spec.rb
diff --git a/spec/permissions_spec.rb b/spec/riak/permissions_spec.rb
similarity index 100%
rename from spec/permissions_spec.rb
rename to spec/riak/permissions_spec.rb
diff --git a/spec/riak_spec.rb b/spec/riak/riak_spec.rb
similarity index 100%
rename from spec/riak_spec.rb
rename to spec/riak/riak_spec.rb

From 990ea9cf288e07a0a0aa4e6c60f9c54a5c89c9b5 Mon Sep 17 00:00:00 2001
From: Garret Alfert <garret@5apps.com>
Date: Wed, 20 Jan 2016 15:33:35 -0500
Subject: [PATCH 02/28] Save object metadata in redis

---
 Gemfile                     |  1 +
 Gemfile.lock                |  2 ++
 lib/remote_storage/swift.rb | 17 +++++++++++++++-
 spec/spec_helper.rb         |  9 +++++++++
 spec/swift/app_spec.rb      | 40 +++++++++++++++++++++++++++++++++++++
 5 files changed, 68 insertions(+), 1 deletion(-)
 create mode 100644 spec/swift/app_spec.rb

diff --git a/Gemfile b/Gemfile
index f5ee5c7..694b773 100644
--- a/Gemfile
+++ b/Gemfile
@@ -13,6 +13,7 @@ group :test do
   gem 'rake'
   gem 'purdytest', :require => false
   gem 'm'
+  gem 'minitest-stub_any_instance'
 end
 
 group :staging, :production do
diff --git a/Gemfile.lock b/Gemfile.lock
index d234198..9d3cc8f 100644
--- a/Gemfile.lock
+++ b/Gemfile.lock
@@ -50,6 +50,7 @@ GEM
     method_source (0.8.2)
     mime-types (2.6.1)
     minitest (5.7.0)
+    minitest-stub_any_instance (1.0.1)
     multi_json (1.11.1)
     multipart-post (2.0.0)
     net-scp (1.0.4)
@@ -109,6 +110,7 @@ DEPENDENCIES
   fog
   m
   mime-types (~> 2.6.1)
+  minitest-stub_any_instance
   purdytest
   rainbows
   rake
diff --git a/lib/remote_storage/swift.rb b/lib/remote_storage/swift.rb
index 5fabbab..c3f5c2f 100644
--- a/lib/remote_storage/swift.rb
+++ b/lib/remote_storage/swift.rb
@@ -125,7 +125,16 @@ module RemoteStorage
 
       res = do_put_request(url, data, content_type)
 
-      if update_dir_objects(user, directory)
+      # TODO get last modified from response and add to metadata
+      metadata = {
+        etag: res.headers[:etag],
+        size: data.length,
+        type: content_type
+      }
+
+      if update_metadata_object(user, directory, key, metadata) &&
+         # TODO provide the last modified to use for the dir objects as well
+         update_dir_objects(user, directory)
         server.headers["ETag"] = %Q("#{res.headers[:etag]}")
         server.halt 200
       else
@@ -255,7 +264,13 @@ module RemoteStorage
       parent_directories
     end
 
+    def update_metadata_object(user, directory, key, metadata)
+      key = "users:#{user}:data:#{directory}/#{key}"
+      redis.hmset(key, *metadata)
+    end
+
     def update_dir_objects(user, directory)
+      # TODO use actual last modified time from the document put request
       timestamp = (Time.now.to_f * 1000).to_i
 
       parent_directories_for(directory).each do |dir|
diff --git a/spec/spec_helper.rb b/spec/spec_helper.rb
index ee4daac..585725a 100644
--- a/spec/spec_helper.rb
+++ b/spec/spec_helper.rb
@@ -9,6 +9,10 @@ require 'minitest/autorun'
 require 'rack/test'
 require 'purdytest'
 require 'riak'
+require "redis"
+require "rest_client"
+require "minitest/stub_any_instance"
+require "ostruct"
 
 def app
   LiquorCabinet
@@ -28,6 +32,11 @@ def write_last_response_to_file(filename = "last_response.html")
 end
 
 alias context describe
+
+def redis
+  @redis ||= Redis.new(host: app.settings.redis["host"], port: app.settings.redis["port"])
+end
+
 if app.settings.respond_to? :riak
   ::Riak.disable_list_keys_warnings = true
 
diff --git a/spec/swift/app_spec.rb b/spec/swift/app_spec.rb
new file mode 100644
index 0000000..bc3ec0a
--- /dev/null
+++ b/spec/swift/app_spec.rb
@@ -0,0 +1,40 @@
+require_relative "../spec_helper"
+
+describe "App" do
+  include Rack::Test::Methods
+
+  def app
+    LiquorCabinet
+  end
+
+  it "returns 404 on non-existing routes" do
+    get "/virginmargarita"
+    last_response.status.must_equal 404
+  end
+
+  describe "PUT requests" do
+    context "authorized" do
+      before do
+        redis.sadd "authorizations:phil:amarillo", [":rw"]
+        header "Authorization", "Bearer amarillo"
+
+      end
+
+      it "creates the metadata object in redis" do
+        put_stub = OpenStruct.new(headers: {etag: "bla"})
+        RemoteStorage::Swift.stub_any_instance :has_name_collision?, false do
+          RestClient.stub :put, put_stub do
+            put "/phil/food/aguacate", "si"
+          end
+        end
+
+        metadata = redis.hgetall "users:phil:data:food/aguacate"
+        metadata["size"].must_equal "2"
+        metadata["type"].must_equal "text/plain; charset=utf-8"
+        metadata["etag"].must_equal "bla"
+        metadata["modified"].must_equal nil
+      end
+    end
+  end
+end
+

From cd2c0865e8aa0279b3bbedd65fdc1ae5b12b563f Mon Sep 17 00:00:00 2001
From: Garret Alfert <garret@5apps.com>
Date: Wed, 20 Jan 2016 15:53:22 -0500
Subject: [PATCH 03/28] Save directory metadata in redis (WIP)

---
 lib/remote_storage/swift.rb |  7 +++++--
 spec/swift/app_spec.rb      | 17 +++++++++++++++--
 2 files changed, 20 insertions(+), 4 deletions(-)

diff --git a/lib/remote_storage/swift.rb b/lib/remote_storage/swift.rb
index c3f5c2f..4c42e10 100644
--- a/lib/remote_storage/swift.rb
+++ b/lib/remote_storage/swift.rb
@@ -265,7 +265,7 @@ module RemoteStorage
     end
 
     def update_metadata_object(user, directory, key, metadata)
-      key = "users:#{user}:data:#{directory}/#{key}"
+      key = "rs_meta:#{user}:#{directory}/#{key}"
       redis.hmset(key, *metadata)
     end
 
@@ -274,7 +274,10 @@ module RemoteStorage
       timestamp = (Time.now.to_f * 1000).to_i
 
       parent_directories_for(directory).each do |dir|
-        do_put_request("#{url_for_directory(user, dir)}/", timestamp.to_s, "text/plain")
+        res = do_put_request("#{url_for_directory(user, dir)}/", timestamp.to_s, "text/plain")
+        key = "rs_meta:#{user}:#{dir}/"
+        metadata = {etag: res.headers[:etag], modified: timestamp}
+        redis.hmset(key, *metadata)
       end
 
       true
diff --git a/spec/swift/app_spec.rb b/spec/swift/app_spec.rb
index bc3ec0a..fdbc39b 100644
--- a/spec/swift/app_spec.rb
+++ b/spec/swift/app_spec.rb
@@ -17,7 +17,6 @@ describe "App" do
       before do
         redis.sadd "authorizations:phil:amarillo", [":rw"]
         header "Authorization", "Bearer amarillo"
-
       end
 
       it "creates the metadata object in redis" do
@@ -28,12 +27,26 @@ describe "App" do
           end
         end
 
-        metadata = redis.hgetall "users:phil:data:food/aguacate"
+        metadata = redis.hgetall "rs_meta:phil:food/aguacate"
         metadata["size"].must_equal "2"
         metadata["type"].must_equal "text/plain; charset=utf-8"
         metadata["etag"].must_equal "bla"
         metadata["modified"].must_equal nil
       end
+
+      it "creates the directory objects metadata in redis" do
+        put_stub = OpenStruct.new(headers: {etag: "bla"})
+        RemoteStorage::Swift.stub_any_instance :has_name_collision?, false do
+          RestClient.stub :put, put_stub do
+            put "/phil/food/aguacate", "si"
+          end
+        end
+
+        metadata = redis.hgetall "rs_meta:phil:food/"
+        metadata["etag"].must_equal "bla"
+        metadata["modified"].length.must_equal 13
+        metadata = redis.hgetall "rs_meta:phil:food/"
+      end
     end
   end
 end

From 189d04af250386c2a1cffa7709c8863637d8846b Mon Sep 17 00:00:00 2001
From: Garret Alfert <garret@5apps.com>
Date: Wed, 20 Jan 2016 17:27:00 -0500
Subject: [PATCH 04/28] Save list of directory items in redis

---
 lib/remote_storage/swift.rb | 16 +++++++++++++---
 spec/swift/app_spec.rb      |  7 +++++++
 2 files changed, 20 insertions(+), 3 deletions(-)

diff --git a/lib/remote_storage/swift.rb b/lib/remote_storage/swift.rb
index 4c42e10..197d1f8 100644
--- a/lib/remote_storage/swift.rb
+++ b/lib/remote_storage/swift.rb
@@ -128,7 +128,7 @@ module RemoteStorage
       # TODO get last modified from response and add to metadata
       metadata = {
         etag: res.headers[:etag],
-        size: data.length,
+        size: data.size,
         type: content_type
       }
 
@@ -264,9 +264,18 @@ module RemoteStorage
       parent_directories
     end
 
+    def parent_directory_for(directory)
+      if directory.match(/\//)
+        return directory[0..directory.rindex("/")-1]
+      elsif directory != ""
+        return "/"
+      end
+    end
+
     def update_metadata_object(user, directory, key, metadata)
-      key = "rs_meta:#{user}:#{directory}/#{key}"
-      redis.hmset(key, *metadata)
+      redis_key = "rs_meta:#{user}:#{directory}/#{key}"
+      redis.hmset(redis_key, *metadata)
+      redis.sadd "rs_meta:#{user}:#{directory}/:items", key
     end
 
     def update_dir_objects(user, directory)
@@ -278,6 +287,7 @@ module RemoteStorage
         key = "rs_meta:#{user}:#{dir}/"
         metadata = {etag: res.headers[:etag], modified: timestamp}
         redis.hmset(key, *metadata)
+        redis.sadd "rs_meta:#{user}:#{parent_directory_for(dir)}:items", "#{dir}/"
       end
 
       true
diff --git a/spec/swift/app_spec.rb b/spec/swift/app_spec.rb
index fdbc39b..84bb9ff 100644
--- a/spec/swift/app_spec.rb
+++ b/spec/swift/app_spec.rb
@@ -39,6 +39,7 @@ describe "App" do
         RemoteStorage::Swift.stub_any_instance :has_name_collision?, false do
           RestClient.stub :put, put_stub do
             put "/phil/food/aguacate", "si"
+            put "/phil/food/camaron", "yummi"
           end
         end
 
@@ -46,6 +47,12 @@ describe "App" do
         metadata["etag"].must_equal "bla"
         metadata["modified"].length.must_equal 13
         metadata = redis.hgetall "rs_meta:phil:food/"
+
+        food_items = redis.smembers "rs_meta:phil:food/:items"
+        food_items.must_equal ["camaron", "aguacate"]
+
+        root_items = redis.smembers "rs_meta:phil:/:items"
+        root_items.must_equal ["food/"]
       end
     end
   end

From 972378e67f3d6a009fcc98fd41c903096a17762a Mon Sep 17 00:00:00 2001
From: Garret Alfert <garret@5apps.com>
Date: Fri, 22 Jan 2016 14:46:41 -0500
Subject: [PATCH 05/28] Don't care for order of array when comparing

---
 spec/swift/app_spec.rb | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/spec/swift/app_spec.rb b/spec/swift/app_spec.rb
index 84bb9ff..a7e25c8 100644
--- a/spec/swift/app_spec.rb
+++ b/spec/swift/app_spec.rb
@@ -49,7 +49,9 @@ describe "App" do
         metadata = redis.hgetall "rs_meta:phil:food/"
 
         food_items = redis.smembers "rs_meta:phil:food/:items"
-        food_items.must_equal ["camaron", "aguacate"]
+        food_items.each do |food_item|
+          ["camaron", "aguacate"].must_include food_item
+        end
 
         root_items = redis.smembers "rs_meta:phil:/:items"
         root_items.must_equal ["food/"]

From 02e5d0b5ab693508fb01c20b9ad07503a7a2e7a5 Mon Sep 17 00:00:00 2001
From: Garret Alfert <garret@5apps.com>
Date: Fri, 22 Jan 2016 14:49:57 -0500
Subject: [PATCH 06/28] Fix relative path in riak specs

---
 spec/riak/app_spec.rb         | 2 +-
 spec/riak/directories_spec.rb | 2 +-
 spec/riak/permissions_spec.rb | 2 +-
 spec/riak/riak_spec.rb        | 2 +-
 4 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/spec/riak/app_spec.rb b/spec/riak/app_spec.rb
index 1eb0ef1..411c6c4 100644
--- a/spec/riak/app_spec.rb
+++ b/spec/riak/app_spec.rb
@@ -1,4 +1,4 @@
-require_relative "spec_helper"
+require_relative "../spec_helper"
 
 describe "App" do
   include Rack::Test::Methods
diff --git a/spec/riak/directories_spec.rb b/spec/riak/directories_spec.rb
index ea62080..6551a7f 100644
--- a/spec/riak/directories_spec.rb
+++ b/spec/riak/directories_spec.rb
@@ -1,4 +1,4 @@
-require_relative "spec_helper"
+require_relative "../spec_helper"
 
 describe "Directories" do
   include Rack::Test::Methods
diff --git a/spec/riak/permissions_spec.rb b/spec/riak/permissions_spec.rb
index 6652c85..6250a2e 100644
--- a/spec/riak/permissions_spec.rb
+++ b/spec/riak/permissions_spec.rb
@@ -1,4 +1,4 @@
-require_relative "spec_helper"
+require_relative "../spec_helper"
 
 describe "Permissions" do
   include Rack::Test::Methods
diff --git a/spec/riak/riak_spec.rb b/spec/riak/riak_spec.rb
index b3116f2..40a457a 100644
--- a/spec/riak/riak_spec.rb
+++ b/spec/riak/riak_spec.rb
@@ -1,4 +1,4 @@
-require_relative "spec_helper"
+require_relative "../spec_helper"
 
 describe "App with Riak backend" do
   include Rack::Test::Methods

From 25c170021e852a6c301d750ab17ceafc4b207cfd Mon Sep 17 00:00:00 2001
From: Garret Alfert <garret@5apps.com>
Date: Fri, 22 Jan 2016 16:04:28 -0500
Subject: [PATCH 07/28] Delete metadata from Redis when deleting objects

---
 lib/remote_storage/swift.rb | 13 +++++-
 spec/swift/app_spec.rb      | 82 ++++++++++++++++++++++++++++++++++++-
 2 files changed, 93 insertions(+), 2 deletions(-)

diff --git a/lib/remote_storage/swift.rb b/lib/remote_storage/swift.rb
index 197d1f8..913f683 100644
--- a/lib/remote_storage/swift.rb
+++ b/lib/remote_storage/swift.rb
@@ -152,6 +152,7 @@ module RemoteStorage
       end
 
       do_delete_request(url)
+      delete_metadata_objects(user, directory, key)
       delete_dir_objects(user, directory)
 
       server.halt 200
@@ -299,13 +300,23 @@ module RemoteStorage
       false
     end
 
+    def delete_metadata_objects(user, directory, key)
+      redis_key = "rs_meta:#{user}:#{directory}/#{key}"
+      redis.del(redis_key)
+      redis.srem "rs_meta:#{user}:#{directory}/:items", key
+    end
+
     def delete_dir_objects(user, directory)
       parent_directories_for(directory).each do |dir|
         if dir_empty?(user, dir)
           do_delete_request("#{url_for_directory(user, dir)}/")
+          redis.del "rs_meta:#{user}:#{directory}/"
+          redis.srem "rs_meta:#{user}:#{parent_directory_for(dir)}:items", "#{dir}/"
         else
           timestamp = (Time.now.to_f * 1000).to_i
-          do_put_request("#{url_for_directory(user, dir)}/", timestamp.to_s, "text/plain")
+          res = do_put_request("#{url_for_directory(user, dir)}/", timestamp.to_s, "text/plain")
+          metadata = {etag: res.headers[:etag], modified: timestamp}
+          redis.hmset("rs_meta:#{user}:#{dir}/", *metadata)
         end
       end
     end
diff --git a/spec/swift/app_spec.rb b/spec/swift/app_spec.rb
index a7e25c8..17d1555 100644
--- a/spec/swift/app_spec.rb
+++ b/spec/swift/app_spec.rb
@@ -46,7 +46,6 @@ describe "App" do
         metadata = redis.hgetall "rs_meta:phil:food/"
         metadata["etag"].must_equal "bla"
         metadata["modified"].length.must_equal 13
-        metadata = redis.hgetall "rs_meta:phil:food/"
 
         food_items = redis.smembers "rs_meta:phil:food/:items"
         food_items.each do |food_item|
@@ -58,5 +57,86 @@ describe "App" do
       end
     end
   end
+
+  describe "DELETE requests" do
+    context "authorized" do
+      before do
+        redis.sadd "authorizations:phil:amarillo", [":rw"]
+        header "Authorization", "Bearer amarillo"
+
+        put_stub = OpenStruct.new(headers: {etag: "bla"})
+        RemoteStorage::Swift.stub_any_instance :has_name_collision?, false do
+          RestClient.stub :put, put_stub do
+            put "/phil/food/aguacate", "si"
+            put "/phil/food/camaron", "yummi"
+          end
+        end
+      end
+
+      it "deletes the metadata object in redis" do
+        put_stub = OpenStruct.new(headers: {etag: "bla"})
+        RemoteStorage::Swift.stub_any_instance :dir_empty?, false do
+          RestClient.stub :put, put_stub do
+            RestClient.stub :delete, "" do
+              delete "/phil/food/aguacate"
+            end
+          end
+        end
+
+        metadata = redis.hgetall "rs_meta:phil:food/aguacate"
+        metadata.must_be_empty
+      end
+
+      it "deletes the directory objects metadata in redis" do
+        old_metadata = redis.hgetall "rs_meta:phil:food/"
+
+        put_stub = OpenStruct.new(headers: {etag: "newetag"})
+        RemoteStorage::Swift.stub_any_instance :dir_empty?, false do
+          RestClient.stub :put, put_stub do
+            RestClient.stub :delete, "" do
+              delete "/phil/food/aguacate"
+            end
+          end
+        end
+
+        metadata = redis.hgetall "rs_meta:phil:food/"
+        metadata["etag"].must_equal "newetag"
+        metadata["modified"].length.must_equal 13
+        metadata["modified"].wont_equal old_metadata["modified"]
+
+        food_items = redis.smembers "rs_meta:phil:food/:items"
+        food_items.must_equal ["camaron"]
+
+        root_items = redis.smembers "rs_meta:phil:/:items"
+        root_items.must_equal ["food/"]
+      end
+
+      it "deletes the parent directory objects metadata when deleting all items" do
+        put_stub = OpenStruct.new(headers: {etag: "bla"})
+        RemoteStorage::Swift.stub_any_instance :dir_empty?, false do
+          RestClient.stub :put, put_stub do
+            RestClient.stub :delete, "" do
+              delete "/phil/food/aguacate"
+            end
+          end
+        end
+
+        RemoteStorage::Swift.stub_any_instance :dir_empty?, true do
+          RestClient.stub :delete, "" do
+            delete "/phil/food/camaron"
+          end
+        end
+
+        metadata = redis.hgetall "rs_meta:phil:food/"
+        metadata.must_be_empty
+
+        food_items = redis.smembers "rs_meta:phil:food/:items"
+        food_items.must_be_empty
+
+        root_items = redis.smembers "rs_meta:phil:/:items"
+        root_items.must_be_empty
+      end
+    end
+  end
 end
 

From f73c286ce9207e2fa85b12c594a620d5f2473ad3 Mon Sep 17 00:00:00 2001
From: Garret Alfert <garret@5apps.com>
Date: Sat, 23 Jan 2016 12:34:08 -0500
Subject: [PATCH 08/28] Purge all keys from redis before every test run

---
 spec/spec_helper.rb    |  6 ++++++
 spec/swift/app_spec.rb | 10 ++++++++++
 2 files changed, 16 insertions(+)

diff --git a/spec/spec_helper.rb b/spec/spec_helper.rb
index 585725a..c92c47f 100644
--- a/spec/spec_helper.rb
+++ b/spec/spec_helper.rb
@@ -37,6 +37,12 @@ def redis
   @redis ||= Redis.new(host: app.settings.redis["host"], port: app.settings.redis["port"])
 end
 
+def purge_redis
+  redis.keys("*").each do |key|
+    redis.del key
+  end
+end
+
 if app.settings.respond_to? :riak
   ::Riak.disable_list_keys_warnings = true
 
diff --git a/spec/swift/app_spec.rb b/spec/swift/app_spec.rb
index 17d1555..9afbdf9 100644
--- a/spec/swift/app_spec.rb
+++ b/spec/swift/app_spec.rb
@@ -13,6 +13,11 @@ describe "App" do
   end
 
   describe "PUT requests" do
+
+    before do
+      purge_redis
+    end
+
     context "authorized" do
       before do
         redis.sadd "authorizations:phil:amarillo", [":rw"]
@@ -59,6 +64,11 @@ describe "App" do
   end
 
   describe "DELETE requests" do
+
+    before do
+      purge_redis
+    end
+
     context "authorized" do
       before do
         redis.sadd "authorizations:phil:amarillo", [":rw"]

From dfc8a5909642b21226ec898ce35c4d4ea73ec995 Mon Sep 17 00:00:00 2001
From: Garret Alfert <garret@5apps.com>
Date: Sat, 23 Jan 2016 13:20:42 -0500
Subject: [PATCH 09/28] Fix redis keys and content for nested directories

---
 lib/remote_storage/swift.rb | 14 ++++++++++++--
 1 file changed, 12 insertions(+), 2 deletions(-)

diff --git a/lib/remote_storage/swift.rb b/lib/remote_storage/swift.rb
index 913f683..58a6a5f 100644
--- a/lib/remote_storage/swift.rb
+++ b/lib/remote_storage/swift.rb
@@ -265,9 +265,17 @@ module RemoteStorage
       parent_directories
     end
 
+    def top_directory(directory)
+      if directory.match(/\//)
+        directory.split("/").last
+      elsif directory != ""
+        return directory
+      end
+    end
+
     def parent_directory_for(directory)
       if directory.match(/\//)
-        return directory[0..directory.rindex("/")-1]
+        return directory[0..directory.rindex("/")]
       elsif directory != ""
         return "/"
       end
@@ -277,6 +285,8 @@ module RemoteStorage
       redis_key = "rs_meta:#{user}:#{directory}/#{key}"
       redis.hmset(redis_key, *metadata)
       redis.sadd "rs_meta:#{user}:#{directory}/:items", key
+
+      true
     end
 
     def update_dir_objects(user, directory)
@@ -288,7 +298,7 @@ module RemoteStorage
         key = "rs_meta:#{user}:#{dir}/"
         metadata = {etag: res.headers[:etag], modified: timestamp}
         redis.hmset(key, *metadata)
-        redis.sadd "rs_meta:#{user}:#{parent_directory_for(dir)}:items", "#{dir}/"
+        redis.sadd "rs_meta:#{user}:#{parent_directory_for(dir)}:items", "#{top_directory(dir)}/"
       end
 
       true

From 599865cf3f2dbed5f88148fd034f1b16047c3dd5 Mon Sep 17 00:00:00 2001
From: Garret Alfert <garret@5apps.com>
Date: Sat, 23 Jan 2016 13:26:38 -0500
Subject: [PATCH 10/28] Generate directory listing from Redis metadata

---
 lib/remote_storage/swift.rb | 42 +++++++++++++++++++++++++++
 spec/swift/app_spec.rb      | 58 +++++++++++++++++++++++++++++++++++++
 2 files changed, 100 insertions(+)

diff --git a/lib/remote_storage/swift.rb b/lib/remote_storage/swift.rb
index 58a6a5f..94ccafc 100644
--- a/lib/remote_storage/swift.rb
+++ b/lib/remote_storage/swift.rb
@@ -75,6 +75,48 @@ module RemoteStorage
     end
 
     def get_directory_listing(user, directory)
+      # TODO add ETag header
+      # TODO check IF_NONE_MATCH header
+
+      server.headers["Content-Type"] = "application/json"
+
+      listing = {
+        "@context" => "http://remotestorage.io/spec/folder-description",
+        "items"    => {}
+      }
+
+      items = redis.smembers "rs_meta:#{user}:#{directory}/:items"
+
+      items.sort.each do |name|
+        redis_key = if directory.empty?
+                      "rs_meta:phil:#{name}"
+                    else
+                      "rs_meta:phil:#{directory}/#{name}"
+                    end
+        metadata = redis.hgetall redis_key
+
+        if name[-1] == "/" # It's a directory
+          listing["items"].merge!({
+            name => {
+              "ETag" => metadata["etag"]
+            }
+          })
+        else # It's a file
+          listing["items"].merge!({
+            name => {
+              "ETag"           => metadata["etag"],
+              "Content-Type"   => metadata["type"],
+              "Content-Length" => metadata["size"].to_i
+            }
+          });
+        end
+
+      end
+
+      listing.to_json
+    end
+
+    def get_directory_listing_from_swift(user, directory)
       is_root_listing = directory.empty?
 
       server.headers["Content-Type"] = "application/json"
diff --git a/spec/swift/app_spec.rb b/spec/swift/app_spec.rb
index 9afbdf9..08bb344 100644
--- a/spec/swift/app_spec.rb
+++ b/spec/swift/app_spec.rb
@@ -148,5 +148,63 @@ describe "App" do
       end
     end
   end
+
+  describe "GET requests" do
+
+    before do
+      purge_redis
+    end
+
+    context "authorized" do
+      before do
+        redis.sadd "authorizations:phil:amarillo", [":rw"]
+        header "Authorization", "Bearer amarillo"
+
+        put_stub = OpenStruct.new(headers: {etag: "bla"})
+        RemoteStorage::Swift.stub_any_instance :has_name_collision?, false do
+          RestClient.stub :put, put_stub do
+            put "/phil/food/aguacate", "si"
+            put "/phil/food/camaron", "yummi"
+            put "/phil/food/desunyos/bolon", "wow"
+          end
+        end
+      end
+
+      describe "directory listings" do
+
+        it "contains all items in the directory" do
+          get "/phil/food/"
+
+          last_response.status.must_equal 200
+          last_response.content_type.must_equal "application/json"
+
+          content = JSON.parse(last_response.body)
+          content["@context"].must_equal "http://remotestorage.io/spec/folder-description"
+          content["items"]["aguacate"].wont_be_nil
+          content["items"]["aguacate"]["Content-Type"].must_equal "text/plain; charset=utf-8"
+          content["items"]["aguacate"]["Content-Length"].must_equal 2
+          content["items"]["aguacate"]["ETag"].must_equal "bla"
+          content["items"]["camaron"].wont_be_nil
+          content["items"]["camaron"]["Content-Type"].must_equal "text/plain; charset=utf-8"
+          content["items"]["camaron"]["Content-Length"].must_equal 5
+          content["items"]["camaron"]["ETag"].must_equal "bla"
+          content["items"]["desunyos/"].wont_be_nil
+          content["items"]["desunyos/"]["ETag"].must_equal "bla"
+        end
+
+        it "contains all items in the root directory" do
+          get "phil/"
+
+          last_response.status.must_equal 200
+          last_response.content_type.must_equal "application/json"
+
+          content = JSON.parse(last_response.body)
+          content["items"]["food/"].wont_be_nil
+          content["items"]["food/"]["ETag"].must_equal "bla"
+        end
+
+      end
+    end
+  end
 end
 

From c676c1a6a8fe0efc1e24db8b3ee0c26fa1c2d089 Mon Sep 17 00:00:00 2001
From: Garret Alfert <garret@5apps.com>
Date: Sat, 23 Jan 2016 14:28:14 -0500
Subject: [PATCH 11/28] Add metadata entry for root directory

---
 lib/remote_storage/swift.rb | 6 ++++++
 spec/swift/app_spec.rb      | 4 ++++
 2 files changed, 10 insertions(+)

diff --git a/lib/remote_storage/swift.rb b/lib/remote_storage/swift.rb
index 94ccafc..ceb65a4 100644
--- a/lib/remote_storage/swift.rb
+++ b/lib/remote_storage/swift.rb
@@ -304,6 +304,8 @@ module RemoteStorage
         directories.pop
       end
 
+      parent_directories << "" # add empty string for the root directory
+
       parent_directories
     end
 
@@ -336,6 +338,7 @@ module RemoteStorage
       timestamp = (Time.now.to_f * 1000).to_i
 
       parent_directories_for(directory).each do |dir|
+        # TODO check if we can actually do a put request to the root dir
         res = do_put_request("#{url_for_directory(user, dir)}/", timestamp.to_s, "text/plain")
         key = "rs_meta:#{user}:#{dir}/"
         metadata = {etag: res.headers[:etag], modified: timestamp}
@@ -346,6 +349,7 @@ module RemoteStorage
       true
     rescue
       parent_directories_for(directory).each do |dir|
+        # TODO check if we can actually do a delete request to the root dir
         do_delete_request("#{url_for_directory(user, dir)}/") rescue false
       end
 
@@ -361,11 +365,13 @@ module RemoteStorage
     def delete_dir_objects(user, directory)
       parent_directories_for(directory).each do |dir|
         if dir_empty?(user, dir)
+          # TODO check if we can actually do a delete request to the root dir
           do_delete_request("#{url_for_directory(user, dir)}/")
           redis.del "rs_meta:#{user}:#{directory}/"
           redis.srem "rs_meta:#{user}:#{parent_directory_for(dir)}:items", "#{dir}/"
         else
           timestamp = (Time.now.to_f * 1000).to_i
+        # TODO check if we can actually do a put request to the root dir
           res = do_put_request("#{url_for_directory(user, dir)}/", timestamp.to_s, "text/plain")
           metadata = {etag: res.headers[:etag], modified: timestamp}
           redis.hmset("rs_meta:#{user}:#{dir}/", *metadata)
diff --git a/spec/swift/app_spec.rb b/spec/swift/app_spec.rb
index 08bb344..924a259 100644
--- a/spec/swift/app_spec.rb
+++ b/spec/swift/app_spec.rb
@@ -48,6 +48,10 @@ describe "App" do
           end
         end
 
+        metadata = redis.hgetall "rs_meta:phil:/"
+        metadata["etag"].must_equal "bla"
+        metadata["modified"].length.must_equal 13
+
         metadata = redis.hgetall "rs_meta:phil:food/"
         metadata["etag"].must_equal "bla"
         metadata["modified"].length.must_equal 13

From bec9f7a6cce21f597e24bfafbd3aaf3ae0a09dc0 Mon Sep 17 00:00:00 2001
From: Garret Alfert <garret@5apps.com>
Date: Sat, 23 Jan 2016 14:29:18 -0500
Subject: [PATCH 12/28] Set ETag header for directory listings

---
 lib/remote_storage/swift.rb | 4 +++-
 spec/swift/app_spec.rb      | 7 +++++++
 2 files changed, 10 insertions(+), 1 deletion(-)

diff --git a/lib/remote_storage/swift.rb b/lib/remote_storage/swift.rb
index ceb65a4..216b7ab 100644
--- a/lib/remote_storage/swift.rb
+++ b/lib/remote_storage/swift.rb
@@ -75,10 +75,12 @@ module RemoteStorage
     end
 
     def get_directory_listing(user, directory)
-      # TODO add ETag header
       # TODO check IF_NONE_MATCH header
+      etag = redis.hget "rs_meta:#{user}:#{directory}/", "etag"
+
 
       server.headers["Content-Type"] = "application/json"
+      server.headers["ETag"] = %Q("#{etag}")
 
       listing = {
         "@context" => "http://remotestorage.io/spec/folder-description",
diff --git a/spec/swift/app_spec.rb b/spec/swift/app_spec.rb
index 924a259..877b0f7 100644
--- a/spec/swift/app_spec.rb
+++ b/spec/swift/app_spec.rb
@@ -176,6 +176,13 @@ describe "App" do
 
       describe "directory listings" do
 
+        it "has an ETag in the header" do
+          get "/phil/food/"
+
+          last_response.status.must_equal 200
+          last_response.headers["ETag"].must_equal "\"bla\""
+        end
+
         it "contains all items in the directory" do
           get "/phil/food/"
 

From 15196ca4b9d9c8733e97142e9c4f2aaf58b5a3f8 Mon Sep 17 00:00:00 2001
From: Garret Alfert <garret@5apps.com>
Date: Sat, 23 Jan 2016 14:29:37 -0500
Subject: [PATCH 13/28] Check IF_NONE_MATCH header for directory listings

---
 lib/remote_storage/swift.rb | 3 ++-
 spec/swift/app_spec.rb      | 7 +++++++
 2 files changed, 9 insertions(+), 1 deletion(-)

diff --git a/lib/remote_storage/swift.rb b/lib/remote_storage/swift.rb
index 216b7ab..2581ab5 100644
--- a/lib/remote_storage/swift.rb
+++ b/lib/remote_storage/swift.rb
@@ -75,9 +75,10 @@ module RemoteStorage
     end
 
     def get_directory_listing(user, directory)
-      # TODO check IF_NONE_MATCH header
       etag = redis.hget "rs_meta:#{user}:#{directory}/", "etag"
 
+      none_match = (server.env["HTTP_IF_NONE_MATCH"] || "").split(",").map(&:strip)
+      server.halt 304 if none_match.include? etag
 
       server.headers["Content-Type"] = "application/json"
       server.headers["ETag"] = %Q("#{etag}")
diff --git a/spec/swift/app_spec.rb b/spec/swift/app_spec.rb
index 877b0f7..9022913 100644
--- a/spec/swift/app_spec.rb
+++ b/spec/swift/app_spec.rb
@@ -183,6 +183,13 @@ describe "App" do
           last_response.headers["ETag"].must_equal "\"bla\""
         end
 
+        it "responds with 304 when IF_NONE_MATCH header contains the ETag" do
+          header "If-None-Match", "bla"
+          get "/phil/food/"
+
+          last_response.status.must_equal 304
+        end
+
         it "contains all items in the directory" do
           get "/phil/food/"
 

From 737be34e73947b959bfff1e48d7ecddb79d35d04 Mon Sep 17 00:00:00 2001
From: Garret Alfert <garret@5apps.com>
Date: Tue, 26 Jan 2016 20:21:45 +0100
Subject: [PATCH 14/28] Configure the source of dir listing (redis or swift)

---
 config.yml.example          | 1 +
 lib/remote_storage/swift.rb | 8 ++++++++
 2 files changed, 9 insertions(+)

diff --git a/config.yml.example b/config.yml.example
index f0ed36d..5f65092 100644
--- a/config.yml.example
+++ b/config.yml.example
@@ -20,6 +20,7 @@ development: &defaults
   # redis:
   #   host: localhost
   #   port: 6379
+  # use_redis_dir_listing: true
 
 test:
   <<: *defaults
diff --git a/lib/remote_storage/swift.rb b/lib/remote_storage/swift.rb
index 2581ab5..80a8406 100644
--- a/lib/remote_storage/swift.rb
+++ b/lib/remote_storage/swift.rb
@@ -75,6 +75,14 @@ module RemoteStorage
     end
 
     def get_directory_listing(user, directory)
+      if settings.use_redis_dir_listing
+        get_directory_listing_from_redis(user, directory)
+      else
+        get_directory_listing_from_swift
+      end
+    end
+
+    def get_directory_listing_from_redis(user, directory)
       etag = redis.hget "rs_meta:#{user}:#{directory}/", "etag"
 
       none_match = (server.env["HTTP_IF_NONE_MATCH"] || "").split(",").map(&:strip)

From 0745354b8a4f789df6fdc8e1899a67f8955157ef Mon Sep 17 00:00:00 2001
From: Garret Alfert <garret@5apps.com>
Date: Tue, 26 Jan 2016 20:32:29 +0100
Subject: [PATCH 15/28] Retrieve directory listing from Redis via Lua

---
 lib/remote_storage/swift.rb | 70 ++++++++++++++++++++++---------------
 1 file changed, 41 insertions(+), 29 deletions(-)

diff --git a/lib/remote_storage/swift.rb b/lib/remote_storage/swift.rb
index 80a8406..9738f36 100644
--- a/lib/remote_storage/swift.rb
+++ b/lib/remote_storage/swift.rb
@@ -82,6 +82,46 @@ module RemoteStorage
       end
     end
 
+    def get_directory_listing_from_redis_via_lua(user, directory)
+      lua_script = <<-EOF
+        local user = ARGV[1]
+        local directory = ARGV[2]
+        local items = redis.call("smembers", "rs_meta:"..user..":"..directory.."/:items")
+        local listing = {}
+
+        for index, name in pairs(items) do
+          local redis_key = "rs_meta:"..user..":"
+          if directory == "" then
+            redis_key = redis_key..name
+          else
+            redis_key = redis_key..directory.."/"..name
+          end
+
+          local metadata_values = redis.call("hgetall", redis_key)
+          local metadata = {}
+
+          -- redis returns hashes as a single list of alternating keys and values
+          -- this collates it into a table
+          for idx = 1, #metadata_values, 2 do
+            metadata[metadata_values[idx]] = metadata_values[idx + 1]
+          end
+
+          listing[name] = {}
+          if string.sub(name, -1) == "/" then
+            listing[name]["ETag"] = metadata["etag"]
+          else
+            listing[name]["ETag"]           = metadata["etag"]
+            listing[name]["Content-Type"]   = metadata["type"]
+            listing[name]["Content-Length"] = tonumber(metadata["size"])
+          end
+        end
+
+        return cjson.encode(listing)
+      EOF
+
+      JSON.parse(redis.eval(lua_script, nil, [user, directory]))
+    end
+
     def get_directory_listing_from_redis(user, directory)
       etag = redis.hget "rs_meta:#{user}:#{directory}/", "etag"
 
@@ -93,37 +133,9 @@ module RemoteStorage
 
       listing = {
         "@context" => "http://remotestorage.io/spec/folder-description",
-        "items"    => {}
+        "items"    => get_directory_listing_from_redis_via_lua(user, directory)
       }
 
-      items = redis.smembers "rs_meta:#{user}:#{directory}/:items"
-
-      items.sort.each do |name|
-        redis_key = if directory.empty?
-                      "rs_meta:phil:#{name}"
-                    else
-                      "rs_meta:phil:#{directory}/#{name}"
-                    end
-        metadata = redis.hgetall redis_key
-
-        if name[-1] == "/" # It's a directory
-          listing["items"].merge!({
-            name => {
-              "ETag" => metadata["etag"]
-            }
-          })
-        else # It's a file
-          listing["items"].merge!({
-            name => {
-              "ETag"           => metadata["etag"],
-              "Content-Type"   => metadata["type"],
-              "Content-Length" => metadata["size"].to_i
-            }
-          });
-        end
-
-      end
-
       listing.to_json
     end
 

From d973771192df9bbcd8431ae9b103bf79f2d479b1 Mon Sep 17 00:00:00 2001
From: Garret Alfert <garret@5apps.com>
Date: Tue, 26 Jan 2016 20:33:43 +0100
Subject: [PATCH 16/28] Remove some duplication

---
 lib/remote_storage/swift.rb | 7 ++-----
 1 file changed, 2 insertions(+), 5 deletions(-)

diff --git a/lib/remote_storage/swift.rb b/lib/remote_storage/swift.rb
index 9738f36..71d0572 100644
--- a/lib/remote_storage/swift.rb
+++ b/lib/remote_storage/swift.rb
@@ -106,11 +106,8 @@ module RemoteStorage
             metadata[metadata_values[idx]] = metadata_values[idx + 1]
           end
 
-          listing[name] = {}
-          if string.sub(name, -1) == "/" then
-            listing[name]["ETag"] = metadata["etag"]
-          else
-            listing[name]["ETag"]           = metadata["etag"]
+          listing[name] = {["ETag"] = metadata["etag"]}
+          if string.sub(name, -1) ~= "/" then
             listing[name]["Content-Type"]   = metadata["type"]
             listing[name]["Content-Length"] = tonumber(metadata["size"])
           end

From 4e7c8f68bbfbe53a5f1d9d2f471687933ba7c5e4 Mon Sep 17 00:00:00 2001
From: Garret Alfert <garret@5apps.com>
Date: Thu, 28 Jan 2016 17:48:20 +0100
Subject: [PATCH 17/28] Add missing arguments to method call

---
 lib/remote_storage/swift.rb | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/remote_storage/swift.rb b/lib/remote_storage/swift.rb
index 71d0572..98eb1df 100644
--- a/lib/remote_storage/swift.rb
+++ b/lib/remote_storage/swift.rb
@@ -78,7 +78,7 @@ module RemoteStorage
       if settings.use_redis_dir_listing
         get_directory_listing_from_redis(user, directory)
       else
-        get_directory_listing_from_swift
+        get_directory_listing_from_swift(user, directory)
       end
     end
 

From 4ca67c7ea9e5b7b81a593c6f0495f52186d45c7e Mon Sep 17 00:00:00 2001
From: Garret Alfert <garret@5apps.com>
Date: Thu, 28 Jan 2016 19:06:24 +0100
Subject: [PATCH 18/28] Do collision detection via Redis metadata

---
 lib/remote_storage/swift.rb | 52 +++++++++++++++++++++++++++++++++++++
 spec/swift/app_spec.rb      | 40 ++++++++++++++++++++++++++++
 2 files changed, 92 insertions(+)

diff --git a/lib/remote_storage/swift.rb b/lib/remote_storage/swift.rb
index 98eb1df..6074492 100644
--- a/lib/remote_storage/swift.rb
+++ b/lib/remote_storage/swift.rb
@@ -289,6 +289,58 @@ module RemoteStorage
     end
 
     def has_name_collision?(user, directory, key)
+      if settings.use_redis_dir_listing
+        has_name_collision_via_redis?(user, directory, key)
+      else
+        has_name_collision_via_swift?(user, directory, key)
+      end
+    end
+
+    def has_name_collision_via_redis?(user, directory, key)
+      lua_script = <<-EOF
+        local user = ARGV[1]
+        local directory = ARGV[2]
+        local key = ARGV[3]
+
+        -- build table with parent directories from remaining arguments
+        local parent_dir_count = #ARGV - 3
+        local parent_directories = {}
+        for i = 4, 4 + parent_dir_count do
+          table.insert(parent_directories, ARGV[i])
+        end
+
+        -- check for existing directory with the same name as the document
+        local redis_key = "rs_meta:"..user..":"
+        if directory == "" then
+          redis_key = redis_key..key.."/"
+        else
+          redis_key = redis_key..directory.."/"..key.."/"
+        end
+        if redis.call("hget", redis_key, "etag") then
+          return true
+        end
+
+        for index, dir in pairs(parent_directories) do
+          if redis.call("hget", "rs_meta:"..user..":"..dir.."/", "etag") then
+            -- the directory already exists, no need to do further checks
+            return false
+          else
+            -- check for existing document with same name as directory
+            if redis.call("hget", "rs_meta:"..user..":"..dir, "etag") then
+              return true
+            end
+          end
+        end
+
+        return false
+      EOF
+
+      parent_directories = parent_directories_for(directory)
+
+      redis.eval(lua_script, nil, [user, directory, key, *parent_directories])
+    end
+
+    def has_name_collision_via_swift?(user, directory, key)
       # check for existing directory with the same name as the document
       url = url_for_key(user, directory, key)
       do_head_request("#{url}/") do |res|
diff --git a/spec/swift/app_spec.rb b/spec/swift/app_spec.rb
index 9022913..9d20293 100644
--- a/spec/swift/app_spec.rb
+++ b/spec/swift/app_spec.rb
@@ -64,6 +64,46 @@ describe "App" do
         root_items = redis.smembers "rs_meta:phil:/:items"
         root_items.must_equal ["food/"]
       end
+
+      describe "name collision checks" do
+        it "is successful when there is no name collision" do
+          put_stub = OpenStruct.new(headers: {etag: "bla"})
+          RestClient.stub :put, put_stub do
+            put "/phil/food/aguacate", "si"
+          end
+
+          last_response.status.must_equal 200
+
+          metadata = redis.hgetall "rs_meta:phil:food/aguacate"
+          metadata["size"].must_equal "2"
+        end
+
+        it "conflicts when there is a directory with same name as document" do
+          put_stub = OpenStruct.new(headers: {etag: "bla"})
+          RestClient.stub :put, put_stub do
+            put "/phil/food/aguacate", "si"
+            put "/phil/food", "wontwork"
+          end
+
+          last_response.status.must_equal 409
+
+          metadata = redis.hgetall "rs_meta:phil:food"
+          metadata.must_be_empty
+        end
+
+        it "conflicts when there is a document with same name as directory" do
+          put_stub = OpenStruct.new(headers: {etag: "bla"})
+          RestClient.stub :put, put_stub do
+            put "/phil/food/aguacate", "si"
+            put "/phil/food/aguacate/empanado", "wontwork"
+          end
+
+          last_response.status.must_equal 409
+
+          metadata = redis.hgetall "rs_meta:phil:food/aguacate/empanado"
+          metadata.must_be_empty
+        end
+      end
     end
   end
 

From 16dcc56fbadf8d174df29eb175697d95e354dc0e Mon Sep 17 00:00:00 2001
From: Garret Alfert <garret@5apps.com>
Date: Fri, 29 Jan 2016 15:16:24 +0100
Subject: [PATCH 19/28] No need to stub has_name_collision? anymore

---
 spec/swift/app_spec.rb | 32 ++++++++++++--------------------
 1 file changed, 12 insertions(+), 20 deletions(-)

diff --git a/spec/swift/app_spec.rb b/spec/swift/app_spec.rb
index 9d20293..6b2a876 100644
--- a/spec/swift/app_spec.rb
+++ b/spec/swift/app_spec.rb
@@ -26,10 +26,8 @@ describe "App" do
 
       it "creates the metadata object in redis" do
         put_stub = OpenStruct.new(headers: {etag: "bla"})
-        RemoteStorage::Swift.stub_any_instance :has_name_collision?, false do
-          RestClient.stub :put, put_stub do
-            put "/phil/food/aguacate", "si"
-          end
+        RestClient.stub :put, put_stub do
+          put "/phil/food/aguacate", "si"
         end
 
         metadata = redis.hgetall "rs_meta:phil:food/aguacate"
@@ -41,11 +39,9 @@ describe "App" do
 
       it "creates the directory objects metadata in redis" do
         put_stub = OpenStruct.new(headers: {etag: "bla"})
-        RemoteStorage::Swift.stub_any_instance :has_name_collision?, false do
-          RestClient.stub :put, put_stub do
-            put "/phil/food/aguacate", "si"
-            put "/phil/food/camaron", "yummi"
-          end
+        RestClient.stub :put, put_stub do
+          put "/phil/food/aguacate", "si"
+          put "/phil/food/camaron", "yummi"
         end
 
         metadata = redis.hgetall "rs_meta:phil:/"
@@ -119,11 +115,9 @@ describe "App" do
         header "Authorization", "Bearer amarillo"
 
         put_stub = OpenStruct.new(headers: {etag: "bla"})
-        RemoteStorage::Swift.stub_any_instance :has_name_collision?, false do
-          RestClient.stub :put, put_stub do
-            put "/phil/food/aguacate", "si"
-            put "/phil/food/camaron", "yummi"
-          end
+        RestClient.stub :put, put_stub do
+          put "/phil/food/aguacate", "si"
+          put "/phil/food/camaron", "yummi"
         end
       end
 
@@ -205,12 +199,10 @@ describe "App" do
         header "Authorization", "Bearer amarillo"
 
         put_stub = OpenStruct.new(headers: {etag: "bla"})
-        RemoteStorage::Swift.stub_any_instance :has_name_collision?, false do
-          RestClient.stub :put, put_stub do
-            put "/phil/food/aguacate", "si"
-            put "/phil/food/camaron", "yummi"
-            put "/phil/food/desunyos/bolon", "wow"
-          end
+        RestClient.stub :put, put_stub do
+          put "/phil/food/aguacate", "si"
+          put "/phil/food/camaron", "yummi"
+          put "/phil/food/desunyos/bolon", "wow"
         end
       end
 

From a4673e9661a744810e8756f08397c91edc8d7430 Mon Sep 17 00:00:00 2001
From: Garret Alfert <garret@5apps.com>
Date: Fri, 29 Jan 2016 16:18:09 +0100
Subject: [PATCH 20/28] Use directory backend config from redis instead of
 config file

---
 config.yml.example          |  1 -
 lib/remote_storage/swift.rb |  8 ++++++--
 spec/spec_helper.rb         |  2 +-
 spec/swift/app_spec.rb      | 31 +++++++++++++++++++++++++++++++
 4 files changed, 38 insertions(+), 4 deletions(-)

diff --git a/config.yml.example b/config.yml.example
index 5f65092..f0ed36d 100644
--- a/config.yml.example
+++ b/config.yml.example
@@ -20,7 +20,6 @@ development: &defaults
   # redis:
   #   host: localhost
   #   port: 6379
-  # use_redis_dir_listing: true
 
 test:
   <<: *defaults
diff --git a/lib/remote_storage/swift.rb b/lib/remote_storage/swift.rb
index 6074492..535947e 100644
--- a/lib/remote_storage/swift.rb
+++ b/lib/remote_storage/swift.rb
@@ -75,7 +75,7 @@ module RemoteStorage
     end
 
     def get_directory_listing(user, directory)
-      if settings.use_redis_dir_listing
+      if directory_backend(user).match /new/
         get_directory_listing_from_redis(user, directory)
       else
         get_directory_listing_from_swift(user, directory)
@@ -289,7 +289,7 @@ module RemoteStorage
     end
 
     def has_name_collision?(user, directory, key)
-      if settings.use_redis_dir_listing
+      if directory_backend(user).match /new/
         has_name_collision_via_redis?(user, directory, key)
       else
         has_name_collision_via_swift?(user, directory, key)
@@ -518,6 +518,10 @@ module RemoteStorage
       @redis ||= Redis.new(host: settings.redis["host"], port: settings.redis["port"])
     end
 
+    def directory_backend(user)
+      @directory_backend ||= redis.get("rs_config:dir_backend:#{user}") || "legacy"
+    end
+
     def etag_for(body)
       objects = JSON.parse(body)
 
diff --git a/spec/spec_helper.rb b/spec/spec_helper.rb
index c92c47f..8cec109 100644
--- a/spec/spec_helper.rb
+++ b/spec/spec_helper.rb
@@ -38,7 +38,7 @@ def redis
 end
 
 def purge_redis
-  redis.keys("*").each do |key|
+  redis.keys("rs_*").each do |key|
     redis.del key
   end
 end
diff --git a/spec/swift/app_spec.rb b/spec/swift/app_spec.rb
index 6b2a876..b4b9d38 100644
--- a/spec/swift/app_spec.rb
+++ b/spec/swift/app_spec.rb
@@ -16,6 +16,7 @@ describe "App" do
 
     before do
       purge_redis
+      redis.set "rs_config:dir_backend:phil", "new"
     end
 
     context "authorized" do
@@ -107,6 +108,7 @@ describe "App" do
 
     before do
       purge_redis
+      redis.set "rs_config:dir_backend:phil", "new"
     end
 
     context "authorized" do
@@ -191,9 +193,11 @@ describe "App" do
 
     before do
       purge_redis
+      redis.set "rs_config:dir_backend:phil", "new"
     end
 
     context "authorized" do
+
       before do
         redis.sadd "authorizations:phil:amarillo", [":rw"]
         header "Authorization", "Bearer amarillo"
@@ -255,6 +259,33 @@ describe "App" do
 
       end
     end
+
+    context "with legacy directory backend" do
+
+      before do
+        redis.sadd "authorizations:phil:amarillo", [":rw"]
+        header "Authorization", "Bearer amarillo"
+
+        put_stub = OpenStruct.new(headers: {etag: "bla"})
+        RestClient.stub :put, put_stub do
+          put "/phil/food/aguacate", "si"
+          put "/phil/food/camaron", "yummi"
+        end
+
+        redis.set "rs_config:dir_backend:phil", "legacy"
+      end
+
+      it "serves directory listing from Swift backend" do
+        RemoteStorage::Swift.stub_any_instance :get_directory_listing_from_swift, "directory listing" do
+          get "/phil/food/"
+        end
+
+        last_response.status.must_equal 200
+        last_response.body.must_equal "directory listing"
+      end
+
+    end
+
   end
 end
 

From 573dadf3657de3e8bc9873893d236d65bf3b2250 Mon Sep 17 00:00:00 2001
From: Garret Alfert <garret@5apps.com>
Date: Fri, 29 Jan 2016 16:18:49 +0100
Subject: [PATCH 21/28] Respond with 503 when directory backend is locked

---
 lib/remote_storage/swift.rb |  1 +
 spec/swift/app_spec.rb      | 32 ++++++++++++++++++++++++++++++++
 2 files changed, 33 insertions(+)

diff --git a/lib/remote_storage/swift.rb b/lib/remote_storage/swift.rb
index 535947e..b815ca3 100644
--- a/lib/remote_storage/swift.rb
+++ b/lib/remote_storage/swift.rb
@@ -28,6 +28,7 @@ module RemoteStorage
 
       server.halt 401 unless permission
       if ["PUT", "DELETE"].include? request_method
+        server.halt 503 if directory_backend(user).match /locked/
         server.halt 401 unless permission == "rw"
       end
     end
diff --git a/spec/swift/app_spec.rb b/spec/swift/app_spec.rb
index b4b9d38..ac0b41a 100644
--- a/spec/swift/app_spec.rb
+++ b/spec/swift/app_spec.rb
@@ -101,6 +101,38 @@ describe "App" do
           metadata.must_be_empty
         end
       end
+
+      describe "directory backend configuration" do
+        context "locked new backed" do
+          before do
+            redis.set "rs_config:dir_backend:phil", "new-locked"
+          end
+
+          it "responds with 503" do
+            put "/phil/food/aguacate", "si"
+
+            last_response.status.must_equal 503
+
+            metadata = redis.hgetall "rs_meta:phil:food/aguacate"
+            metadata.must_be_empty
+          end
+        end
+
+        context "locked legacy backend" do
+          before do
+            redis.set "rs_config:dir_backend:phil", "legacy-locked"
+          end
+
+          it "responds with 503" do
+            put "/phil/food/aguacate", "si"
+
+            last_response.status.must_equal 503
+
+            metadata = redis.hgetall "rs_meta:phil:food/aguacate"
+            metadata.must_be_empty
+          end
+        end
+      end
     end
   end
 

From 7084a50d1b21093a8a2556f0e506a47ebbdb8d33 Mon Sep 17 00:00:00 2001
From: Garret Alfert <garret@5apps.com>
Date: Fri, 29 Jan 2016 16:45:17 +0100
Subject: [PATCH 22/28] Make dir_empty? method use Redis metadata

---
 lib/remote_storage/swift.rb |  8 ++++++--
 spec/swift/app_spec.rb      | 27 ++++++++-------------------
 2 files changed, 14 insertions(+), 21 deletions(-)

diff --git a/lib/remote_storage/swift.rb b/lib/remote_storage/swift.rb
index b815ca3..4a64d13 100644
--- a/lib/remote_storage/swift.rb
+++ b/lib/remote_storage/swift.rb
@@ -453,8 +453,12 @@ module RemoteStorage
     end
 
     def dir_empty?(user, dir)
-      do_get_request("#{container_url_for(user)}/?format=plain&limit=1&path=#{escape(dir)}/") do |res|
-        return res.headers[:content_length] == "0"
+      if directory_backend(user).match /new/
+        redis.smembers("rs_meta:#{user}:#{dir}/:items").empty?
+      else
+        do_get_request("#{container_url_for(user)}/?format=plain&limit=1&path=#{escape(dir)}/") do |res|
+          return res.headers[:content_length] == "0"
+        end
       end
     end
 
diff --git a/spec/swift/app_spec.rb b/spec/swift/app_spec.rb
index ac0b41a..dc51ee4 100644
--- a/spec/swift/app_spec.rb
+++ b/spec/swift/app_spec.rb
@@ -157,11 +157,9 @@ describe "App" do
 
       it "deletes the metadata object in redis" do
         put_stub = OpenStruct.new(headers: {etag: "bla"})
-        RemoteStorage::Swift.stub_any_instance :dir_empty?, false do
-          RestClient.stub :put, put_stub do
-            RestClient.stub :delete, "" do
-              delete "/phil/food/aguacate"
-            end
+        RestClient.stub :put, put_stub do
+          RestClient.stub :delete, "" do
+            delete "/phil/food/aguacate"
           end
         end
 
@@ -173,11 +171,9 @@ describe "App" do
         old_metadata = redis.hgetall "rs_meta:phil:food/"
 
         put_stub = OpenStruct.new(headers: {etag: "newetag"})
-        RemoteStorage::Swift.stub_any_instance :dir_empty?, false do
-          RestClient.stub :put, put_stub do
-            RestClient.stub :delete, "" do
-              delete "/phil/food/aguacate"
-            end
+        RestClient.stub :put, put_stub do
+          RestClient.stub :delete, "" do
+            delete "/phil/food/aguacate"
           end
         end
 
@@ -195,16 +191,9 @@ describe "App" do
 
       it "deletes the parent directory objects metadata when deleting all items" do
         put_stub = OpenStruct.new(headers: {etag: "bla"})
-        RemoteStorage::Swift.stub_any_instance :dir_empty?, false do
-          RestClient.stub :put, put_stub do
-            RestClient.stub :delete, "" do
-              delete "/phil/food/aguacate"
-            end
-          end
-        end
-
-        RemoteStorage::Swift.stub_any_instance :dir_empty?, true do
+        RestClient.stub :put, put_stub do
           RestClient.stub :delete, "" do
+            delete "/phil/food/aguacate"
             delete "/phil/food/camaron"
           end
         end

From 9ba85446efa1656c69ba7c61123e10d1d334bc63 Mon Sep 17 00:00:00 2001
From: Garret Alfert <garret@5apps.com>
Date: Mon, 1 Feb 2016 14:58:19 +0100
Subject: [PATCH 23/28] Use parentheses to fix syntax warnings

---
 lib/remote_storage/swift.rb | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/lib/remote_storage/swift.rb b/lib/remote_storage/swift.rb
index 4a64d13..a53ee2a 100644
--- a/lib/remote_storage/swift.rb
+++ b/lib/remote_storage/swift.rb
@@ -28,7 +28,7 @@ module RemoteStorage
 
       server.halt 401 unless permission
       if ["PUT", "DELETE"].include? request_method
-        server.halt 503 if directory_backend(user).match /locked/
+        server.halt 503 if directory_backend(user).match(/locked/)
         server.halt 401 unless permission == "rw"
       end
     end
@@ -76,7 +76,7 @@ module RemoteStorage
     end
 
     def get_directory_listing(user, directory)
-      if directory_backend(user).match /new/
+      if directory_backend(user).match(/new/)
         get_directory_listing_from_redis(user, directory)
       else
         get_directory_listing_from_swift(user, directory)
@@ -290,7 +290,7 @@ module RemoteStorage
     end
 
     def has_name_collision?(user, directory, key)
-      if directory_backend(user).match /new/
+      if directory_backend(user).match(/new/)
         has_name_collision_via_redis?(user, directory, key)
       else
         has_name_collision_via_swift?(user, directory, key)
@@ -453,7 +453,7 @@ module RemoteStorage
     end
 
     def dir_empty?(user, dir)
-      if directory_backend(user).match /new/
+      if directory_backend(user).match(/new/)
         redis.smembers("rs_meta:#{user}:#{dir}/:items").empty?
       else
         do_get_request("#{container_url_for(user)}/?format=plain&limit=1&path=#{escape(dir)}/") do |res|

From 409628c9e726fa837c7e4615d67463a10c0ecef6 Mon Sep 17 00:00:00 2001
From: Garret Alfert <garret@5apps.com>
Date: Mon, 1 Feb 2016 18:04:35 +0100
Subject: [PATCH 24/28] Trying to fix specs when run on Travis

---
 config.yml.example            |  5 +++++
 spec/riak/directories_spec.rb |  4 ++--
 spec/riak/riak_spec.rb        | 10 +++++-----
 spec/spec_helper.rb           | 14 ++++++++------
 4 files changed, 20 insertions(+), 13 deletions(-)

diff --git a/config.yml.example b/config.yml.example
index f0ed36d..b4e3ac3 100644
--- a/config.yml.example
+++ b/config.yml.example
@@ -32,6 +32,11 @@ test:
       cs_binaries: rs.binaries.test
       authorizations: rs_authorizations_test
       opslog: rs_opslog_test
+  swift:
+    host: "https://swift.example.com"
+  redis:
+    host: localhost
+    port: 6379
 
 staging:
   <<: *defaults
diff --git a/spec/riak/directories_spec.rb b/spec/riak/directories_spec.rb
index 6551a7f..eea4f82 100644
--- a/spec/riak/directories_spec.rb
+++ b/spec/riak/directories_spec.rb
@@ -251,7 +251,7 @@ describe "Directories" do
         context "charset given in content-type header" do
           before do
             header "Content-Type", "image/jpeg; charset=binary"
-            filename = File.join(File.expand_path(File.dirname(__FILE__)), "fixtures", "rockrule.jpeg")
+            filename = File.join(File.expand_path(File.dirname(__FILE__)), "..", "fixtures", "rockrule.jpeg")
             @image = File.open(filename, "r").read
             put "/jimmy/tasks/jaypeg.jpg", @image
           end
@@ -273,7 +273,7 @@ describe "Directories" do
         context "no charset in content-type header" do
           before do
             header "Content-Type", "image/jpeg"
-            filename = File.join(File.expand_path(File.dirname(__FILE__)), "fixtures", "rockrule.jpeg")
+            filename = File.join(File.expand_path(File.dirname(__FILE__)), "..", "fixtures", "rockrule.jpeg")
             @image = File.open(filename, "r").read
             put "/jimmy/tasks/jaypeg.jpg", @image
           end
diff --git a/spec/riak/riak_spec.rb b/spec/riak/riak_spec.rb
index 40a457a..19c2d69 100644
--- a/spec/riak/riak_spec.rb
+++ b/spec/riak/riak_spec.rb
@@ -446,7 +446,7 @@ describe "App with Riak backend" do
           context "binary charset in content-type header" do
             before do
               header "Content-Type", "image/jpeg; charset=binary"
-              filename = File.join(File.expand_path(File.dirname(__FILE__)), "fixtures", "rockrule.jpeg")
+              filename = File.join(File.expand_path(File.dirname(__FILE__)), "..", "fixtures", "rockrule.jpeg")
               @image = File.open(filename, "r").read
               put "/jimmy/documents/jaypeg", @image
             end
@@ -502,7 +502,7 @@ describe "App with Riak backend" do
             context "overwriting existing file with same file" do
               before do
                 header "Content-Type", "image/jpeg; charset=binary"
-                filename = File.join(File.expand_path(File.dirname(__FILE__)), "fixtures", "rockrule.jpeg")
+                filename = File.join(File.expand_path(File.dirname(__FILE__)), "..", "fixtures", "rockrule.jpeg")
                 @image = File.open(filename, "r").read
                 put "/jimmy/documents/jaypeg", @image
               end
@@ -518,7 +518,7 @@ describe "App with Riak backend" do
             context "overwriting existing file with different file" do
               before do
                 header "Content-Type", "image/jpeg; charset=binary"
-                filename = File.join(File.expand_path(File.dirname(__FILE__)), "fixtures", "rockrule.jpeg")
+                filename = File.join(File.expand_path(File.dirname(__FILE__)), "..", "fixtures", "rockrule.jpeg")
                 @image = File.open(filename, "r").read
                 put "/jimmy/documents/jaypeg", @image+"foo"
               end
@@ -540,7 +540,7 @@ describe "App with Riak backend" do
           context "no binary charset in content-type header" do
             before do
               header "Content-Type", "image/jpeg"
-              filename = File.join(File.expand_path(File.dirname(__FILE__)), "fixtures", "rockrule.jpeg")
+              filename = File.join(File.expand_path(File.dirname(__FILE__)), "..", "fixtures", "rockrule.jpeg")
               @image = File.open(filename, "r").read
               put "/jimmy/documents/jaypeg", @image
             end
@@ -705,7 +705,7 @@ describe "App with Riak backend" do
         context "binary data" do
           before do
             header "Content-Type", "image/jpeg; charset=binary"
-            filename = File.join(File.expand_path(File.dirname(__FILE__)), "fixtures", "rockrule.jpeg")
+            filename = File.join(File.expand_path(File.dirname(__FILE__)), "..", "fixtures", "rockrule.jpeg")
             @image = File.open(filename, "r").read
             put "/jimmy/documents/jaypeg", @image
 
diff --git a/spec/spec_helper.rb b/spec/spec_helper.rb
index 8cec109..4975f52 100644
--- a/spec/spec_helper.rb
+++ b/spec/spec_helper.rb
@@ -33,13 +33,15 @@ end
 
 alias context describe
 
-def redis
-  @redis ||= Redis.new(host: app.settings.redis["host"], port: app.settings.redis["port"])
-end
+if app.settings.respond_to? :redis
+  def redis
+    @redis ||= Redis.new(host: app.settings.redis["host"], port: app.settings.redis["port"])
+  end
 
-def purge_redis
-  redis.keys("rs_*").each do |key|
-    redis.del key
+  def purge_redis
+    redis.keys("rs_*").each do |key|
+      redis.del key
+    end
   end
 end
 

From 536df233369537099f7564edb4cb3196b1da88cb Mon Sep 17 00:00:00 2001
From: Garret Alfert <garret@5apps.com>
Date: Tue, 2 Feb 2016 00:10:47 +0100
Subject: [PATCH 25/28] Make Travis create a Swift token file

---
 .travis.yml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.travis.yml b/.travis.yml
index 135619f..0f78e31 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -7,6 +7,7 @@ before_install:
   - sh .travis/install_riakcs.sh
 before_script:
   - cp config.yml.example config.yml
+  - mkdir -p tmp && echo "swifttoken" > tmp/swift_token.txt
 script: bundle exec rake test
 branches:
   only:

From 954f046dd9d15f80ce9882f256065f573c908214 Mon Sep 17 00:00:00 2001
From: Garret Alfert <garret@5apps.com>
Date: Tue, 2 Feb 2016 00:19:15 +0100
Subject: [PATCH 26/28] Make Travis install Redis

---
 .travis.yml | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/.travis.yml b/.travis.yml
index 0f78e31..dbf71dc 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -1,6 +1,8 @@
 language: ruby
 rvm:
   - 2.2.4
+services:
+  - redis-server
 sudo: required
 dist: trusty
 before_install:

From 2516e9090ec4a79574dc9bb692f44b26849d8c44 Mon Sep 17 00:00:00 2001
From: Garret Alfert <garret@5apps.com>
Date: Tue, 2 Feb 2016 00:57:53 +0100
Subject: [PATCH 27/28] Make Travis only run Swift specs for now

---
 .travis.yml        |  2 +-
 config.yml.example | 18 +++++++++---------
 2 files changed, 10 insertions(+), 10 deletions(-)

diff --git a/.travis.yml b/.travis.yml
index b883498..d3d7d89 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -10,7 +10,7 @@ before_install:
 before_script:
   - cp config.yml.example config.yml
   - mkdir -p tmp && echo "swifttoken" > tmp/swift_token.txt
-script: bundle exec rake test
+script: ruby spec/swift/*
 branches:
   only:
     - master
diff --git a/config.yml.example b/config.yml.example
index b4e3ac3..e2aac55 100644
--- a/config.yml.example
+++ b/config.yml.example
@@ -23,15 +23,15 @@ development: &defaults
 
 test:
   <<: *defaults
-  riak:
-    <<: *riak_defaults
-    buckets:
-      data: rs_data_test
-      directories: rs_directories_test
-      binaries: rs_binaries_test
-      cs_binaries: rs.binaries.test
-      authorizations: rs_authorizations_test
-      opslog: rs_opslog_test
+  # riak:
+  #   <<: *riak_defaults
+  #   buckets:
+  #     data: rs_data_test
+  #     directories: rs_directories_test
+  #     binaries: rs_binaries_test
+  #     cs_binaries: rs.binaries.test
+  #     authorizations: rs_authorizations_test
+  #     opslog: rs_opslog_test
   swift:
     host: "https://swift.example.com"
   redis:

From d7a71d039f29fd67c16b0e60c16ea6f355d7b0f0 Mon Sep 17 00:00:00 2001
From: Garret Alfert <garret@5apps.com>
Date: Tue, 2 Feb 2016 01:04:49 +0100
Subject: [PATCH 28/28] Comment out the Riak settings from example config

---
 config.yml.example | 26 +++++++++++++-------------
 1 file changed, 13 insertions(+), 13 deletions(-)

diff --git a/config.yml.example b/config.yml.example
index e2aac55..6fb3307 100644
--- a/config.yml.example
+++ b/config.yml.example
@@ -1,18 +1,18 @@
 development: &defaults
   maintenance: false
-  riak: &riak_defaults
-    host: localhost
-    http_port: 8098
-    riak_cs:
-      credentials_file: "cs_credentials.json"
-      endpoint: "http://cs.example.com:8080"
-    buckets:
-      data: rs_data
-      directories: rs_directories
-      binaries: rs_binaries
-      cs_binaries: rs.binaries
-      authorizations: rs_authorizations
-      opslog: rs_opslog
+  # riak: &riak_defaults
+  #   host: localhost
+  #   http_port: 8098
+  #   riak_cs:
+  #     credentials_file: "cs_credentials.json"
+  #     endpoint: "http://cs.example.com:8080"
+  #   buckets:
+  #     data: rs_data
+  #     directories: rs_directories
+  #     binaries: rs_binaries
+  #     cs_binaries: rs.binaries
+  #     authorizations: rs_authorizations
+  #     opslog: rs_opslog
   # # uncomment this section and comment the riak one
   # swift: &swift_defaults
   #   host: "https://swift.example.com"