Support directory listings

lib/remote_storage/riak.rb

@@ -1,5 +1,6 @@
 require "riak"
 require "json"
+require "cgi"
 
 module RemoteStorage
   module Riak
@@ -12,6 +13,10 @@ module RemoteStorage
       @data_bucket ||= client.bucket("user_data")
     end
 
+    def directory_bucket
+      @directory_bucket ||= client.bucket("rs_directories")
+    end
+
     def authorize_request(user, category, token)
       request_method = env["REQUEST_METHOD"]
       return true if category.split("/").first == "public" && request_method == "GET"
@@ -41,6 +46,19 @@ module RemoteStorage
       halt 404
     end
 
+    def get_directory_listing(user, directory)
+      directory_object = directory_bucket.get("#{user}:#{directory}")
+      headers["Content-Type"] = "application/json"
+      headers["Last-Modified"] = directory_object.last_modified.to_s(:rfc822)
+
+      listing = directory_listing(user, directory)
+
+      return listing.to_json
+    rescue ::Riak::HTTPFailedRequest
+      headers["Content-Type"] = "application/json"
+      return "{}"
+    end
+
     def put_data(user, category, key, data, content_type=nil)
       object = data_bucket.new("#{user}:#{category}:#{key}")
       object.content_type = content_type || "text/plain; charset=utf-8"
@@ -50,14 +68,33 @@ module RemoteStorage
       else
         object.raw_data = data
       end
-      object.indexes.merge!({:user_id_bin => [user]})
+      object.indexes.merge!({:user_id_bin => [user],
+                             :directory_bin => [category]})
       object.store
+
+      update_directory_object(user, category)
     rescue ::Riak::HTTPFailedRequest
       halt 422
     end
 
+    def update_directory_object(user, category)
+      if category.match /\//
+        parent_directory = category[0..category.rindex("/")-1]
+      end
+      directory = directory_bucket.new("#{user}:#{category}")
+      directory.raw_data = ""
+      directory.indexes.merge!({:user_id_bin => [user]})
+      if parent_directory
+        directory.indexes.merge!({:directory_bin => [parent_directory]})
+      end
+      directory.store
+    end
+
     def delete_data(user, category, key)
       riak_response = data_bucket.delete("#{user}:#{category}:#{key}")
+      if directory_entries(user, category).empty?
+        directory_bucket.delete "#{user}:#{category}"
+      end
       halt riak_response[:code]
     rescue ::Riak::HTTPFailedRequest
       halt 404
@@ -88,5 +125,59 @@ module RemoteStorage
       permission
     end
 
+    def directory_listing(user, directory)
+      listing = {}
+
+      sub_directories(user, directory).each do |entry|
+        timestamp = DateTime.rfc2822(entry["last_modified"]).to_time.to_i
+        listing.merge!({ "#{entry["name"]}/" => timestamp })
+      end
+
+      directory_entries(user, directory).each do |entry|
+        timestamp = DateTime.rfc2822(entry["last_modified"]).to_time.to_i
+        listing.merge!({ entry["name"] => timestamp })
+      end
+
+      listing
+    end
+
+    def directory_entries(user, directory)
+      map_query = <<-EOH
+        function(v){
+          keys = v.key.split(':');
+          key_name = keys[keys.length-1];
+          last_modified_date = v.values[0]['metadata']['X-Riak-Last-Modified'];
+          return [{
+            name: key_name,
+            last_modified: last_modified_date,
+          }];
+        }
+      EOH
+      objects = ::Riak::MapReduce.new(client).
+        index("user_data", "user_id_bin", user).
+        index("user_data", "directory_bin", directory).
+        map(map_query, :keep => true).
+        run
+    end
+
+    def sub_directories(user, directory)
+      map_query = <<-EOH
+        function(v){
+          keys = v.key.split(':');
+          key_name = keys[keys.length-1];
+          last_modified_date = v.values[0]['metadata']['X-Riak-Last-Modified'];
+          return [{
+            name: key_name,
+            last_modified: last_modified_date,
+          }];
+        }
+      EOH
+      objects = ::Riak::MapReduce.new(client).
+        index("rs_directories", "user_id_bin", user).
+        index("rs_directories", "directory_bin", directory).
+        map(map_query, :keep => true).
+        run
+    end
+
   end
 end

liquor-cabinet.rb

@@ -29,20 +29,26 @@ class LiquorCabinet < Sinatra::Base
     disable :logging
   end
 
-  before "/:user/*/:key" do
-    headers 'Access-Control-Allow-Origin' => '*',
-            'Access-Control-Allow-Methods' => 'GET, PUT, DELETE',
-            'Access-Control-Allow-Headers' => 'Authorization, Content-Type, Origin'
-    headers['Access-Control-Allow-Origin'] = env["HTTP_ORIGIN"] if env["HTTP_ORIGIN"]
+  ["/:user/*/:key", "/:user/*/"].each do |path|
+    before path do
+      headers 'Access-Control-Allow-Origin' => '*',
+              'Access-Control-Allow-Methods' => 'GET, PUT, DELETE',
+              'Access-Control-Allow-Headers' => 'Authorization, Content-Type, Origin'
+      headers['Access-Control-Allow-Origin'] = env["HTTP_ORIGIN"] if env["HTTP_ORIGIN"]
 
-    @user, @category, @key = params[:user], params[:splat].first, params[:key]
-    token = env["HTTP_AUTHORIZATION"] ? env["HTTP_AUTHORIZATION"].split(" ")[1] : ""
+      @user, @directory, @key = params[:user], params[:splat].first, params[:key]
+      token = env["HTTP_AUTHORIZATION"] ? env["HTTP_AUTHORIZATION"].split(" ")[1] : ""
 
-    authorize_request(@user, @category, token) unless request.options?
+      authorize_request(@user, @directory, token) unless request.options?
+    end
   end
 
   get "/:user/*/:key" do
-    get_data(@user, @category, @key)
+    get_data(@user, @directory, @key)
+  end
+
+  get "/:user/*/" do
+    get_directory_listing(@user, @directory)
   end
 
   put "/:user/*/:key" do
@@ -54,11 +60,11 @@ class LiquorCabinet < Sinatra::Base
       content_type = env['CONTENT_TYPE']
     end
 
-    put_data(@user, @category, @key, data, content_type)
+    put_data(@user, @directory, @key, data, content_type)
   end
 
   delete "/:user/*/:key" do
-    delete_data(@user, @category, @key)
+    delete_data(@user, @directory, @key)
   end
 
   options "/:user/*/:key" do

spec/directories_spec.rb

@@ -0,0 +1,153 @@
+require_relative "spec_helper"
+
+describe "Directories" do
+  include Rack::Test::Methods
+  include RemoteStorage::Riak
+
+  before do
+    purge_all_buckets
+
+    auth = auth_bucket.new("jimmy:123")
+    auth.data = ["documents:r", "tasks:rw"]
+    auth.store
+
+    header "Authorization", "Bearer 123"
+  end
+
+  describe "GET listing" do
+
+    before do
+      put "/jimmy/tasks/home/foo", "do the laundry"
+      put "/jimmy/tasks/home/bar", "do the laundry"
+    end
+
+    it "lists the objects with a timestamp of the last modification" do
+      get "/jimmy/tasks/home/"
+
+      # File.open("stacktrace.html", "w") do |f|
+      #   f.write last_response.body
+      # end
+      last_response.status.must_equal 200
+      last_response.content_type.must_equal "application/json"
+
+      content = JSON.parse(last_response.body)
+      content["foo"].to_s.must_match /\d+/
+      content["foo"].to_s.length.must_be :>=, 10
+    end
+
+    it "has a Last-Modifier header set" do
+      get "/jimmy/tasks/home/"
+
+      last_response.headers["Last-Modified"].wont_be_nil
+    end
+
+    it "has CORS headers set" do
+      get "/jimmy/tasks/home/"
+
+      last_response.headers["Access-Control-Allow-Origin"].must_equal "*"
+      last_response.headers["Access-Control-Allow-Methods"].must_equal "GET, PUT, DELETE"
+      last_response.headers["Access-Control-Allow-Headers"].must_equal "Authorization, Content-Type, Origin"
+    end
+
+    context "with sub-directories" do
+      before do
+        put "/jimmy/tasks/home/laundry", "do the laundry"
+      end
+    end
+
+    it "lists the containing objects as well as the direct sub-directories" do
+      get "/jimmy/tasks/"
+
+      last_response.status.must_equal 200
+      content = JSON.parse(last_response.body)
+      # p content
+      # dir = directory_bucket.get("jimmy:tasks/home")
+      # puts dir.indexes.inspect
+      content["home/"].to_s.must_match /\d+/
+    end
+
+    context "for a sub-directory" do
+      before do
+        put "/jimmy/tasks/home/laundry", "do the laundry"
+      end
+
+      it "lists the objects with timestamp" do
+        get "/jimmy/tasks/home/"
+
+        last_response.status.must_equal 200
+
+        content = JSON.parse(last_response.body)
+        content["laundry"].to_s.must_match /\d+/
+        content["laundry"].to_s.length.must_be :>=, 10
+      end
+    end
+
+    describe "for an empty or absent directory" do
+      it "returns an empty listing" do
+        get "/jimmy/documents/notfound/"
+
+        last_response.status.must_equal 200
+        last_response.body.must_equal "{}"
+      end
+    end
+  end
+
+  describe "directory object" do
+    describe "PUT file" do
+      context "no existing directory object" do
+        it "creates a new directory object" do
+          put "/jimmy/tasks/home/trash", "take out the trash"
+
+          object = directory_bucket.get("jimmy:tasks/home")
+          object.last_modified.wont_be_nil
+        end
+
+        it "sets the correct index for the directory object" do
+          put "/jimmy/tasks/home/trash", "take out the trash"
+
+          object = directory_bucket.get("jimmy:tasks/home")
+          object.indexes["directory_bin"].must_include "tasks"
+        end
+      end
+
+      context "existing directory object" do
+        before do
+          @directory = directory_bucket.new("jimmy:tasks/home")
+          @directory.content_type = "text/plain"
+          @directory.raw_data = ""
+          @directory.store
+          @old_timestamp = @directory.reload.last_modified
+        end
+
+        it "updates the timestamp of the directory" do
+          sleep 1
+          put "/jimmy/tasks/home/trash", "take out the trash"
+
+          @directory.reload
+          @directory.last_modified.must_be :>, @old_timestamp
+        end
+      end
+    end
+
+    describe "DELETE file" do
+      context "last file in directory" do
+        before do
+          directory_bucket.delete("jimmy:tasks")
+          put "/jimmy/tasks/trash", "take out the trash"
+        end
+
+        it "deletes the directory object" do
+          delete "/jimmy/tasks/trash"
+
+          last_response.status.must_equal 204
+
+          lambda {
+            directory_bucket.get("jimmy:tasks")
+          }.must_raise Riak::HTTPFailedRequest
+        end
+      end
+    end
+
+  end
+
+end

spec/spec_helper.rb

@@ -14,3 +14,33 @@ ENV["RACK_ENV"] = "test"
 config = File.read(File.expand_path('../config.yml', File.dirname(__FILE__)))
 riak_config = YAML.load(config)[ENV['RACK_ENV']]['riak'].symbolize_keys
 set :riak_config, riak_config
+
+::Riak.disable_list_keys_warnings = true
+
+def app
+  LiquorCabinet
+end
+
+def storage_client
+  @storage_client ||= ::Riak::Client.new(settings.riak_config)
+end
+
+def data_bucket
+  @data_bucket ||= storage_client.bucket("user_data")
+end
+
+def auth_bucket
+  @auth_bucket ||= storage_client.bucket("authorizations")
+end
+
+def directory_bucket
+  @directory_bucket ||= storage_client.bucket("rs_directories")
+end
+
+def purge_all_buckets
+  [data_bucket, directory_bucket, auth_bucket].each do |bucket|
+    bucket.keys.each {|key| bucket.delete key}
+  end
+end
+
+alias context describe