diff --git a/lib/remote_storage/riak.rb b/lib/remote_storage/riak.rb index 5925fd7..944dd2c 100644 --- a/lib/remote_storage/riak.rb +++ b/lib/remote_storage/riak.rb @@ -1,5 +1,6 @@ require "riak" require "json" +require "cgi" module RemoteStorage module Riak @@ -12,6 +13,10 @@ module RemoteStorage @data_bucket ||= client.bucket("user_data") end + def directory_bucket + @directory_bucket ||= client.bucket("rs_directories") + end + def authorize_request(user, category, token) request_method = env["REQUEST_METHOD"] return true if category.split("/").first == "public" && request_method == "GET" @@ -41,6 +46,19 @@ module RemoteStorage halt 404 end + def get_directory_listing(user, directory) + directory_object = directory_bucket.get("#{user}:#{directory}") + headers["Content-Type"] = "application/json" + headers["Last-Modified"] = directory_object.last_modified.to_s(:rfc822) + + listing = directory_listing(user, directory) + + return listing.to_json + rescue ::Riak::HTTPFailedRequest + headers["Content-Type"] = "application/json" + return "{}" + end + def put_data(user, category, key, data, content_type=nil) object = data_bucket.new("#{user}:#{category}:#{key}") object.content_type = content_type || "text/plain; charset=utf-8" @@ -50,14 +68,33 @@ module RemoteStorage else object.raw_data = data end - object.indexes.merge!({:user_id_bin => [user]}) + object.indexes.merge!({:user_id_bin => [user], + :directory_bin => [category]}) object.store + + update_directory_object(user, category) rescue ::Riak::HTTPFailedRequest halt 422 end + def update_directory_object(user, category) + if category.match /\// + parent_directory = category[0..category.rindex("/")-1] + end + directory = directory_bucket.new("#{user}:#{category}") + directory.raw_data = "" + directory.indexes.merge!({:user_id_bin => [user]}) + if parent_directory + directory.indexes.merge!({:directory_bin => [parent_directory]}) + end + directory.store + end + def delete_data(user, category, key) riak_response = data_bucket.delete("#{user}:#{category}:#{key}") + if directory_entries(user, category).empty? + directory_bucket.delete "#{user}:#{category}" + end halt riak_response[:code] rescue ::Riak::HTTPFailedRequest halt 404 @@ -88,5 +125,59 @@ module RemoteStorage permission end + def directory_listing(user, directory) + listing = {} + + sub_directories(user, directory).each do |entry| + timestamp = DateTime.rfc2822(entry["last_modified"]).to_time.to_i + listing.merge!({ "#{entry["name"]}/" => timestamp }) + end + + directory_entries(user, directory).each do |entry| + timestamp = DateTime.rfc2822(entry["last_modified"]).to_time.to_i + listing.merge!({ entry["name"] => timestamp }) + end + + listing + end + + def directory_entries(user, directory) + map_query = <<-EOH + function(v){ + keys = v.key.split(':'); + key_name = keys[keys.length-1]; + last_modified_date = v.values[0]['metadata']['X-Riak-Last-Modified']; + return [{ + name: key_name, + last_modified: last_modified_date, + }]; + } + EOH + objects = ::Riak::MapReduce.new(client). + index("user_data", "user_id_bin", user). + index("user_data", "directory_bin", directory). + map(map_query, :keep => true). + run + end + + def sub_directories(user, directory) + map_query = <<-EOH + function(v){ + keys = v.key.split(':'); + key_name = keys[keys.length-1]; + last_modified_date = v.values[0]['metadata']['X-Riak-Last-Modified']; + return [{ + name: key_name, + last_modified: last_modified_date, + }]; + } + EOH + objects = ::Riak::MapReduce.new(client). + index("rs_directories", "user_id_bin", user). + index("rs_directories", "directory_bin", directory). + map(map_query, :keep => true). + run + end + end end diff --git a/liquor-cabinet.rb b/liquor-cabinet.rb index 3c836cb..d223bad 100644 --- a/liquor-cabinet.rb +++ b/liquor-cabinet.rb @@ -29,20 +29,26 @@ class LiquorCabinet < Sinatra::Base disable :logging end - before "/:user/*/:key" do - headers 'Access-Control-Allow-Origin' => '*', - 'Access-Control-Allow-Methods' => 'GET, PUT, DELETE', - 'Access-Control-Allow-Headers' => 'Authorization, Content-Type, Origin' - headers['Access-Control-Allow-Origin'] = env["HTTP_ORIGIN"] if env["HTTP_ORIGIN"] + ["/:user/*/:key", "/:user/*/"].each do |path| + before path do + headers 'Access-Control-Allow-Origin' => '*', + 'Access-Control-Allow-Methods' => 'GET, PUT, DELETE', + 'Access-Control-Allow-Headers' => 'Authorization, Content-Type, Origin' + headers['Access-Control-Allow-Origin'] = env["HTTP_ORIGIN"] if env["HTTP_ORIGIN"] - @user, @category, @key = params[:user], params[:splat].first, params[:key] - token = env["HTTP_AUTHORIZATION"] ? env["HTTP_AUTHORIZATION"].split(" ")[1] : "" + @user, @directory, @key = params[:user], params[:splat].first, params[:key] + token = env["HTTP_AUTHORIZATION"] ? env["HTTP_AUTHORIZATION"].split(" ")[1] : "" - authorize_request(@user, @category, token) unless request.options? + authorize_request(@user, @directory, token) unless request.options? + end end get "/:user/*/:key" do - get_data(@user, @category, @key) + get_data(@user, @directory, @key) + end + + get "/:user/*/" do + get_directory_listing(@user, @directory) end put "/:user/*/:key" do @@ -54,11 +60,11 @@ class LiquorCabinet < Sinatra::Base content_type = env['CONTENT_TYPE'] end - put_data(@user, @category, @key, data, content_type) + put_data(@user, @directory, @key, data, content_type) end delete "/:user/*/:key" do - delete_data(@user, @category, @key) + delete_data(@user, @directory, @key) end options "/:user/*/:key" do diff --git a/spec/directories_spec.rb b/spec/directories_spec.rb new file mode 100644 index 0000000..dbaac25 --- /dev/null +++ b/spec/directories_spec.rb @@ -0,0 +1,153 @@ +require_relative "spec_helper" + +describe "Directories" do + include Rack::Test::Methods + include RemoteStorage::Riak + + before do + purge_all_buckets + + auth = auth_bucket.new("jimmy:123") + auth.data = ["documents:r", "tasks:rw"] + auth.store + + header "Authorization", "Bearer 123" + end + + describe "GET listing" do + + before do + put "/jimmy/tasks/home/foo", "do the laundry" + put "/jimmy/tasks/home/bar", "do the laundry" + end + + it "lists the objects with a timestamp of the last modification" do + get "/jimmy/tasks/home/" + + # File.open("stacktrace.html", "w") do |f| + # f.write last_response.body + # end + last_response.status.must_equal 200 + last_response.content_type.must_equal "application/json" + + content = JSON.parse(last_response.body) + content["foo"].to_s.must_match /\d+/ + content["foo"].to_s.length.must_be :>=, 10 + end + + it "has a Last-Modifier header set" do + get "/jimmy/tasks/home/" + + last_response.headers["Last-Modified"].wont_be_nil + end + + it "has CORS headers set" do + get "/jimmy/tasks/home/" + + last_response.headers["Access-Control-Allow-Origin"].must_equal "*" + last_response.headers["Access-Control-Allow-Methods"].must_equal "GET, PUT, DELETE" + last_response.headers["Access-Control-Allow-Headers"].must_equal "Authorization, Content-Type, Origin" + end + + context "with sub-directories" do + before do + put "/jimmy/tasks/home/laundry", "do the laundry" + end + end + + it "lists the containing objects as well as the direct sub-directories" do + get "/jimmy/tasks/" + + last_response.status.must_equal 200 + content = JSON.parse(last_response.body) + # p content + # dir = directory_bucket.get("jimmy:tasks/home") + # puts dir.indexes.inspect + content["home/"].to_s.must_match /\d+/ + end + + context "for a sub-directory" do + before do + put "/jimmy/tasks/home/laundry", "do the laundry" + end + + it "lists the objects with timestamp" do + get "/jimmy/tasks/home/" + + last_response.status.must_equal 200 + + content = JSON.parse(last_response.body) + content["laundry"].to_s.must_match /\d+/ + content["laundry"].to_s.length.must_be :>=, 10 + end + end + + describe "for an empty or absent directory" do + it "returns an empty listing" do + get "/jimmy/documents/notfound/" + + last_response.status.must_equal 200 + last_response.body.must_equal "{}" + end + end + end + + describe "directory object" do + describe "PUT file" do + context "no existing directory object" do + it "creates a new directory object" do + put "/jimmy/tasks/home/trash", "take out the trash" + + object = directory_bucket.get("jimmy:tasks/home") + object.last_modified.wont_be_nil + end + + it "sets the correct index for the directory object" do + put "/jimmy/tasks/home/trash", "take out the trash" + + object = directory_bucket.get("jimmy:tasks/home") + object.indexes["directory_bin"].must_include "tasks" + end + end + + context "existing directory object" do + before do + @directory = directory_bucket.new("jimmy:tasks/home") + @directory.content_type = "text/plain" + @directory.raw_data = "" + @directory.store + @old_timestamp = @directory.reload.last_modified + end + + it "updates the timestamp of the directory" do + sleep 1 + put "/jimmy/tasks/home/trash", "take out the trash" + + @directory.reload + @directory.last_modified.must_be :>, @old_timestamp + end + end + end + + describe "DELETE file" do + context "last file in directory" do + before do + directory_bucket.delete("jimmy:tasks") + put "/jimmy/tasks/trash", "take out the trash" + end + + it "deletes the directory object" do + delete "/jimmy/tasks/trash" + + last_response.status.must_equal 204 + + lambda { + directory_bucket.get("jimmy:tasks") + }.must_raise Riak::HTTPFailedRequest + end + end + end + + end + +end diff --git a/spec/spec_helper.rb b/spec/spec_helper.rb index 55c4968..6fb97c6 100644 --- a/spec/spec_helper.rb +++ b/spec/spec_helper.rb @@ -14,3 +14,33 @@ ENV["RACK_ENV"] = "test" config = File.read(File.expand_path('../config.yml', File.dirname(__FILE__))) riak_config = YAML.load(config)[ENV['RACK_ENV']]['riak'].symbolize_keys set :riak_config, riak_config + +::Riak.disable_list_keys_warnings = true + +def app + LiquorCabinet +end + +def storage_client + @storage_client ||= ::Riak::Client.new(settings.riak_config) +end + +def data_bucket + @data_bucket ||= storage_client.bucket("user_data") +end + +def auth_bucket + @auth_bucket ||= storage_client.bucket("authorizations") +end + +def directory_bucket + @directory_bucket ||= storage_client.bucket("rs_directories") +end + +def purge_all_buckets + [data_bucket, directory_bucket, auth_bucket].each do |bucket| + bucket.keys.each {|key| bucket.delete key} + end +end + +alias context describe