From f852e7719f5417d79bba62bf4335b1f954976223 Mon Sep 17 00:00:00 2001 From: Garret Alfert Date: Thu, 8 Aug 2013 17:13:05 +0200 Subject: [PATCH 01/13] Requests for empty directories respond with 404 (refs #26) --- lib/remote_storage/riak.rb | 3 +-- spec/directories_spec.rb | 3 +-- spec/permissions_spec.rb | 10 +++++----- 3 files changed, 7 insertions(+), 9 deletions(-) diff --git a/lib/remote_storage/riak.rb b/lib/remote_storage/riak.rb index cec809c..76afbf1 100644 --- a/lib/remote_storage/riak.rb +++ b/lib/remote_storage/riak.rb @@ -75,8 +75,7 @@ module RemoteStorage return listing.to_json rescue ::Riak::HTTPFailedRequest - server.headers["Content-Type"] = "application/json" - return "{}" + server.halt 404 end def put_data(user, directory, key, data, content_type=nil) diff --git a/spec/directories_spec.rb b/spec/directories_spec.rb index 3292017..4e290c0 100644 --- a/spec/directories_spec.rb +++ b/spec/directories_spec.rb @@ -187,8 +187,7 @@ describe "Directories" do it "returns an empty listing" do get "/jimmy/documents/notfound/" - last_response.status.must_equal 200 - last_response.body.must_equal "{}" + last_response.status.must_equal 404 end end diff --git a/spec/permissions_spec.rb b/spec/permissions_spec.rb index d4f7bbf..8f13a16 100644 --- a/spec/permissions_spec.rb +++ b/spec/permissions_spec.rb @@ -345,7 +345,7 @@ describe "Permissions" do it "allows GET requests" do get "/jimmy/public/tasks/" - last_response.status.must_equal 200 + last_response.status.must_equal 404 end it "allows PUT requests" do @@ -403,19 +403,19 @@ describe "Permissions" do end it "allows GET requests" do - get "/jimmy/tasks/" + get "/jimmy/public/tasks/" - last_response.status.must_equal 200 + last_response.status.must_equal 404 end it "disallows PUT requests" do - put "/jimmy/tasks/foo", "some text" + put "/jimmy/public/tasks/foo", "some text" last_response.status.must_equal 403 end it "disallows DELETE requests" do - delete "/jimmy/tasks/hello" + delete "/jimmy/public/tasks/hello" last_response.status.must_equal 403 end From 5dd1d26b202b7cfaf8d4661a8ff059353e60cb0e Mon Sep 17 00:00:00 2001 From: Garret Alfert Date: Fri, 9 Aug 2013 10:56:04 +0200 Subject: [PATCH 02/13] Set and check ETag headers --- lib/remote_storage/riak.rb | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/lib/remote_storage/riak.rb b/lib/remote_storage/riak.rb index 76afbf1..19c2035 100644 --- a/lib/remote_storage/riak.rb +++ b/lib/remote_storage/riak.rb @@ -40,8 +40,11 @@ module RemoteStorage def get_data(user, directory, key) object = data_bucket.get("#{user}:#{directory}:#{key}") + server.halt 304 if server.env["HTTP_IF_NONE_MATCH"] == object.etag + server.headers["Content-Type"] = object.content_type server.headers["Last-Modified"] = last_modified_date_for(object) + server.headers["ETag"] = object.etag if binary_key = object.meta["binary_key"] object = cs_binary_bucket.files.get(binary_key[0]) @@ -66,10 +69,14 @@ module RemoteStorage def get_directory_listing(user, directory) directory_object = directory_bucket.get("#{user}:#{directory}") + + server.halt 304 if server.env["HTTP_IF_NONE_MATCH"] == directory_object.etag + timestamp = directory_object.data.to_i timestamp /= 1000 if timestamp.to_s.length == 13 server.headers["Content-Type"] = "application/json" server.headers["Last-Modified"] = Time.at(timestamp).to_s(:rfc822) + server.headers["ETag"] = directory_object.etag listing = directory_listing(user, directory) @@ -81,6 +88,10 @@ module RemoteStorage def put_data(user, directory, key, data, content_type=nil) object = build_data_object(user, directory, key, data, content_type) + if required_match = server.env["HTTP_IF_MATCH"] + server.halt 412 unless required_match == object.etag + end + object_exists = !object.raw_data.nil? existing_object_size = object_size(object) @@ -102,6 +113,7 @@ module RemoteStorage update_all_directory_objects(user, directory, timestamp) + server.headers["ETag"] = object.etag server.halt 200 rescue ::Riak::HTTPFailedRequest server.halt 422 @@ -110,6 +122,11 @@ module RemoteStorage def delete_data(user, directory, key) object = data_bucket.get("#{user}:#{directory}:#{key}") existing_object_size = object_size(object) + etag = object.etag + + if match_requirement = server.env["HTTP_IF_MATCH"] + server.halt 412 unless match_requirement == etag + end if binary_key = object.meta["binary_key"] object = cs_binary_bucket.files.get(binary_key[0]) @@ -125,6 +142,7 @@ module RemoteStorage timestamp = (Time.now.to_f * 1000).to_i delete_or_update_directory_objects(user, directory, timestamp) + server.headers["ETag"] = etag server.halt riak_response[:code] rescue ::Riak::HTTPFailedRequest server.halt 404 From 189d202d669270305f91351e167c4b1f4f60a381 Mon Sep 17 00:00:00 2001 From: Garret Alfert Date: Fri, 25 Oct 2013 23:09:04 +0200 Subject: [PATCH 03/13] Specs for ETag headers --- lib/remote_storage/riak.rb | 4 +-- spec/directories_spec.rb | 47 ++++++++++++++++++++++++++++++++++ spec/riak_spec.rb | 52 ++++++++++++++++++++++++++++---------- 3 files changed, 87 insertions(+), 16 deletions(-) diff --git a/lib/remote_storage/riak.rb b/lib/remote_storage/riak.rb index 19c2035..27ff1a3 100644 --- a/lib/remote_storage/riak.rb +++ b/lib/remote_storage/riak.rb @@ -124,8 +124,8 @@ module RemoteStorage existing_object_size = object_size(object) etag = object.etag - if match_requirement = server.env["HTTP_IF_MATCH"] - server.halt 412 unless match_requirement == etag + if required_match = server.env["HTTP_IF_MATCH"] + server.halt 412 unless required_match == etag end if binary_key = object.meta["binary_key"] diff --git a/spec/directories_spec.rb b/spec/directories_spec.rb index 4e290c0..56763be 100644 --- a/spec/directories_spec.rb +++ b/spec/directories_spec.rb @@ -44,6 +44,18 @@ describe "Directories" do last_modified.day.must_equal now.day end + it "has an ETag header set" do + get "/jimmy/tasks/" + + last_response.status.must_equal 200 + last_response.headers["ETag"].wont_be_nil + + # check that ETag stays the same + etag = last_response.headers["ETag"] + get "/jimmy/tasks/" + last_response.headers["ETag"].must_equal etag + end + it "has CORS headers set" do get "/jimmy/tasks/" @@ -55,6 +67,9 @@ describe "Directories" do context "with sub-directories" do before do + get "/jimmy/tasks/" + @old_etag = last_response.headers["ETag"] + put "/jimmy/tasks/home/laundry", "do the laundry" end @@ -71,6 +86,13 @@ describe "Directories" do content["home/"].to_s.length.must_equal 13 end + it "updates the ETag of the parent directory" do + get "/jimmy/tasks/" + + last_response.headers["ETag"].wont_be_nil + last_response.headers["ETag"].wont_equal @old_etag + end + context "for a different user" do before do auth = auth_bucket.new("alice:321") @@ -260,6 +282,13 @@ describe "Directories" do content["tasks/"].must_be_kind_of Integer content["tasks/"].to_s.length.must_equal 13 end + + it "has an ETag header set" do + get "/jimmy/" + + last_response.status.must_equal 200 + last_response.headers["ETag"].wont_be_nil + end end context "for the public directory" do @@ -280,6 +309,13 @@ describe "Directories" do content = JSON.parse(last_response.body) content.must_include "5apps" end + + it "has an ETag header set" do + get "/jimmy/public/bookmarks/" + + last_response.status.must_equal 200 + last_response.headers["ETag"].wont_be_nil + end end context "when directly authorized for the public directory" do @@ -449,6 +485,17 @@ describe "Directories" do directory_bucket.get("jimmy:").wont_be_nil end + it "updates the ETag header of the parent directory" do + get "/jimmy/tasks/" + @old_etag = last_response.headers["ETag"] + + delete "/jimmy/tasks/home/trash" + + get "/jimmy/tasks/" + last_response.headers["ETag"].wont_be_nil + last_response.headers["ETag"].wont_equal @old_etag + end + describe "timestamps" do before do @old_timestamp = (2.seconds.ago.to_f * 1000).to_i diff --git a/spec/riak_spec.rb b/spec/riak_spec.rb index c34a601..d913064 100644 --- a/spec/riak_spec.rb +++ b/spec/riak_spec.rb @@ -32,6 +32,11 @@ describe "App with Riak backend" do last_modified.year.must_equal now.year last_modified.day.must_equal now.day end + + it "has an ETag header set" do + last_response.status.must_equal 200 + last_response.headers["ETag"].wont_be_nil + end end describe "GET data with custom content type" do @@ -102,6 +107,10 @@ describe "App with Riak backend" do data_bucket.get("jimmy:documents:bar").content_type.must_equal "text/plain; charset=utf-8" end + it "sets the ETag header" do + last_response.headers["ETag"].wont_be_nil + end + it "indexes the data set" do indexes = data_bucket.get("jimmy:documents:bar").indexes indexes["user_id_bin"].must_be_kind_of Set @@ -197,15 +206,18 @@ describe "App with Riak backend" do describe "with existing content" do before do put "/jimmy/documents/archive/foo", "lorem ipsum" - put "/jimmy/documents/archive/foo", "some awesome content" end it "saves the value" do + put "/jimmy/documents/archive/foo", "some awesome content" + last_response.status.must_equal 200 data_bucket.get("jimmy:documents/archive:foo").data.must_equal "some awesome content" end it "logs the operations" do + put "/jimmy/documents/archive/foo", "some awesome content" + objects = [] opslog_bucket.keys.each { |k| objects << opslog_bucket.get(k) rescue nil } @@ -220,21 +232,29 @@ describe "App with Riak backend" do update_entry.indexes["user_id_bin"].must_include "jimmy" end - describe "when no serializer is registered for the given content-type" do - before do - header "Content-Type", "text/html; charset=UTF-8" - put "/jimmy/documents/html", '' - put "/jimmy/documents/html", '' - end + it "changes the ETag header" do + old_etag = last_response.headers["ETag"] + put "/jimmy/documents/archive/foo", "some awesome content" - it "saves the value" do - last_response.status.must_equal 200 - data_bucket.get("jimmy:documents:html").raw_data.must_equal "" - end + last_response.headers["ETag"].wont_be_nil + last_response.headers["ETag"].wont_equal old_etag + end + end - it "uses the requested content type" do - data_bucket.get("jimmy:documents:html").content_type.must_equal "text/html; charset=UTF-8" - end + describe "exsting content without serializer registered for the given content-type" do + before do + header "Content-Type", "text/html; charset=UTF-8" + put "/jimmy/documents/html", '' + put "/jimmy/documents/html", '' + end + + it "saves the value" do + last_response.status.must_equal 200 + data_bucket.get("jimmy:documents:html").raw_data.must_equal "" + end + + it "uses the requested content type" do + data_bucket.get("jimmy:documents:html").content_type.must_equal "text/html; charset=UTF-8" end end @@ -396,6 +416,10 @@ describe "App with Riak backend" do log_entry.indexes["user_id_bin"].must_include "jimmy" end + it "sets the ETag header" do + last_response.headers["ETag"].wont_be_nil + end + context "non-existing object" do before do delete "/jimmy/documents/foozius" From 67435157edd64036e84b204c4c0548be7972e8d8 Mon Sep 17 00:00:00 2001 From: Garret Alfert Date: Sat, 26 Oct 2013 03:31:54 +0200 Subject: [PATCH 04/13] Specs for If-Match header --- spec/riak_spec.rb | 93 ++++++++++++++++++++++++++++++++++++++--------- 1 file changed, 75 insertions(+), 18 deletions(-) diff --git a/spec/riak_spec.rb b/spec/riak_spec.rb index d913064..f7f3b8c 100644 --- a/spec/riak_spec.rb +++ b/spec/riak_spec.rb @@ -239,6 +239,29 @@ describe "App with Riak backend" do last_response.headers["ETag"].wont_be_nil last_response.headers["ETag"].wont_equal old_etag end + + describe "when If-Match header is set" do + it "allows the request if the header matches the current ETag" do + old_etag = last_response.headers["ETag"] + header "If-Match", old_etag + + put "/jimmy/documents/archive/foo", "some awesome content" + last_response.status.must_equal 200 + + get "/jimmy/documents/archive/foo" + last_response.body.must_equal "some awesome content" + end + + it "fails the request if the header does not match the current ETag" do + header "If-Match", "WONTMATCH" + + put "/jimmy/documents/archive/foo", "some awesome content" + last_response.status.must_equal 412 + + get "/jimmy/documents/archive/foo" + last_response.body.must_equal "lorem ipsum" + end + end end describe "exsting content without serializer registered for the given content-type" do @@ -396,28 +419,33 @@ describe "App with Riak backend" do describe "DELETE" do before do header "Authorization", "Bearer 123" - delete "/jimmy/documents/foo" end - it "removes the key" do - last_response.status.must_equal 204 - lambda { - data_bucket.get("jimmy:documents:foo") - }.must_raise Riak::HTTPFailedRequest - end + describe "basics" do + before do + delete "/jimmy/documents/foo" + end - it "logs the operation" do - objects = [] - opslog_bucket.keys.each { |k| objects << opslog_bucket.get(k) rescue nil } + it "removes the key" do + last_response.status.must_equal 204 + lambda { + data_bucket.get("jimmy:documents:foo") + }.must_raise Riak::HTTPFailedRequest + end - log_entry = objects.select{|o| o.data["count"] == -1}.first - log_entry.data["size"].must_equal(-22) - log_entry.data["category"].must_equal "documents" - log_entry.indexes["user_id_bin"].must_include "jimmy" - end + it "logs the operation" do + objects = [] + opslog_bucket.keys.each { |k| objects << opslog_bucket.get(k) rescue nil } - it "sets the ETag header" do - last_response.headers["ETag"].wont_be_nil + log_entry = objects.select{|o| o.data["count"] == -1}.first + log_entry.data["size"].must_equal(-22) + log_entry.data["category"].must_equal "documents" + log_entry.indexes["user_id_bin"].must_include "jimmy" + end + + it "sets the ETag header" do + last_response.headers["ETag"].wont_be_nil + end end context "non-existing object" do @@ -425,10 +453,39 @@ describe "App with Riak backend" do delete "/jimmy/documents/foozius" end + it "responds with 404" do + last_response.status.must_equal 404 + end + it "doesn't log the operation" do objects = [] opslog_bucket.keys.each { |k| objects << opslog_bucket.get(k) rescue nil } - objects.select{|o| o.data["count"] == -1}.size.must_equal 1 + objects.select{|o| o.data["count"] == -1}.size.must_equal 0 + end + end + + context "when an If-Match header is given" do + it "allows the request if it matches the current ETag" do + get "/jimmy/documents/foo" + old_etag = last_response.headers["ETag"] + header "If-Match", old_etag + + delete "/jimmy/documents/foo" + last_response.status.must_equal 204 + + get "/jimmy/documents/foo" + last_response.status.must_equal 404 + end + + it "fails the request if it does not match the current ETag" do + header "If-Match", "WONTMATCH" + + delete "/jimmy/documents/foo" + last_response.status.must_equal 412 + + get "/jimmy/documents/foo" + last_response.status.must_equal 200 + last_response.body.must_equal "some private text data" end end From f61eef717ea9776805a132f90556a5e23994cf29 Mon Sep 17 00:00:00 2001 From: Garret Alfert Date: Sat, 26 Oct 2013 04:37:33 +0200 Subject: [PATCH 05/13] Don't overwrite existing data when If-None-Match is "*" (refs #26) --- lib/remote_storage/riak.rb | 2 ++ spec/riak_spec.rb | 21 +++++++++++++++++++++ 2 files changed, 23 insertions(+) diff --git a/lib/remote_storage/riak.rb b/lib/remote_storage/riak.rb index 27ff1a3..94be481 100644 --- a/lib/remote_storage/riak.rb +++ b/lib/remote_storage/riak.rb @@ -95,6 +95,8 @@ module RemoteStorage object_exists = !object.raw_data.nil? existing_object_size = object_size(object) + server.halt 412 if object_exists && server.env["HTTP_IF_NONE_MATCH"] == "*" + timestamp = (Time.now.to_f * 1000).to_i object.meta["timestamp"] = timestamp diff --git a/spec/riak_spec.rb b/spec/riak_spec.rb index f7f3b8c..f70cc72 100644 --- a/spec/riak_spec.rb +++ b/spec/riak_spec.rb @@ -262,6 +262,27 @@ describe "App with Riak backend" do last_response.body.must_equal "lorem ipsum" end end + + describe "when If-None-Match header is set" do + before do + header "If-None-Match", "*" + end + + it "fails when the document already exists" do + put "/jimmy/documents/archive/foo", "some awesome content" + + last_response.status.must_equal 412 + + get "/jimmy/documents/archive/foo" + last_response.body.must_equal "lorem ipsum" + end + + it "succeeds when the document does not exist" do + put "/jimmy/documents/archive/bar", "my little content" + + last_response.status.must_equal 200 + end + end end describe "exsting content without serializer registered for the given content-type" do From f7eac9b411beb722771bc86dd07acfac76156539 Mon Sep 17 00:00:00 2001 From: Garret Alfert Date: Sat, 26 Oct 2013 04:57:44 +0200 Subject: [PATCH 06/13] Specs for If-None-Match on GET requests --- spec/directories_spec.rb | 24 ++++++++++++++++++++++++ spec/riak_spec.rb | 25 ++++++++++++++++++++++++- 2 files changed, 48 insertions(+), 1 deletion(-) diff --git a/spec/directories_spec.rb b/spec/directories_spec.rb index 56763be..8b7e806 100644 --- a/spec/directories_spec.rb +++ b/spec/directories_spec.rb @@ -65,6 +65,30 @@ describe "Directories" do last_response.headers["Access-Control-Allow-Headers"].must_equal "Authorization, Content-Type, Origin" end + context "when If-None-Match header is set" do + before do + get "/jimmy/tasks/" + + @etag = last_response.headers["ETag"] + end + + it "responds with 'not modified' when it matches the current ETag" do + header "If-None-Match", @etag + get "/jimmy/tasks/" + + last_response.status.must_equal 304 + last_response.body.must_be_empty + end + + it "responds normally when it does not match the current ETag" do + header "If-None-Match", "FOO" + get "/jimmy/tasks/" + + last_response.status.must_equal 200 + last_response.body.wont_be_empty + end + end + context "with sub-directories" do before do get "/jimmy/tasks/" diff --git a/spec/riak_spec.rb b/spec/riak_spec.rb index f70cc72..d908150 100644 --- a/spec/riak_spec.rb +++ b/spec/riak_spec.rb @@ -63,19 +63,42 @@ describe "App with Riak backend" do object.data = "some private text data" object.store + @etag = object.etag + auth = auth_bucket.new("jimmy:123") auth.data = ["documents", "public"] auth.store end describe "GET" do - it "returns the value" do + before do header "Authorization", "Bearer 123" + end + + it "returns the value" do get "/jimmy/documents/foo" last_response.status.must_equal 200 last_response.body.must_equal "some private text data" end + + describe "when If-None-Match header is set" do + it "responds with 'not modified' when it matches the current ETag" do + header "If-None-Match", @etag + get "/jimmy/documents/foo" + + last_response.status.must_equal 304 + last_response.body.must_be_empty + end + + it "responds normally when it does not match the current ETag" do + header "If-None-Match", "FOO" + get "/jimmy/documents/foo" + + last_response.status.must_equal 200 + last_response.body.must_equal "some private text data" + end + end end describe "GET nonexisting key" do From 3b5f99ee0d83635afe7f02eb4f8f46f86e322491 Mon Sep 17 00:00:00 2001 From: Garret Alfert Date: Sun, 27 Oct 2013 19:08:15 +0100 Subject: [PATCH 07/13] Refactor --- spec/directories_spec.rb | 13 ++++--------- 1 file changed, 4 insertions(+), 9 deletions(-) diff --git a/spec/directories_spec.rb b/spec/directories_spec.rb index 8b7e806..c3db0cb 100644 --- a/spec/directories_spec.rb +++ b/spec/directories_spec.rb @@ -382,9 +382,11 @@ describe "Directories" do describe "directory object" do describe "PUT file" do context "no existing directory object" do - it "creates a new directory object" do + before do put "/jimmy/tasks/home/trash", "take out the trash" + end + it "creates a new directory object" do object = data_bucket.get("jimmy:tasks/home:trash") directory = directory_bucket.get("jimmy:tasks/home") @@ -393,15 +395,11 @@ describe "Directories" do end it "sets the correct index for the directory object" do - put "/jimmy/tasks/home/trash", "take out the trash" - object = directory_bucket.get("jimmy:tasks/home") object.indexes["directory_bin"].must_include "tasks" end it "creates directory objects for the parent directories" do - put "/jimmy/tasks/home/trash", "take out the trash" - object = directory_bucket.get("jimmy:tasks") object.indexes["directory_bin"].must_include "/" object.data.wont_be_nil @@ -414,10 +412,7 @@ describe "Directories" do context "existing directory object" do before do - directory = directory_bucket.new("jimmy:tasks/home") - directory.content_type = "text/plain" - directory.data = (2.seconds.ago.to_f * 1000).to_i - directory.store + put "/jimmy/tasks/home/trash", "collect some trash" end it "updates the timestamp of the directory" do From 5215cdc5e0f06bff4b381d71790ed285680fcbd2 Mon Sep 17 00:00:00 2001 From: Garret Alfert Date: Sun, 27 Oct 2013 19:35:44 +0100 Subject: [PATCH 08/13] Spec for updating ETags of all parent directories on delete --- spec/directories_spec.rb | 20 +++++++++++++++++--- 1 file changed, 17 insertions(+), 3 deletions(-) diff --git a/spec/directories_spec.rb b/spec/directories_spec.rb index c3db0cb..ea4e287 100644 --- a/spec/directories_spec.rb +++ b/spec/directories_spec.rb @@ -504,15 +504,29 @@ describe "Directories" do directory_bucket.get("jimmy:").wont_be_nil end - it "updates the ETag header of the parent directory" do + it "updates the ETag headers of all parent directories" do + get "/jimmy/tasks/home/" + home_etag = last_response.headers["ETag"] + get "/jimmy/tasks/" - @old_etag = last_response.headers["ETag"] + tasks_etag = last_response.headers["ETag"] + + get "/jimmy/" + root_etag = last_response.headers["ETag"] delete "/jimmy/tasks/home/trash" + get "/jimmy/tasks/home/" + last_response.headers["ETag"].wont_be_nil + last_response.headers["ETag"].wont_equal home_etag + get "/jimmy/tasks/" last_response.headers["ETag"].wont_be_nil - last_response.headers["ETag"].wont_equal @old_etag + last_response.headers["ETag"].wont_equal tasks_etag + + get "/jimmy/" + last_response.headers["ETag"].wont_be_nil + last_response.headers["ETag"].wont_equal root_etag end describe "timestamps" do From b907f02f374123f3f288466bf39176ad565a7f5d Mon Sep 17 00:00:00 2001 From: Garret Alfert Date: Sun, 27 Oct 2013 20:15:40 +0100 Subject: [PATCH 09/13] Specs for ETag headers for binary files --- lib/remote_storage/riak.rb | 2 +- spec/riak_spec.rb | 30 ++++++++++++++++-------------- 2 files changed, 17 insertions(+), 15 deletions(-) diff --git a/lib/remote_storage/riak.rb b/lib/remote_storage/riak.rb index 94be481..df0b4c7 100644 --- a/lib/remote_storage/riak.rb +++ b/lib/remote_storage/riak.rb @@ -108,7 +108,7 @@ module RemoteStorage new_object_size = object.raw_data.size end - response = object.store + object.store log_count = object_exists ? 0 : 1 log_operation(user, directory, log_count, new_object_size, existing_object_size) diff --git a/spec/riak_spec.rb b/spec/riak_spec.rb index d908150..65bbfae 100644 --- a/spec/riak_spec.rb +++ b/spec/riak_spec.rb @@ -369,13 +369,23 @@ describe "App with Riak backend" do last_response.body.must_equal @image end - # it "indexes the binary set" do - # indexes = binary_bucket.get("jimmy:documents:jaypeg").indexes - # indexes["user_id_bin"].must_be_kind_of Set - # indexes["user_id_bin"].must_include "jimmy" + it "responds with an ETag header" do + last_response.headers["ETag"].wont_be_nil + etag = last_response.headers["ETag"] - # indexes["directory_bin"].must_include "documents" - # end + get "/jimmy/documents/jaypeg" + + last_response.headers["ETag"].wont_be_nil + last_response.headers["ETag"].must_equal etag + end + + it "changes the ETag when updating the file" do + old_etag = last_response.headers["ETag"] + put "/jimmy/documents/jaypeg", @image + + last_response.headers["ETag"].wont_be_nil + last_response.headers["ETag"].wont_equal old_etag + end it "logs the operation" do objects = [] @@ -409,14 +419,6 @@ describe "App with Riak backend" do last_response.status.must_equal 200 last_response.body.must_equal @image end - - # it "indexes the binary set" do - # indexes = binary_bucket.get("jimmy:documents:jaypeg").indexes - # indexes["user_id_bin"].must_be_kind_of Set - # indexes["user_id_bin"].must_include "jimmy" - - # indexes["directory_bin"].must_include "documents" - # end end end From 2a2cc0a3ec15f8ec88c26a799a43a49967337798 Mon Sep 17 00:00:00 2001 From: Garret Alfert Date: Sun, 27 Oct 2013 20:38:27 +0100 Subject: [PATCH 10/13] Allow If-Match and If-None-Match headers via CORS --- liquor-cabinet.rb | 2 +- spec/directories_spec.rb | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/liquor-cabinet.rb b/liquor-cabinet.rb index 1df2e25..d2d1c70 100644 --- a/liquor-cabinet.rb +++ b/liquor-cabinet.rb @@ -42,7 +42,7 @@ class LiquorCabinet < Sinatra::Base before path do headers 'Access-Control-Allow-Origin' => '*', 'Access-Control-Allow-Methods' => 'GET, PUT, DELETE', - 'Access-Control-Allow-Headers' => 'Authorization, Content-Type, Origin' + 'Access-Control-Allow-Headers' => 'Authorization, Content-Type, Origin, If-Match, If-None-Match', headers['Access-Control-Allow-Origin'] = env["HTTP_ORIGIN"] if env["HTTP_ORIGIN"] headers['Cache-Control'] = 'no-cache' diff --git a/spec/directories_spec.rb b/spec/directories_spec.rb index ea4e287..9af309c 100644 --- a/spec/directories_spec.rb +++ b/spec/directories_spec.rb @@ -62,7 +62,7 @@ describe "Directories" do last_response.status.must_equal 200 last_response.headers["Access-Control-Allow-Origin"].must_equal "*" last_response.headers["Access-Control-Allow-Methods"].must_equal "GET, PUT, DELETE" - last_response.headers["Access-Control-Allow-Headers"].must_equal "Authorization, Content-Type, Origin" + last_response.headers["Access-Control-Allow-Headers"].must_equal "Authorization, Content-Type, Origin, If-Match, If-None-Match" end context "when If-None-Match header is set" do @@ -437,7 +437,7 @@ describe "Directories" do last_response.headers["Access-Control-Allow-Origin"].must_equal "*" last_response.headers["Access-Control-Allow-Methods"].must_equal "GET, PUT, DELETE" - last_response.headers["Access-Control-Allow-Headers"].must_equal "Authorization, Content-Type, Origin" + last_response.headers["Access-Control-Allow-Headers"].must_equal "Authorization, Content-Type, Origin, If-Match, If-None-Match" end context "sub-directories" do @@ -448,7 +448,7 @@ describe "Directories" do last_response.headers["Access-Control-Allow-Origin"].must_equal "*" last_response.headers["Access-Control-Allow-Methods"].must_equal "GET, PUT, DELETE" - last_response.headers["Access-Control-Allow-Headers"].must_equal "Authorization, Content-Type, Origin" + last_response.headers["Access-Control-Allow-Headers"].must_equal "Authorization, Content-Type, Origin, If-Match, If-None-Match" end end @@ -460,7 +460,7 @@ describe "Directories" do last_response.headers["Access-Control-Allow-Origin"].must_equal "*" last_response.headers["Access-Control-Allow-Methods"].must_equal "GET, PUT, DELETE" - last_response.headers["Access-Control-Allow-Headers"].must_equal "Authorization, Content-Type, Origin" + last_response.headers["Access-Control-Allow-Headers"].must_equal "Authorization, Content-Type, Origin, If-Match, If-None-Match" end end end From a470b37c2c0e6942690dea33896dc812f3c38761 Mon Sep 17 00:00:00 2001 From: Garret Alfert Date: Sun, 27 Oct 2013 20:52:13 +0100 Subject: [PATCH 11/13] Allow to expose ETag header via CORS --- liquor-cabinet.rb | 1 + spec/directories_spec.rb | 4 ++++ 2 files changed, 5 insertions(+) diff --git a/liquor-cabinet.rb b/liquor-cabinet.rb index d2d1c70..71eecd1 100644 --- a/liquor-cabinet.rb +++ b/liquor-cabinet.rb @@ -43,6 +43,7 @@ class LiquorCabinet < Sinatra::Base headers 'Access-Control-Allow-Origin' => '*', 'Access-Control-Allow-Methods' => 'GET, PUT, DELETE', 'Access-Control-Allow-Headers' => 'Authorization, Content-Type, Origin, If-Match, If-None-Match', + 'Access-Control-Expose-Headers' => 'ETag' headers['Access-Control-Allow-Origin'] = env["HTTP_ORIGIN"] if env["HTTP_ORIGIN"] headers['Cache-Control'] = 'no-cache' diff --git a/spec/directories_spec.rb b/spec/directories_spec.rb index 9af309c..704f33b 100644 --- a/spec/directories_spec.rb +++ b/spec/directories_spec.rb @@ -63,6 +63,7 @@ describe "Directories" do last_response.headers["Access-Control-Allow-Origin"].must_equal "*" last_response.headers["Access-Control-Allow-Methods"].must_equal "GET, PUT, DELETE" last_response.headers["Access-Control-Allow-Headers"].must_equal "Authorization, Content-Type, Origin, If-Match, If-None-Match" + last_response.headers["Access-Control-Expose-Headers"].must_equal "ETag" end context "when If-None-Match header is set" do @@ -438,6 +439,7 @@ describe "Directories" do last_response.headers["Access-Control-Allow-Origin"].must_equal "*" last_response.headers["Access-Control-Allow-Methods"].must_equal "GET, PUT, DELETE" last_response.headers["Access-Control-Allow-Headers"].must_equal "Authorization, Content-Type, Origin, If-Match, If-None-Match" + last_response.headers["Access-Control-Expose-Headers"].must_equal "ETag" end context "sub-directories" do @@ -449,6 +451,7 @@ describe "Directories" do last_response.headers["Access-Control-Allow-Origin"].must_equal "*" last_response.headers["Access-Control-Allow-Methods"].must_equal "GET, PUT, DELETE" last_response.headers["Access-Control-Allow-Headers"].must_equal "Authorization, Content-Type, Origin, If-Match, If-None-Match" + last_response.headers["Access-Control-Expose-Headers"].must_equal "ETag" end end @@ -461,6 +464,7 @@ describe "Directories" do last_response.headers["Access-Control-Allow-Origin"].must_equal "*" last_response.headers["Access-Control-Allow-Methods"].must_equal "GET, PUT, DELETE" last_response.headers["Access-Control-Allow-Headers"].must_equal "Authorization, Content-Type, Origin, If-Match, If-None-Match" + last_response.headers["Access-Control-Expose-Headers"].must_equal "ETag" end end end From 39dac0a5edd28b3e262e2913958380d72695f5db Mon Sep 17 00:00:00 2001 From: Garret Alfert Date: Sun, 27 Oct 2013 21:11:15 +0100 Subject: [PATCH 12/13] Remove some duplication --- lib/remote_storage/riak.rb | 40 +++++++++++++++++--------------------- 1 file changed, 18 insertions(+), 22 deletions(-) diff --git a/lib/remote_storage/riak.rb b/lib/remote_storage/riak.rb index df0b4c7..3ca5b62 100644 --- a/lib/remote_storage/riak.rb +++ b/lib/remote_storage/riak.rb @@ -260,12 +260,7 @@ module RemoteStorage end def directory_entries(user, directory) - directory = "/" if directory == "" - - user_keys = data_bucket.get_index("user_id_bin", user) - directory_keys = data_bucket.get_index("directory_bin", directory) - - all_keys = user_keys & directory_keys + all_keys = user_directory_keys(user, directory, data_bucket) return [] if all_keys.empty? map_query = <<-EOH @@ -283,23 +278,11 @@ module RemoteStorage } EOH - map_reduce = ::Riak::MapReduce.new(client) - all_keys.each do |key| - map_reduce.add(data_bucket.name, key) - end - - map_reduce. - map(map_query, :keep => true). - run + run_map_reduce(data_bucket, all_keys, map_query) end def sub_directories(user, directory) - directory = "/" if directory == "" - - user_keys = directory_bucket.get_index("user_id_bin", user) - directory_keys = directory_bucket.get_index("directory_bin", directory) - - all_keys = user_keys & directory_keys + all_keys = user_directory_keys(user, directory, directory_bucket) return [] if all_keys.empty? map_query = <<-EOH @@ -314,9 +297,22 @@ module RemoteStorage } EOH + run_map_reduce(directory_bucket, all_keys, map_query) + end + + def user_directory_keys(user, directory, bucket) + directory = "/" if directory == "" + + user_keys = bucket.get_index("user_id_bin", user) + directory_keys = bucket.get_index("directory_bin", directory) + + user_keys & directory_keys + end + + def run_map_reduce(bucket, keys, map_query) map_reduce = ::Riak::MapReduce.new(client) - all_keys.each do |key| - map_reduce.add(directory_bucket.name, key) + keys.each do |key| + map_reduce.add(bucket.name, key) end map_reduce. From 256b3c426e0b23f594dfaa502a90f8ef8a753ac5 Mon Sep 17 00:00:00 2001 From: Garret Alfert Date: Sun, 27 Oct 2013 22:09:47 +0100 Subject: [PATCH 13/13] Use ETags as version in directory listings --- lib/remote_storage/riak.rb | 12 +++++++++--- spec/directories_spec.rb | 37 ++++++++++++++++++++++--------------- 2 files changed, 31 insertions(+), 18 deletions(-) diff --git a/lib/remote_storage/riak.rb b/lib/remote_storage/riak.rb index 3ca5b62..768de46 100644 --- a/lib/remote_storage/riak.rb +++ b/lib/remote_storage/riak.rb @@ -241,8 +241,9 @@ module RemoteStorage sub_directories(user, directory).each do |entry| directory_name = entry["name"].split("/").last timestamp = entry["timestamp"].to_i + etag = entry["etag"] - listing.merge!({ "#{directory_name}/" => timestamp }) + listing.merge!({ "#{directory_name}/" => etag }) end directory_entries(user, directory).each do |entry| @@ -252,8 +253,9 @@ module RemoteStorage else DateTime.rfc2822(entry["last_modified"]).to_time.to_i end + etag = entry["etag"] - listing.merge!({ entry_name => timestamp }) + listing.merge!({ entry_name => etag }) end listing @@ -270,10 +272,12 @@ module RemoteStorage key_name = keys.join(':'); last_modified_date = v.values[0]['metadata']['X-Riak-Last-Modified']; timestamp = v.values[0]['metadata']['X-Riak-Meta']['X-Riak-Meta-Timestamp']; + etag = v.values[0]['metadata']['X-Riak-VTag']; return [{ name: key_name, last_modified: last_modified_date, timestamp: timestamp, + etag: etag }]; } EOH @@ -289,10 +293,12 @@ module RemoteStorage function(v){ keys = v.key.split(':'); key_name = keys[keys.length-1]; - timestamp = v.values[0]['data'] + timestamp = v.values[0]['data']; + etag = v.values[0]['metadata']['X-Riak-VTag']; return [{ name: key_name, timestamp: timestamp, + etag: etag }]; } EOH diff --git a/spec/directories_spec.rb b/spec/directories_spec.rb index 704f33b..1e5f656 100644 --- a/spec/directories_spec.rb +++ b/spec/directories_spec.rb @@ -19,17 +19,18 @@ describe "Directories" do put "/jimmy/tasks/http%3A%2F%2F5apps.com", "prettify design" end - it "lists the objects with a timestamp of the last modification" do + it "lists the objects with their version" do get "/jimmy/tasks/" last_response.status.must_equal 200 last_response.content_type.must_equal "application/json" + foo = data_bucket.get("jimmy:tasks:foo") + content = JSON.parse(last_response.body) content.must_include "http://5apps.com" content.must_include "foo" - content["foo"].must_be_kind_of Integer - content["foo"].to_s.length.must_equal 13 + content["foo"].must_equal foo.etag.gsub(/"/, "") end it "has a Last-Modifier header set" do @@ -103,12 +104,13 @@ describe "Directories" do last_response.status.must_equal 200 + home = directory_bucket.get("jimmy:tasks/home") + content = JSON.parse(last_response.body) content.must_include "foo" content.must_include "http://5apps.com" content.must_include "home/" - content["home/"].must_be_kind_of Integer - content["home/"].to_s.length.must_equal 13 + content["home/"].must_equal home.etag.gsub(/"/, "") end it "updates the ETag of the parent directory" do @@ -149,10 +151,11 @@ describe "Directories" do last_response.status.must_equal 200 + projects = directory_bucket.get("jimmy:tasks/private/projects") + content = JSON.parse(last_response.body) content.must_include "projects/" - content["projects/"].must_be_kind_of Integer - content["projects/"].to_s.length.must_equal 13 + content["projects/"].must_equal projects.etag.gsub(/"/, "") end it "updates the timestamps of the existing directory objects" do @@ -184,10 +187,11 @@ describe "Directories" do last_response.status.must_equal 200 + jaypeg = data_bucket.get("jimmy:tasks:jaypeg.jpg") + content = JSON.parse(last_response.body) content.must_include "jaypeg.jpg" - content["jaypeg.jpg"].must_be_kind_of Integer - content["jaypeg.jpg"].to_s.length.must_equal 13 + content["jaypeg.jpg"].must_equal jaypeg.etag.gsub(/"/, "") end end @@ -204,10 +208,11 @@ describe "Directories" do last_response.status.must_equal 200 + jaypeg = data_bucket.get("jimmy:tasks:jaypeg.jpg") + content = JSON.parse(last_response.body) content.must_include "jaypeg.jpg" - content["jaypeg.jpg"].must_be_kind_of Integer - content["jaypeg.jpg"].to_s.length.must_equal 13 + content["jaypeg.jpg"].must_equal jaypeg.etag.gsub(/"/, "") end end end @@ -223,10 +228,11 @@ describe "Directories" do last_response.status.must_equal 200 + laundry = data_bucket.get("jimmy:tasks/home:laundry") + content = JSON.parse(last_response.body) content.must_include "laundry" - content["laundry"].must_be_kind_of Integer - content["laundry"].to_s.length.must_equal 13 + content["laundry"].must_equal laundry.etag.gsub(/"/, "") end end @@ -300,12 +306,13 @@ describe "Directories" do last_response.status.must_equal 200 + tasks = directory_bucket.get("jimmy:tasks") + content = JSON.parse(last_response.body) content.must_include "root-1" content.must_include "root-2" content.must_include "tasks/" - content["tasks/"].must_be_kind_of Integer - content["tasks/"].to_s.length.must_equal 13 + content["tasks/"].must_equal tasks.etag.gsub(/"/, "") end it "has an ETag header set" do