From fac033cff5f840968365161585b1debf495fc473 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Greg=20Kar=C3=A9kinian?= Date: Thu, 8 Mar 2018 12:59:03 +0100 Subject: [PATCH 01/29] Use all keys from the config file's redis section in the specs You can pass a db index, and the code was using it, but not the specs --- spec/spec_helper.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/spec/spec_helper.rb b/spec/spec_helper.rb index dfc955f..8c46f3a 100644 --- a/spec/spec_helper.rb +++ b/spec/spec_helper.rb @@ -35,7 +35,7 @@ alias context describe if app.settings.respond_to? :redis def redis - @redis ||= Redis.new(host: app.settings.redis["host"], port: app.settings.redis["port"]) + @redis ||= Redis.new(app.settings.redis.symbolize_keys) end def purge_redis From 60c508fba9114cd4d7c977422e0053ea8ca282f5 Mon Sep 17 00:00:00 2001 From: Sebastian Kippe Date: Mon, 12 Mar 2018 16:32:00 +0200 Subject: [PATCH 02/29] Add Last-Modified date to document items in listings Not in RS spec draft yet, see https://github.com/remotestorage/spec/issues/158 for discussion. --- lib/remote_storage/swift.rb | 9 ++++++++- spec/swift/app_spec.rb | 1 + 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/lib/remote_storage/swift.rb b/lib/remote_storage/swift.rb index 9a531ff..d104f11 100644 --- a/lib/remote_storage/swift.rb +++ b/lib/remote_storage/swift.rb @@ -126,13 +126,20 @@ module RemoteStorage if string.sub(name, -1) ~= "/" then listing[name]["Content-Type"] = metadata["t"] listing[name]["Content-Length"] = tonumber(metadata["s"]) + listing[name]["Last-Modified"] = tonumber(metadata["m"]) end end return cjson.encode(listing) EOF - JSON.parse(redis.eval(lua_script, nil, [user, directory])) + items = JSON.parse(redis.eval(lua_script, nil, [user, directory])) + + items.reject{|k, _| k.end_with? "/"}.each do |_, v| + v["Last-Modified"] = Time.at(v["Last-Modified"]/1000).httpdate + end + + items end def put_data(user, directory, key, data, content_type) diff --git a/spec/swift/app_spec.rb b/spec/swift/app_spec.rb index ae29074..ae207c7 100644 --- a/spec/swift/app_spec.rb +++ b/spec/swift/app_spec.rb @@ -152,6 +152,7 @@ describe "App" do content["items"]["bamboo.txt"]["ETag"].must_equal "bla" content["items"]["bamboo.txt"]["Content-Type"].must_equal "text/plain; charset=utf-8" content["items"]["bamboo.txt"]["Content-Length"].must_equal 8 + content["items"]["bamboo.txt"]["Last-Modified"].must_equal "Fri, 04 Mar 2016 12:20:18 GMT" end end From 21061324dce31eed4447f9b4f2a55913ae0f1935 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Greg=20Kar=C3=A9kinian?= Date: Sun, 15 Apr 2018 20:03:49 +0200 Subject: [PATCH 03/29] Do not install Riak CS, run Travis builds on Trusty containers --- .travis.yml | 4 ---- 1 file changed, 4 deletions(-) diff --git a/.travis.yml b/.travis.yml index f7fa338..0f38df9 100644 --- a/.travis.yml +++ b/.travis.yml @@ -5,7 +5,6 @@ rvm: services: - redis-server before_install: - - sh .travis/install_riakcs.sh - gem install bundler before_script: - cp config.yml.example config.yml @@ -22,6 +21,3 @@ notifications: - http://hook-juggler.herokuapp.com/hooks/travis on_success: always on_failure: always -# Force legacy Blue Box build for now -sudo: required -group: legacy From ac6bec48c40309ea050871f384ac7c69b89cf040 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Greg=20Kar=C3=A9kinian?= Date: Mon, 16 Apr 2018 11:54:32 +0200 Subject: [PATCH 04/29] Remove all Riak code and its dependencies Closes #75 --- Gemfile | 2 - Gemfile.lock | 40 +- README.md | 4 +- config.yml.example | 24 +- lib/remote_storage/riak.rb | 531 ----------------------- liquor-cabinet.rb | 5 +- spec/riak/app_spec.rb | 15 - spec/riak/directories_spec.rb | 640 ---------------------------- spec/riak/permissions_spec.rb | 424 ------------------- spec/riak/riak_spec.rb | 775 ---------------------------------- spec/spec_helper.rb | 72 ---- 11 files changed, 5 insertions(+), 2527 deletions(-) delete mode 100644 lib/remote_storage/riak.rb delete mode 100644 spec/riak/app_spec.rb delete mode 100644 spec/riak/directories_spec.rb delete mode 100644 spec/riak/permissions_spec.rb delete mode 100644 spec/riak/riak_spec.rb diff --git a/Gemfile b/Gemfile index 49b11a4..6f8b5eb 100644 --- a/Gemfile +++ b/Gemfile @@ -3,8 +3,6 @@ source "https://rubygems.org" gem "sinatra" gem "sinatra-contrib" gem "activesupport" -gem "riak-client", git: "https://github.com/5apps/riak-ruby-client", branch: "invalid_uri_error" -gem "fog-aws" gem "rest-client", "~> 2.1.0.rc1" # Fixes a memory leak in Ruby 2.4 gem "redis" # Remove require when we can update to 3.0, which sets the new storage diff --git a/Gemfile.lock b/Gemfile.lock index e794749..5365411 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -1,15 +1,3 @@ -GIT - remote: https://github.com/5apps/riak-ruby-client - revision: 5f21df86b14339aeb252374851d29ad813cca1dd - branch: invalid_uri_error - specs: - riak-client (1.4.0) - beefcake (~> 0.3.7) - builder (>= 2.1.2) - i18n (>= 0.4.0) - innertube (~> 1.0.2) - multi_json (~> 1.0) - GEM remote: https://rubygems.org/ specs: @@ -19,36 +7,15 @@ GEM minitest (~> 5.1) tzinfo (~> 1.1) backports (3.8.0) - beefcake (0.3.7) - builder (3.2.3) concurrent-ruby (1.0.5) domain_name (0.5.20170404) unf (>= 0.0.5, < 1.0.0) - excon (0.57.1) faraday (0.12.1) multipart-post (>= 1.2, < 3) - fog-aws (1.4.0) - fog-core (~> 1.38) - fog-json (~> 1.0) - fog-xml (~> 0.1) - ipaddress (~> 0.8) - fog-core (1.44.3) - builder - excon (~> 0.49) - formatador (~> 0.2) - fog-json (1.0.2) - fog-core (~> 1.0) - multi_json (~> 1.10) - fog-xml (0.1.3) - fog-core - nokogiri (>= 1.5.11, < 2.0.0) - formatador (0.2.5) http-accept (1.7.0) http-cookie (1.0.3) domain_name (~> 0.5) i18n (0.8.4) - innertube (1.0.2) - ipaddress (0.8.3) kgio (2.11.0) m (1.5.1) method_source (>= 0.6.7) @@ -57,15 +24,12 @@ GEM mime-types (3.1) mime-types-data (~> 3.2015) mime-types-data (3.2016.0521) - mini_portile2 (2.2.0) minitest (5.10.2) minitest-stub_any_instance (1.0.1) multi_json (1.12.1) multipart-post (2.0.0) mustermann (1.0.0) netrc (0.11.0) - nokogiri (1.8.0) - mini_portile2 (~> 2.2.0) purdytest (2.0.0) minitest (~> 5.5) rack (2.0.3) @@ -115,7 +79,6 @@ PLATFORMS DEPENDENCIES activesupport - fog-aws m mime-types minitest-stub_any_instance @@ -125,10 +88,9 @@ DEPENDENCIES rake redis rest-client (~> 2.1.0.rc1) - riak-client! sentry-raven sinatra sinatra-contrib BUNDLED WITH - 1.15.1 + 1.16.0 diff --git a/README.md b/README.md index aa52a0a..9f3af01 100644 --- a/README.md +++ b/README.md @@ -4,8 +4,8 @@ Liquor Cabinet is where Frank stores all his stuff. It's a remoteStorage-compatible storage provider API, based on Sinatra and currently -using Riak as backend. You can use it on its own, or e.g. mount it from a Rails -application. +using Openstack Swift as backend. You can use it on its own, or e.g. mount it +from a Rails application. It's merely implementing the storage API, not including the Webfinger and OAuth parts of remoteStorage. You have to set the authorization keys/values in the diff --git a/config.yml.example b/config.yml.example index 6fb3307..b44c579 100644 --- a/config.yml.example +++ b/config.yml.example @@ -1,19 +1,6 @@ development: &defaults maintenance: false - # riak: &riak_defaults - # host: localhost - # http_port: 8098 - # riak_cs: - # credentials_file: "cs_credentials.json" - # endpoint: "http://cs.example.com:8080" - # buckets: - # data: rs_data - # directories: rs_directories - # binaries: rs_binaries - # cs_binaries: rs.binaries - # authorizations: rs_authorizations - # opslog: rs_opslog - # # uncomment this section and comment the riak one + # # uncomment this section # swift: &swift_defaults # host: "https://swift.example.com" # # Redis is needed for the swift backend @@ -23,15 +10,6 @@ development: &defaults test: <<: *defaults - # riak: - # <<: *riak_defaults - # buckets: - # data: rs_data_test - # directories: rs_directories_test - # binaries: rs_binaries_test - # cs_binaries: rs.binaries.test - # authorizations: rs_authorizations_test - # opslog: rs_opslog_test swift: host: "https://swift.example.com" redis: diff --git a/lib/remote_storage/riak.rb b/lib/remote_storage/riak.rb deleted file mode 100644 index 17bd69b..0000000 --- a/lib/remote_storage/riak.rb +++ /dev/null @@ -1,531 +0,0 @@ -require "riak" -require "json" -require "cgi" -require "active_support/core_ext/time/conversions" -require "active_support/core_ext/numeric/time" - -module RemoteStorage - class Riak - - ::Riak.url_decoding = true - - attr_accessor :settings, :server, :cs_credentials - - def initialize(settings, server) - self.settings = settings - self.server = server - - credentials = File.read(settings.riak['riak_cs']['credentials_file']) - self.cs_credentials = JSON.parse(credentials) - end - - def authorize_request(user, directory, token, listing=false) - request_method = server.env["REQUEST_METHOD"] - - if directory.split("/").first == "public" - return true if ["GET", "HEAD"].include?(request_method) && !listing - end - - authorizations = auth_bucket.get("#{user}:#{token}").data - permission = directory_permission(authorizations, directory) - - server.halt 401 unless permission - if ["PUT", "DELETE"].include? request_method - server.halt 401 unless permission == "rw" - end - rescue ::Riak::HTTPFailedRequest - server.halt 401 - end - - def get_head(user, directory, key) - object = data_bucket.get("#{user}:#{directory}:#{key}") - set_object_response_headers(object) - server.halt 200 - rescue ::Riak::HTTPFailedRequest - server.halt 404 - end - - def get_data(user, directory, key) - object = data_bucket.get("#{user}:#{directory}:#{key}") - - set_object_response_headers(object) - - none_match = (server.env["HTTP_IF_NONE_MATCH"] || "").split(",").map(&:strip) - server.halt 304 if none_match.include? object.etag - - if binary_key = object.meta["binary_key"] - object = cs_binary_bucket.files.get(binary_key[0]) - - case object.content_type[/^[^;\s]+/] - when "application/json" - return object.body.to_json - else - return object.body - end - end - - case object.content_type[/^[^;\s]+/] - when "application/json" - return object.data.to_json - else - data = serializer_for(object.content_type) ? object.data : object.raw_data - - # Never return nil, always turn data into a string - return data.nil? ? '' : data - end - rescue ::Riak::HTTPFailedRequest - server.halt 404 - end - - def get_head_directory_listing(user, directory) - directory_object = directory_bucket.get("#{user}:#{directory}") - set_directory_response_headers(directory_object) - server.halt 200 - rescue ::Riak::HTTPFailedRequest - server.halt 404 - end - - def get_directory_listing(user, directory) - directory_object = directory_bucket.get("#{user}:#{directory}") - - set_directory_response_headers(directory_object) - - none_match = (server.env["HTTP_IF_NONE_MATCH"] || "").split(",").map(&:strip) - server.halt 304 if none_match.include? directory_object.etag - - listing = directory_listing(user, directory) - - return listing.to_json - rescue ::Riak::HTTPFailedRequest - server.halt 404 - end - - def put_data(user, directory, key, data, content_type=nil) - server.halt 409 if has_name_collision?(user, directory, key) - - object = build_data_object(user, directory, key, data, content_type) - - if required_match = server.env["HTTP_IF_MATCH"] - server.halt 412 unless required_match == object.etag - end - - object_exists = !object.raw_data.nil? || !object.meta["binary_key"].nil? - existing_object_size = object_size(object) - - server.halt 412 if object_exists && server.env["HTTP_IF_NONE_MATCH"] == "*" - - timestamp = (Time.now.to_f * 1000).to_i - object.meta["timestamp"] = timestamp - - if binary_data?(object.content_type, data) - save_binary_data(object, data) or server.halt 422 - new_object_size = data.size - else - set_object_data(object, data) or server.halt 422 - new_object_size = object.raw_data.size - end - - object.store - - log_count = object_exists ? 0 : 1 - log_operation(user, directory, log_count, new_object_size, existing_object_size) - - update_all_directory_objects(user, directory, timestamp) - - server.headers["ETag"] = object.etag - server.halt object_exists ? 200 : 201 - rescue ::Riak::HTTPFailedRequest - server.halt 422 - end - - def delete_data(user, directory, key) - object = data_bucket.get("#{user}:#{directory}:#{key}") - existing_object_size = object_size(object) - etag = object.etag - - if required_match = server.env["HTTP_IF_MATCH"] - server.halt 412 unless required_match == etag - end - - if binary_key = object.meta["binary_key"] - object = cs_binary_bucket.files.get(binary_key[0]) - object.destroy - end - - riak_response = data_bucket.delete("#{user}:#{directory}:#{key}") - - if riak_response[:code] != 404 - log_operation(user, directory, -1, 0, existing_object_size) - end - - timestamp = (Time.now.to_f * 1000).to_i - delete_or_update_directory_objects(user, directory, timestamp) - - server.halt 200 - rescue ::Riak::HTTPFailedRequest - server.halt 404 - end - - private - - def set_object_response_headers(object) - server.headers["Content-Type"] = object.content_type - server.headers["ETag"] = object.etag - server.headers["Content-Length"] = object_size(object).to_s - end - - def set_directory_response_headers(directory_object) - server.headers["Content-Type"] = "application/json" - server.headers["ETag"] = directory_object.etag - end - - def extract_category(directory) - if directory.match(/^public\//) - "public/#{directory.split('/')[1]}" - else - directory.split('/').first - end - end - - def build_data_object(user, directory, key, data, content_type=nil) - object = data_bucket.get_or_new("#{user}:#{directory}:#{key}") - - object.content_type = content_type || "text/plain; charset=utf-8" - - directory_index = directory == "" ? "/" : directory - object.indexes.merge!({:user_id_bin => [user], - :directory_bin => [directory_index]}) - - object - end - - def log_operation(user, directory, count, new_size=0, old_size=0) - size = (-old_size + new_size) - return if count == 0 && size == 0 - - log_entry = opslog_bucket.new - log_entry.content_type = "application/json" - log_entry.data = { - "count" => count, - "size" => size, - "category" => extract_category(directory) - } - log_entry.indexes.merge!({:user_id_bin => [user]}) - log_entry.store - end - - def object_size(object) - if binary_key = object.meta["binary_key"] - response = cs_client.head_object cs_binary_bucket.key, binary_key[0] - response.headers["Content-Length"].to_i - else - object.raw_data.nil? ? 0 : object.raw_data.size - end - end - - def escape(string) - ::Riak.escaper.escape(string).gsub("+", "%20").gsub('/', "%2F") - end - - # A URI object that can be used with HTTP backend methods - def riak_uri(bucket, key) - URI.parse "http://#{settings.riak["host"]}:#{settings.riak["http_port"]}/riak/#{bucket}/#{key}" - end - - def serializer_for(content_type) - ::Riak::Serializers[content_type[/^[^;\s]+/]] - end - - def directory_permission(authorizations, directory) - authorizations = authorizations.map do |auth| - auth.index(":") ? auth.split(":") : [auth, "rw"] - end - authorizations = Hash[*authorizations.flatten] - - permission = authorizations[""] - - authorizations.each do |key, value| - if directory.match(/^(public\/)?#{key}(\/|$)/) - if permission.nil? || permission == "r" - permission = value - end - return permission if permission == "rw" - end - end - - permission - end - - def directory_listing(user, directory) - listing = { - "@context" => "http://remotestorage.io/spec/folder-description", - "items" => {} - } - - sub_directories(user, directory).each do |entry| - directory_name = entry["name"].split("/").last - etag = entry["etag"] - - listing["items"].merge!({ "#{directory_name}/" => { "ETag" => etag }}) - end - - directory_entries(user, directory).each do |entry| - entry_name = entry["name"] - etag = entry["etag"] - content_type = entry["contentType"] - content_length = entry["contentLength"].to_i - - listing["items"].merge!({ - entry_name => { - "ETag" => etag, - "Content-Type" => content_type, - "Content-Length" => content_length - } - }) - end - - listing - end - - def directory_entries(user, directory) - all_keys = user_directory_keys(user, directory, data_bucket) - return [] if all_keys.empty? - - map_query = <<-EOH - function(v){ - var metadata = v.values[0]['metadata']; - var dir_name = metadata['index']['directory_bin']; - if (dir_name === '/') { - dir_name = ''; - } - var name = v.key.match(/^[^:]*:(.*)/)[1]; // strip username from key - name = name.replace(dir_name + ':', ''); // strip directory from key - var etag = metadata['X-Riak-VTag']; - var contentType = metadata['content-type']; - var contentLength = metadata['X-Riak-Meta']['X-Riak-Meta-Content_length'] || 0; - - return [{ - name: name, - etag: etag, - contentType: contentType, - contentLength: contentLength - }]; - } - EOH - - run_map_reduce(data_bucket, all_keys, map_query) - end - - def sub_directories(user, directory) - all_keys = user_directory_keys(user, directory, directory_bucket) - return [] if all_keys.empty? - - map_query = <<-EOH - function(v){ - var name = v.key.match(/^[^:]*:(.*)/)[1]; // strip username from key - var etag = v.values[0]['metadata']['X-Riak-VTag']; - - return [{ - name: name, - etag: etag - }]; - } - EOH - - run_map_reduce(directory_bucket, all_keys, map_query) - end - - def user_directory_keys(user, directory, bucket) - directory = "/" if directory == "" - - user_keys = bucket.get_index("user_id_bin", user) - directory_keys = bucket.get_index("directory_bin", directory) - - user_keys & directory_keys - end - - def run_map_reduce(bucket, keys, map_query) - map_reduce = ::Riak::MapReduce.new(client) - keys.each do |key| - map_reduce.add(bucket.name, key) - end - - map_reduce. - map(map_query, :keep => true). - run - end - - def update_all_directory_objects(user, directory, timestamp) - parent_directories_for(directory).each do |parent_directory| - update_directory_object(user, parent_directory, timestamp) - end - end - - def update_directory_object(user, directory, timestamp) - if directory.match(/\//) - parent_directory = directory[0..directory.rindex("/")-1] - elsif directory != "" - parent_directory = "/" - end - - directory_object = directory_bucket.new("#{user}:#{directory}") - directory_object.content_type = "text/plain; charset=utf-8" - directory_object.data = timestamp.to_s - directory_object.indexes.merge!({:user_id_bin => [user]}) - if parent_directory - directory_object.indexes.merge!({:directory_bin => [parent_directory]}) - end - directory_object.store - end - - def delete_or_update_directory_objects(user, directory, timestamp) - parent_directories_for(directory).each do |parent_directory| - existing_files = directory_entries(user, parent_directory) - existing_subdirectories = sub_directories(user, parent_directory) - - if existing_files.empty? && existing_subdirectories.empty? - directory_bucket.delete "#{user}:#{parent_directory}" - else - update_directory_object(user, parent_directory, timestamp) - end - end - end - - def set_object_data(object, data) - if object.content_type[/^[^;\s]+/] == "application/json" - data = "{}" if data.blank? - data = JSON.parse(data) - end - - object.meta["content_length"] = data.size - - if serializer_for(object.content_type) - object.data = data - else - object.raw_data = data - end - rescue JSON::ParserError - return false - end - - def save_binary_data(object, data) - cs_binary_object = cs_binary_bucket.files.create( - :key => object.key, - :body => data, - :content_type => object.content_type - ) - - object.meta["binary_key"] = cs_binary_object.key - object.meta["content_length"] = cs_binary_object.content_length - object.raw_data = "" - end - - def binary_data?(content_type, data) - return true if content_type[/[^;\s]+$/] == "charset=binary" - - original_encoding = data.encoding - data.force_encoding("UTF-8") - is_binary = !data.valid_encoding? - data.force_encoding(original_encoding) - - is_binary - end - - def parent_directories_for(directory) - directories = directory.split("/") - parent_directories = [] - - while directories.any? - parent_directories << directories.join("/") - directories.pop - end - - parent_directories << "" - end - - def has_name_collision?(user, directory, key) - parent_directories = parent_directories_for(directory).reverse - parent_directories.shift # remove root dir entry - - # check for existing documents with the same name as one of the parent directories - parent_directories.each do |dir| - begin - parts = dir.split("/") - document_key = parts.pop - directory_name = parts.join("/") - data_bucket.get("#{user}:#{directory_name}:#{document_key}") - return true - rescue ::Riak::HTTPFailedRequest - end - end - - # check for an existing directory with same name as document - begin - directory_bucket.get("#{user}:#{directory}/#{key}") - return true - rescue ::Riak::HTTPFailedRequest - end - - false - end - - def client - @client ||= ::Riak::Client.new(:host => settings.riak['host'], - :http_port => settings.riak['http_port']) - end - - def data_bucket - @data_bucket ||= begin - bucket = client.bucket(settings.riak['buckets']['data']) - bucket.allow_mult = false - bucket - end - end - - def directory_bucket - @directory_bucket ||= begin - bucket = client.bucket(settings.riak['buckets']['directories']) - bucket.allow_mult = false - bucket - end - end - - def auth_bucket - @auth_bucket ||= begin - bucket = client.bucket(settings.riak['buckets']['authorizations']) - bucket.allow_mult = false - bucket - end - end - - def binary_bucket - @binary_bucket ||= begin - bucket = client.bucket(settings.riak['buckets']['binaries']) - bucket.allow_mult = false - bucket - end - end - - def opslog_bucket - @opslog_bucket ||= begin - bucket = client.bucket(settings.riak['buckets']['opslog']) - bucket.allow_mult = false - bucket - end - end - - def cs_client - @cs_client ||= Fog::Storage.new({ - :provider => 'AWS', - :aws_access_key_id => cs_credentials['key_id'], - :aws_secret_access_key => cs_credentials['key_secret'], - :endpoint => settings.riak['riak_cs']['endpoint'] - }) - end - - def cs_binary_bucket - @cs_binary_bucket ||= cs_client.directories.create(:key => settings.riak['buckets']['cs_binaries']) - end - - end -end diff --git a/liquor-cabinet.rb b/liquor-cabinet.rb index c350c90..d9a41fb 100644 --- a/liquor-cabinet.rb +++ b/liquor-cabinet.rb @@ -4,7 +4,6 @@ require "json" require "sinatra/base" require 'sinatra/config_file' require "sinatra/reloader" -require "remote_storage/riak" require "remote_storage/swift" class LiquorCabinet < Sinatra::Base @@ -129,9 +128,7 @@ class LiquorCabinet < Sinatra::Base def storage @storage ||= begin - if settings.respond_to? :riak - RemoteStorage::Riak.new(settings, self) - elsif settings.respond_to? :swift + if settings.respond_to? :swift RemoteStorage::Swift.new(settings, self) else puts <<-EOF diff --git a/spec/riak/app_spec.rb b/spec/riak/app_spec.rb deleted file mode 100644 index 411c6c4..0000000 --- a/spec/riak/app_spec.rb +++ /dev/null @@ -1,15 +0,0 @@ -require_relative "../spec_helper" - -describe "App" do - include Rack::Test::Methods - - def app - LiquorCabinet - end - - it "returns 404 on non-existing routes" do - get "/virginmargarita" - last_response.status.must_equal 404 - end - -end diff --git a/spec/riak/directories_spec.rb b/spec/riak/directories_spec.rb deleted file mode 100644 index eea4f82..0000000 --- a/spec/riak/directories_spec.rb +++ /dev/null @@ -1,640 +0,0 @@ -require_relative "../spec_helper" - -describe "Directories" do - include Rack::Test::Methods - - before do - purge_all_buckets - - auth = auth_bucket.new("jimmy:123") - auth.data = [":r", "documents:r", "tasks:rw"] - auth.store - - header "Authorization", "Bearer 123" - end - - describe "HEAD listing" do - before do - put "/jimmy/tasks/foo", "do the laundry" - put "/jimmy/tasks/http%3A%2F%2F5apps.com", "prettify design" - - head "/jimmy/tasks/" - end - - it "has an empty body" do - last_response.status.must_equal 200 - last_response.body.must_equal "" - end - - it "has an ETag header set" do - last_response.status.must_equal 200 - last_response.headers["ETag"].wont_be_nil - - # check that ETag stays the same - etag = last_response.headers["ETag"] - get "/jimmy/tasks/" - last_response.headers["ETag"].must_equal etag - end - - it "has CORS headers set" do - last_response.status.must_equal 200 - last_response.headers["Access-Control-Allow-Origin"].must_equal "*" - last_response.headers["Access-Control-Allow-Methods"].must_equal "GET, PUT, DELETE" - last_response.headers["Access-Control-Allow-Headers"].must_equal "Authorization, Content-Type, Origin, If-Match, If-None-Match" - last_response.headers["Access-Control-Expose-Headers"].must_equal "ETag, Content-Length" - end - - context "for an empty or absent directory" do - it "responds with 404" do - head "/jimmy/documents/" - - last_response.status.must_equal 404 - end - end - end - - describe "GET listing" do - before do - put "/jimmy/tasks/foo", "do the laundry" - put "/jimmy/tasks/http%3A%2F%2F5apps.com", "prettify design" - put "/jimmy/tasks/%3A/foo%3Abar%40foo.org", "hello world" - end - - it "lists the objects with version, length and content-type" do - get "/jimmy/tasks/" - - last_response.status.must_equal 200 - last_response.content_type.must_equal "application/json" - - foo = data_bucket.get("jimmy:tasks:foo") - - content = JSON.parse(last_response.body) - content["items"]["http://5apps.com"].wont_be_nil - content["items"][":/"].wont_be_nil - content["items"]["foo"].wont_be_nil - content["items"]["foo"]["ETag"].must_equal foo.etag.gsub(/"/, "") - content["items"]["foo"]["Content-Type"].must_equal "text/plain" - content["items"]["foo"]["Content-Length"].must_equal 14 - end - - it "has an ETag header set" do - get "/jimmy/tasks/" - - last_response.status.must_equal 200 - last_response.headers["ETag"].wont_be_nil - - # check that ETag stays the same - etag = last_response.headers["ETag"] - get "/jimmy/tasks/" - last_response.headers["ETag"].must_equal etag - end - - it "has CORS headers set" do - get "/jimmy/tasks/" - - last_response.status.must_equal 200 - last_response.headers["Access-Control-Allow-Origin"].must_equal "*" - last_response.headers["Access-Control-Allow-Methods"].must_equal "GET, PUT, DELETE" - last_response.headers["Access-Control-Allow-Headers"].must_equal "Authorization, Content-Type, Origin, If-Match, If-None-Match" - last_response.headers["Access-Control-Expose-Headers"].must_equal "ETag, Content-Length" - end - - it "has caching headers set" do - get "/jimmy/tasks/" - - last_response.status.must_equal 200 - last_response.headers["Expires"].must_equal "0" - end - - it "doesn't choke on colons in the directory name" do - get "/jimmy/tasks/%3A/" - - last_response.status.must_equal 200 - last_response.content_type.must_equal "application/json" - - content = JSON.parse(last_response.body) - content["items"]["foo:bar@foo.org"].wont_be_nil - end - - context "when If-None-Match header is set" do - before do - get "/jimmy/tasks/" - - @etag = last_response.headers["ETag"] - end - - it "responds with 'not modified' when it matches the current ETag" do - header "If-None-Match", @etag - get "/jimmy/tasks/" - - last_response.status.must_equal 304 - last_response.body.must_be_empty - last_response.headers["ETag"].must_equal @etag - end - - it "responds normally when it does not match the current ETag" do - header "If-None-Match", "FOO" - get "/jimmy/tasks/" - - last_response.status.must_equal 200 - last_response.body.wont_be_empty - end - end - - describe "when If-None-Match header is set with multiple revisions" do - before do - get "/jimmy/tasks/" - - @etag = last_response.headers["ETag"] - end - - it "responds with 'not modified' when it contains the current ETag" do - header "If-None-Match", "DEADBEEF,#{@etag} ,F00BA4" - get "/jimmy/tasks/" - - last_response.status.must_equal 304 - last_response.body.must_be_empty - last_response.headers["ETag"].must_equal @etag - end - - it "responds normally when it does not contain the current ETag" do - header "If-None-Match", "FOO,BAR" - get "/jimmy/tasks/" - - last_response.status.must_equal 200 - last_response.body.wont_be_empty - end - end - - context "with sub-directories" do - before do - get "/jimmy/tasks/" - @old_etag = last_response.headers["ETag"] - - put "/jimmy/tasks/home/laundry", "do the laundry" - end - - it "lists the containing objects as well as the direct sub-directories" do - get "/jimmy/tasks/" - - last_response.status.must_equal 200 - - home = directory_bucket.get("jimmy:tasks/home") - - content = JSON.parse(last_response.body) - content["items"]["foo"].wont_be_nil - content["items"]["http://5apps.com"].wont_be_nil - content["items"]["home/"].wont_be_nil - content["items"]["home/"]["ETag"].must_equal home.etag.gsub(/"/, "") - end - - it "updates the ETag of the parent directory" do - get "/jimmy/tasks/" - - last_response.headers["ETag"].wont_be_nil - last_response.headers["ETag"].wont_equal @old_etag - end - - context "for a different user" do - before do - auth = auth_bucket.new("alice:321") - auth.data = [":r", "documents:r", "tasks:rw"] - auth.store - - header "Authorization", "Bearer 321" - - put "/alice/tasks/homework", "write an essay" - end - - it "does not list the directories of jimmy" do - get "/alice/tasks/" - - last_response.status.must_equal 200 - - content = JSON.parse(last_response.body) - content["items"]["/"].must_be_nil - content["items"]["tasks/"].must_be_nil - content["items"]["home/"].must_be_nil - content["items"]["homework"].wont_be_nil - end - end - - context "sub-directories without objects" do - it "lists the direct sub-directories" do - put "/jimmy/tasks/private/projects/world-domination/start", "write a manifesto" - get "/jimmy/tasks/private/" - - last_response.status.must_equal 200 - - projects = directory_bucket.get("jimmy:tasks/private/projects") - - content = JSON.parse(last_response.body) - content["items"]["projects/"]["ETag"].must_equal projects.etag.gsub(/"/, "") - end - - it "updates the timestamps of the existing directory objects" do - directory = directory_bucket.new("jimmy:tasks") - directory.content_type = "text/plain" - directory.data = (2.seconds.ago.to_f * 1000).to_i - directory.store - - put "/jimmy/tasks/private/projects/world-domination/start", "write a manifesto" - - object = data_bucket.get("jimmy:tasks/private/projects/world-domination:start") - directory = directory_bucket.get("jimmy:tasks") - - directory.data.to_i.must_equal object.meta['timestamp'][0].to_i - end - end - - context "with binary data" do - context "charset given in content-type header" do - before do - header "Content-Type", "image/jpeg; charset=binary" - filename = File.join(File.expand_path(File.dirname(__FILE__)), "..", "fixtures", "rockrule.jpeg") - @image = File.open(filename, "r").read - put "/jimmy/tasks/jaypeg.jpg", @image - end - - it "lists the binary files" do - get "/jimmy/tasks/" - - last_response.status.must_equal 200 - - jaypeg = data_bucket.get("jimmy:tasks:jaypeg.jpg") - - content = JSON.parse(last_response.body) - content["items"]["jaypeg.jpg"]["ETag"].must_equal jaypeg.etag.gsub(/"/, "") - content["items"]["jaypeg.jpg"]["Content-Type"].must_equal "image/jpeg" - content["items"]["jaypeg.jpg"]["Content-Length"].must_equal 16044 - end - end - - context "no charset in content-type header" do - before do - header "Content-Type", "image/jpeg" - filename = File.join(File.expand_path(File.dirname(__FILE__)), "..", "fixtures", "rockrule.jpeg") - @image = File.open(filename, "r").read - put "/jimmy/tasks/jaypeg.jpg", @image - end - - it "lists the binary files" do - get "/jimmy/tasks/" - - last_response.status.must_equal 200 - - jaypeg = data_bucket.get("jimmy:tasks:jaypeg.jpg") - - content = JSON.parse(last_response.body) - content["items"]["jaypeg.jpg"]["ETag"].must_equal jaypeg.etag.gsub(/"/, "") - content["items"]["jaypeg.jpg"]["Content-Type"].must_equal "image/jpeg" - content["items"]["jaypeg.jpg"]["Content-Length"].must_equal 16044 - end - end - end - end - - context "for a sub-directory" do - before do - put "/jimmy/tasks/home/laundry", "do the laundry" - end - - it "lists the objects with timestamp" do - get "/jimmy/tasks/home/" - - last_response.status.must_equal 200 - - laundry = data_bucket.get("jimmy:tasks/home:laundry") - - content = JSON.parse(last_response.body) - content["items"]["laundry"]["ETag"].must_equal laundry.etag.gsub(/"/, "") - end - end - - context "for an empty or absent directory" do - it "returns an empty listing" do - get "/jimmy/documents/notfound/" - - last_response.status.must_equal 404 - end - end - - context "special characters in directory name" do - before do - put "/jimmy/tasks/foo~bar/task1", "some task" - end - - it "lists the directory in the parent directory" do - get "/jimmy/tasks/" - - last_response.status.must_equal 200 - - content = JSON.parse(last_response.body) - content["items"]["foo~bar/"].wont_be_nil - end - - it "lists the containing objects" do - get "/jimmy/tasks/foo~bar/" - - last_response.status.must_equal 200 - - content = JSON.parse(last_response.body) - content["items"]["task1"].wont_be_nil - end - - it "returns the requested object" do - get "/jimmy/tasks/foo~bar/task1" - - last_response.status.must_equal 200 - - last_response.body.must_equal "some task" - end - end - - context "special characters in object name" do - before do - put "/jimmy/tasks/bla~blub", "some task" - end - - it "lists the containing object" do - get "/jimmy/tasks/" - - last_response.status.must_equal 200 - - content = JSON.parse(last_response.body) - content["items"]["bla~blub"].wont_be_nil - end - end - - context "for the root directory" do - before do - auth = auth_bucket.new("jimmy:123") - auth.data = [":rw"] - auth.store - - put "/jimmy/root-1", "Put my root down" - put "/jimmy/root-2", "Back to the roots" - end - - it "lists the containing objects and direct sub-directories" do - get "/jimmy/" - - last_response.status.must_equal 200 - - tasks = directory_bucket.get("jimmy:tasks") - - content = JSON.parse(last_response.body) - content["items"]["root-1"].wont_be_nil - content["items"]["root-2"].wont_be_nil - content["items"]["tasks/"].wont_be_nil - content["items"]["tasks/"]["ETag"].must_equal tasks.etag.gsub(/"/, "") - end - - it "has an ETag header set" do - get "/jimmy/" - - last_response.status.must_equal 200 - last_response.headers["ETag"].wont_be_nil - end - end - - context "for the public directory" do - before do - auth = auth_bucket.new("jimmy:123") - auth.data = ["documents:r", "bookmarks:rw"] - auth.store - - put "/jimmy/public/bookmarks/5apps", "http://5apps.com" - end - - context "when authorized for the category" do - it "lists the files" do - get "/jimmy/public/bookmarks/" - - last_response.status.must_equal 200 - - content = JSON.parse(last_response.body) - content["items"]["5apps"].wont_be_nil - end - - it "has an ETag header set" do - get "/jimmy/public/bookmarks/" - - last_response.status.must_equal 200 - last_response.headers["ETag"].wont_be_nil - end - end - - context "when directly authorized for the public directory" do - before do - auth = auth_bucket.new("jimmy:123") - auth.data = ["documents:r", "public/bookmarks:rw"] - auth.store - end - - it "lists the files" do - get "/jimmy/public/bookmarks/" - - last_response.status.must_equal 200 - - content = JSON.parse(last_response.body) - content["items"]["5apps"].wont_be_nil - end - end - - context "when not authorized" do - before do - auth_bucket.delete("jimmy:123") - end - - it "does not allow a directory listing of the public root" do - get "/jimmy/public/" - - last_response.status.must_equal 401 - end - - it "does not allow a directory listing of a sub-directory" do - get "/jimmy/public/bookmarks/" - - last_response.status.must_equal 401 - end - end - end - end - - describe "directory object" do - describe "PUT file" do - context "no existing directory object" do - before do - put "/jimmy/tasks/home/trash", "take out the trash" - end - - it "creates a new directory object" do - object = data_bucket.get("jimmy:tasks/home:trash") - directory = directory_bucket.get("jimmy:tasks/home") - - directory.data.wont_be_nil - directory.data.to_i.must_equal object.meta['timestamp'][0].to_i - end - - it "sets the correct index for the directory object" do - object = directory_bucket.get("jimmy:tasks/home") - object.indexes["directory_bin"].must_include "tasks" - end - - it "creates directory objects for the parent directories" do - object = directory_bucket.get("jimmy:tasks") - object.indexes["directory_bin"].must_include "/" - object.data.wont_be_nil - - object = directory_bucket.get("jimmy:") - object.indexes["directory_bin"].must_be_empty - object.data.wont_be_nil - end - end - - context "existing directory object" do - before do - put "/jimmy/tasks/home/trash", "collect some trash" - end - - it "updates the timestamp of the directory" do - put "/jimmy/tasks/home/trash", "take out the trash" - - last_response.status.must_equal 200 - - object = data_bucket.get("jimmy:tasks/home:trash") - directory = directory_bucket.get("jimmy:tasks/home") - - directory.data.to_i.must_equal object.meta['timestamp'][0].to_i - end - end - end - end - - describe "OPTIONS listing" do - it "has CORS headers set" do - options "/jimmy/tasks/" - - last_response.status.must_equal 200 - - last_response.headers["Access-Control-Allow-Origin"].must_equal "*" - last_response.headers["Access-Control-Allow-Methods"].must_equal "GET, PUT, DELETE" - last_response.headers["Access-Control-Allow-Headers"].must_equal "Authorization, Content-Type, Origin, If-Match, If-None-Match" - last_response.headers["Access-Control-Expose-Headers"].must_equal "ETag, Content-Length" - end - - context "sub-directories" do - it "has CORS headers set" do - options "/jimmy/tasks/foo/bar/" - - last_response.status.must_equal 200 - - last_response.headers["Access-Control-Allow-Origin"].must_equal "*" - last_response.headers["Access-Control-Allow-Methods"].must_equal "GET, PUT, DELETE" - last_response.headers["Access-Control-Allow-Headers"].must_equal "Authorization, Content-Type, Origin, If-Match, If-None-Match" - last_response.headers["Access-Control-Expose-Headers"].must_equal "ETag, Content-Length" - end - end - - context "root directory" do - it "has CORS headers set" do - options "/jimmy/" - - last_response.status.must_equal 200 - - last_response.headers["Access-Control-Allow-Origin"].must_equal "*" - last_response.headers["Access-Control-Allow-Methods"].must_equal "GET, PUT, DELETE" - last_response.headers["Access-Control-Allow-Headers"].must_equal "Authorization, Content-Type, Origin, If-Match, If-None-Match" - last_response.headers["Access-Control-Expose-Headers"].must_equal "ETag, Content-Length" - end - end - end - - describe "DELETE file" do - context "last file in directory" do - before do - put "/jimmy/tasks/home/trash", "take out the trash" - end - - it "deletes the directory objects for all empty parent directories" do - delete "/jimmy/tasks/home/trash" - - last_response.status.must_equal 200 - - lambda { - directory_bucket.get("jimmy:tasks/home") - }.must_raise Riak::HTTPFailedRequest - - lambda { - directory_bucket.get("jimmy:tasks") - }.must_raise Riak::HTTPFailedRequest - - lambda { - directory_bucket.get("jimmy:") - }.must_raise Riak::HTTPFailedRequest - end - end - - context "with additional files in directory" do - before do - put "/jimmy/tasks/home/trash", "take out the trash" - put "/jimmy/tasks/home/laundry/washing", "wash the clothes" - end - - it "does not delete the directory objects for the parent directories" do - delete "/jimmy/tasks/home/trash" - - directory_bucket.get("jimmy:tasks/home").wont_be_nil - directory_bucket.get("jimmy:tasks").wont_be_nil - directory_bucket.get("jimmy:").wont_be_nil - end - - it "updates the ETag headers of all parent directories" do - get "/jimmy/tasks/home/" - home_etag = last_response.headers["ETag"] - - get "/jimmy/tasks/" - tasks_etag = last_response.headers["ETag"] - - get "/jimmy/" - root_etag = last_response.headers["ETag"] - - delete "/jimmy/tasks/home/trash" - - get "/jimmy/tasks/home/" - last_response.headers["ETag"].wont_be_nil - last_response.headers["ETag"].wont_equal home_etag - - get "/jimmy/tasks/" - last_response.headers["ETag"].wont_be_nil - last_response.headers["ETag"].wont_equal tasks_etag - - get "/jimmy/" - last_response.headers["ETag"].wont_be_nil - last_response.headers["ETag"].wont_equal root_etag - end - - describe "timestamps" do - before do - @old_timestamp = (2.seconds.ago.to_f * 1000).to_i - - ["tasks/home", "tasks", ""].each do |dir| - directory = directory_bucket.get("jimmy:#{dir}") - directory.data = @old_timestamp.to_s - directory.store - end - end - - it "updates the timestamp for the parent directories" do - delete "/jimmy/tasks/home/trash" - - directory_bucket.get("jimmy:tasks/home").data.to_i.must_be :>, @old_timestamp - directory_bucket.get("jimmy:tasks").data.to_i.must_be :>, @old_timestamp - directory_bucket.get("jimmy:").data.to_i.must_be :>, @old_timestamp - end - end - end - end - -end diff --git a/spec/riak/permissions_spec.rb b/spec/riak/permissions_spec.rb deleted file mode 100644 index 6250a2e..0000000 --- a/spec/riak/permissions_spec.rb +++ /dev/null @@ -1,424 +0,0 @@ -require_relative "../spec_helper" - -describe "Permissions" do - include Rack::Test::Methods - - before do - purge_all_buckets - end - - describe "GET" do - context "public data" do - before do - object = data_bucket.new("jimmy:public:foo") - object.content_type = "text/plain" - object.data = "some text data" - object.store - - object = data_bucket.new("jimmy:public/documents:foo") - object.content_type = "text/plain" - object.data = "some text data" - object.store - end - - it "returns the value on all get requests" do - get "/jimmy/public/foo" - - last_response.status.must_equal 200 - last_response.body.must_equal "some text data" - end - - it "returns the value from a sub-directory" do - get "/jimmy/public/documents/foo" - - last_response.status.must_equal 200 - last_response.body.must_equal "some text data" - end - end - - context "private data" do - before do - object = data_bucket.new("jimmy:documents:foo") - object.content_type = "text/plain" - object.data = "some private, authorized text data" - object.store - - object = data_bucket.new("jimmy:documents/very/interesting:text") - object.content_type = "text/plain" - object.data = "some very interesting writing" - object.store - - object = data_bucket.new("jimmy:confidential:bar") - object.content_type = "text/plain" - object.data = "some private, non-authorized text data" - object.store - - auth = auth_bucket.new("jimmy:123") - auth.data = ["documents:r", "tasks:rw"] - auth.store - - header "Authorization", "Bearer 123" - end - - context "when authorized" do - it "returns the value for a key in a top-level directory" do - get "/jimmy/documents/foo" - - last_response.status.must_equal 200 - last_response.body.must_equal "some private, authorized text data" - end - - it "returns the value for a key in a sub-directory" do - get "/jimmy/documents/very/interesting/text" - - last_response.status.must_equal 200 - last_response.body.must_equal "some very interesting writing" - end - end - - context "when not authorized" do - it "returns a 401 for a key in a top-level directory" do - get "/jimmy/confidential/bar" - - last_response.status.must_equal 401 - end - end - end - end - - describe "PUT" do - before do - auth = auth_bucket.new("jimmy:123") - auth.data = ["documents:r", "contacts:rw", "tasks:r", "tasks/home:rw"] - auth.store - - header "Authorization", "Bearer 123" - end - - context "to a top-level directory" do - it "saves the value when there are write permissions" do - put "/jimmy/contacts/1", "John Doe" - - last_response.status.must_equal 201 - data_bucket.get("jimmy:contacts:1").data.must_equal "John Doe" - end - - it "returns a 401 when there are read permissions only" do - put "/jimmy/documents/foo", "some text" - - last_response.status.must_equal 401 - end - end - - context "to a sub-directory" do - it "saves the value when there are direct write permissions" do - put "/jimmy/tasks/home/1", "take out the trash" - - last_response.status.must_equal 201 - data_bucket.get("jimmy:tasks/home:1").data.must_equal "take out the trash" - end - - it "saves the value when there are write permissions for a parent directory" do - put "/jimmy/contacts/family/1", "Bobby Brother" - - last_response.status.must_equal 201 - data_bucket.get("jimmy:contacts/family:1").data.must_equal "Bobby Brother" - end - - it "returns a 401 when there are read permissions only" do - put "/jimmy/documents/business/1", "some text" - - last_response.status.must_equal 401 - end - end - - context "to the public directory" do - context "when authorized for the corresponding category" do - it "saves the value" do - put "/jimmy/public/contacts/foo", "Foo Bar" - - last_response.status.must_equal 201 - data_bucket.get("jimmy:public/contacts:foo").data.must_equal "Foo Bar" - end - - it "saves the value to a sub-directory" do - put "/jimmy/public/contacts/family/foo", "Foo Bar" - - last_response.status.must_equal 201 - data_bucket.get("jimmy:public/contacts/family:foo").data.must_equal "Foo Bar" - end - end - - context "when not authorized for the corresponding category" do - it "returns a 401" do - put "/jimmy/public/documents/foo", "Foo Bar" - - last_response.status.must_equal 401 - end - end - end - end - - describe "DELETE" do - before do - auth = auth_bucket.new("jimmy:123") - auth.data = ["documents:r", "tasks:rw"] - auth.store - - header "Authorization", "Bearer 123" - end - - context "when authorized" do - before do - object = data_bucket.new("jimmy:tasks:1") - object.content_type = "text/plain" - object.data = "do the laundry" - object.store - - object = data_bucket.new("jimmy:tasks/home:1") - object.content_type = "text/plain" - object.data = "take out the trash" - object.store - end - - it "removes the key from a top-level directory" do - delete "/jimmy/tasks/1" - - last_response.status.must_equal 200 - lambda { - data_bucket.get("jimmy:tasks:1") - }.must_raise Riak::HTTPFailedRequest - end - - it "removes the key from a top-level directory" do - delete "/jimmy/tasks/home/1" - - last_response.status.must_equal 200 - lambda { - data_bucket.get("jimmy:tasks/home:1") - }.must_raise Riak::HTTPFailedRequest - end - - context "public directory" do - before do - object = data_bucket.new("jimmy:public/tasks:open") - object.content_type = "text/plain" - object.data = "hello world" - object.store - end - - it "removes the key" do - delete "/jimmy/public/tasks/open" - - last_response.status.must_equal 200 - lambda { - data_bucket.get("jimmy:public/tasks:open") - }.must_raise Riak::HTTPFailedRequest - end - end - end - - context "when not authorized" do - before do - object = data_bucket.new("jimmy:documents:private") - object.content_type = "text/plain" - object.data = "some private, authorized text data" - object.store - - object = data_bucket.new("jimmy:documents/business:foo") - object.content_type = "text/plain" - object.data = "some private, authorized text data" - object.store - end - - it "returns a 401 for a key in a top-level directory" do - delete "/jimmy/documents/private" - - last_response.status.must_equal 401 - end - - it "returns a 401 for a key in a sub-directory" do - delete "/jimmy/documents/business/foo" - - last_response.status.must_equal 401 - end - - context "public directory" do - before do - object = data_bucket.new("jimmy:public/documents:foo") - object.content_type = "text/plain" - object.data = "some private, authorized text data" - object.store - end - - it "returns a 401" do - delete "/jimmy/public/documents/foo" - - last_response.status.must_equal 401 - end - end - end - end - - describe "global permissions" do - before do - object = data_bucket.new("jimmy:documents/very/interesting:text") - object.content_type = "text/plain" - object.data = "some very interesting writing" - object.store - end - - context "write all" do - before do - auth = auth_bucket.new("jimmy:123") - auth.data = [":rw", "documents:r"] - auth.store - - header "Authorization", "Bearer 123" - end - - it "allows GET requests" do - get "/jimmy/documents/very/interesting/text" - - last_response.status.must_equal 200 - last_response.body.must_equal "some very interesting writing" - end - - it "allows PUT requests" do - put "/jimmy/contacts/1", "John Doe" - - last_response.status.must_equal 201 - data_bucket.get("jimmy:contacts:1").data.must_equal "John Doe" - end - - it "allows DELETE requests" do - delete "/jimmy/documents/very/interesting/text" - - last_response.status.must_equal 200 - lambda { - data_bucket.get("jimmy:documents/very/interesting:text") - }.must_raise Riak::HTTPFailedRequest - end - - context "root directory" do - before do - object = data_bucket.new("jimmy::root") - object.content_type = "text/plain" - object.data = "Back to the roots" - object.store - end - - it "allows GET requests" do - get "/jimmy/root" - - last_response.status.must_equal 200 - last_response.body.must_equal "Back to the roots" - end - - it "allows PUT requests" do - put "/jimmy/1", "Gonna kick it root down" - - last_response.status.must_equal 201 - data_bucket.get("jimmy::1").data.must_equal "Gonna kick it root down" - end - - it "allows DELETE requests" do - delete "/jimmy/root" - - last_response.status.must_equal 200 - lambda { - data_bucket.get("jimmy::root") - }.must_raise Riak::HTTPFailedRequest - end - end - - context "public directory" do - before do - object = data_bucket.new("jimmy:public/tasks:hello") - object.content_type = "text/plain" - object.data = "Hello World" - object.store - end - - it "allows GET requests" do - get "/jimmy/public/tasks/" - - last_response.status.must_equal 404 - end - - it "allows PUT requests" do - put "/jimmy/public/1", "Hello World" - - last_response.status.must_equal 201 - data_bucket.get("jimmy:public:1").data.must_equal "Hello World" - end - - it "allows DELETE requests" do - delete "/jimmy/public/tasks/hello" - - last_response.status.must_equal 200 - lambda { - data_bucket.get("jimmy:public/tasks:hello") - }.must_raise Riak::HTTPFailedRequest - end - end - end - - context "read all" do - before do - auth = auth_bucket.new("jimmy:123") - auth.data = [":r", "contacts:rw"] - auth.store - - header "Authorization", "Bearer 123" - end - - it "allows GET requests" do - get "/jimmy/documents/very/interesting/text" - - last_response.status.must_equal 200 - last_response.body.must_equal "some very interesting writing" - end - - it "disallows PUT requests" do - put "/jimmy/documents/foo", "some text" - - last_response.status.must_equal 401 - end - - it "disallows DELETE requests" do - delete "/jimmy/documents/very/interesting/text" - - last_response.status.must_equal 401 - end - - context "public directory" do - before do - object = data_bucket.new("jimmy:public/tasks:hello") - object.content_type = "text/plain" - object.data = "Hello World" - object.store - end - - it "allows GET requests" do - get "/jimmy/public/tasks/" - - last_response.status.must_equal 404 - end - - it "disallows PUT requests" do - put "/jimmy/public/tasks/foo", "some text" - - last_response.status.must_equal 401 - end - - it "disallows DELETE requests" do - delete "/jimmy/public/tasks/hello" - - last_response.status.must_equal 401 - end - end - end - end - -end diff --git a/spec/riak/riak_spec.rb b/spec/riak/riak_spec.rb deleted file mode 100644 index 19c2d69..0000000 --- a/spec/riak/riak_spec.rb +++ /dev/null @@ -1,775 +0,0 @@ -require_relative "../spec_helper" - -describe "App with Riak backend" do - include Rack::Test::Methods - - before do - purge_all_buckets - end - - describe "HEAD public data" do - before do - object = data_bucket.new("jimmy:public:foo") - object.content_type = "text/plain" - object.data = "some text data" - object.store - - head "/jimmy/public/foo" - end - - it "works" do - last_response.status.must_equal 200 - last_response.body.must_equal "" - last_response.headers["ETag"].wont_be_nil - last_response.headers["Content-Length"].must_equal "14" - end - end - - describe "GET public data" do - describe "file with content" do - before do - object = data_bucket.new("jimmy:public:foo") - object.content_type = "text/plain" - object.data = "some text data" - object.store - - get "/jimmy/public/foo" - end - - it "works" do - last_response.status.must_equal 200 - last_response.body.must_equal "some text data" - last_response.headers["ETag"].wont_be_nil - last_response.headers["Content-Length"].must_equal "14" - last_response.headers["Expires"].must_equal "0" - end - - describe "empty file" do - before do - object = data_bucket.new("jimmy:public:empty") - object.content_type = "text/plain" - object.data = "" - object.store - - get "/jimmy/public/empty" - end - - it "returns an empty body" do - last_response.status.must_equal 200 - # Rack::MockRequest turns the body into a string. We can't use - # `last_response.body` to check for nil, because: - # >> [nil].join - # => "" - last_response.body.must_equal '' - last_response.headers["Content-Length"].must_equal '0' - end - end - end - - describe "GET data with custom content type" do - before do - object = data_bucket.new("jimmy:public:magic") - object.content_type = "text/magic" - object.raw_data = "some text data" - object.store - end - - it "returns the value with the correct content type" do - get "/jimmy/public/magic" - - last_response.status.must_equal 200 - last_response.content_type.must_equal "text/magic" - last_response.body.must_equal "some text data" - end - end - - describe "private data" do - before do - object = data_bucket.new("jimmy:documents:foo") - object.content_type = "text/plain" - object.data = "some private text data" - object.store - - @etag = object.etag - - auth = auth_bucket.new("jimmy:123") - auth.data = ["documents", "public"] - auth.store - end - - describe "HEAD" do - before do - header "Authorization", "Bearer 123" - head "/jimmy/documents/foo" - end - - it "works" do - last_response.status.must_equal 200 - last_response.body.must_equal "" - last_response.headers["ETag"].wont_be_nil - last_response.headers["Content-Length"].must_equal "22" - end - end - - describe "HEAD nonexisting key" do - it "returns a 404" do - header "Authorization", "Bearer 123" - head "/jimmy/documents/somestupidkey" - - last_response.status.must_equal 404 - end - end - - describe "GET" do - before do - header "Authorization", "Bearer 123" - end - - it "returns the value" do - get "/jimmy/documents/foo" - - last_response.status.must_equal 200 - last_response.body.must_equal "some private text data" - end - - describe "when If-None-Match header is set" do - it "responds with 'not modified' when it matches the current ETag" do - header "If-None-Match", @etag - get "/jimmy/documents/foo" - - last_response.status.must_equal 304 - last_response.body.must_be_empty - last_response.headers["ETag"].must_equal @etag - end - - it "responds normally when it does not match the current ETag" do - header "If-None-Match", "FOO" - get "/jimmy/documents/foo" - - last_response.status.must_equal 200 - last_response.body.must_equal "some private text data" - end - end - - describe "when If-None-Match header is set with multiple revisions" do - it "responds with 'not modified' when it contains the current ETag" do - header "If-None-Match", "DEADBEEF,#{@etag},F00BA4" - get "/jimmy/documents/foo" - - last_response.status.must_equal 304 - last_response.body.must_be_empty - last_response.headers["ETag"].must_equal @etag - end - - it "responds normally when it does not contain the current ETag" do - header "If-None-Match", "FOO,BAR" - get "/jimmy/documents/foo" - - last_response.status.must_equal 200 - last_response.body.must_equal "some private text data" - end - end - end - - describe "GET nonexisting key" do - it "returns a 404" do - header "Authorization", "Bearer 123" - get "/jimmy/documents/somestupidkey" - - last_response.status.must_equal 404 - end - end - - describe "PUT" do - before do - header "Authorization", "Bearer 123" - end - - describe "with implicit content type" do - before do - put "/jimmy/documents/bar", "another text" - end - - it "saves the value" do - last_response.status.must_equal 201 - last_response.body.must_equal "" - data_bucket.get("jimmy:documents:bar").data.must_equal "another text" - end - - it "stores the data as plain text with utf-8 encoding" do - data_bucket.get("jimmy:documents:bar").content_type.must_equal "text/plain; charset=utf-8" - end - - it "sets the ETag header" do - last_response.headers["ETag"].wont_be_nil - end - - it "indexes the data set" do - indexes = data_bucket.get("jimmy:documents:bar").indexes - indexes["user_id_bin"].must_be_kind_of Set - indexes["user_id_bin"].must_include "jimmy" - - indexes["directory_bin"].must_include "documents" - end - - it "logs the operation" do - objects = [] - opslog_bucket.keys.each { |k| objects << opslog_bucket.get(k) rescue nil } - - log_entry = objects.select{|o| o.data["count"] == 1}.first - log_entry.data["size"].must_equal 12 - log_entry.data["category"].must_equal "documents" - log_entry.indexes["user_id_bin"].must_include "jimmy" - end - end - - describe "with explicit content type" do - before do - header "Content-Type", "application/json" - put "/jimmy/documents/jason", '{"foo": "bar", "unhosted": 1}' - end - - it "saves the value (as JSON)" do - last_response.status.must_equal 201 - data_bucket.get("jimmy:documents:jason").data.must_be_kind_of Hash - data_bucket.get("jimmy:documents:jason").data.must_equal({"foo" => "bar", "unhosted" => 1}) - end - - it "uses the requested content type" do - data_bucket.get("jimmy:documents:jason").content_type.must_equal "application/json" - end - - it "delivers the data correctly" do - header "Authorization", "Bearer 123" - get "/jimmy/documents/jason" - - last_response.body.must_equal '{"foo":"bar","unhosted":1}' - last_response.content_type.must_equal "application/json" - end - end - - describe "with arbitrary content type" do - before do - header "Content-Type", "text/magic" - put "/jimmy/documents/magic", "pure magic" - end - - it "saves the value" do - last_response.status.must_equal 201 - data_bucket.get("jimmy:documents:magic").raw_data.must_equal "pure magic" - end - - it "uses the requested content type" do - data_bucket.get("jimmy:documents:magic").content_type.must_equal "text/magic" - end - - it "delivers the data correctly" do - header "Authorization", "Bearer 123" - get "/jimmy/documents/magic" - - last_response.body.must_equal "pure magic" - last_response.content_type.must_equal "text/magic" - end - end - - describe "with content type containing the encoding" do - before do - header "Content-Type", "application/json; charset=UTF-8" - put "/jimmy/documents/jason", '{"foo": "bar", "unhosted": 1}' - end - - it "saves the value (as JSON)" do - last_response.status.must_equal 201 - data_bucket.get("jimmy:documents:jason").data.must_be_kind_of Hash - data_bucket.get("jimmy:documents:jason").data.must_equal({"foo" => "bar", "unhosted" => 1}) - end - - it "uses the requested content type" do - data_bucket.get("jimmy:documents:jason").content_type.must_equal "application/json; charset=UTF-8" - end - - it "delivers the data correctly" do - get "/jimmy/documents/jason" - - last_response.body.must_equal '{"foo":"bar","unhosted":1}' - last_response.content_type.must_equal "application/json; charset=UTF-8" - end - end - - describe "naming collisions between documents and directories" do - before do - put "/jimmy/documents/archive/document", "lorem ipsum" - end - - it "responds with 409 when directory with same name already exists" do - put "/jimmy/documents/archive", "some awesome content" - - last_response.status.must_equal 409 - - lambda { - data_bucket.get("jimmy:documents/archive") - }.must_raise Riak::HTTPFailedRequest - end - - it "responds with 409 when there is an existing document with same name as one of the directories" do - put "/jimmy/documents/archive/document/subdir/doc", "some awesome content" - - last_response.status.must_equal 409 - - lambda { - data_bucket.get("jimmy:documents/archive/document/subdir/doc") - }.must_raise Riak::HTTPFailedRequest - end - end - - describe "with existing content" do - before do - put "/jimmy/documents/archive/foo", "lorem ipsum" - end - - it "saves the value" do - put "/jimmy/documents/archive/foo", "some awesome content" - - last_response.status.must_equal 200 - data_bucket.get("jimmy:documents/archive:foo").data.must_equal "some awesome content" - end - - it "logs the operations" do - put "/jimmy/documents/archive/foo", "some awesome content" - - objects = [] - opslog_bucket.keys.each { |k| objects << opslog_bucket.get(k) rescue nil } - - create_entry = objects.select{|o| o.data["count"] == 1}.first - create_entry.data["size"].must_equal 11 - create_entry.data["category"].must_equal "documents" - create_entry.indexes["user_id_bin"].must_include "jimmy" - - update_entry = objects.select{|o| o.data["count"] == 0}.first - update_entry.data["size"].must_equal 9 - update_entry.data["category"].must_equal "documents" - update_entry.indexes["user_id_bin"].must_include "jimmy" - end - - it "changes the ETag header" do - old_etag = last_response.headers["ETag"] - put "/jimmy/documents/archive/foo", "some awesome content" - - last_response.headers["ETag"].wont_be_nil - last_response.headers["ETag"].wont_equal old_etag - end - - describe "when If-Match header is set" do - it "allows the request if the header matches the current ETag" do - old_etag = last_response.headers["ETag"] - header "If-Match", old_etag - - put "/jimmy/documents/archive/foo", "some awesome content" - last_response.status.must_equal 200 - - get "/jimmy/documents/archive/foo" - last_response.body.must_equal "some awesome content" - end - - it "fails the request if the header does not match the current ETag" do - header "If-Match", "WONTMATCH" - - put "/jimmy/documents/archive/foo", "some awesome content" - last_response.status.must_equal 412 - - get "/jimmy/documents/archive/foo" - last_response.body.must_equal "lorem ipsum" - end - end - - describe "when If-None-Match header is set" do - before do - header "If-None-Match", "*" - end - - it "fails when the document already exists" do - put "/jimmy/documents/archive/foo", "some awesome content" - - last_response.status.must_equal 412 - - get "/jimmy/documents/archive/foo" - last_response.body.must_equal "lorem ipsum" - end - - it "succeeds when the document does not exist" do - put "/jimmy/documents/archive/bar", "my little content" - - last_response.status.must_equal 201 - end - end - end - - describe "exsting content without serializer registered for the given content-type" do - before do - header "Content-Type", "text/html; charset=UTF-8" - put "/jimmy/documents/html", '' - put "/jimmy/documents/html", '' - end - - it "saves the value" do - last_response.status.must_equal 200 - data_bucket.get("jimmy:documents:html").raw_data.must_equal "" - end - - it "uses the requested content type" do - data_bucket.get("jimmy:documents:html").content_type.must_equal "text/html; charset=UTF-8" - end - end - - describe "public data" do - before do - put "/jimmy/public/documents/notes/foo", "note to self" - end - - it "saves the value" do - last_response.status.must_equal 201 - data_bucket.get("jimmy:public/documents/notes:foo").data.must_equal "note to self" - end - - it "logs the operation" do - objects = [] - opslog_bucket.keys.each { |k| objects << opslog_bucket.get(k) rescue nil } - - log_entry = objects.select{|o| o.data["count"] == 1}.first - log_entry.data["size"].must_equal 12 - log_entry.data["category"].must_equal "public/documents" - log_entry.indexes["user_id_bin"].must_include "jimmy" - end - end - - context "with binary data" do - context "binary charset in content-type header" do - before do - header "Content-Type", "image/jpeg; charset=binary" - filename = File.join(File.expand_path(File.dirname(__FILE__)), "..", "fixtures", "rockrule.jpeg") - @image = File.open(filename, "r").read - put "/jimmy/documents/jaypeg", @image - end - - it "uses the requested content type" do - get "/jimmy/documents/jaypeg" - - last_response.status.must_equal 200 - last_response.content_type.must_equal "image/jpeg; charset=binary" - end - - it "delivers the data correctly" do - get "/jimmy/documents/jaypeg" - - last_response.status.must_equal 200 - last_response.body.must_equal @image - end - - it "responds with an ETag header" do - last_response.headers["ETag"].wont_be_nil - etag = last_response.headers["ETag"] - - get "/jimmy/documents/jaypeg" - - last_response.headers["ETag"].wont_be_nil - last_response.headers["ETag"].must_equal etag - end - - it "responds with a Content-Length header" do - get "/jimmy/documents/jaypeg" - - last_response.headers["Content-Length"].must_equal "16044" - end - - it "changes the ETag when updating the file" do - old_etag = last_response.headers["ETag"] - put "/jimmy/documents/jaypeg", @image - - last_response.headers["ETag"].wont_be_nil - last_response.headers["ETag"].wont_equal old_etag - end - - it "logs the operation" do - objects = [] - opslog_bucket.keys.each { |k| objects << opslog_bucket.get(k) rescue nil } - - log_entry = objects.select{|o| o.data["count"] == 1}.first - log_entry.data["size"].must_equal 16044 - log_entry.data["category"].must_equal "documents" - log_entry.indexes["user_id_bin"].must_include "jimmy" - end - - context "overwriting existing file with same file" do - before do - header "Content-Type", "image/jpeg; charset=binary" - filename = File.join(File.expand_path(File.dirname(__FILE__)), "..", "fixtures", "rockrule.jpeg") - @image = File.open(filename, "r").read - put "/jimmy/documents/jaypeg", @image - end - - it "doesn't log the operation" do - objects = [] - opslog_bucket.keys.each { |k| objects << opslog_bucket.get(k) rescue nil } - - objects.size.must_equal 1 - end - end - - context "overwriting existing file with different file" do - before do - header "Content-Type", "image/jpeg; charset=binary" - filename = File.join(File.expand_path(File.dirname(__FILE__)), "..", "fixtures", "rockrule.jpeg") - @image = File.open(filename, "r").read - put "/jimmy/documents/jaypeg", @image+"foo" - end - - it "logs the operation changing only the size" do - objects = [] - opslog_bucket.keys.each { |k| objects << opslog_bucket.get(k) rescue nil } - - objects.size.must_equal 2 - - log_entry = objects.select{|o| o.data["count"] == 0}.first - log_entry.data["size"].must_equal 3 - log_entry.data["category"].must_equal "documents" - log_entry.indexes["user_id_bin"].must_include "jimmy" - end - end - end - - context "no binary charset in content-type header" do - before do - header "Content-Type", "image/jpeg" - filename = File.join(File.expand_path(File.dirname(__FILE__)), "..", "fixtures", "rockrule.jpeg") - @image = File.open(filename, "r").read - put "/jimmy/documents/jaypeg", @image - end - - it "uses the requested content type" do - get "/jimmy/documents/jaypeg" - - last_response.status.must_equal 200 - last_response.content_type.must_equal "image/jpeg" - end - - it "delivers the data correctly" do - get "/jimmy/documents/jaypeg" - - last_response.status.must_equal 200 - last_response.body.must_equal @image - end - end - end - - context "with escaped key" do - before do - put "/jimmy/documents/bar%3Abaz/http%3A%2F%2F5apps.com", "super website" - end - - it "delivers the data correctly" do - header "Authorization", "Bearer 123" - get "/jimmy/documents/bar%3Abaz/http%3A%2F%2F5apps.com" - - last_response.body.must_equal 'super website' - end - end - - context "with unescaped key" do - before do - put "/jimmy/documents/bar:baz/john@doe.com", "John Doe" - end - - it "lists the document in the directory" do - get "/jimmy/documents/bar:baz/" - - content = JSON.parse(last_response.body) - content["items"]["john@doe.com"].wont_be_nil - end - - it "delivers the data correctly" do - get "/jimmy/documents/bar:baz/john@doe.com" - - last_response.body.must_equal "John Doe" - end - end - - context "escaped square brackets in key" do - before do - put "/jimmy/documents/gracehopper%5B1%5D.jpg", "super image" - end - - it "delivers the data correctly" do - header "Authorization", "Bearer 123" - get "/jimmy/documents/gracehopper%5B1%5D.jpg" - - last_response.body.must_equal "super image" - end - end - - context "invalid JSON" do - context "empty body" do - before do - header "Content-Type", "application/json" - put "/jimmy/documents/jason", "" - end - - it "saves an empty JSON object" do - last_response.status.must_equal 201 - data_bucket.get("jimmy:documents:jason").data.must_be_kind_of Hash - data_bucket.get("jimmy:documents:jason").data.must_equal({}) - end - end - - context "unparsable JSON" do - before do - header "Content-Type", "application/json" - put "/jimmy/documents/jason", "foo" - end - - it "returns a 422" do - last_response.status.must_equal 422 - end - end - end - end - - describe "DELETE" do - before do - header "Authorization", "Bearer 123" - end - - describe "basics" do - before do - delete "/jimmy/documents/foo" - end - - it "removes the key" do - last_response.status.must_equal 200 - lambda { - data_bucket.get("jimmy:documents:foo") - }.must_raise Riak::HTTPFailedRequest - end - - it "logs the operation" do - objects = [] - opslog_bucket.keys.each { |k| objects << opslog_bucket.get(k) rescue nil } - - log_entry = objects.select{|o| o.data["count"] == -1}.first - log_entry.data["size"].must_equal(-22) - log_entry.data["category"].must_equal "documents" - log_entry.indexes["user_id_bin"].must_include "jimmy" - end - end - - context "non-existing object" do - before do - delete "/jimmy/documents/foozius" - end - - it "responds with 404" do - last_response.status.must_equal 404 - end - - it "doesn't log the operation" do - objects = [] - opslog_bucket.keys.each { |k| objects << opslog_bucket.get(k) rescue nil } - objects.select{|o| o.data["count"] == -1}.size.must_equal 0 - end - end - - context "when an If-Match header is given" do - it "allows the request if it matches the current ETag" do - get "/jimmy/documents/foo" - old_etag = last_response.headers["ETag"] - header "If-Match", old_etag - - delete "/jimmy/documents/foo" - last_response.status.must_equal 200 - - get "/jimmy/documents/foo" - last_response.status.must_equal 404 - end - - it "fails the request if it does not match the current ETag" do - header "If-Match", "WONTMATCH" - - delete "/jimmy/documents/foo" - last_response.status.must_equal 412 - - get "/jimmy/documents/foo" - last_response.status.must_equal 200 - last_response.body.must_equal "some private text data" - end - end - - context "binary data" do - before do - header "Content-Type", "image/jpeg; charset=binary" - filename = File.join(File.expand_path(File.dirname(__FILE__)), "..", "fixtures", "rockrule.jpeg") - @image = File.open(filename, "r").read - put "/jimmy/documents/jaypeg", @image - - delete "/jimmy/documents/jaypeg" - end - - it "removes the main object" do - last_response.status.must_equal 200 - lambda { - data_bucket.get("jimmy:documents:jaypeg") - }.must_raise Riak::HTTPFailedRequest - end - - it "removes the binary object" do - last_response.status.must_equal 200 - - binary = cs_binary_bucket.files.get("jimmy:documents:jaypeg") - binary.must_be_nil - end - - it "logs the operation" do - objects = [] - opslog_bucket.keys.each { |k| objects << opslog_bucket.get(k) rescue nil } - - log_entry = objects.select{|o| o.data["count"] == -1 && o.data["size"] == -16044}.first - log_entry.data["category"].must_equal "documents" - log_entry.indexes["user_id_bin"].must_include "jimmy" - end - end - end - end - - describe "unauthorized access" do - before do - auth = auth_bucket.new("jimmy:123") - auth.data = ["documents", "public"] - auth.store - - header "Authorization", "Bearer 321" - end - - describe "GET" do - it "returns a 401" do - get "/jimmy/documents/foo" - - last_response.status.must_equal 401 - end - end - - describe "PUT" do - it "returns a 401" do - put "/jimmy/documents/foo", "some text" - - last_response.status.must_equal 401 - end - end - - describe "DELETE" do - it "returns a 401" do - delete "/jimmy/documents/foo" - - last_response.status.must_equal 401 - end - end - end - end -end diff --git a/spec/spec_helper.rb b/spec/spec_helper.rb index 8c46f3a..b08c6b5 100644 --- a/spec/spec_helper.rb +++ b/spec/spec_helper.rb @@ -8,7 +8,6 @@ require_relative '../liquor-cabinet' require 'minitest/autorun' require 'rack/test' require 'purdytest' -require 'riak' require "redis" require "rest_client" require "minitest/stub_any_instance" @@ -44,74 +43,3 @@ if app.settings.respond_to? :redis end end end - -if app.settings.respond_to? :riak - ::Riak.disable_list_keys_warnings = true - - def client - @client ||= ::Riak::Client.new(:host => app.settings.riak['host'], - :http_port => app.settings.riak['http_port']) - end - - def data_bucket - @data_bucket ||= begin - bucket = client.bucket(app.settings.riak['buckets']['data']) - bucket.allow_mult = false - bucket - end - end - - def directory_bucket - @directory_bucket ||= begin - bucket = client.bucket(app.settings.riak['buckets']['directories']) - bucket.allow_mult = false - bucket - end - end - - def auth_bucket - @auth_bucket ||= begin - bucket = client.bucket(app.settings.riak['buckets']['authorizations']) - bucket.allow_mult = false - bucket - end - end - - def opslog_bucket - @opslog_bucket ||= begin - bucket = client.bucket(app.settings.riak['buckets']['opslog']) - bucket.allow_mult = false - bucket - end - end - - def cs_credentials - @cs_credentials ||= begin - credentials = File.read(app.settings.riak['riak_cs']['credentials_file']) - JSON.parse(credentials) - end - end - - def cs_client - @cs_client ||= Fog::Storage.new({ - :provider => 'AWS', - :aws_access_key_id => cs_credentials['key_id'], - :aws_secret_access_key => cs_credentials['key_secret'], - :endpoint => app.settings.riak['riak_cs']['endpoint'] - }) - end - - def cs_binary_bucket - @cs_binary_bucket ||= cs_client.directories.create(:key => app.settings.riak['buckets']['cs_binaries']) - end - - def purge_all_buckets - [data_bucket, directory_bucket, auth_bucket, opslog_bucket].each do |bucket| - bucket.keys.each {|key| bucket.delete key} - end - - cs_binary_bucket.files.each do |file| - file.destroy - end - end -end From 24b5bf003551782bcbdf6827a5fe0f49d41dd87d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Greg=20Kar=C3=A9kinian?= Date: Mon, 16 Apr 2018 12:17:28 +0200 Subject: [PATCH 05/29] Update gem dependencies --- Gemfile.lock | 57 ++++++++++++++++++++++++++-------------------------- 1 file changed, 29 insertions(+), 28 deletions(-) diff --git a/Gemfile.lock b/Gemfile.lock index 5365411..a3a4cca 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -1,76 +1,77 @@ GEM remote: https://rubygems.org/ specs: - activesupport (5.1.2) + activesupport (5.2.0) concurrent-ruby (~> 1.0, >= 1.0.2) - i18n (~> 0.7) + i18n (>= 0.7, < 2) minitest (~> 5.1) tzinfo (~> 1.1) - backports (3.8.0) + backports (3.11.2) concurrent-ruby (1.0.5) domain_name (0.5.20170404) unf (>= 0.0.5, < 1.0.0) - faraday (0.12.1) + faraday (0.14.0) multipart-post (>= 1.2, < 3) http-accept (1.7.0) http-cookie (1.0.3) domain_name (~> 0.5) - i18n (0.8.4) - kgio (2.11.0) + i18n (1.0.0) + concurrent-ruby (~> 1.0) + kgio (2.11.2) m (1.5.1) method_source (>= 0.6.7) rake (>= 0.9.2.2) - method_source (0.8.2) + method_source (0.9.0) mime-types (3.1) mime-types-data (~> 3.2015) mime-types-data (3.2016.0521) - minitest (5.10.2) - minitest-stub_any_instance (1.0.1) - multi_json (1.12.1) + minitest (5.11.3) + minitest-stub_any_instance (1.0.2) + multi_json (1.13.1) multipart-post (2.0.0) - mustermann (1.0.0) + mustermann (1.0.2) netrc (0.11.0) purdytest (2.0.0) minitest (~> 5.5) - rack (2.0.3) - rack-protection (2.0.0) + rack (2.0.4) + rack-protection (2.0.1) rack - rack-test (0.6.3) - rack (>= 1.0) + rack-test (1.0.0) + rack (>= 1.0, < 3) rainbows (5.1.1) kgio (~> 2.5) rack (>= 1.1, < 3.0) unicorn (~> 5.1) - raindrops (0.18.0) - rake (12.0.0) - redis (3.3.3) + raindrops (0.19.0) + rake (12.3.1) + redis (4.0.1) rest-client (2.1.0.rc1) http-accept (>= 1.7.0, < 2.0) http-cookie (>= 1.0.2, < 2.0) mime-types (>= 1.16, < 4.0) netrc (~> 0.8) - sentry-raven (2.5.3) + sentry-raven (2.7.2) faraday (>= 0.7.6, < 1.0) - sinatra (2.0.0) + sinatra (2.0.1) mustermann (~> 1.0) rack (~> 2.0) - rack-protection (= 2.0.0) + rack-protection (= 2.0.1) tilt (~> 2.0) - sinatra-contrib (2.0.0) + sinatra-contrib (2.0.1) backports (>= 2.0) multi_json mustermann (~> 1.0) - rack-protection (= 2.0.0) - sinatra (= 2.0.0) + rack-protection (= 2.0.1) + sinatra (= 2.0.1) tilt (>= 1.3, < 3) thread_safe (0.3.6) - tilt (2.0.7) - tzinfo (1.2.3) + tilt (2.0.8) + tzinfo (1.2.5) thread_safe (~> 0.1) unf (0.1.4) unf_ext - unf_ext (0.0.7.4) - unicorn (5.3.0) + unf_ext (0.0.7.5) + unicorn (5.4.0) kgio (~> 2.6) raindrops (~> 0.7) From 86dc45f4446abff4fecb7ad7be0c84bbbbe64f5f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Greg=20Kar=C3=A9kinian?= Date: Thu, 26 Apr 2018 13:17:27 +0200 Subject: [PATCH 06/29] Add support for S3 through the REST API This is currently using the old S3 authentication (https://s3.amazonaws.com/doc/s3-developer-guide/RESTAuthentication.html), in order to support Exoscale Storage Refs #112 --- .travis.yml | 9 +- config.yml.example.s3 | 31 + ...ig.yml.example => config.yml.example.swift | 0 lib/remote_storage/rest_provider.rb | 492 ++++++++++ lib/remote_storage/s3_rest.rb | 110 +++ lib/remote_storage/swift.rb | 452 +-------- liquor-cabinet.rb | 3 + spec/s3/app_spec.rb | 871 ++++++++++++++++++ 8 files changed, 1521 insertions(+), 447 deletions(-) create mode 100644 config.yml.example.s3 rename config.yml.example => config.yml.example.swift (100%) create mode 100644 lib/remote_storage/rest_provider.rb create mode 100644 lib/remote_storage/s3_rest.rb create mode 100644 spec/s3/app_spec.rb diff --git a/.travis.yml b/.travis.yml index 0f38df9..6c1356e 100644 --- a/.travis.yml +++ b/.travis.yml @@ -4,12 +4,10 @@ rvm: - 2.4.1 services: - redis-server -before_install: - - gem install bundler before_script: - - cp config.yml.example config.yml + - cp config.yml.example.$BACKEND config.yml - mkdir -p tmp && echo "swifttoken" > tmp/swift_token.txt -script: ruby spec/swift/* +script: ruby spec/$BACKEND/* branches: only: - master @@ -21,3 +19,6 @@ notifications: - http://hook-juggler.herokuapp.com/hooks/travis on_success: always on_failure: always +env: + - BACKEND=s3 + - BACKEND=swift diff --git a/config.yml.example.s3 b/config.yml.example.s3 new file mode 100644 index 0000000..0acb9fd --- /dev/null +++ b/config.yml.example.s3 @@ -0,0 +1,31 @@ +development: &defaults + maintenance: false + # # uncomment this section + # s3: + # endpoint: "https://some-endpoint" + # region: "region" + # access_key_id: "" + # secret_key_id: "" + # bucket: "test-bucket" + # # Redis is needed for the swift backend + # redis: + # host: localhost + # port: 6379 + +test: + <<: *defaults + s3: + endpoint: "https://some-endpoint" + region: "region" + access_key_id: "" + secret_key_id: "" + bucket: "test-bucket" + redis: + host: localhost + port: 6379 + +staging: + <<: *defaults + +production: + <<: *defaults diff --git a/config.yml.example b/config.yml.example.swift similarity index 100% rename from config.yml.example rename to config.yml.example.swift diff --git a/lib/remote_storage/rest_provider.rb b/lib/remote_storage/rest_provider.rb new file mode 100644 index 0000000..c353389 --- /dev/null +++ b/lib/remote_storage/rest_provider.rb @@ -0,0 +1,492 @@ +require "rest_client" +require "json" +require "cgi" +require "active_support/core_ext/time/conversions" +require "active_support/core_ext/numeric/time" +require "active_support/core_ext/hash" +require "redis" +require "digest/md5" + +module RemoteStorage + module RestProvider + + attr_accessor :settings, :server + + def initialize(settings, server) + @settings = settings + @server = server + end + + def authorize_request(user, directory, token, listing=false) + request_method = server.env["REQUEST_METHOD"] + + if directory.split("/").first == "public" + return true if ["GET", "HEAD"].include?(request_method) && !listing + end + + server.halt 401, "Unauthorized" if token.nil? || token.empty? + + authorizations = redis.smembers("authorizations:#{user}:#{token}") + permission = directory_permission(authorizations, directory) + + server.halt 401, "Unauthorized" unless permission + if ["PUT", "DELETE"].include? request_method + server.halt 401, "Unauthorized" unless permission == "rw" + end + end + + def get_head(user, directory, key) + url = url_for_key(user, directory, key) + + res = do_head_request(url) + + set_response_headers(res) + rescue RestClient::ResourceNotFound + server.halt 404 + end + + def get_data(user, directory, key) + url = url_for_key(user, directory, key) + + res = do_get_request(url) + + set_response_headers(res) + + none_match = (server.env["HTTP_IF_NONE_MATCH"] || "").split(",") + .map(&:strip) + .map { |s| s.gsub(/^"?W\//, "") } + server.halt 304 if none_match.include? format_etag(res.headers[:etag]) + + return res.body + rescue RestClient::ResourceNotFound + server.halt 404, "Not Found" + end + + def get_head_directory_listing(user, directory) + get_directory_listing(user, directory) + + "" # just return empty body, headers are set by get_directory_listing + end + + def get_directory_listing(user, directory) + etag = redis.hget "rs:m:#{user}:#{directory}/", "e" + + server.headers["Content-Type"] = "application/ld+json" + + none_match = (server.env["HTTP_IF_NONE_MATCH"] || "").split(",") + .map(&:strip) + .map { |s| s.gsub(/^"?W\//, "") } + + if etag + server.halt 304 if none_match.include? %Q("#{etag}") + + items = get_directory_listing_from_redis_via_lua(user, directory) + else + etag = etag_for(user, directory) + items = {} + + server.halt 304 if none_match.include? %Q("#{etag}") + end + + server.headers["ETag"] = %Q("#{etag}") + + listing = { + "@context" => "http://remotestorage.io/spec/folder-description", + "items" => items + } + + listing.to_json + end + + def put_data(user, directory, key, data, content_type) + server.halt 400 if server.env["HTTP_CONTENT_RANGE"] + server.halt 409, "Conflict" if has_name_collision?(user, directory, key) + + existing_metadata = redis.hgetall redis_metadata_object_key(user, directory, key) + url = url_for_key(user, directory, key) + + if required_match = server.env["HTTP_IF_MATCH"] + required_match = required_match.gsub(/^"?W\//, "") + unless required_match == %Q("#{existing_metadata["e"]}") + + # get actual metadata and compare in case redis metadata became out of sync + begin + head_res = do_head_request(url) + # The file doesn't exist, return 412 + rescue RestClient::ResourceNotFound + server.halt 412, "Precondition Failed" + end + + if required_match == format_etag(head_res.headers[:etag]) + # log previous size difference that was missed ealier because of redis failure + log_size_difference(user, existing_metadata["s"], head_res.headers[:content_length]) + else + server.halt 412, "Precondition Failed" + end + end + end + if server.env["HTTP_IF_NONE_MATCH"] == "*" + server.halt 412, "Precondition Failed" unless existing_metadata.empty? + end + + etag, timestamp = do_put_request_and_return_etag_and_last_modified(url, data, content_type) + + metadata = { + e: etag, + s: data.size, + t: content_type, + m: timestamp + } + + if update_metadata_object(user, directory, key, metadata) + if metadata_changed?(existing_metadata, metadata) + update_dir_objects(user, directory, timestamp, checksum_for(data)) + log_size_difference(user, existing_metadata["s"], metadata[:s]) + end + + server.headers["ETag"] = %Q("#{etag}") + server.halt existing_metadata.empty? ? 201 : 200 + else + server.halt 500 + end + end + + def delete_data(user, directory, key) + url = url_for_key(user, directory, key) + not_found = false + + existing_metadata = redis.hgetall "rs:m:#{user}:#{directory}/#{key}" + + if required_match = server.env["HTTP_IF_MATCH"] + unless required_match.gsub(/^"?W\//, "") == %Q("#{existing_metadata["e"]}") + server.halt 412, "Precondition Failed" + end + end + + not_found = !try_to_delete(url) + + log_size_difference(user, existing_metadata["s"], 0) + delete_metadata_objects(user, directory, key) + delete_dir_objects(user, directory) + + if not_found + server.halt 404, "Not Found" + else + server.headers["Etag"] = %Q("#{existing_metadata["e"]}") + server.halt 200 + end + end + + private + + # Implement this method in your class that includes this module. For example + # %Q("#{etag}") if the ETag does not already have quotes around it + def format_etag(etag) + NotImplementedError + end + + def base_url + NotImplementedError + end + + def container_url_for(user) + NotImplementedError + end + + def default_headers + raise NotImplementedError + end + + def set_response_headers(response) + server.headers["ETag"] = format_etag(response.headers[:etag]) + server.headers["Content-Type"] = response.headers[:content_type] + server.headers["Content-Length"] = response.headers[:content_length] + server.headers["Last-Modified"] = response.headers[:last_modified] + end + + def extract_category(directory) + if directory.match(/^public\//) + "public/#{directory.split('/')[1]}" + else + directory.split('/').first + end + end + + def directory_permission(authorizations, directory) + authorizations = authorizations.map do |auth| + auth.index(":") ? auth.split(":") : [auth, "rw"] + end + authorizations = Hash[*authorizations.flatten] + + permission = authorizations[""] + + authorizations.each do |key, value| + if directory.match(/^(public\/)?#{key}(\/|$)/) + if permission.nil? || permission == "r" + permission = value + end + return permission if permission == "rw" + end + end + + permission + end + + def has_name_collision?(user, directory, key) + lua_script = <<-EOF + local user = ARGV[1] + local directory = ARGV[2] + local key = ARGV[3] + + -- build table with parent directories from remaining arguments + local parent_dir_count = #ARGV - 3 + local parent_directories = {} + for i = 4, 4 + parent_dir_count do + table.insert(parent_directories, ARGV[i]) + end + + -- check for existing directory with the same name as the document + local redis_key = "rs:m:"..user..":" + if directory == "" then + redis_key = redis_key..key.."/" + else + redis_key = redis_key..directory.."/"..key.."/" + end + if redis.call("hget", redis_key, "e") then + return true + end + + for index, dir in pairs(parent_directories) do + if redis.call("hget", "rs:m:"..user..":"..dir.."/", "e") then + -- the directory already exists, no need to do further checks + return false + else + -- check for existing document with same name as directory + if redis.call("hget", "rs:m:"..user..":"..dir, "e") then + return true + end + end + end + + return false + EOF + + parent_directories = parent_directories_for(directory) + + redis.eval(lua_script, nil, [user, directory, key, *parent_directories]) + end + + def metadata_changed?(old_metadata, new_metadata) + # check metadata relevant to the directory listing + # ie. the timestamp (m) is not relevant, because it's not used in + # the listing + return old_metadata["e"] != new_metadata[:e] || + old_metadata["s"] != new_metadata[:s].to_s || + old_metadata["t"] != new_metadata[:t] + end + + def timestamp_for(date) + return DateTime.parse(date).strftime("%Q").to_i + end + + def log_size_difference(user, old_size, new_size) + delta = new_size.to_i - old_size.to_i + redis.incrby "rs:s:#{user}", delta + end + + def checksum_for(data) + Digest::MD5.hexdigest(data) + end + + def parent_directories_for(directory) + directories = directory.split("/") + parent_directories = [] + + while directories.any? + parent_directories << directories.join("/") + directories.pop + end + + parent_directories << "" # add empty string for the root directory + + parent_directories + end + + def top_directory(directory) + if directory.match(/\//) + directory.split("/").last + elsif directory != "" + return directory + end + end + + def parent_directory_for(directory) + if directory.match(/\//) + return directory[0..directory.rindex("/")] + elsif directory != "" + return "/" + end + end + + def update_metadata_object(user, directory, key, metadata) + redis_key = redis_metadata_object_key(user, directory, key) + redis.hmset(redis_key, *metadata) + redis.sadd "rs:m:#{user}:#{directory}/:items", key + + true + end + + def update_dir_objects(user, directory, timestamp, checksum) + parent_directories_for(directory).each do |dir| + etag = etag_for(dir, timestamp, checksum) + + key = "rs:m:#{user}:#{dir}/" + metadata = {e: etag, m: timestamp} + redis.hmset(key, *metadata) + redis.sadd "rs:m:#{user}:#{parent_directory_for(dir)}:items", "#{top_directory(dir)}/" + end + end + + def delete_metadata_objects(user, directory, key) + redis.del redis_metadata_object_key(user, directory, key) + redis.srem "rs:m:#{user}:#{directory}/:items", key + end + + def delete_dir_objects(user, directory) + timestamp = (Time.now.to_f * 1000).to_i + + parent_directories_for(directory).each do |dir| + if dir_empty?(user, dir) + redis.del "rs:m:#{user}:#{dir}/" + redis.srem "rs:m:#{user}:#{parent_directory_for(dir)}:items", "#{top_directory(dir)}/" + else + etag = etag_for(dir, timestamp) + + metadata = {e: etag, m: timestamp} + redis.hmset("rs:m:#{user}:#{dir}/", *metadata) + end + end + end + + def dir_empty?(user, dir) + redis.smembers("rs:m:#{user}:#{dir}/:items").empty? + end + + def redis_metadata_object_key(user, directory, key) + "rs:m:#{user}:#{[directory, key].delete_if(&:empty?).join("/")}" + end + + def url_for_key(user, directory, key) + File.join [container_url_for(user), escape(directory), escape(key)].compact + end + + def do_put_request(url, data, content_type) + deal_with_unauthorized_requests do + RestClient.put(url, data, default_headers.merge({content_type: content_type})) + end + end + + def do_put_request_and_return_etag_and_last_modified(url, data, content_type) + res = do_put_request(url, data, content_type) + + return [res.headers[:etag], timestamp_for(res.headers[:last_modified])] + end + + def do_get_request(url, &block) + deal_with_unauthorized_requests do + RestClient.get(url, default_headers, &block) + end + end + + def do_head_request(url, &block) + deal_with_unauthorized_requests do + RestClient.head(url, default_headers, &block) + end + end + + def do_delete_request(url) + deal_with_unauthorized_requests do + RestClient.delete(url, default_headers) + end + end + + def escape(url) + # We want spaces to turn into %20 and slashes to stay slashes + CGI::escape(url).gsub('+', '%20').gsub('%2F', '/') + end + + def redis + @redis ||= Redis.new(settings.redis.symbolize_keys) + end + + def etag_for(*args) + Digest::MD5.hexdigest args.join(":") + end + + def deal_with_unauthorized_requests(&block) + begin + block.call + rescue RestClient::Unauthorized => ex + Raven.capture_exception(ex) + server.halt 500 + end + end + + def try_to_delete(url) + found = true + + begin + do_delete_request(url) + rescue RestClient::ResourceNotFound + found = false + end + + found + end + + def get_directory_listing_from_redis_via_lua(user, directory) + lua_script = <<-EOF + local user = ARGV[1] + local directory = ARGV[2] + local items = redis.call("smembers", "rs:m:"..user..":"..directory.."/:items") + local listing = {} + + for index, name in pairs(items) do + local redis_key = "rs:m:"..user..":" + if directory == "" then + redis_key = redis_key..name + else + redis_key = redis_key..directory.."/"..name + end + + local metadata_values = redis.call("hgetall", redis_key) + local metadata = {} + + -- redis returns hashes as a single list of alternating keys and values + -- this collates it into a table + for idx = 1, #metadata_values, 2 do + metadata[metadata_values[idx]] = metadata_values[idx + 1] + end + + listing[name] = {["ETag"] = metadata["e"]} + if string.sub(name, -1) ~= "/" then + listing[name]["Content-Type"] = metadata["t"] + listing[name]["Content-Length"] = tonumber(metadata["s"]) + listing[name]["Last-Modified"] = tonumber(metadata["m"]) + end + end + + return cjson.encode(listing) + EOF + + items = JSON.parse(redis.eval(lua_script, nil, [user, directory])) + + items.reject{|k, _| k.end_with? "/"}.each do |_, v| + v["Last-Modified"] = Time.at(v["Last-Modified"]/1000).httpdate + end + + items + end + + end +end diff --git a/lib/remote_storage/s3_rest.rb b/lib/remote_storage/s3_rest.rb new file mode 100644 index 0000000..5f3db7d --- /dev/null +++ b/lib/remote_storage/s3_rest.rb @@ -0,0 +1,110 @@ +require "remote_storage/rest_provider" +require "digest" +require "base64" +require "openssl" +require "webrick/httputils" + +module RemoteStorage + class S3Rest + include RestProvider + + private + + # S3 already wraps the ETag around quotes + def format_etag(etag) + etag + end + + def do_put_request(url, data, content_type) + deal_with_unauthorized_requests do + md5 = Digest::MD5.base64digest(data) + authorization_headers = authorization_headers_for("PUT", md5, content_type, url) + RestClient.put(url, data, authorization_headers.merge({ "Content-Type" => content_type, "Content-Md5" => md5})) + end + end + + # S3 does not return a Last-Modified response header on PUTs + def do_put_request_and_return_etag_and_last_modified(url, data, content_type) + res = do_put_request(url, data, content_type) + head_res = do_head_request(url) + + return [res.headers[:etag].delete('"'), timestamp_for(head_res.headers[:last_modified])] + end + + def do_get_request(url, &block) + deal_with_unauthorized_requests do + authorization_headers = authorization_headers_for("GET", "", "", url) + RestClient.get(url, authorization_headers, &block) + end + end + + def do_head_request(url, &block) + deal_with_unauthorized_requests do + authorization_headers = authorization_headers_for("HEAD", "", "", url) + RestClient.head(url, authorization_headers, &block) + end + end + + def do_delete_request(url) + deal_with_unauthorized_requests do + authorization_headers = authorization_headers_for("DELETE", "", "", url) + RestClient.delete(url, authorization_headers) + end + end + + def try_to_delete(url) + found = true + + begin + do_head_request(url) + rescue RestClient::ResourceNotFound + found = false + end + + do_delete_request(url) if found + + return found + end + + # This is using the S3 authorizations, not the newer AW V4 Signatures + # (https://s3.amazonaws.com/doc/s3-developer-guide/RESTAuthentication.html) + def authorization_headers_for(http_verb, md5, content_type, url) + url = File.join("/", url.gsub(base_url, "")) + date = Time.now.httpdate + signed_data = signature(http_verb, md5, content_type, date, url) + { "Authorization" => "AWS #{credentials[:access_key_id]}:#{signed_data}", + "Date" => date} + end + + def credentials + @credentials ||= { access_key_id: settings.s3["access_key_id"], secret_key_id: settings.s3["secret_key_id"] } + end + + def digest(secret, string_to_sign) + Base64.encode64(hmac(secret, string_to_sign)).strip + end + + def hmac(key, value) + OpenSSL::HMAC.digest(OpenSSL::Digest.new('sha1'), key, value) + end + + def uri_escape(s) + WEBrick::HTTPUtils.escape(s).gsub('%5B', '[').gsub('%5D', ']') + end + + def signature(http_verb, md5, content_type, date, url) + string_to_sign = [http_verb, md5, content_type, date, url].join "\n" + signature = digest(credentials[:secret_key_id], string_to_sign) + uri_escape(signature) + end + + def base_url + @base_url ||= settings.s3["endpoint"] + end + + def container_url_for(user) + "#{base_url}#{settings.s3["bucket"]}/#{user}" + end + end + +end diff --git a/lib/remote_storage/swift.rb b/lib/remote_storage/swift.rb index d104f11..cdef119 100644 --- a/lib/remote_storage/swift.rb +++ b/lib/remote_storage/swift.rb @@ -1,466 +1,32 @@ require "rest_client" -require "json" -require "cgi" require "active_support/core_ext/time/conversions" require "active_support/core_ext/numeric/time" require "active_support/core_ext/hash" -require "redis" -require "digest/md5" +require "remote_storage/rest_provider" module RemoteStorage class Swift - - attr_accessor :settings, :server - - def initialize(settings, server) - @settings = settings - @server = server - end - - def authorize_request(user, directory, token, listing=false) - request_method = server.env["REQUEST_METHOD"] - - if directory.split("/").first == "public" - return true if ["GET", "HEAD"].include?(request_method) && !listing - end - - server.halt 401, "Unauthorized" if token.nil? || token.empty? - - authorizations = redis.smembers("authorizations:#{user}:#{token}") - permission = directory_permission(authorizations, directory) - - server.halt 401, "Unauthorized" unless permission - if ["PUT", "DELETE"].include? request_method - server.halt 401, "Unauthorized" unless permission == "rw" - end - end - - def get_head(user, directory, key) - url = url_for_key(user, directory, key) - - res = do_head_request(url) - - set_response_headers(res) - rescue RestClient::ResourceNotFound - server.halt 404 - end - - def get_data(user, directory, key) - url = url_for_key(user, directory, key) - - res = do_get_request(url) - - set_response_headers(res) - - none_match = (server.env["HTTP_IF_NONE_MATCH"] || "").split(",") - .map(&:strip) - .map { |s| s.gsub(/^"?W\//, "") } - server.halt 304 if none_match.include? %Q("#{res.headers[:etag]}") - - return res.body - rescue RestClient::ResourceNotFound - server.halt 404, "Not Found" - end - - def get_head_directory_listing(user, directory) - get_directory_listing(user, directory) - - "" # just return empty body, headers are set by get_directory_listing - end - - def get_directory_listing(user, directory) - etag = redis.hget "rs:m:#{user}:#{directory}/", "e" - - server.headers["Content-Type"] = "application/ld+json" - - none_match = (server.env["HTTP_IF_NONE_MATCH"] || "").split(",") - .map(&:strip) - .map { |s| s.gsub(/^"?W\//, "") } - - if etag - server.halt 304 if none_match.include? %Q("#{etag}") - - items = get_directory_listing_from_redis_via_lua(user, directory) - else - etag = etag_for(user, directory) - items = {} - - server.halt 304 if none_match.include? %Q("#{etag}") - end - - server.headers["ETag"] = %Q("#{etag}") - - listing = { - "@context" => "http://remotestorage.io/spec/folder-description", - "items" => items - } - - listing.to_json - end - - def get_directory_listing_from_redis_via_lua(user, directory) - lua_script = <<-EOF - local user = ARGV[1] - local directory = ARGV[2] - local items = redis.call("smembers", "rs:m:"..user..":"..directory.."/:items") - local listing = {} - - for index, name in pairs(items) do - local redis_key = "rs:m:"..user..":" - if directory == "" then - redis_key = redis_key..name - else - redis_key = redis_key..directory.."/"..name - end - - local metadata_values = redis.call("hgetall", redis_key) - local metadata = {} - - -- redis returns hashes as a single list of alternating keys and values - -- this collates it into a table - for idx = 1, #metadata_values, 2 do - metadata[metadata_values[idx]] = metadata_values[idx + 1] - end - - listing[name] = {["ETag"] = metadata["e"]} - if string.sub(name, -1) ~= "/" then - listing[name]["Content-Type"] = metadata["t"] - listing[name]["Content-Length"] = tonumber(metadata["s"]) - listing[name]["Last-Modified"] = tonumber(metadata["m"]) - end - end - - return cjson.encode(listing) - EOF - - items = JSON.parse(redis.eval(lua_script, nil, [user, directory])) - - items.reject{|k, _| k.end_with? "/"}.each do |_, v| - v["Last-Modified"] = Time.at(v["Last-Modified"]/1000).httpdate - end - - items - end - - def put_data(user, directory, key, data, content_type) - server.halt 400 if server.env["HTTP_CONTENT_RANGE"] - server.halt 409, "Conflict" if has_name_collision?(user, directory, key) - - existing_metadata = redis.hgetall redis_metadata_object_key(user, directory, key) - url = url_for_key(user, directory, key) - - if required_match = server.env["HTTP_IF_MATCH"] - required_match = required_match.gsub(/^"?W\//, "") - unless required_match == %Q("#{existing_metadata["e"]}") - - # get actual metadata and compare in case redis metadata became out of sync - begin - head_res = do_head_request(url) - # The file doesn't exist in Orbit, return 412 - rescue RestClient::ResourceNotFound - server.halt 412, "Precondition Failed" - end - - if required_match == %Q("#{head_res.headers[:etag]}") - # log previous size difference that was missed ealier because of redis failure - log_size_difference(user, existing_metadata["s"], head_res.headers[:content_length]) - else - server.halt 412, "Precondition Failed" - end - end - end - if server.env["HTTP_IF_NONE_MATCH"] == "*" - server.halt 412, "Precondition Failed" unless existing_metadata.empty? - end - - res = do_put_request(url, data, content_type) - - timestamp = timestamp_for(res.headers[:last_modified]) - - metadata = { - e: res.headers[:etag], - s: data.size, - t: content_type, - m: timestamp - } - - if update_metadata_object(user, directory, key, metadata) - if metadata_changed?(existing_metadata, metadata) - update_dir_objects(user, directory, timestamp, checksum_for(data)) - log_size_difference(user, existing_metadata["s"], metadata[:s]) - end - - server.headers["ETag"] = %Q("#{res.headers[:etag]}") - server.halt existing_metadata.empty? ? 201 : 200 - else - server.halt 500 - end - end - - def log_size_difference(user, old_size, new_size) - delta = new_size.to_i - old_size.to_i - redis.incrby "rs:s:#{user}", delta - end - - def checksum_for(data) - Digest::MD5.hexdigest(data) - end - - def delete_data(user, directory, key) - url = url_for_key(user, directory, key) - not_found = false - - existing_metadata = redis.hgetall "rs:m:#{user}:#{directory}/#{key}" - - if required_match = server.env["HTTP_IF_MATCH"] - unless required_match.gsub(/^"?W\//, "") == %Q("#{existing_metadata["e"]}") - server.halt 412, "Precondition Failed" - end - end - - begin - do_delete_request(url) - rescue RestClient::ResourceNotFound - not_found = true - end - - log_size_difference(user, existing_metadata["s"], 0) - delete_metadata_objects(user, directory, key) - delete_dir_objects(user, directory) - - if not_found - server.halt 404, "Not Found" - else - server.headers["Etag"] = %Q("#{existing_metadata["e"]}") - server.halt 200 - end - end + include RestProvider private - def set_response_headers(response) - server.headers["ETag"] = %Q("#{response.headers[:etag]}") - server.headers["Content-Type"] = response.headers[:content_type] - server.headers["Content-Length"] = response.headers[:content_length] - server.headers["Last-Modified"] = response.headers[:last_modified] - end - - def extract_category(directory) - if directory.match(/^public\//) - "public/#{directory.split('/')[1]}" - else - directory.split('/').first - end - end - - def directory_permission(authorizations, directory) - authorizations = authorizations.map do |auth| - auth.index(":") ? auth.split(":") : [auth, "rw"] - end - authorizations = Hash[*authorizations.flatten] - - permission = authorizations[""] - - authorizations.each do |key, value| - if directory.match(/^(public\/)?#{key}(\/|$)/) - if permission.nil? || permission == "r" - permission = value - end - return permission if permission == "rw" - end - end - - permission - end - - def has_name_collision?(user, directory, key) - lua_script = <<-EOF - local user = ARGV[1] - local directory = ARGV[2] - local key = ARGV[3] - - -- build table with parent directories from remaining arguments - local parent_dir_count = #ARGV - 3 - local parent_directories = {} - for i = 4, 4 + parent_dir_count do - table.insert(parent_directories, ARGV[i]) - end - - -- check for existing directory with the same name as the document - local redis_key = "rs:m:"..user..":" - if directory == "" then - redis_key = redis_key..key.."/" - else - redis_key = redis_key..directory.."/"..key.."/" - end - if redis.call("hget", redis_key, "e") then - return true - end - - for index, dir in pairs(parent_directories) do - if redis.call("hget", "rs:m:"..user..":"..dir.."/", "e") then - -- the directory already exists, no need to do further checks - return false - else - -- check for existing document with same name as directory - if redis.call("hget", "rs:m:"..user..":"..dir, "e") then - return true - end - end - end - - return false - EOF - - parent_directories = parent_directories_for(directory) - - redis.eval(lua_script, nil, [user, directory, key, *parent_directories]) - end - - def metadata_changed?(old_metadata, new_metadata) - # check metadata relevant to the directory listing - # ie. the timestamp (m) is not relevant, because it's not used in - # the listing - return old_metadata["e"] != new_metadata[:e] || - old_metadata["s"] != new_metadata[:s].to_s || - old_metadata["t"] != new_metadata[:t] - end - - def timestamp_for(date) - return DateTime.parse(date).strftime("%Q").to_i - end - - def parent_directories_for(directory) - directories = directory.split("/") - parent_directories = [] - - while directories.any? - parent_directories << directories.join("/") - directories.pop - end - - parent_directories << "" # add empty string for the root directory - - parent_directories - end - - def top_directory(directory) - if directory.match(/\//) - directory.split("/").last - elsif directory != "" - return directory - end - end - - def parent_directory_for(directory) - if directory.match(/\//) - return directory[0..directory.rindex("/")] - elsif directory != "" - return "/" - end - end - - def update_metadata_object(user, directory, key, metadata) - redis_key = redis_metadata_object_key(user, directory, key) - redis.hmset(redis_key, *metadata) - redis.sadd "rs:m:#{user}:#{directory}/:items", key - - true - end - - def update_dir_objects(user, directory, timestamp, checksum) - parent_directories_for(directory).each do |dir| - etag = etag_for(dir, timestamp, checksum) - - key = "rs:m:#{user}:#{dir}/" - metadata = {e: etag, m: timestamp} - redis.hmset(key, *metadata) - redis.sadd "rs:m:#{user}:#{parent_directory_for(dir)}:items", "#{top_directory(dir)}/" - end - end - - def delete_metadata_objects(user, directory, key) - redis.del redis_metadata_object_key(user, directory, key) - redis.srem "rs:m:#{user}:#{directory}/:items", key - end - - def delete_dir_objects(user, directory) - timestamp = (Time.now.to_f * 1000).to_i - - parent_directories_for(directory).each do |dir| - if dir_empty?(user, dir) - redis.del "rs:m:#{user}:#{dir}/" - redis.srem "rs:m:#{user}:#{parent_directory_for(dir)}:items", "#{top_directory(dir)}/" - else - etag = etag_for(dir, timestamp) - - metadata = {e: etag, m: timestamp} - redis.hmset("rs:m:#{user}:#{dir}/", *metadata) - end - end - end - - def dir_empty?(user, dir) - redis.smembers("rs:m:#{user}:#{dir}/:items").empty? - end - - def redis_metadata_object_key(user, directory, key) - "rs:m:#{user}:#{[directory, key].delete_if(&:empty?).join("/")}" - end - - def container_url_for(user) - "#{base_url}/rs:documents:#{settings.environment.to_s}/#{user}" - end - - def url_for_key(user, directory, key) - File.join [container_url_for(user), escape(directory), escape(key)].compact + # Add quotes around the ETag + def format_etag(etag) + %Q("#{etag}") end def base_url @base_url ||= settings.swift["host"] end + def container_url_for(user) + "#{base_url}/rs:documents:#{settings.environment.to_s}/#{user}" + end + def default_headers {"x-auth-token" => swift_token} end - def do_put_request(url, data, content_type) - deal_with_unauthorized_requests do - RestClient.put(url, data, default_headers.merge({content_type: content_type})) - end - end - - def do_get_request(url, &block) - deal_with_unauthorized_requests do - RestClient.get(url, default_headers, &block) - end - end - - def do_head_request(url, &block) - deal_with_unauthorized_requests do - RestClient.head(url, default_headers, &block) - end - end - - def do_delete_request(url) - deal_with_unauthorized_requests do - RestClient.delete(url, default_headers) - end - end - - def escape(url) - # We want spaces to turn into %20 and slashes to stay slashes - CGI::escape(url).gsub('+', '%20').gsub('%2F', '/') - end - - def redis - @redis ||= Redis.new(settings.redis.symbolize_keys) - end - - def etag_for(*args) - Digest::MD5.hexdigest args.join(":") - end - def reload_swift_token server.logger.debug "Reloading swift token. Old token: #{settings.swift_token}" # Remove the line break from the token file. The line break that the diff --git a/liquor-cabinet.rb b/liquor-cabinet.rb index d9a41fb..2d21522 100644 --- a/liquor-cabinet.rb +++ b/liquor-cabinet.rb @@ -5,6 +5,7 @@ require "sinatra/base" require 'sinatra/config_file' require "sinatra/reloader" require "remote_storage/swift" +require "remote_storage/s3_rest" class LiquorCabinet < Sinatra::Base @@ -130,6 +131,8 @@ class LiquorCabinet < Sinatra::Base @storage ||= begin if settings.respond_to? :swift RemoteStorage::Swift.new(settings, self) + elsif settings.respond_to? :s3 + RemoteStorage::S3Rest.new(settings, self) else puts <<-EOF You need to set one storage backend in your config.yml file. diff --git a/spec/s3/app_spec.rb b/spec/s3/app_spec.rb new file mode 100644 index 0000000..3842a3b --- /dev/null +++ b/spec/s3/app_spec.rb @@ -0,0 +1,871 @@ +require_relative "../spec_helper" + +describe "App" do + include Rack::Test::Methods + + def app + LiquorCabinet + end + + it "returns 404 on non-existing routes" do + get "/virginmargarita" + last_response.status.must_equal 404 + end + + describe "PUT requests" do + + before do + purge_redis + end + + context "authorized" do + before do + redis.sadd "authorizations:phil:amarillo", [":rw"] + header "Authorization", "Bearer amarillo" + end + + it "creates the metadata object in redis" do + put_stub = OpenStruct.new(headers: { + etag: '"bla"' + }) + + RestClient.stub :put, put_stub do + RestClient.stub :head, OpenStruct.new(headers: { last_modified: "Fri, 04 Mar 2016 12:20:18 GMT" }) do + put "/phil/food/aguacate", "si" + end + end + + metadata = redis.hgetall "rs:m:phil:food/aguacate" + metadata["s"].must_equal "2" + metadata["t"].must_equal "text/plain; charset=utf-8" + metadata["e"].must_equal "bla" + metadata["m"].length.must_equal 13 + end + + it "creates the directory objects metadata in redis" do + put_stub = OpenStruct.new(headers: { + etag: '"bla"' + }) + get_stub = OpenStruct.new(body: "rootbody") + + RestClient.stub :put, put_stub do + RestClient.stub :head, OpenStruct.new(headers: { last_modified: "Fri, 04 Mar 2016 12:20:18 GMT" }) do + RestClient.stub :get, get_stub do + RemoteStorage::S3Rest.stub_any_instance :etag_for, "newetag" do + put "/phil/food/aguacate", "si" + put "/phil/food/camaron", "yummi" + end + end + end + end + + metadata = redis.hgetall "rs:m:phil:/" + metadata["e"].must_equal "newetag" + metadata["m"].length.must_equal 13 + + metadata = redis.hgetall "rs:m:phil:food/" + metadata["e"].must_equal "newetag" + metadata["m"].length.must_equal 13 + + food_items = redis.smembers "rs:m:phil:food/:items" + food_items.each do |food_item| + ["camaron", "aguacate"].must_include food_item + end + + root_items = redis.smembers "rs:m:phil:/:items" + root_items.must_equal ["food/"] + end + + context "response code" do + before do + @put_stub = OpenStruct.new(headers: { + etag: '"bla"' + }) + end + + it "is 201 for newly created objects" do + RestClient.stub :put, @put_stub do + RestClient.stub :head, OpenStruct.new(headers: { last_modified: "Fri, 04 Mar 2016 12:20:18 GMT" }) do + put "/phil/food/aguacate", "muy deliciosa" + end + end + + last_response.status.must_equal 201 + end + + it "is 200 for updated objects" do + RestClient.stub :put, @put_stub do + RestClient.stub :head, OpenStruct.new(headers: { last_modified: "Fri, 04 Mar 2016 12:20:18 GMT" }) do + put "/phil/food/aguacate", "deliciosa" + put "/phil/food/aguacate", "muy deliciosa" + end + end + + last_response.status.must_equal 200 + end + end + + context "logging usage size" do + before do + @put_stub = OpenStruct.new(headers: { + etag: '"bla"' + }) + @head_stub = OpenStruct.new(headers: { last_modified: "Fri, 04 Mar 2016 12:20:18 GMT" }) + end + + it "logs the complete size when creating new objects" do + RestClient.stub :put, @put_stub do + RestClient.stub :head, @head_stub do + put "/phil/food/aguacate", "1234567890" + end + end + + size_log = redis.get "rs:s:phil" + size_log.must_equal "10" + end + + it "logs the size difference when updating existing objects" do + RestClient.stub :put, @put_stub do + RestClient.stub :head, @head_stub do + put "/phil/food/camaron", "1234567890" + put "/phil/food/aguacate", "1234567890" + put "/phil/food/aguacate", "123" + end + end + + size_log = redis.get "rs:s:phil" + size_log.must_equal "13" + end + end + + describe "objects in root dir" do + before do + put_stub = OpenStruct.new(headers: { + etag: '"bla"', + last_modified: "Fri, 04 Mar 2016 12:20:18 GMT" + }) + + RestClient.stub :put, put_stub do + RestClient.stub :head, OpenStruct.new(headers: { last_modified: "Fri, 04 Mar 2016 12:20:18 GMT" }) do + put "/phil/bamboo.txt", "shir kan" + end + end + end + + it "are listed in the directory listing with all metadata" do + get "phil/" + + last_response.status.must_equal 200 + last_response.content_type.must_equal "application/ld+json" + + content = JSON.parse(last_response.body) + content["items"]["bamboo.txt"].wont_be_nil + content["items"]["bamboo.txt"]["ETag"].must_equal "bla" + content["items"]["bamboo.txt"]["Content-Type"].must_equal "text/plain; charset=utf-8" + content["items"]["bamboo.txt"]["Content-Length"].must_equal 8 + content["items"]["bamboo.txt"]["Last-Modified"].must_equal "Fri, 04 Mar 2016 12:20:18 GMT" + end + end + + describe "name collision checks" do + it "is successful when there is no name collision" do + put_stub = OpenStruct.new(headers: { + etag: '"bla"', + last_modified: "Fri, 04 Mar 2016 12:20:18 GMT" + }) + get_stub = OpenStruct.new(body: "rootbody") + + RestClient.stub :put, put_stub do + RestClient.stub :head, OpenStruct.new(headers: { last_modified: "Fri, 04 Mar 2016 12:20:18 GMT" }) do + RestClient.stub :get, get_stub do + RemoteStorage::S3Rest.stub_any_instance :etag_for, "rootetag" do + put "/phil/food/aguacate", "si" + end + end + end + end + + last_response.status.must_equal 201 + + metadata = redis.hgetall "rs:m:phil:food/aguacate" + metadata["s"].must_equal "2" + end + + it "conflicts when there is a directory with same name as document" do + put_stub = OpenStruct.new(headers: { + etag: '"bla"' + }) + + RestClient.stub :put, put_stub do + RestClient.stub :head, OpenStruct.new(headers: { last_modified: "Fri, 04 Mar 2016 12:20:18 GMT" }) do + put "/phil/food/aguacate", "si" + put "/phil/food", "wontwork" + end + end + + last_response.status.must_equal 409 + last_response.body.must_equal "Conflict" + + metadata = redis.hgetall "rs:m:phil:food" + metadata.must_be_empty + end + + it "conflicts when there is a document with same name as directory" do + put_stub = OpenStruct.new(headers: { + etag: '"bla"' + }) + + RestClient.stub :put, put_stub do + RestClient.stub :head, OpenStruct.new(headers: { last_modified: "Fri, 04 Mar 2016 12:20:18 GMT" }) do + put "/phil/food/aguacate", "si" + put "/phil/food/aguacate/empanado", "wontwork" + end + end + + last_response.status.must_equal 409 + + metadata = redis.hgetall "rs:m:phil:food/aguacate/empanado" + metadata.must_be_empty + end + + it "returns 400 when a Content-Range header is sent" do + header "Content-Range", "bytes 0-3/3" + + put "/phil/food/aguacate", "si" + + last_response.status.must_equal 400 + end + end + + describe "If-Match header" do + before do + put_stub = OpenStruct.new(headers: { + etag: '"oldetag"' + }) + + RestClient.stub :put, put_stub do + RestClient.stub :head, OpenStruct.new(headers: { last_modified: "Fri, 04 Mar 2016 12:20:18 GMT" }) do + put "/phil/food/aguacate", "si" + end + end + end + + it "allows the request if the header matches the current ETag" do + header "If-Match", "\"oldetag\"" + + put_stub = OpenStruct.new(headers: { + etag: '"newetag"' + }) + + RestClient.stub :put, put_stub do + RestClient.stub :head, OpenStruct.new(headers: { last_modified: "Fri, 04 Mar 2016 12:20:18 GMT" }) do + put "/phil/food/aguacate", "aye" + end + end + + last_response.status.must_equal 200 + last_response.headers["Etag"].must_equal "\"newetag\"" + end + + it "allows the request if the header contains a weak ETAG matching the current ETag" do + header "If-Match", "W/\"oldetag\"" + + put_stub = OpenStruct.new(headers: { + etag: '"newetag"' + }) + + RestClient.stub :put, put_stub do + RestClient.stub :head, OpenStruct.new(headers: { last_modified: "Fri, 04 Mar 2016 12:20:18 GMT" }) do + put "/phil/food/aguacate", "aye" + end + end + + last_response.status.must_equal 200 + last_response.headers["Etag"].must_equal "\"newetag\"" + end + + it "allows the request if the header contains a weak ETAG with leading quote matching the current ETag" do + header "If-Match", "\"W/\"oldetag\"" + + put_stub = OpenStruct.new(headers: { + etag: '"newetag"', + }) + + RestClient.stub :put, put_stub do + RestClient.stub :head, OpenStruct.new(headers: { last_modified: "Fri, 04 Mar 2016 12:20:18 GMT" }) do + put "/phil/food/aguacate", "aye" + end + end + + last_response.status.must_equal 200 + last_response.headers["Etag"].must_equal "\"newetag\"" + end + + it "fails the request if the header does not match the current ETag" do + header "If-Match", "someotheretag" + + head_stub = OpenStruct.new(headers: { + etag: '"oldetag"', + last_modified: "Fri, 04 Mar 2016 12:20:18 GMT", + content_type: "text/plain", + content_length: 23 + }) + + RestClient.stub :head, head_stub do + put "/phil/food/aguacate", "aye" + end + + last_response.status.must_equal 412 + last_response.body.must_equal "Precondition Failed" + end + + it "allows the request if redis metadata became out of sync" do + header "If-Match", "\"existingetag\"" + + head_stub = OpenStruct.new(headers: { + etag: '"existingetag"', + last_modified: "Fri, 04 Mar 2016 12:20:18 GMT", + content_type: "text/plain", + content_length: 23 + }) + + put_stub = OpenStruct.new(headers: { + etag: '"newetag"' + }) + + RestClient.stub :head, head_stub do + RestClient.stub :put, put_stub do + put "/phil/food/aguacate", "aye" + end + end + + last_response.status.must_equal 200 + end + end + + describe "If-None-Match header set to '*'" do + it "succeeds when the document doesn't exist yet" do + put_stub = OpenStruct.new(headers: { + etag: '"someetag"' + }) + + header "If-None-Match", "*" + + RestClient.stub :put, put_stub do + RestClient.stub :head, OpenStruct.new(headers: { last_modified: "Fri, 04 Mar 2016 12:20:18 GMT" }) do + put "/phil/food/aguacate", "si" + end + end + + last_response.status.must_equal 201 + end + + it "fails the request if the document already exists" do + put_stub = OpenStruct.new(headers: { + etag: '"someetag"' + }) + + RestClient.stub :put, put_stub do + RestClient.stub :head, OpenStruct.new(headers: { last_modified: "Fri, 04 Mar 2016 12:20:18 GMT" }) do + put "/phil/food/aguacate", "si" + end + end + + header "If-None-Match", "*" + RestClient.stub :put, put_stub do + RestClient.stub :head, OpenStruct.new(headers: { last_modified: "Fri, 04 Mar 2016 12:20:18 GMT" }) do + put "/phil/food/aguacate", "si" + end + end + + last_response.status.must_equal 412 + last_response.body.must_equal "Precondition Failed" + end + end + end + + end + + describe "DELETE requests" do + + before do + purge_redis + end + + context "not authorized" do + describe "with no token" do + it "says it's not authorized" do + delete "/phil/food/aguacate" + + last_response.status.must_equal 401 + last_response.body.must_equal "Unauthorized" + end + end + + describe "with empty token" do + it "says it's not authorized" do + header "Authorization", "Bearer " + delete "/phil/food/aguacate" + + last_response.status.must_equal 401 + last_response.body.must_equal "Unauthorized" + end + end + + describe "with wrong token" do + it "says it's not authorized" do + header "Authorization", "Bearer wrongtoken" + delete "/phil/food/aguacate" + + last_response.status.must_equal 401 + last_response.body.must_equal "Unauthorized" + end + end + + end + + context "authorized" do + before do + redis.sadd "authorizations:phil:amarillo", [":rw"] + header "Authorization", "Bearer amarillo" + + put_stub = OpenStruct.new(headers: { + etag: '"bla"' + }) + + RestClient.stub :put, put_stub do + RestClient.stub :head, OpenStruct.new(headers: { last_modified: "Fri, 04 Mar 2016 12:20:18 GMT" }) do + put "/phil/food/aguacate", "si" + put "/phil/food/camaron", "yummi" + put "/phil/food/desayunos/bolon", "wow" + end + end + end + + it "decreases the size log by size of deleted object" do + RestClient.stub :delete, "" do + RemoteStorage::S3Rest.stub_any_instance :etag_for, "rootetag" do + RestClient.stub :head, OpenStruct.new(headers: { last_modified: "Fri, 04 Mar 2016 12:20:18 GMT" }) do + delete "/phil/food/aguacate" + end + end + end + + size_log = redis.get "rs:s:phil" + size_log.must_equal "8" + end + + it "deletes the metadata object in redis" do + RestClient.stub :delete, "" do + RemoteStorage::S3Rest.stub_any_instance :etag_for, "rootetag" do + RestClient.stub :head, OpenStruct.new(headers: { last_modified: "Fri, 04 Mar 2016 12:20:18 GMT" }) do + delete "/phil/food/aguacate" + end + end + end + + metadata = redis.hgetall "rs:m:phil:food/aguacate" + metadata.must_be_empty + end + + it "deletes the directory objects metadata in redis" do + old_metadata = redis.hgetall "rs:m:phil:food/" + + RestClient.stub :delete, "" do + RemoteStorage::S3Rest.stub_any_instance :etag_for, "newetag" do + RestClient.stub :head, OpenStruct.new(headers: { last_modified: "Fri, 04 Mar 2016 12:20:18 GMT" }) do + delete "/phil/food/aguacate" + end + end + end + + metadata = redis.hgetall "rs:m:phil:food/" + metadata["e"].must_equal "newetag" + metadata["m"].length.must_equal 13 + metadata["m"].wont_equal old_metadata["m"] + + food_items = redis.smembers "rs:m:phil:food/:items" + food_items.sort.must_equal ["camaron", "desayunos/"] + + root_items = redis.smembers "rs:m:phil:/:items" + root_items.must_equal ["food/"] + end + + it "deletes the parent directory objects metadata when deleting all items" do + RestClient.stub :delete, "" do + RemoteStorage::S3Rest.stub_any_instance :etag_for, "rootetag" do + RestClient.stub :head, OpenStruct.new(headers: { last_modified: "Fri, 04 Mar 2016 12:20:18 GMT" }) do + delete "/phil/food/aguacate" + delete "/phil/food/camaron" + delete "/phil/food/desayunos/bolon" + end + end + end + + redis.smembers("rs:m:phil:food/desayunos:items").must_be_empty + redis.hgetall("rs:m:phil:food/desayunos/").must_be_empty + + redis.smembers("rs:m:phil:food/:items").must_be_empty + redis.hgetall("rs:m:phil:food/").must_be_empty + + redis.smembers("rs:m:phil:/:items").must_be_empty + end + + it "responds with the ETag of the deleted item in the header" do + RestClient.stub :delete, "" do + RestClient.stub :head, OpenStruct.new(headers: { last_modified: "Fri, 04 Mar 2016 12:20:18 GMT" }) do + delete "/phil/food/aguacate" + end + end + + last_response.headers["ETag"].must_equal "\"bla\"" + end + + context "when item doesn't exist" do + before do + purge_redis + + put_stub = OpenStruct.new(headers: { + etag: '"bla"' + }) + + RestClient.stub :put, put_stub do + RestClient.stub :head, OpenStruct.new(headers: { last_modified: "Fri, 04 Mar 2016 12:20:18 GMT" }) do + put "/phil/food/steak", "si" + end + end + + raises_exception = ->(url, headers) { raise RestClient::ResourceNotFound.new } + RestClient.stub :head, raises_exception do + delete "/phil/food/steak" + end + end + + it "returns a 404" do + last_response.status.must_equal 404 + last_response.body.must_equal "Not Found" + end + + it "deletes any metadata that might still exist" do + raises_exception = ->(url, headers) { raise RestClient::ResourceNotFound.new } + RestClient.stub :head, raises_exception do + delete "/phil/food/steak" + end + + metadata = redis.hgetall "rs:m:phil:food/steak" + metadata.must_be_empty + + redis.smembers("rs:m:phil:food/:items").must_be_empty + redis.hgetall("rs:m:phil:food/").must_be_empty + + redis.smembers("rs:m:phil:/:items").must_be_empty + end + end + + describe "If-Match header" do + it "succeeds when the header matches the current ETag" do + header "If-Match", "\"bla\"" + + RestClient.stub :delete, "" do + RestClient.stub :head, OpenStruct.new(headers: { last_modified: "Fri, 04 Mar 2016 12:20:18 GMT" }) do + delete "/phil/food/aguacate" + end + end + + last_response.status.must_equal 200 + end + + it "succeeds when the header contains a weak ETAG matching the current ETag" do + header "If-Match", "W/\"bla\"" + + RestClient.stub :delete, "" do + RestClient.stub :head, OpenStruct.new(headers: { last_modified: "Fri, 04 Mar 2016 12:20:18 GMT" }) do + delete "/phil/food/aguacate" + end + end + + last_response.status.must_equal 200 + end + + it "fails the request if it does not match the current ETag" do + header "If-Match", "someotheretag" + + RestClient.stub :delete, "" do + RestClient.stub :head, OpenStruct.new(headers: { etag: '"someetag"' }) do + delete "/phil/food/aguacate" + end + end + + last_response.status.must_equal 412 + last_response.body.must_equal "Precondition Failed" + end + end + end + end + + describe "GET requests" do + + before do + purge_redis + end + + context "not authorized" do + + describe "without token" do + it "says it's not authorized" do + get "/phil/food/" + + last_response.status.must_equal 401 + last_response.body.must_equal "Unauthorized" + end + end + + describe "with wrong token" do + it "says it's not authorized" do + header "Authorization", "Bearer wrongtoken" + get "/phil/food/" + + last_response.status.must_equal 401 + last_response.body.must_equal "Unauthorized" + end + end + + end + + context "authorized" do + + before do + redis.sadd "authorizations:phil:amarillo", [":rw"] + header "Authorization", "Bearer amarillo" + + put_stub = OpenStruct.new(headers: { + etag: '"bla"' + }) + + RestClient.stub :put, put_stub do + RestClient.stub :head, OpenStruct.new(headers: { last_modified: "Fri, 04 Mar 2016 12:20:18 GMT" }) do + put "/phil/food/aguacate", "si" + put "/phil/food/camaron", "yummi" + put "/phil/food/desayunos/bolon", "wow" + end + end + end + + describe "documents" do + + it "returns the required response headers" do + get_stub = OpenStruct.new(body: "si", headers: { + etag: '"0815etag"', + last_modified: "Fri, 04 Mar 2016 12:20:18 GMT", + content_type: "text/plain; charset=utf-8", + content_length: 2 + }) + + RestClient.stub :get, get_stub do + get "/phil/food/aguacate" + end + + last_response.status.must_equal 200 + last_response.headers["ETag"].must_equal "\"0815etag\"" + last_response.headers["Cache-Control"].must_equal "no-cache" + last_response.headers["Content-Type"].must_equal "text/plain; charset=utf-8" + end + + it "returns a 404 when data doesn't exist" do + raises_exception = ->(url, headers) { raise RestClient::ResourceNotFound.new } + RestClient.stub :get, raises_exception do + get "/phil/food/steak" + end + + last_response.status.must_equal 404 + last_response.body.must_equal "Not Found" + end + + it "responds with 304 when IF_NONE_MATCH header contains the ETag" do + header "If-None-Match", "\"0815etag\"" + + get_stub = OpenStruct.new(body: "si", headers: { + etag: '"0815etag"', + last_modified: "Fri, 04 Mar 2016 12:20:18 GMT", + content_type: "text/plain; charset=utf-8", + content_length: 2 + }) + + RestClient.stub :get, get_stub do + get "/phil/food/aguacate" + end + + last_response.status.must_equal 304 + end + + it "responds with 304 when IF_NONE_MATCH header contains weak ETAG matching the current ETag" do + header "If-None-Match", "W/\"0815etag\"" + + get_stub = OpenStruct.new(body: "si", headers: { + etag: '"0815etag"', + last_modified: "Fri, 04 Mar 2016 12:20:18 GMT", + content_type: "text/plain; charset=utf-8", + content_length: 2 + }) + + RestClient.stub :get, get_stub do + get "/phil/food/aguacate" + end + + last_response.status.must_equal 304 + end + + end + + describe "directory listings" do + + it "returns the correct ETag header" do + get "/phil/food/" + + last_response.status.must_equal 200 + last_response.headers["ETag"].must_equal "\"f9f85fbf5aa1fa378fd79ac8aa0a457d\"" + end + + it "returns a Cache-Control header with value 'no-cache'" do + get "/phil/food/" + + last_response.status.must_equal 200 + last_response.headers["Cache-Control"].must_equal "no-cache" + end + + it "responds with 304 when IF_NONE_MATCH header contains the ETag" do + header "If-None-Match", "\"f9f85fbf5aa1fa378fd79ac8aa0a457d\"" + get "/phil/food/" + + last_response.status.must_equal 304 + end + + it "responds with 304 when IF_NONE_MATCH header contains weak ETAG matching the ETag" do + header "If-None-Match", "W/\"f9f85fbf5aa1fa378fd79ac8aa0a457d\"" + get "/phil/food/" + + last_response.status.must_equal 304 + end + + it "contains all items in the directory" do + get "/phil/food/" + + last_response.status.must_equal 200 + last_response.content_type.must_equal "application/ld+json" + + content = JSON.parse(last_response.body) + content["@context"].must_equal "http://remotestorage.io/spec/folder-description" + content["items"]["aguacate"].wont_be_nil + content["items"]["aguacate"]["Content-Type"].must_equal "text/plain; charset=utf-8" + content["items"]["aguacate"]["Content-Length"].must_equal 2 + content["items"]["aguacate"]["ETag"].must_equal "bla" + content["items"]["camaron"].wont_be_nil + content["items"]["camaron"]["Content-Type"].must_equal "text/plain; charset=utf-8" + content["items"]["camaron"]["Content-Length"].must_equal 5 + content["items"]["camaron"]["ETag"].must_equal "bla" + content["items"]["desayunos/"].wont_be_nil + content["items"]["desayunos/"]["ETag"].must_equal "dd36e3cfe52b5f33421150b289a7d48d" + end + + it "contains all items in the root directory" do + get "phil/" + + last_response.status.must_equal 200 + last_response.content_type.must_equal "application/ld+json" + + content = JSON.parse(last_response.body) + content["items"]["food/"].wont_be_nil + content["items"]["food/"]["ETag"].must_equal "f9f85fbf5aa1fa378fd79ac8aa0a457d" + end + + it "responds with an empty directory liting when directory doesn't exist" do + get "phil/some-non-existing-dir/" + + last_response.status.must_equal 200 + last_response.content_type.must_equal "application/ld+json" + + content = JSON.parse(last_response.body) + content["items"].must_equal({}) + end + + end + end + + end + + describe "HEAD requests" do + + before do + purge_redis + end + + context "not authorized" do + + describe "without token" do + it "says it's not authorized" do + head "/phil/food/camarones" + + last_response.status.must_equal 401 + last_response.body.must_be_empty + end + end + + describe "with wrong token" do + it "says it's not authorized" do + header "Authorization", "Bearer wrongtoken" + head "/phil/food/camarones" + + last_response.status.must_equal 401 + last_response.body.must_be_empty + end + end + + end + + context "authorized" do + + before do + redis.sadd "authorizations:phil:amarillo", [":rw"] + header "Authorization", "Bearer amarillo" + + put_stub = OpenStruct.new(headers: { + etag: "bla" + }) + + RestClient.stub :put, put_stub do + RestClient.stub :head, OpenStruct.new(headers: { last_modified: "Fri, 04 Mar 2016 12:20:18 GMT" }) do + put "/phil/food/aguacate", "si" + put "/phil/food/camaron", "yummi" + put "/phil/food/desayunos/bolon", "wow" + end + end + end + + describe "directory listings" do + it "returns the correct header information" do + get "/phil/food/" + + last_response.status.must_equal 200 + last_response.content_type.must_equal "application/ld+json" + last_response.headers["ETag"].must_equal "\"f9f85fbf5aa1fa378fd79ac8aa0a457d\"" + end + end + + describe "documents" do + it "returns a 404 when the document doesn't exist" do + raises_exception = ->(url, headers) { raise RestClient::ResourceNotFound.new } + RestClient.stub :head, raises_exception do + head "/phil/food/steak" + end + + last_response.status.must_equal 404 + last_response.body.must_be_empty + end + end + + end + + end + +end + From f083022e6db54778171e916f330971fb77fc7ae4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Greg=20Kar=C3=A9kinian?= Date: Mon, 16 Apr 2018 16:40:04 +0200 Subject: [PATCH 07/29] Get the ETag from Redis on a GET request This prevents doing a network request when we would return a 304 anyway --- lib/remote_storage/rest_provider.rb | 11 ++++++----- spec/s3/app_spec.rb | 28 +++++----------------------- spec/swift/app_spec.rb | 29 ++++++----------------------- 3 files changed, 17 insertions(+), 51 deletions(-) diff --git a/lib/remote_storage/rest_provider.rb b/lib/remote_storage/rest_provider.rb index c353389..9034533 100644 --- a/lib/remote_storage/rest_provider.rb +++ b/lib/remote_storage/rest_provider.rb @@ -46,17 +46,18 @@ module RemoteStorage end def get_data(user, directory, key) + none_match = (server.env["HTTP_IF_NONE_MATCH"] || "").split(",") + .map(&:strip) + .map { |s| s.gsub(/^"?W\//, "") } + etag = redis.hget redis_metadata_object_key(user, directory, key), "e" + server.halt 304 if none_match.include? %Q("#{etag}") + url = url_for_key(user, directory, key) res = do_get_request(url) set_response_headers(res) - none_match = (server.env["HTTP_IF_NONE_MATCH"] || "").split(",") - .map(&:strip) - .map { |s| s.gsub(/^"?W\//, "") } - server.halt 304 if none_match.include? format_etag(res.headers[:etag]) - return res.body rescue RestClient::ResourceNotFound server.halt 404, "Not Found" diff --git a/spec/s3/app_spec.rb b/spec/s3/app_spec.rb index 3842a3b..17188ed 100644 --- a/spec/s3/app_spec.rb +++ b/spec/s3/app_spec.rb @@ -639,7 +639,7 @@ describe "App" do header "Authorization", "Bearer amarillo" put_stub = OpenStruct.new(headers: { - etag: '"bla"' + etag: '"0815etag"' }) RestClient.stub :put, put_stub do @@ -684,16 +684,7 @@ describe "App" do it "responds with 304 when IF_NONE_MATCH header contains the ETag" do header "If-None-Match", "\"0815etag\"" - get_stub = OpenStruct.new(body: "si", headers: { - etag: '"0815etag"', - last_modified: "Fri, 04 Mar 2016 12:20:18 GMT", - content_type: "text/plain; charset=utf-8", - content_length: 2 - }) - - RestClient.stub :get, get_stub do - get "/phil/food/aguacate" - end + get "/phil/food/aguacate" last_response.status.must_equal 304 end @@ -701,16 +692,7 @@ describe "App" do it "responds with 304 when IF_NONE_MATCH header contains weak ETAG matching the current ETag" do header "If-None-Match", "W/\"0815etag\"" - get_stub = OpenStruct.new(body: "si", headers: { - etag: '"0815etag"', - last_modified: "Fri, 04 Mar 2016 12:20:18 GMT", - content_type: "text/plain; charset=utf-8", - content_length: 2 - }) - - RestClient.stub :get, get_stub do - get "/phil/food/aguacate" - end + get "/phil/food/aguacate" last_response.status.must_equal 304 end @@ -758,11 +740,11 @@ describe "App" do content["items"]["aguacate"].wont_be_nil content["items"]["aguacate"]["Content-Type"].must_equal "text/plain; charset=utf-8" content["items"]["aguacate"]["Content-Length"].must_equal 2 - content["items"]["aguacate"]["ETag"].must_equal "bla" + content["items"]["aguacate"]["ETag"].must_equal "0815etag" content["items"]["camaron"].wont_be_nil content["items"]["camaron"]["Content-Type"].must_equal "text/plain; charset=utf-8" content["items"]["camaron"]["Content-Length"].must_equal 5 - content["items"]["camaron"]["ETag"].must_equal "bla" + content["items"]["camaron"]["ETag"].must_equal "0815etag" content["items"]["desayunos/"].wont_be_nil content["items"]["desayunos/"]["ETag"].must_equal "dd36e3cfe52b5f33421150b289a7d48d" end diff --git a/spec/swift/app_spec.rb b/spec/swift/app_spec.rb index ae207c7..1cb6aae 100644 --- a/spec/swift/app_spec.rb +++ b/spec/swift/app_spec.rb @@ -597,7 +597,7 @@ describe "App" do header "Authorization", "Bearer amarillo" put_stub = OpenStruct.new(headers: { - etag: "bla", + etag: "0815etag", last_modified: "Fri, 04 Mar 2016 12:20:18 GMT" }) @@ -639,18 +639,10 @@ describe "App" do end it "responds with 304 when IF_NONE_MATCH header contains the ETag" do + header "If-None-Match", "\"0815etag\"" - get_stub = OpenStruct.new(body: "si", headers: { - etag: "0815etag", - last_modified: "Fri, 04 Mar 2016 12:20:18 GMT", - content_type: "text/plain; charset=utf-8", - content_length: 2 - }) - - RestClient.stub :get, get_stub do - get "/phil/food/aguacate" - end + get "/phil/food/aguacate" last_response.status.must_equal 304 end @@ -658,16 +650,7 @@ describe "App" do it "responds with 304 when IF_NONE_MATCH header contains weak ETAG matching the current ETag" do header "If-None-Match", "W/\"0815etag\"" - get_stub = OpenStruct.new(body: "si", headers: { - etag: "0815etag", - last_modified: "Fri, 04 Mar 2016 12:20:18 GMT", - content_type: "text/plain; charset=utf-8", - content_length: 2 - }) - - RestClient.stub :get, get_stub do - get "/phil/food/aguacate" - end + get "/phil/food/aguacate" last_response.status.must_equal 304 end @@ -715,11 +698,11 @@ describe "App" do content["items"]["aguacate"].wont_be_nil content["items"]["aguacate"]["Content-Type"].must_equal "text/plain; charset=utf-8" content["items"]["aguacate"]["Content-Length"].must_equal 2 - content["items"]["aguacate"]["ETag"].must_equal "bla" + content["items"]["aguacate"]["ETag"].must_equal "0815etag" content["items"]["camaron"].wont_be_nil content["items"]["camaron"]["Content-Type"].must_equal "text/plain; charset=utf-8" content["items"]["camaron"]["Content-Length"].must_equal 5 - content["items"]["camaron"]["ETag"].must_equal "bla" + content["items"]["camaron"]["ETag"].must_equal "0815etag" content["items"]["desayunos/"].wont_be_nil content["items"]["desayunos/"]["ETag"].must_equal "dd36e3cfe52b5f33421150b289a7d48d" end From 7bd45543f6a0e51d9dc96ad9bbecb57e05dad8af Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Greg=20Kar=C3=A9kinian?= Date: Mon, 16 Apr 2018 17:23:53 +0200 Subject: [PATCH 08/29] Set headers from the Redis metadata on a GET that results in a 304 Also add specs to check for the response headers --- lib/remote_storage/rest_provider.rb | 8 ++++++-- spec/s3/app_spec.rb | 4 ++++ spec/swift/app_spec.rb | 4 ++++ 3 files changed, 14 insertions(+), 2 deletions(-) diff --git a/lib/remote_storage/rest_provider.rb b/lib/remote_storage/rest_provider.rb index 9034533..37ebe10 100644 --- a/lib/remote_storage/rest_provider.rb +++ b/lib/remote_storage/rest_provider.rb @@ -49,8 +49,12 @@ module RemoteStorage none_match = (server.env["HTTP_IF_NONE_MATCH"] || "").split(",") .map(&:strip) .map { |s| s.gsub(/^"?W\//, "") } - etag = redis.hget redis_metadata_object_key(user, directory, key), "e" - server.halt 304 if none_match.include? %Q("#{etag}") + existing_metadata = redis.hgetall redis_metadata_object_key(user, directory, key) + if none_match.include? %Q("#{existing_metadata["e"]}") + server.headers["ETag"] = %Q("#{existing_metadata["e"]}") + server.headers["Last-Modified"] = Time.at(existing_metadata["m"].to_i / 1000).httpdate + server.halt 304 + end url = url_for_key(user, directory, key) diff --git a/spec/s3/app_spec.rb b/spec/s3/app_spec.rb index 17188ed..8f818f1 100644 --- a/spec/s3/app_spec.rb +++ b/spec/s3/app_spec.rb @@ -687,6 +687,8 @@ describe "App" do get "/phil/food/aguacate" last_response.status.must_equal 304 + last_response.headers["ETag"].must_equal "\"0815etag\"" + last_response.headers["Last-Modified"].must_equal "Fri, 04 Mar 2016 12:20:18 GMT" end it "responds with 304 when IF_NONE_MATCH header contains weak ETAG matching the current ETag" do @@ -695,6 +697,8 @@ describe "App" do get "/phil/food/aguacate" last_response.status.must_equal 304 + last_response.headers["ETag"].must_equal "\"0815etag\"" + last_response.headers["Last-Modified"].must_equal "Fri, 04 Mar 2016 12:20:18 GMT" end end diff --git a/spec/swift/app_spec.rb b/spec/swift/app_spec.rb index 1cb6aae..c400af4 100644 --- a/spec/swift/app_spec.rb +++ b/spec/swift/app_spec.rb @@ -645,6 +645,8 @@ describe "App" do get "/phil/food/aguacate" last_response.status.must_equal 304 + last_response.headers["ETag"].must_equal "\"0815etag\"" + last_response.headers["Last-Modified"].must_equal "Fri, 04 Mar 2016 12:20:18 GMT" end it "responds with 304 when IF_NONE_MATCH header contains weak ETAG matching the current ETag" do @@ -653,6 +655,8 @@ describe "App" do get "/phil/food/aguacate" last_response.status.must_equal 304 + last_response.headers["ETag"].must_equal "\"0815etag\"" + last_response.headers["Last-Modified"].must_equal "Fri, 04 Mar 2016 12:20:18 GMT" end end From 21dad2aba7cb8b2e81ca6aacd55271dcabc643a9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Greg=20Kar=C3=A9kinian?= Date: Mon, 16 Apr 2018 22:08:36 +0200 Subject: [PATCH 09/29] Get the metadata from Redis on a HEAD request No need to hit the storage backend, we can set the response headers from the data in Redis --- lib/remote_storage/rest_provider.rb | 25 ++++++++++------ spec/s3/app_spec.rb | 45 ++++++++++++++++++++++------- spec/swift/app_spec.rb | 44 ++++++++++++++++++++++------ 3 files changed, 86 insertions(+), 28 deletions(-) diff --git a/lib/remote_storage/rest_provider.rb b/lib/remote_storage/rest_provider.rb index 37ebe10..0e95577 100644 --- a/lib/remote_storage/rest_provider.rb +++ b/lib/remote_storage/rest_provider.rb @@ -36,23 +36,30 @@ module RemoteStorage end def get_head(user, directory, key) - url = url_for_key(user, directory, key) + none_match = (server.env["HTTP_IF_NONE_MATCH"] || "").split(",") + .map(&:strip) + .map { |s| s.gsub(/^"?W\//, "") } + metadata = redis.hgetall redis_metadata_object_key(user, directory, key) - res = do_head_request(url) + server.halt 404 if metadata.empty? - set_response_headers(res) - rescue RestClient::ResourceNotFound - server.halt 404 + # Set the response headers for a 304 or 200 response + server.headers["ETag"] = %Q("#{metadata["e"]}") + server.headers["Last-Modified"] = Time.at(metadata["m"].to_i / 1000).httpdate + + if none_match.include? %Q("#{metadata["e"]}") + server.halt 304 + end end def get_data(user, directory, key) none_match = (server.env["HTTP_IF_NONE_MATCH"] || "").split(",") .map(&:strip) .map { |s| s.gsub(/^"?W\//, "") } - existing_metadata = redis.hgetall redis_metadata_object_key(user, directory, key) - if none_match.include? %Q("#{existing_metadata["e"]}") - server.headers["ETag"] = %Q("#{existing_metadata["e"]}") - server.headers["Last-Modified"] = Time.at(existing_metadata["m"].to_i / 1000).httpdate + metadata = redis.hgetall redis_metadata_object_key(user, directory, key) + if none_match.include? %Q("#{metadata["e"]}") + server.headers["ETag"] = %Q("#{metadata["e"]}") + server.headers["Last-Modified"] = Time.at(metadata["m"].to_i / 1000).httpdate server.halt 304 end diff --git a/spec/s3/app_spec.rb b/spec/s3/app_spec.rb index 8f818f1..6ab49ec 100644 --- a/spec/s3/app_spec.rb +++ b/spec/s3/app_spec.rb @@ -815,7 +815,7 @@ describe "App" do header "Authorization", "Bearer amarillo" put_stub = OpenStruct.new(headers: { - etag: "bla" + etag: "0815etag" }) RestClient.stub :put, put_stub do @@ -838,20 +838,45 @@ describe "App" do end describe "documents" do - it "returns a 404 when the document doesn't exist" do - raises_exception = ->(url, headers) { raise RestClient::ResourceNotFound.new } - RestClient.stub :head, raises_exception do + context "when the document doesn't exist" do + it "returns a 404" do head "/phil/food/steak" + + last_response.status.must_equal 404 + last_response.body.must_be_empty + end + end + + context "when the document exists" do + it "returns the required response headers" do + head "/phil/food/aguacate" + + last_response.status.must_equal 200 + last_response.headers["ETag"].must_equal "\"0815etag\"" + last_response.headers["Cache-Control"].must_equal "no-cache" end - last_response.status.must_equal 404 - last_response.body.must_be_empty + it "responds with 304 when IF_NONE_MATCH header contains the ETag" do + header "If-None-Match", "\"0815etag\"" + + head "/phil/food/aguacate" + + last_response.status.must_equal 304 + last_response.headers["ETag"].must_equal "\"0815etag\"" + last_response.headers["Last-Modified"].must_equal "Fri, 04 Mar 2016 12:20:18 GMT" + end + + it "responds with 304 when IF_NONE_MATCH header contains weak ETAG matching the current ETag" do + header "If-None-Match", "W/\"0815etag\"" + + head "/phil/food/aguacate" + + last_response.status.must_equal 304 + last_response.headers["ETag"].must_equal "\"0815etag\"" + last_response.headers["Last-Modified"].must_equal "Fri, 04 Mar 2016 12:20:18 GMT" + end end end - end - end - end - diff --git a/spec/swift/app_spec.rb b/spec/swift/app_spec.rb index c400af4..16311d9 100644 --- a/spec/swift/app_spec.rb +++ b/spec/swift/app_spec.rb @@ -773,7 +773,7 @@ describe "App" do header "Authorization", "Bearer amarillo" put_stub = OpenStruct.new(headers: { - etag: "bla", + etag: "0815etag", last_modified: "Fri, 04 Mar 2016 12:20:18 GMT" }) @@ -795,20 +795,46 @@ describe "App" do end describe "documents" do - it "returns a 404 when the document doesn't exist" do - raises_exception = ->(url, headers) { raise RestClient::ResourceNotFound.new } - RestClient.stub :head, raises_exception do + context "when the document doesn't exist" do + it "returns a 404" do head "/phil/food/steak" + + last_response.status.must_equal 404 + last_response.body.must_be_empty + end + end + + context "when the document exists" do + it "returns the required response headers" do + head "/phil/food/aguacate" + + last_response.status.must_equal 200 + last_response.headers["ETag"].must_equal "\"0815etag\"" + last_response.headers["Cache-Control"].must_equal "no-cache" end - last_response.status.must_equal 404 - last_response.body.must_be_empty + it "responds with 304 when IF_NONE_MATCH header contains the ETag" do + header "If-None-Match", "\"0815etag\"" + + head "/phil/food/aguacate" + + last_response.status.must_equal 304 + last_response.headers["ETag"].must_equal "\"0815etag\"" + last_response.headers["Last-Modified"].must_equal "Fri, 04 Mar 2016 12:20:18 GMT" + end + + it "responds with 304 when IF_NONE_MATCH header contains weak ETAG matching the current ETag" do + header "If-None-Match", "W/\"0815etag\"" + + head "/phil/food/aguacate" + + last_response.status.must_equal 304 + last_response.headers["ETag"].must_equal "\"0815etag\"" + last_response.headers["Last-Modified"].must_equal "Fri, 04 Mar 2016 12:20:18 GMT" + end end end - end - end - end From 21f3a9f60f974efefe9488a94e2d5c0a6078e724 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Greg=20Kar=C3=A9kinian?= Date: Thu, 19 Apr 2018 18:18:05 +0200 Subject: [PATCH 10/29] Remove all duplication the specs S3 and Swift now run the same specs. The only difference is the before block that defines the stubbed HTTP requests and the responses from the Swift and S3 servers --- Gemfile | 1 + Gemfile.lock | 12 + spec/s3/app_spec.rb | 918 ++-------------------------------------- spec/shared_examples.rb | 635 +++++++++++++++++++++++++++ spec/spec_helper.rb | 36 +- spec/swift/app_spec.rb | 872 ++------------------------------------ 6 files changed, 762 insertions(+), 1712 deletions(-) create mode 100644 spec/shared_examples.rb diff --git a/Gemfile b/Gemfile index 6f8b5eb..293f7e3 100644 --- a/Gemfile +++ b/Gemfile @@ -15,6 +15,7 @@ group :test do gem 'purdytest', :require => false gem 'm' gem 'minitest-stub_any_instance' + gem 'webmock' end group :staging, :production do diff --git a/Gemfile.lock b/Gemfile.lock index a3a4cca..e31610b 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -6,12 +6,17 @@ GEM i18n (>= 0.7, < 2) minitest (~> 5.1) tzinfo (~> 1.1) + addressable (2.5.2) + public_suffix (>= 2.0.2, < 4.0) backports (3.11.2) concurrent-ruby (1.0.5) + crack (0.4.3) + safe_yaml (~> 1.0.0) domain_name (0.5.20170404) unf (>= 0.0.5, < 1.0.0) faraday (0.14.0) multipart-post (>= 1.2, < 3) + hashdiff (0.3.7) http-accept (1.7.0) http-cookie (1.0.3) domain_name (~> 0.5) @@ -31,6 +36,7 @@ GEM multipart-post (2.0.0) mustermann (1.0.2) netrc (0.11.0) + public_suffix (3.0.2) purdytest (2.0.0) minitest (~> 5.5) rack (2.0.4) @@ -50,6 +56,7 @@ GEM http-cookie (>= 1.0.2, < 2.0) mime-types (>= 1.16, < 4.0) netrc (~> 0.8) + safe_yaml (1.0.4) sentry-raven (2.7.2) faraday (>= 0.7.6, < 1.0) sinatra (2.0.1) @@ -74,6 +81,10 @@ GEM unicorn (5.4.0) kgio (~> 2.6) raindrops (~> 0.7) + webmock (3.3.0) + addressable (>= 2.3.6) + crack (>= 0.3.2) + hashdiff PLATFORMS ruby @@ -92,6 +103,7 @@ DEPENDENCIES sentry-raven sinatra sinatra-contrib + webmock BUNDLED WITH 1.16.0 diff --git a/spec/s3/app_spec.rb b/spec/s3/app_spec.rb index 6ab49ec..9874edc 100644 --- a/spec/s3/app_spec.rb +++ b/spec/s3/app_spec.rb @@ -1,882 +1,56 @@ require_relative "../spec_helper" describe "App" do - include Rack::Test::Methods - - def app - LiquorCabinet + def container_url_for(user) + "#{app.settings.s3["endpoint"]}#{app.settings.s3["bucket"]}/#{user}" end - it "returns 404 on non-existing routes" do - get "/virginmargarita" - last_response.status.must_equal 404 + def storage_class + RemoteStorage::S3Rest end - describe "PUT requests" do - - before do - purge_redis - end - - context "authorized" do - before do - redis.sadd "authorizations:phil:amarillo", [":rw"] - header "Authorization", "Bearer amarillo" - end - - it "creates the metadata object in redis" do - put_stub = OpenStruct.new(headers: { - etag: '"bla"' - }) - - RestClient.stub :put, put_stub do - RestClient.stub :head, OpenStruct.new(headers: { last_modified: "Fri, 04 Mar 2016 12:20:18 GMT" }) do - put "/phil/food/aguacate", "si" - end - end - - metadata = redis.hgetall "rs:m:phil:food/aguacate" - metadata["s"].must_equal "2" - metadata["t"].must_equal "text/plain; charset=utf-8" - metadata["e"].must_equal "bla" - metadata["m"].length.must_equal 13 - end - - it "creates the directory objects metadata in redis" do - put_stub = OpenStruct.new(headers: { - etag: '"bla"' - }) - get_stub = OpenStruct.new(body: "rootbody") - - RestClient.stub :put, put_stub do - RestClient.stub :head, OpenStruct.new(headers: { last_modified: "Fri, 04 Mar 2016 12:20:18 GMT" }) do - RestClient.stub :get, get_stub do - RemoteStorage::S3Rest.stub_any_instance :etag_for, "newetag" do - put "/phil/food/aguacate", "si" - put "/phil/food/camaron", "yummi" - end - end - end - end - - metadata = redis.hgetall "rs:m:phil:/" - metadata["e"].must_equal "newetag" - metadata["m"].length.must_equal 13 - - metadata = redis.hgetall "rs:m:phil:food/" - metadata["e"].must_equal "newetag" - metadata["m"].length.must_equal 13 - - food_items = redis.smembers "rs:m:phil:food/:items" - food_items.each do |food_item| - ["camaron", "aguacate"].must_include food_item - end - - root_items = redis.smembers "rs:m:phil:/:items" - root_items.must_equal ["food/"] - end - - context "response code" do - before do - @put_stub = OpenStruct.new(headers: { - etag: '"bla"' - }) - end - - it "is 201 for newly created objects" do - RestClient.stub :put, @put_stub do - RestClient.stub :head, OpenStruct.new(headers: { last_modified: "Fri, 04 Mar 2016 12:20:18 GMT" }) do - put "/phil/food/aguacate", "muy deliciosa" - end - end - - last_response.status.must_equal 201 - end - - it "is 200 for updated objects" do - RestClient.stub :put, @put_stub do - RestClient.stub :head, OpenStruct.new(headers: { last_modified: "Fri, 04 Mar 2016 12:20:18 GMT" }) do - put "/phil/food/aguacate", "deliciosa" - put "/phil/food/aguacate", "muy deliciosa" - end - end - - last_response.status.must_equal 200 - end - end - - context "logging usage size" do - before do - @put_stub = OpenStruct.new(headers: { - etag: '"bla"' - }) - @head_stub = OpenStruct.new(headers: { last_modified: "Fri, 04 Mar 2016 12:20:18 GMT" }) - end - - it "logs the complete size when creating new objects" do - RestClient.stub :put, @put_stub do - RestClient.stub :head, @head_stub do - put "/phil/food/aguacate", "1234567890" - end - end - - size_log = redis.get "rs:s:phil" - size_log.must_equal "10" - end - - it "logs the size difference when updating existing objects" do - RestClient.stub :put, @put_stub do - RestClient.stub :head, @head_stub do - put "/phil/food/camaron", "1234567890" - put "/phil/food/aguacate", "1234567890" - put "/phil/food/aguacate", "123" - end - end - - size_log = redis.get "rs:s:phil" - size_log.must_equal "13" - end - end - - describe "objects in root dir" do - before do - put_stub = OpenStruct.new(headers: { - etag: '"bla"', - last_modified: "Fri, 04 Mar 2016 12:20:18 GMT" - }) - - RestClient.stub :put, put_stub do - RestClient.stub :head, OpenStruct.new(headers: { last_modified: "Fri, 04 Mar 2016 12:20:18 GMT" }) do - put "/phil/bamboo.txt", "shir kan" - end - end - end - - it "are listed in the directory listing with all metadata" do - get "phil/" - - last_response.status.must_equal 200 - last_response.content_type.must_equal "application/ld+json" - - content = JSON.parse(last_response.body) - content["items"]["bamboo.txt"].wont_be_nil - content["items"]["bamboo.txt"]["ETag"].must_equal "bla" - content["items"]["bamboo.txt"]["Content-Type"].must_equal "text/plain; charset=utf-8" - content["items"]["bamboo.txt"]["Content-Length"].must_equal 8 - content["items"]["bamboo.txt"]["Last-Modified"].must_equal "Fri, 04 Mar 2016 12:20:18 GMT" - end - end - - describe "name collision checks" do - it "is successful when there is no name collision" do - put_stub = OpenStruct.new(headers: { - etag: '"bla"', - last_modified: "Fri, 04 Mar 2016 12:20:18 GMT" - }) - get_stub = OpenStruct.new(body: "rootbody") - - RestClient.stub :put, put_stub do - RestClient.stub :head, OpenStruct.new(headers: { last_modified: "Fri, 04 Mar 2016 12:20:18 GMT" }) do - RestClient.stub :get, get_stub do - RemoteStorage::S3Rest.stub_any_instance :etag_for, "rootetag" do - put "/phil/food/aguacate", "si" - end - end - end - end - - last_response.status.must_equal 201 - - metadata = redis.hgetall "rs:m:phil:food/aguacate" - metadata["s"].must_equal "2" - end - - it "conflicts when there is a directory with same name as document" do - put_stub = OpenStruct.new(headers: { - etag: '"bla"' - }) - - RestClient.stub :put, put_stub do - RestClient.stub :head, OpenStruct.new(headers: { last_modified: "Fri, 04 Mar 2016 12:20:18 GMT" }) do - put "/phil/food/aguacate", "si" - put "/phil/food", "wontwork" - end - end - - last_response.status.must_equal 409 - last_response.body.must_equal "Conflict" - - metadata = redis.hgetall "rs:m:phil:food" - metadata.must_be_empty - end - - it "conflicts when there is a document with same name as directory" do - put_stub = OpenStruct.new(headers: { - etag: '"bla"' - }) - - RestClient.stub :put, put_stub do - RestClient.stub :head, OpenStruct.new(headers: { last_modified: "Fri, 04 Mar 2016 12:20:18 GMT" }) do - put "/phil/food/aguacate", "si" - put "/phil/food/aguacate/empanado", "wontwork" - end - end - - last_response.status.must_equal 409 - - metadata = redis.hgetall "rs:m:phil:food/aguacate/empanado" - metadata.must_be_empty - end - - it "returns 400 when a Content-Range header is sent" do - header "Content-Range", "bytes 0-3/3" - - put "/phil/food/aguacate", "si" - - last_response.status.must_equal 400 - end - end - - describe "If-Match header" do - before do - put_stub = OpenStruct.new(headers: { - etag: '"oldetag"' - }) - - RestClient.stub :put, put_stub do - RestClient.stub :head, OpenStruct.new(headers: { last_modified: "Fri, 04 Mar 2016 12:20:18 GMT" }) do - put "/phil/food/aguacate", "si" - end - end - end - - it "allows the request if the header matches the current ETag" do - header "If-Match", "\"oldetag\"" - - put_stub = OpenStruct.new(headers: { - etag: '"newetag"' - }) - - RestClient.stub :put, put_stub do - RestClient.stub :head, OpenStruct.new(headers: { last_modified: "Fri, 04 Mar 2016 12:20:18 GMT" }) do - put "/phil/food/aguacate", "aye" - end - end - - last_response.status.must_equal 200 - last_response.headers["Etag"].must_equal "\"newetag\"" - end - - it "allows the request if the header contains a weak ETAG matching the current ETag" do - header "If-Match", "W/\"oldetag\"" - - put_stub = OpenStruct.new(headers: { - etag: '"newetag"' - }) - - RestClient.stub :put, put_stub do - RestClient.stub :head, OpenStruct.new(headers: { last_modified: "Fri, 04 Mar 2016 12:20:18 GMT" }) do - put "/phil/food/aguacate", "aye" - end - end - - last_response.status.must_equal 200 - last_response.headers["Etag"].must_equal "\"newetag\"" - end - - it "allows the request if the header contains a weak ETAG with leading quote matching the current ETag" do - header "If-Match", "\"W/\"oldetag\"" - - put_stub = OpenStruct.new(headers: { - etag: '"newetag"', - }) - - RestClient.stub :put, put_stub do - RestClient.stub :head, OpenStruct.new(headers: { last_modified: "Fri, 04 Mar 2016 12:20:18 GMT" }) do - put "/phil/food/aguacate", "aye" - end - end - - last_response.status.must_equal 200 - last_response.headers["Etag"].must_equal "\"newetag\"" - end - - it "fails the request if the header does not match the current ETag" do - header "If-Match", "someotheretag" - - head_stub = OpenStruct.new(headers: { - etag: '"oldetag"', - last_modified: "Fri, 04 Mar 2016 12:20:18 GMT", - content_type: "text/plain", - content_length: 23 - }) - - RestClient.stub :head, head_stub do - put "/phil/food/aguacate", "aye" - end - - last_response.status.must_equal 412 - last_response.body.must_equal "Precondition Failed" - end - - it "allows the request if redis metadata became out of sync" do - header "If-Match", "\"existingetag\"" - - head_stub = OpenStruct.new(headers: { - etag: '"existingetag"', - last_modified: "Fri, 04 Mar 2016 12:20:18 GMT", - content_type: "text/plain", - content_length: 23 - }) - - put_stub = OpenStruct.new(headers: { - etag: '"newetag"' - }) - - RestClient.stub :head, head_stub do - RestClient.stub :put, put_stub do - put "/phil/food/aguacate", "aye" - end - end - - last_response.status.must_equal 200 - end - end - - describe "If-None-Match header set to '*'" do - it "succeeds when the document doesn't exist yet" do - put_stub = OpenStruct.new(headers: { - etag: '"someetag"' - }) - - header "If-None-Match", "*" - - RestClient.stub :put, put_stub do - RestClient.stub :head, OpenStruct.new(headers: { last_modified: "Fri, 04 Mar 2016 12:20:18 GMT" }) do - put "/phil/food/aguacate", "si" - end - end - - last_response.status.must_equal 201 - end - - it "fails the request if the document already exists" do - put_stub = OpenStruct.new(headers: { - etag: '"someetag"' - }) - - RestClient.stub :put, put_stub do - RestClient.stub :head, OpenStruct.new(headers: { last_modified: "Fri, 04 Mar 2016 12:20:18 GMT" }) do - put "/phil/food/aguacate", "si" - end - end - - header "If-None-Match", "*" - RestClient.stub :put, put_stub do - RestClient.stub :head, OpenStruct.new(headers: { last_modified: "Fri, 04 Mar 2016 12:20:18 GMT" }) do - put "/phil/food/aguacate", "si" - end - end - - last_response.status.must_equal 412 - last_response.body.must_equal "Precondition Failed" - end - end - end - + before do + stub_request(:put, "#{container_url_for("phil")}/food/aguacate"). + to_return(status: 200, headers: { etag: '"0815etag"' }) + stub_request(:put, "#{container_url_for("phil")}/food/aguacate"). + with(body: "si"). + to_return(status: 200, headers: { etag: '"0815etag"' }) + stub_request(:put, "#{container_url_for("phil")}/food/aguacate"). + with(body: "aye"). + to_return(status: 200, headers: { etag: '"0915etag"' }) + stub_request(:put, "#{container_url_for("phil")}/food/aguacate"). + with(body: "deliciosa"). + to_return(status: 200, headers: { etag: '"0815etag"' }) + stub_request(:put, "#{container_url_for("phil")}/food/aguacate"). + with(body: "muy deliciosa"). + to_return(status: 200, headers: { etag: '"0815etag"' }) + stub_request(:head, "#{container_url_for("phil")}/food/aguacate"). + to_return(status: 200, headers: { last_modified: "Fri, 04 Mar 2016 12:20:18 GMT" }) + stub_request(:delete, "#{container_url_for("phil")}/food/aguacate"). + to_return(status: 200, headers: { etag: '"0815etag"' }) + stub_request(:put, "#{container_url_for("phil")}/food/camaron"). + to_return(status: 200, headers: { etag: '"0816etag"' }) + stub_request(:head, "#{container_url_for("phil")}/food/camaron"). + to_return(status: 200, headers: { last_modified: "Fri, 04 Mar 2016 12:20:18 GMT" }) + stub_request(:delete, "#{container_url_for("phil")}/food/camaron"). + to_return(status: 200, headers: { etag: '"0816etag"' }) + stub_request(:put, "#{container_url_for("phil")}/food/desayunos/bolon"). + to_return(status: 200, headers: { etag: '"0817etag"' }) + stub_request(:head, "#{container_url_for("phil")}/food/desayunos/bolon"). + to_return(status: 200, headers: { last_modified: "Fri, 04 Mar 2016 12:20:18 GMT" }) + stub_request(:delete, "#{container_url_for("phil")}/food/desayunos/bolon"). + to_return(status: 200, headers: { etag: '"0817etag"' }) + stub_request(:get, "#{container_url_for("phil")}/food/aguacate"). + to_return(status: 200, body: "rootbody", headers: { etag: '"0817etag"', content_type: "text/plain; charset=utf-8" }) + stub_request(:put, "#{container_url_for("phil")}/bamboo.txt"). + to_return(status: 200, headers: { etag: '"0818etag"' }) + stub_request(:head, "#{container_url_for("phil")}/bamboo.txt"). + to_return(status: 200, headers: { last_modified: "Fri, 04 Mar 2016 12:20:18 GMT" }) + stub_request(:head, "#{container_url_for("phil")}/food/steak"). + to_return(status: 404) + stub_request(:get, "#{container_url_for("phil")}/food/steak"). + to_return(status: 404) end - describe "DELETE requests" do - - before do - purge_redis - end - - context "not authorized" do - describe "with no token" do - it "says it's not authorized" do - delete "/phil/food/aguacate" - - last_response.status.must_equal 401 - last_response.body.must_equal "Unauthorized" - end - end - - describe "with empty token" do - it "says it's not authorized" do - header "Authorization", "Bearer " - delete "/phil/food/aguacate" - - last_response.status.must_equal 401 - last_response.body.must_equal "Unauthorized" - end - end - - describe "with wrong token" do - it "says it's not authorized" do - header "Authorization", "Bearer wrongtoken" - delete "/phil/food/aguacate" - - last_response.status.must_equal 401 - last_response.body.must_equal "Unauthorized" - end - end - - end - - context "authorized" do - before do - redis.sadd "authorizations:phil:amarillo", [":rw"] - header "Authorization", "Bearer amarillo" - - put_stub = OpenStruct.new(headers: { - etag: '"bla"' - }) - - RestClient.stub :put, put_stub do - RestClient.stub :head, OpenStruct.new(headers: { last_modified: "Fri, 04 Mar 2016 12:20:18 GMT" }) do - put "/phil/food/aguacate", "si" - put "/phil/food/camaron", "yummi" - put "/phil/food/desayunos/bolon", "wow" - end - end - end - - it "decreases the size log by size of deleted object" do - RestClient.stub :delete, "" do - RemoteStorage::S3Rest.stub_any_instance :etag_for, "rootetag" do - RestClient.stub :head, OpenStruct.new(headers: { last_modified: "Fri, 04 Mar 2016 12:20:18 GMT" }) do - delete "/phil/food/aguacate" - end - end - end - - size_log = redis.get "rs:s:phil" - size_log.must_equal "8" - end - - it "deletes the metadata object in redis" do - RestClient.stub :delete, "" do - RemoteStorage::S3Rest.stub_any_instance :etag_for, "rootetag" do - RestClient.stub :head, OpenStruct.new(headers: { last_modified: "Fri, 04 Mar 2016 12:20:18 GMT" }) do - delete "/phil/food/aguacate" - end - end - end - - metadata = redis.hgetall "rs:m:phil:food/aguacate" - metadata.must_be_empty - end - - it "deletes the directory objects metadata in redis" do - old_metadata = redis.hgetall "rs:m:phil:food/" - - RestClient.stub :delete, "" do - RemoteStorage::S3Rest.stub_any_instance :etag_for, "newetag" do - RestClient.stub :head, OpenStruct.new(headers: { last_modified: "Fri, 04 Mar 2016 12:20:18 GMT" }) do - delete "/phil/food/aguacate" - end - end - end - - metadata = redis.hgetall "rs:m:phil:food/" - metadata["e"].must_equal "newetag" - metadata["m"].length.must_equal 13 - metadata["m"].wont_equal old_metadata["m"] - - food_items = redis.smembers "rs:m:phil:food/:items" - food_items.sort.must_equal ["camaron", "desayunos/"] - - root_items = redis.smembers "rs:m:phil:/:items" - root_items.must_equal ["food/"] - end - - it "deletes the parent directory objects metadata when deleting all items" do - RestClient.stub :delete, "" do - RemoteStorage::S3Rest.stub_any_instance :etag_for, "rootetag" do - RestClient.stub :head, OpenStruct.new(headers: { last_modified: "Fri, 04 Mar 2016 12:20:18 GMT" }) do - delete "/phil/food/aguacate" - delete "/phil/food/camaron" - delete "/phil/food/desayunos/bolon" - end - end - end - - redis.smembers("rs:m:phil:food/desayunos:items").must_be_empty - redis.hgetall("rs:m:phil:food/desayunos/").must_be_empty - - redis.smembers("rs:m:phil:food/:items").must_be_empty - redis.hgetall("rs:m:phil:food/").must_be_empty - - redis.smembers("rs:m:phil:/:items").must_be_empty - end - - it "responds with the ETag of the deleted item in the header" do - RestClient.stub :delete, "" do - RestClient.stub :head, OpenStruct.new(headers: { last_modified: "Fri, 04 Mar 2016 12:20:18 GMT" }) do - delete "/phil/food/aguacate" - end - end - - last_response.headers["ETag"].must_equal "\"bla\"" - end - - context "when item doesn't exist" do - before do - purge_redis - - put_stub = OpenStruct.new(headers: { - etag: '"bla"' - }) - - RestClient.stub :put, put_stub do - RestClient.stub :head, OpenStruct.new(headers: { last_modified: "Fri, 04 Mar 2016 12:20:18 GMT" }) do - put "/phil/food/steak", "si" - end - end - - raises_exception = ->(url, headers) { raise RestClient::ResourceNotFound.new } - RestClient.stub :head, raises_exception do - delete "/phil/food/steak" - end - end - - it "returns a 404" do - last_response.status.must_equal 404 - last_response.body.must_equal "Not Found" - end - - it "deletes any metadata that might still exist" do - raises_exception = ->(url, headers) { raise RestClient::ResourceNotFound.new } - RestClient.stub :head, raises_exception do - delete "/phil/food/steak" - end - - metadata = redis.hgetall "rs:m:phil:food/steak" - metadata.must_be_empty - - redis.smembers("rs:m:phil:food/:items").must_be_empty - redis.hgetall("rs:m:phil:food/").must_be_empty - - redis.smembers("rs:m:phil:/:items").must_be_empty - end - end - - describe "If-Match header" do - it "succeeds when the header matches the current ETag" do - header "If-Match", "\"bla\"" - - RestClient.stub :delete, "" do - RestClient.stub :head, OpenStruct.new(headers: { last_modified: "Fri, 04 Mar 2016 12:20:18 GMT" }) do - delete "/phil/food/aguacate" - end - end - - last_response.status.must_equal 200 - end - - it "succeeds when the header contains a weak ETAG matching the current ETag" do - header "If-Match", "W/\"bla\"" - - RestClient.stub :delete, "" do - RestClient.stub :head, OpenStruct.new(headers: { last_modified: "Fri, 04 Mar 2016 12:20:18 GMT" }) do - delete "/phil/food/aguacate" - end - end - - last_response.status.must_equal 200 - end - - it "fails the request if it does not match the current ETag" do - header "If-Match", "someotheretag" - - RestClient.stub :delete, "" do - RestClient.stub :head, OpenStruct.new(headers: { etag: '"someetag"' }) do - delete "/phil/food/aguacate" - end - end - - last_response.status.must_equal 412 - last_response.body.must_equal "Precondition Failed" - end - end - end - end - - describe "GET requests" do - - before do - purge_redis - end - - context "not authorized" do - - describe "without token" do - it "says it's not authorized" do - get "/phil/food/" - - last_response.status.must_equal 401 - last_response.body.must_equal "Unauthorized" - end - end - - describe "with wrong token" do - it "says it's not authorized" do - header "Authorization", "Bearer wrongtoken" - get "/phil/food/" - - last_response.status.must_equal 401 - last_response.body.must_equal "Unauthorized" - end - end - - end - - context "authorized" do - - before do - redis.sadd "authorizations:phil:amarillo", [":rw"] - header "Authorization", "Bearer amarillo" - - put_stub = OpenStruct.new(headers: { - etag: '"0815etag"' - }) - - RestClient.stub :put, put_stub do - RestClient.stub :head, OpenStruct.new(headers: { last_modified: "Fri, 04 Mar 2016 12:20:18 GMT" }) do - put "/phil/food/aguacate", "si" - put "/phil/food/camaron", "yummi" - put "/phil/food/desayunos/bolon", "wow" - end - end - end - - describe "documents" do - - it "returns the required response headers" do - get_stub = OpenStruct.new(body: "si", headers: { - etag: '"0815etag"', - last_modified: "Fri, 04 Mar 2016 12:20:18 GMT", - content_type: "text/plain; charset=utf-8", - content_length: 2 - }) - - RestClient.stub :get, get_stub do - get "/phil/food/aguacate" - end - - last_response.status.must_equal 200 - last_response.headers["ETag"].must_equal "\"0815etag\"" - last_response.headers["Cache-Control"].must_equal "no-cache" - last_response.headers["Content-Type"].must_equal "text/plain; charset=utf-8" - end - - it "returns a 404 when data doesn't exist" do - raises_exception = ->(url, headers) { raise RestClient::ResourceNotFound.new } - RestClient.stub :get, raises_exception do - get "/phil/food/steak" - end - - last_response.status.must_equal 404 - last_response.body.must_equal "Not Found" - end - - it "responds with 304 when IF_NONE_MATCH header contains the ETag" do - header "If-None-Match", "\"0815etag\"" - - get "/phil/food/aguacate" - - last_response.status.must_equal 304 - last_response.headers["ETag"].must_equal "\"0815etag\"" - last_response.headers["Last-Modified"].must_equal "Fri, 04 Mar 2016 12:20:18 GMT" - end - - it "responds with 304 when IF_NONE_MATCH header contains weak ETAG matching the current ETag" do - header "If-None-Match", "W/\"0815etag\"" - - get "/phil/food/aguacate" - - last_response.status.must_equal 304 - last_response.headers["ETag"].must_equal "\"0815etag\"" - last_response.headers["Last-Modified"].must_equal "Fri, 04 Mar 2016 12:20:18 GMT" - end - - end - - describe "directory listings" do - - it "returns the correct ETag header" do - get "/phil/food/" - - last_response.status.must_equal 200 - last_response.headers["ETag"].must_equal "\"f9f85fbf5aa1fa378fd79ac8aa0a457d\"" - end - - it "returns a Cache-Control header with value 'no-cache'" do - get "/phil/food/" - - last_response.status.must_equal 200 - last_response.headers["Cache-Control"].must_equal "no-cache" - end - - it "responds with 304 when IF_NONE_MATCH header contains the ETag" do - header "If-None-Match", "\"f9f85fbf5aa1fa378fd79ac8aa0a457d\"" - get "/phil/food/" - - last_response.status.must_equal 304 - end - - it "responds with 304 when IF_NONE_MATCH header contains weak ETAG matching the ETag" do - header "If-None-Match", "W/\"f9f85fbf5aa1fa378fd79ac8aa0a457d\"" - get "/phil/food/" - - last_response.status.must_equal 304 - end - - it "contains all items in the directory" do - get "/phil/food/" - - last_response.status.must_equal 200 - last_response.content_type.must_equal "application/ld+json" - - content = JSON.parse(last_response.body) - content["@context"].must_equal "http://remotestorage.io/spec/folder-description" - content["items"]["aguacate"].wont_be_nil - content["items"]["aguacate"]["Content-Type"].must_equal "text/plain; charset=utf-8" - content["items"]["aguacate"]["Content-Length"].must_equal 2 - content["items"]["aguacate"]["ETag"].must_equal "0815etag" - content["items"]["camaron"].wont_be_nil - content["items"]["camaron"]["Content-Type"].must_equal "text/plain; charset=utf-8" - content["items"]["camaron"]["Content-Length"].must_equal 5 - content["items"]["camaron"]["ETag"].must_equal "0815etag" - content["items"]["desayunos/"].wont_be_nil - content["items"]["desayunos/"]["ETag"].must_equal "dd36e3cfe52b5f33421150b289a7d48d" - end - - it "contains all items in the root directory" do - get "phil/" - - last_response.status.must_equal 200 - last_response.content_type.must_equal "application/ld+json" - - content = JSON.parse(last_response.body) - content["items"]["food/"].wont_be_nil - content["items"]["food/"]["ETag"].must_equal "f9f85fbf5aa1fa378fd79ac8aa0a457d" - end - - it "responds with an empty directory liting when directory doesn't exist" do - get "phil/some-non-existing-dir/" - - last_response.status.must_equal 200 - last_response.content_type.must_equal "application/ld+json" - - content = JSON.parse(last_response.body) - content["items"].must_equal({}) - end - - end - end - - end - - describe "HEAD requests" do - - before do - purge_redis - end - - context "not authorized" do - - describe "without token" do - it "says it's not authorized" do - head "/phil/food/camarones" - - last_response.status.must_equal 401 - last_response.body.must_be_empty - end - end - - describe "with wrong token" do - it "says it's not authorized" do - header "Authorization", "Bearer wrongtoken" - head "/phil/food/camarones" - - last_response.status.must_equal 401 - last_response.body.must_be_empty - end - end - - end - - context "authorized" do - - before do - redis.sadd "authorizations:phil:amarillo", [":rw"] - header "Authorization", "Bearer amarillo" - - put_stub = OpenStruct.new(headers: { - etag: "0815etag" - }) - - RestClient.stub :put, put_stub do - RestClient.stub :head, OpenStruct.new(headers: { last_modified: "Fri, 04 Mar 2016 12:20:18 GMT" }) do - put "/phil/food/aguacate", "si" - put "/phil/food/camaron", "yummi" - put "/phil/food/desayunos/bolon", "wow" - end - end - end - - describe "directory listings" do - it "returns the correct header information" do - get "/phil/food/" - - last_response.status.must_equal 200 - last_response.content_type.must_equal "application/ld+json" - last_response.headers["ETag"].must_equal "\"f9f85fbf5aa1fa378fd79ac8aa0a457d\"" - end - end - - describe "documents" do - context "when the document doesn't exist" do - it "returns a 404" do - head "/phil/food/steak" - - last_response.status.must_equal 404 - last_response.body.must_be_empty - end - end - - context "when the document exists" do - it "returns the required response headers" do - head "/phil/food/aguacate" - - last_response.status.must_equal 200 - last_response.headers["ETag"].must_equal "\"0815etag\"" - last_response.headers["Cache-Control"].must_equal "no-cache" - end - - it "responds with 304 when IF_NONE_MATCH header contains the ETag" do - header "If-None-Match", "\"0815etag\"" - - head "/phil/food/aguacate" - - last_response.status.must_equal 304 - last_response.headers["ETag"].must_equal "\"0815etag\"" - last_response.headers["Last-Modified"].must_equal "Fri, 04 Mar 2016 12:20:18 GMT" - end - - it "responds with 304 when IF_NONE_MATCH header contains weak ETAG matching the current ETag" do - header "If-None-Match", "W/\"0815etag\"" - - head "/phil/food/aguacate" - - last_response.status.must_equal 304 - last_response.headers["ETag"].must_equal "\"0815etag\"" - last_response.headers["Last-Modified"].must_equal "Fri, 04 Mar 2016 12:20:18 GMT" - end - end - end - end - end + it_behaves_like 'a REST adapter' end diff --git a/spec/shared_examples.rb b/spec/shared_examples.rb new file mode 100644 index 0000000..ec74e15 --- /dev/null +++ b/spec/shared_examples.rb @@ -0,0 +1,635 @@ +require_relative "./spec_helper" + +shared_examples_for 'a REST adapter' do + include Rack::Test::Methods + + def container_url_for(user) + raise NotImplementedError + end + + def storage_class + raise NotImplementedError + end + + def config_file + raise NotImplementedError + end + + it "returns 404 on non-existing routes" do + get "/virginmargarita" + last_response.status.must_equal 404 + end + + describe "PUT requests" do + + before do + purge_redis + end + + context "authorized" do + before do + redis.sadd "authorizations:phil:amarillo", [":rw"] + header "Authorization", "Bearer amarillo" + end + + it "creates the metadata object in redis" do + put "/phil/food/aguacate", "si" + + metadata = redis.hgetall "rs:m:phil:food/aguacate" + metadata["s"].must_equal "2" + metadata["t"].must_equal "text/plain; charset=utf-8" + metadata["e"].must_equal "0815etag" + metadata["m"].length.must_equal 13 + end + + it "creates the directory objects metadata in redis" do + put "/phil/food/aguacate", "si" + put "/phil/food/camaron", "yummi" + + metadata = redis.hgetall "rs:m:phil:/" + metadata["e"].must_equal "fe2976909daaf074660981ab563fe65d" + metadata["m"].length.must_equal 13 + + metadata = redis.hgetall "rs:m:phil:food/" + metadata["e"].must_equal "926f98ff820f2f9764fd3c60a22865ad" + metadata["m"].length.must_equal 13 + + food_items = redis.smembers "rs:m:phil:food/:items" + food_items.each do |food_item| + ["camaron", "aguacate"].must_include food_item + end + + root_items = redis.smembers "rs:m:phil:/:items" + root_items.must_equal ["food/"] + end + + context "response code" do + it "is 201 for newly created objects" do + put "/phil/food/aguacate", "ci" + + last_response.status.must_equal 201 + end + + it "is 200 for updated objects" do + put "/phil/food/aguacate", "deliciosa" + put "/phil/food/aguacate", "muy deliciosa" + + last_response.status.must_equal 200 + end + end + + context "logging usage size" do + it "logs the complete size when creating new objects" do + put "/phil/food/aguacate", "1234567890" + + size_log = redis.get "rs:s:phil" + size_log.must_equal "10" + end + + it "logs the size difference when updating existing objects" do + put "/phil/food/camaron", "1234567890" + put "/phil/food/aguacate", "1234567890" + put "/phil/food/aguacate", "123" + + size_log = redis.get "rs:s:phil" + size_log.must_equal "13" + end + end + + describe "objects in root dir" do + before do + put "/phil/bamboo.txt", "shir kan" + end + + it "are listed in the directory listing with all metadata" do + get "phil/" + + last_response.status.must_equal 200 + last_response.content_type.must_equal "application/ld+json" + + content = JSON.parse(last_response.body) + content["items"]["bamboo.txt"].wont_be_nil + content["items"]["bamboo.txt"]["ETag"].must_equal "0818etag" + content["items"]["bamboo.txt"]["Content-Type"].must_equal "text/plain; charset=utf-8" + content["items"]["bamboo.txt"]["Content-Length"].must_equal 8 + content["items"]["bamboo.txt"]["Last-Modified"].must_equal "Fri, 04 Mar 2016 12:20:18 GMT" + end + end + + describe "name collision checks" do + it "is successful when there is no name collision" do + put "/phil/food/aguacate", "si" + + last_response.status.must_equal 201 + + metadata = redis.hgetall "rs:m:phil:food/aguacate" + metadata["s"].must_equal "2" + end + + it "conflicts when there is a directory with same name as document" do + put "/phil/food/aguacate", "si" + put "/phil/food", "wontwork" + + last_response.status.must_equal 409 + last_response.body.must_equal "Conflict" + + metadata = redis.hgetall "rs:m:phil:food" + metadata.must_be_empty + end + + it "conflicts when there is a document with same name as directory" do + put "/phil/food/aguacate", "si" + put "/phil/food/aguacate/empanado", "wontwork" + + last_response.status.must_equal 409 + + metadata = redis.hgetall "rs:m:phil:food/aguacate/empanado" + metadata.must_be_empty + end + + it "returns 400 when a Content-Range header is sent" do + header "Content-Range", "bytes 0-3/3" + + put "/phil/food/aguacate", "si" + + last_response.status.must_equal 400 + end + end + + describe "If-Match header" do + before do + put "/phil/food/aguacate", "si" + end + + it "allows the request if the header matches the current ETag" do + header "If-Match", "\"0815etag\"" + + put "/phil/food/aguacate", "aye" + + last_response.status.must_equal 200 + last_response.headers["Etag"].must_equal "\"0915etag\"" + end + + it "allows the request if the header contains a weak ETAG matching the current ETag" do + header "If-Match", "W/\"0815etag\"" + + put "/phil/food/aguacate", "aye" + + last_response.status.must_equal 200 + last_response.headers["Etag"].must_equal "\"0915etag\"" + end + + it "allows the request if the header contains a weak ETAG with leading quote matching the current ETag" do + header "If-Match", "\"W/\"0815etag\"" + + put "/phil/food/aguacate", "aye" + + last_response.status.must_equal 200 + last_response.headers["Etag"].must_equal "\"0915etag\"" + end + + it "fails the request if the header does not match the current ETag" do + header "If-Match", "someotheretag" + + put "/phil/food/aguacate", "aye" + + last_response.status.must_equal 412 + last_response.body.must_equal "Precondition Failed" + end + + it "allows the request if redis metadata became out of sync" do + header "If-Match", "\"0815etag\"" + + put "/phil/food/aguacate", "aye" + + last_response.status.must_equal 200 + end + end + + describe "If-None-Match header set to '*'" do + it "succeeds when the document doesn't exist yet" do + header "If-None-Match", "*" + + put "/phil/food/aguacate", "si" + + last_response.status.must_equal 201 + end + + it "fails the request if the document already exists" do + put "/phil/food/aguacate", "si" + + header "If-None-Match", "*" + put "/phil/food/aguacate", "si" + + last_response.status.must_equal 412 + last_response.body.must_equal "Precondition Failed" + end + end + end + + end + + describe "DELETE requests" do + + before do + purge_redis + end + + context "not authorized" do + describe "with no token" do + it "says it's not authorized" do + delete "/phil/food/aguacate" + + last_response.status.must_equal 401 + last_response.body.must_equal "Unauthorized" + end + end + + describe "with empty token" do + it "says it's not authorized" do + header "Authorization", "Bearer " + delete "/phil/food/aguacate" + + last_response.status.must_equal 401 + last_response.body.must_equal "Unauthorized" + end + end + + describe "with wrong token" do + it "says it's not authorized" do + header "Authorization", "Bearer wrongtoken" + delete "/phil/food/aguacate" + + last_response.status.must_equal 401 + last_response.body.must_equal "Unauthorized" + end + end + + end + + context "authorized" do + before do + redis.sadd "authorizations:phil:amarillo", [":rw"] + header "Authorization", "Bearer amarillo" + + put "/phil/food/aguacate", "si" + put "/phil/food/camaron", "yummi" + put "/phil/food/desayunos/bolon", "wow" + end + + it "decreases the size log by size of deleted object" do + delete "/phil/food/aguacate" + + size_log = redis.get "rs:s:phil" + size_log.must_equal "8" + end + + it "deletes the metadata object in redis" do + delete "/phil/food/aguacate" + + metadata = redis.hgetall "rs:m:phil:food/aguacate" + metadata.must_be_empty + end + + it "deletes the directory objects metadata in redis" do + old_metadata = redis.hgetall "rs:m:phil:food/" + + storage_class.stub_any_instance :etag_for, "newetag" do + delete "/phil/food/aguacate" + end + + metadata = redis.hgetall "rs:m:phil:food/" + metadata["e"].must_equal "newetag" + metadata["m"].length.must_equal 13 + metadata["m"].wont_equal old_metadata["m"] + + food_items = redis.smembers "rs:m:phil:food/:items" + food_items.sort.must_equal ["camaron", "desayunos/"] + + root_items = redis.smembers "rs:m:phil:/:items" + root_items.must_equal ["food/"] + end + + it "deletes the parent directory objects metadata when deleting all items" do + delete "/phil/food/aguacate" + delete "/phil/food/camaron" + delete "/phil/food/desayunos/bolon" + + redis.smembers("rs:m:phil:food/desayunos:items").must_be_empty + redis.hgetall("rs:m:phil:food/desayunos/").must_be_empty + + redis.smembers("rs:m:phil:food/:items").must_be_empty + redis.hgetall("rs:m:phil:food/").must_be_empty + + redis.smembers("rs:m:phil:/:items").must_be_empty + end + + it "responds with the ETag of the deleted item in the header" do + delete "/phil/food/aguacate" + + last_response.headers["ETag"].must_equal "\"0815etag\"" + end + + context "when item doesn't exist" do + before do + purge_redis + + delete "/phil/food/steak" + end + + it "returns a 404" do + last_response.status.must_equal 404 + last_response.body.must_equal "Not Found" + end + + it "deletes any metadata that might still exist" do + delete "/phil/food/steak" + + metadata = redis.hgetall "rs:m:phil:food/steak" + metadata.must_be_empty + + redis.smembers("rs:m:phil:food/:items").must_be_empty + redis.hgetall("rs:m:phil:food/").must_be_empty + + redis.smembers("rs:m:phil:/:items").must_be_empty + end + end + + describe "If-Match header" do + it "succeeds when the header matches the current ETag" do + header "If-Match", "\"0815etag\"" + + delete "/phil/food/aguacate" + + last_response.status.must_equal 200 + end + + it "succeeds when the header contains a weak ETAG matching the current ETag" do + header "If-Match", "W/\"0815etag\"" + + delete "/phil/food/aguacate" + + last_response.status.must_equal 200 + end + + it "fails the request if it does not match the current ETag" do + header "If-Match", "someotheretag" + + delete "/phil/food/aguacate" + + last_response.status.must_equal 412 + last_response.body.must_equal "Precondition Failed" + end + end + end + end + + describe "GET requests" do + + before do + purge_redis + end + + context "not authorized" do + + describe "without token" do + it "says it's not authorized" do + get "/phil/food/" + + last_response.status.must_equal 401 + last_response.body.must_equal "Unauthorized" + end + end + + describe "with wrong token" do + it "says it's not authorized" do + header "Authorization", "Bearer wrongtoken" + get "/phil/food/" + + last_response.status.must_equal 401 + last_response.body.must_equal "Unauthorized" + end + end + + end + + context "authorized" do + + before do + redis.sadd "authorizations:phil:amarillo", [":rw"] + header "Authorization", "Bearer amarillo" + + put "/phil/food/aguacate", "si" + put "/phil/food/camaron", "yummi" + put "/phil/food/desayunos/bolon", "wow" + end + + describe "documents" do + + it "returns the required response headers" do + get "/phil/food/aguacate" + + last_response.status.must_equal 200 + last_response.headers["ETag"].must_equal "\"0817etag\"" + last_response.headers["Cache-Control"].must_equal "no-cache" + last_response.headers["Content-Type"].must_equal "text/plain; charset=utf-8" + end + + it "returns a 404 when data doesn't exist" do + get "/phil/food/steak" + + last_response.status.must_equal 404 + last_response.body.must_equal "Not Found" + end + + it "responds with 304 when IF_NONE_MATCH header contains the ETag" do + header "If-None-Match", "\"0815etag\"" + + get "/phil/food/aguacate" + + last_response.status.must_equal 304 + last_response.headers["ETag"].must_equal "\"0815etag\"" + last_response.headers["Last-Modified"].must_equal "Fri, 04 Mar 2016 12:20:18 GMT" + end + + it "responds with 304 when IF_NONE_MATCH header contains weak ETAG matching the current ETag" do + header "If-None-Match", "W/\"0815etag\"" + + get "/phil/food/aguacate" + + last_response.status.must_equal 304 + last_response.headers["ETag"].must_equal "\"0815etag\"" + last_response.headers["Last-Modified"].must_equal "Fri, 04 Mar 2016 12:20:18 GMT" + end + + end + + describe "directory listings" do + + it "returns the correct ETag header" do + get "/phil/food/" + + last_response.status.must_equal 200 + last_response.headers["ETag"].must_equal "\"f9f85fbf5aa1fa378fd79ac8aa0a457d\"" + end + + it "returns a Cache-Control header with value 'no-cache'" do + get "/phil/food/" + + last_response.status.must_equal 200 + last_response.headers["Cache-Control"].must_equal "no-cache" + end + + it "responds with 304 when IF_NONE_MATCH header contains the ETag" do + header "If-None-Match", "\"f9f85fbf5aa1fa378fd79ac8aa0a457d\"" + get "/phil/food/" + + last_response.status.must_equal 304 + end + + it "responds with 304 when IF_NONE_MATCH header contains weak ETAG matching the ETag" do + header "If-None-Match", "W/\"f9f85fbf5aa1fa378fd79ac8aa0a457d\"" + get "/phil/food/" + + last_response.status.must_equal 304 + end + + it "contains all items in the directory" do + get "/phil/food/" + + last_response.status.must_equal 200 + last_response.content_type.must_equal "application/ld+json" + + content = JSON.parse(last_response.body) + content["@context"].must_equal "http://remotestorage.io/spec/folder-description" + content["items"]["aguacate"].wont_be_nil + content["items"]["aguacate"]["Content-Type"].must_equal "text/plain; charset=utf-8" + content["items"]["aguacate"]["Content-Length"].must_equal 2 + content["items"]["aguacate"]["ETag"].must_equal "0815etag" + content["items"]["camaron"].wont_be_nil + content["items"]["camaron"]["Content-Type"].must_equal "text/plain; charset=utf-8" + content["items"]["camaron"]["Content-Length"].must_equal 5 + content["items"]["camaron"]["ETag"].must_equal "0816etag" + content["items"]["desayunos/"].wont_be_nil + content["items"]["desayunos/"]["ETag"].must_equal "dd36e3cfe52b5f33421150b289a7d48d" + end + + it "contains all items in the root directory" do + get "phil/" + + last_response.status.must_equal 200 + last_response.content_type.must_equal "application/ld+json" + + content = JSON.parse(last_response.body) + content["items"]["food/"].wont_be_nil + content["items"]["food/"]["ETag"].must_equal "f9f85fbf5aa1fa378fd79ac8aa0a457d" + end + + it "responds with an empty directory liting when directory doesn't exist" do + get "phil/some-non-existing-dir/" + + last_response.status.must_equal 200 + last_response.content_type.must_equal "application/ld+json" + + content = JSON.parse(last_response.body) + content["items"].must_equal({}) + end + + end + end + + end + + describe "HEAD requests" do + + before do + purge_redis + end + + context "not authorized" do + + describe "without token" do + it "says it's not authorized" do + head "/phil/food/camarones" + + last_response.status.must_equal 401 + last_response.body.must_be_empty + end + end + + describe "with wrong token" do + it "says it's not authorized" do + header "Authorization", "Bearer wrongtoken" + head "/phil/food/camarones" + + last_response.status.must_equal 401 + last_response.body.must_be_empty + end + end + + end + + context "authorized" do + + before do + redis.sadd "authorizations:phil:amarillo", [":rw"] + header "Authorization", "Bearer amarillo" + + put "/phil/food/aguacate", "si" + put "/phil/food/camaron", "yummi" + put "/phil/food/desayunos/bolon", "wow" + end + + describe "directory listings" do + it "returns the correct header information" do + get "/phil/food/" + + last_response.status.must_equal 200 + last_response.content_type.must_equal "application/ld+json" + last_response.headers["ETag"].must_equal "\"f9f85fbf5aa1fa378fd79ac8aa0a457d\"" + end + end + + describe "documents" do + context "when the document doesn't exist" do + it "returns a 404" do + head "/phil/food/steak" + + last_response.status.must_equal 404 + last_response.body.must_be_empty + end + end + + context "when the document exists" do + it "returns the required response headers" do + head "/phil/food/aguacate" + + last_response.status.must_equal 200 + last_response.headers["ETag"].must_equal "\"0815etag\"" + last_response.headers["Cache-Control"].must_equal "no-cache" + end + + it "responds with 304 when IF_NONE_MATCH header contains the ETag" do + header "If-None-Match", "\"0815etag\"" + + head "/phil/food/aguacate" + + last_response.status.must_equal 304 + last_response.headers["ETag"].must_equal "\"0815etag\"" + last_response.headers["Last-Modified"].must_equal "Fri, 04 Mar 2016 12:20:18 GMT" + end + + it "responds with 304 when IF_NONE_MATCH header contains weak ETAG matching the current ETag" do + header "If-None-Match", "W/\"0815etag\"" + + head "/phil/food/aguacate" + + last_response.status.must_equal 304 + last_response.headers["ETag"].must_equal "\"0815etag\"" + last_response.headers["Last-Modified"].must_equal "Fri, 04 Mar 2016 12:20:18 GMT" + end + end + end + end + end +end diff --git a/spec/spec_helper.rb b/spec/spec_helper.rb index b08c6b5..16a4dad 100644 --- a/spec/spec_helper.rb +++ b/spec/spec_helper.rb @@ -6,12 +6,15 @@ Bundler.require require_relative '../liquor-cabinet' require 'minitest/autorun' +require "minitest/stub_any_instance" require 'rack/test' require 'purdytest' require "redis" require "rest_client" -require "minitest/stub_any_instance" require "ostruct" +require 'webmock/minitest' + +WebMock.disable_net_connect! def app LiquorCabinet @@ -19,17 +22,6 @@ end app.set :environment, :test -def wait_a_second - now = Time.now.to_i - while Time.now.to_i == now; end -end - -def write_last_response_to_file(filename = "last_response.html") - File.open(filename, "w") do |f| - f.write last_response.body - end -end - alias context describe if app.settings.respond_to? :redis @@ -43,3 +35,23 @@ if app.settings.respond_to? :redis end end end + +MiniTest::Spec.class_eval do + def self.shared_examples + @shared_examples ||= {} + end +end + +module MiniTest::Spec::SharedExamples + def shared_examples_for(desc, &block) + MiniTest::Spec.shared_examples[desc] = block + end + + def it_behaves_like(desc) + self.instance_eval(&MiniTest::Spec.shared_examples[desc]) + end +end + +Object.class_eval { include(MiniTest::Spec::SharedExamples) } + +require_relative 'shared_examples' diff --git a/spec/swift/app_spec.rb b/spec/swift/app_spec.rb index 16311d9..a90c2d7 100644 --- a/spec/swift/app_spec.rb +++ b/spec/swift/app_spec.rb @@ -1,840 +1,56 @@ require_relative "../spec_helper" describe "App" do - include Rack::Test::Methods - - def app - LiquorCabinet + def container_url_for(user) + "#{app.settings.swift["host"]}/rs:documents:test/#{user}" end - it "returns 404 on non-existing routes" do - get "/virginmargarita" - last_response.status.must_equal 404 + def storage_class + RemoteStorage::Swift end - describe "PUT requests" do - - before do - purge_redis - end - - context "authorized" do - before do - redis.sadd "authorizations:phil:amarillo", [":rw"] - header "Authorization", "Bearer amarillo" - end - - it "creates the metadata object in redis" do - put_stub = OpenStruct.new(headers: { - etag: "bla", - last_modified: "Fri, 04 Mar 2016 12:20:18 GMT" - }) - - RestClient.stub :put, put_stub do - put "/phil/food/aguacate", "si" - end - - metadata = redis.hgetall "rs:m:phil:food/aguacate" - metadata["s"].must_equal "2" - metadata["t"].must_equal "text/plain; charset=utf-8" - metadata["e"].must_equal "bla" - metadata["m"].length.must_equal 13 - end - - it "creates the directory objects metadata in redis" do - put_stub = OpenStruct.new(headers: { - etag: "bla", - last_modified: "Fri, 04 Mar 2016 12:20:18 GMT" - }) - get_stub = OpenStruct.new(body: "rootbody") - - RestClient.stub :put, put_stub do - RestClient.stub :get, get_stub do - RemoteStorage::Swift.stub_any_instance :etag_for, "newetag" do - put "/phil/food/aguacate", "si" - put "/phil/food/camaron", "yummi" - end - end - end - - metadata = redis.hgetall "rs:m:phil:/" - metadata["e"].must_equal "newetag" - metadata["m"].length.must_equal 13 - - metadata = redis.hgetall "rs:m:phil:food/" - metadata["e"].must_equal "newetag" - metadata["m"].length.must_equal 13 - - food_items = redis.smembers "rs:m:phil:food/:items" - food_items.each do |food_item| - ["camaron", "aguacate"].must_include food_item - end - - root_items = redis.smembers "rs:m:phil:/:items" - root_items.must_equal ["food/"] - end - - context "response code" do - before do - @put_stub = OpenStruct.new(headers: { - etag: "bla", - last_modified: "Fri, 04 Mar 2016 12:20:18 GMT" - }) - end - - it "is 201 for newly created objects" do - RestClient.stub :put, @put_stub do - put "/phil/food/aguacate", "muy deliciosa" - end - - last_response.status.must_equal 201 - end - - it "is 200 for updated objects" do - RestClient.stub :put, @put_stub do - put "/phil/food/aguacate", "deliciosa" - put "/phil/food/aguacate", "muy deliciosa" - end - - last_response.status.must_equal 200 - end - end - - context "logging usage size" do - before do - @put_stub = OpenStruct.new(headers: { - etag: "bla", - last_modified: "Fri, 04 Mar 2016 12:20:18 GMT" - }) - end - - it "logs the complete size when creating new objects" do - RestClient.stub :put, @put_stub do - put "/phil/food/aguacate", "1234567890" - end - - size_log = redis.get "rs:s:phil" - size_log.must_equal "10" - end - - it "logs the size difference when updating existing objects" do - RestClient.stub :put, @put_stub do - put "/phil/food/camaron", "1234567890" - put "/phil/food/aguacate", "1234567890" - put "/phil/food/aguacate", "123" - end - - size_log = redis.get "rs:s:phil" - size_log.must_equal "13" - end - end - - describe "objects in root dir" do - before do - put_stub = OpenStruct.new(headers: { - etag: "bla", - last_modified: "Fri, 04 Mar 2016 12:20:18 GMT" - }) - - RestClient.stub :put, put_stub do - put "/phil/bamboo.txt", "shir kan" - end - end - - it "are listed in the directory listing with all metadata" do - get "phil/" - - last_response.status.must_equal 200 - last_response.content_type.must_equal "application/ld+json" - - content = JSON.parse(last_response.body) - content["items"]["bamboo.txt"].wont_be_nil - content["items"]["bamboo.txt"]["ETag"].must_equal "bla" - content["items"]["bamboo.txt"]["Content-Type"].must_equal "text/plain; charset=utf-8" - content["items"]["bamboo.txt"]["Content-Length"].must_equal 8 - content["items"]["bamboo.txt"]["Last-Modified"].must_equal "Fri, 04 Mar 2016 12:20:18 GMT" - end - end - - describe "name collision checks" do - it "is successful when there is no name collision" do - put_stub = OpenStruct.new(headers: { - etag: "bla", - last_modified: "Fri, 04 Mar 2016 12:20:18 GMT" - }) - get_stub = OpenStruct.new(body: "rootbody") - - RestClient.stub :put, put_stub do - RestClient.stub :get, get_stub do - RemoteStorage::Swift.stub_any_instance :etag_for, "rootetag" do - put "/phil/food/aguacate", "si" - end - end - end - - last_response.status.must_equal 201 - - metadata = redis.hgetall "rs:m:phil:food/aguacate" - metadata["s"].must_equal "2" - end - - it "conflicts when there is a directory with same name as document" do - put_stub = OpenStruct.new(headers: { - etag: "bla", - last_modified: "Fri, 04 Mar 2016 12:20:18 GMT" - }) - - RestClient.stub :put, put_stub do - put "/phil/food/aguacate", "si" - put "/phil/food", "wontwork" - end - - last_response.status.must_equal 409 - last_response.body.must_equal "Conflict" - - metadata = redis.hgetall "rs:m:phil:food" - metadata.must_be_empty - end - - it "conflicts when there is a document with same name as directory" do - put_stub = OpenStruct.new(headers: { - etag: "bla", - last_modified: "Fri, 04 Mar 2016 12:20:18 GMT" - }) - - RestClient.stub :put, put_stub do - put "/phil/food/aguacate", "si" - put "/phil/food/aguacate/empanado", "wontwork" - end - - last_response.status.must_equal 409 - - metadata = redis.hgetall "rs:m:phil:food/aguacate/empanado" - metadata.must_be_empty - end - - it "returns 400 when a Content-Range header is sent" do - header "Content-Range", "bytes 0-3/3" - - put "/phil/food/aguacate", "si" - - last_response.status.must_equal 400 - end - end - - describe "If-Match header" do - before do - put_stub = OpenStruct.new(headers: { - etag: "oldetag", - last_modified: "Fri, 04 Mar 2016 12:20:18 GMT" - }) - - RestClient.stub :put, put_stub do - put "/phil/food/aguacate", "si" - end - end - - it "allows the request if the header matches the current ETag" do - header "If-Match", "\"oldetag\"" - - put_stub = OpenStruct.new(headers: { - etag: "newetag", - last_modified: "Fri, 04 Mar 2016 12:20:18 GMT" - }) - - RestClient.stub :put, put_stub do - put "/phil/food/aguacate", "aye" - end - - last_response.status.must_equal 200 - last_response.headers["Etag"].must_equal "\"newetag\"" - end - - it "allows the request if the header contains a weak ETAG matching the current ETag" do - header "If-Match", "W/\"oldetag\"" - - put_stub = OpenStruct.new(headers: { - etag: "newetag", - last_modified: "Fri, 04 Mar 2016 12:20:18 GMT" - }) - - RestClient.stub :put, put_stub do - put "/phil/food/aguacate", "aye" - end - - last_response.status.must_equal 200 - last_response.headers["Etag"].must_equal "\"newetag\"" - end - - it "allows the request if the header contains a weak ETAG with leading quote matching the current ETag" do - header "If-Match", "\"W/\"oldetag\"" - - put_stub = OpenStruct.new(headers: { - etag: "newetag", - last_modified: "Fri, 04 Mar 2016 12:20:18 GMT" - }) - - RestClient.stub :put, put_stub do - put "/phil/food/aguacate", "aye" - end - - last_response.status.must_equal 200 - last_response.headers["Etag"].must_equal "\"newetag\"" - end - - it "fails the request if the header does not match the current ETag" do - header "If-Match", "someotheretag" - - head_stub = OpenStruct.new(headers: { - etag: "oldetag", - last_modified: "Fri, 04 Mar 2016 12:20:18 GMT", - content_type: "text/plain", - content_length: 23 - }) - - RestClient.stub :head, head_stub do - put "/phil/food/aguacate", "aye" - end - - last_response.status.must_equal 412 - last_response.body.must_equal "Precondition Failed" - end - - it "allows the request if redis metadata became out of sync" do - header "If-Match", "\"existingetag\"" - - head_stub = OpenStruct.new(headers: { - etag: "existingetag", - last_modified: "Fri, 04 Mar 2016 12:20:18 GMT", - content_type: "text/plain", - content_length: 23 - }) - - put_stub = OpenStruct.new(headers: { - etag: "newetag", - last_modified: "Fri, 04 Mar 2016 12:20:18 GMT" - }) - - RestClient.stub :head, head_stub do - RestClient.stub :put, put_stub do - put "/phil/food/aguacate", "aye" - end - end - - last_response.status.must_equal 200 - end - end - - describe "If-None-Match header set to '*'" do - it "succeeds when the document doesn't exist yet" do - put_stub = OpenStruct.new(headers: { - etag: "someetag", - last_modified: "Fri, 04 Mar 2016 12:20:18 GMT" - }) - - header "If-None-Match", "*" - - RestClient.stub :put, put_stub do - put "/phil/food/aguacate", "si" - end - - last_response.status.must_equal 201 - end - - it "fails the request if the document already exists" do - put_stub = OpenStruct.new(headers: { - etag: "someetag", - last_modified: "Fri, 04 Mar 2016 12:20:18 GMT" - }) - - RestClient.stub :put, put_stub do - put "/phil/food/aguacate", "si" - end - - header "If-None-Match", "*" - RestClient.stub :put, put_stub do - put "/phil/food/aguacate", "si" - end - - last_response.status.must_equal 412 - last_response.body.must_equal "Precondition Failed" - end - end - end - + def config_file + "config.yml.example.swift" end - describe "DELETE requests" do - - before do - purge_redis - end - - context "not authorized" do - describe "with no token" do - it "says it's not authorized" do - delete "/phil/food/aguacate" - - last_response.status.must_equal 401 - last_response.body.must_equal "Unauthorized" - end - end - - describe "with empty token" do - it "says it's not authorized" do - header "Authorization", "Bearer " - delete "/phil/food/aguacate" - - last_response.status.must_equal 401 - last_response.body.must_equal "Unauthorized" - end - end - - describe "with wrong token" do - it "says it's not authorized" do - header "Authorization", "Bearer wrongtoken" - delete "/phil/food/aguacate" - - last_response.status.must_equal 401 - last_response.body.must_equal "Unauthorized" - end - end - - end - - context "authorized" do - before do - redis.sadd "authorizations:phil:amarillo", [":rw"] - header "Authorization", "Bearer amarillo" - - put_stub = OpenStruct.new(headers: { - etag: "bla", - last_modified: "Fri, 04 Mar 2016 12:20:18 GMT" - }) - - RestClient.stub :put, put_stub do - put "/phil/food/aguacate", "si" - put "/phil/food/camaron", "yummi" - put "/phil/food/desayunos/bolon", "wow" - end - end - - it "decreases the size log by size of deleted object" do - RestClient.stub :delete, "" do - RemoteStorage::Swift.stub_any_instance :etag_for, "rootetag" do - delete "/phil/food/aguacate" - end - end - - size_log = redis.get "rs:s:phil" - size_log.must_equal "8" - end - - it "deletes the metadata object in redis" do - RestClient.stub :delete, "" do - RemoteStorage::Swift.stub_any_instance :etag_for, "rootetag" do - delete "/phil/food/aguacate" - end - end - - metadata = redis.hgetall "rs:m:phil:food/aguacate" - metadata.must_be_empty - end - - it "deletes the directory objects metadata in redis" do - old_metadata = redis.hgetall "rs:m:phil:food/" - - RestClient.stub :delete, "" do - RemoteStorage::Swift.stub_any_instance :etag_for, "newetag" do - delete "/phil/food/aguacate" - end - end - - metadata = redis.hgetall "rs:m:phil:food/" - metadata["e"].must_equal "newetag" - metadata["m"].length.must_equal 13 - metadata["m"].wont_equal old_metadata["m"] - - food_items = redis.smembers "rs:m:phil:food/:items" - food_items.sort.must_equal ["camaron", "desayunos/"] - - root_items = redis.smembers "rs:m:phil:/:items" - root_items.must_equal ["food/"] - end - - it "deletes the parent directory objects metadata when deleting all items" do - RestClient.stub :delete, "" do - RemoteStorage::Swift.stub_any_instance :etag_for, "rootetag" do - delete "/phil/food/aguacate" - delete "/phil/food/camaron" - delete "/phil/food/desayunos/bolon" - end - end - - redis.smembers("rs:m:phil:food/desayunos:items").must_be_empty - redis.hgetall("rs:m:phil:food/desayunos/").must_be_empty - - redis.smembers("rs:m:phil:food/:items").must_be_empty - redis.hgetall("rs:m:phil:food/").must_be_empty - - redis.smembers("rs:m:phil:/:items").must_be_empty - end - - it "responds with the ETag of the deleted item in the header" do - RestClient.stub :delete, "" do - delete "/phil/food/aguacate" - end - - last_response.headers["ETag"].must_equal "\"bla\"" - end - - context "when item doesn't exist" do - before do - purge_redis - - put_stub = OpenStruct.new(headers: { - etag: "bla", - last_modified: "Fri, 04 Mar 2016 12:20:18 GMT" - }) - - RestClient.stub :put, put_stub do - put "/phil/food/steak", "si" - end - - raises_exception = ->(url, headers) { raise RestClient::ResourceNotFound.new } - RestClient.stub :delete, raises_exception do - delete "/phil/food/steak" - end - end - - it "returns a 404" do - last_response.status.must_equal 404 - last_response.body.must_equal "Not Found" - end - - it "deletes any metadata that might still exist" do - raises_exception = ->(url, headers) { raise RestClient::ResourceNotFound.new } - RestClient.stub :delete, raises_exception do - delete "/phil/food/steak" - end - - metadata = redis.hgetall "rs:m:phil:food/steak" - metadata.must_be_empty - - redis.smembers("rs:m:phil:food/:items").must_be_empty - redis.hgetall("rs:m:phil:food/").must_be_empty - - redis.smembers("rs:m:phil:/:items").must_be_empty - end - end - - describe "If-Match header" do - it "succeeds when the header matches the current ETag" do - header "If-Match", "\"bla\"" - - RestClient.stub :delete, "" do - delete "/phil/food/aguacate" - end - - last_response.status.must_equal 200 - end - - it "succeeds when the header contains a weak ETAG matching the current ETag" do - header "If-Match", "W/\"bla\"" - - RestClient.stub :delete, "" do - delete "/phil/food/aguacate" - end - - last_response.status.must_equal 200 - end - - it "fails the request if it does not match the current ETag" do - header "If-Match", "someotheretag" - - delete "/phil/food/aguacate" - - last_response.status.must_equal 412 - last_response.body.must_equal "Precondition Failed" - end - end - end + before do + stub_request(:put, "#{container_url_for("phil")}/food/aguacate"). + to_return(status: 200, headers: { etag: "0815etag", last_modified: "Fri, 04 Mar 2016 12:20:18 GMT" }) + stub_request(:put, "#{container_url_for("phil")}/food/aguacate"). + with(body: "si"). + to_return(status: 200, headers: { etag: "0815etag", last_modified: "Fri, 04 Mar 2016 12:20:18 GMT" }) + stub_request(:put, "#{container_url_for("phil")}/food/aguacate"). + with(body: "aye"). + to_return(status: 200, headers: { etag: "0915etag", last_modified: "Fri, 04 Mar 2016 12:20:18 GMT" }) + stub_request(:put, "#{container_url_for("phil")}/food/aguacate"). + with(body: "deliciosa"). + to_return(status: 200, headers: { etag: "0815etag", last_modified: "Fri, 04 Mar 2016 12:20:18 GMT" }) + stub_request(:put, "#{container_url_for("phil")}/food/aguacate"). + with(body: "muy deliciosa"). + to_return(status: 200, headers: { etag: "0815etag", last_modified: "Fri, 04 Mar 2016 12:20:18 GMT" }) + stub_request(:head, "#{container_url_for("phil")}/food/aguacate"). + to_return(status: 200, headers: { last_modified: "Fri, 04 Mar 2016 12:20:18 GMT" }) + stub_request(:delete, "#{container_url_for("phil")}/food/aguacate"). + to_return(status: 200, headers: { etag: "0815etag" }) + stub_request(:put, "#{container_url_for("phil")}/food/camaron"). + to_return(status: 200, headers: { etag: "0816etag", last_modified: "Fri, 04 Mar 2016 12:20:18 GMT" }) + stub_request(:delete, "#{container_url_for("phil")}/food/camaron"). + to_return(status: 200, headers: { etag: "0816etag" }) + stub_request(:put, "#{container_url_for("phil")}/food/desayunos/bolon"). + to_return(status: 200, headers: { etag: "0817etag", last_modified: "Fri, 04 Mar 2016 12:20:18 GMT" }) + stub_request(:delete, "#{container_url_for("phil")}/food/desayunos/bolon"). + to_return(status: 200, headers: { etag: "0817etag" }) + stub_request(:get, "#{container_url_for("phil")}/food/aguacate"). + to_return(status: 200, body: "rootbody", headers: { etag: "0817etag", content_type: "text/plain; charset=utf-8" }) + stub_request(:put, "#{container_url_for("phil")}/bamboo.txt"). + to_return(status: 200, headers: { etag: "0818etag", last_modified: "Fri, 04 Mar 2016 12:20:18 GMT" }) + stub_request(:head, "#{container_url_for("phil")}/food/steak"). + to_return(status: 404) + stub_request(:get, "#{container_url_for("phil")}/food/steak"). + to_return(status: 404) + stub_request(:delete, "#{container_url_for("phil")}/food/steak"). + to_return(status: 404) end - describe "GET requests" do - - before do - purge_redis - end - - context "not authorized" do - - describe "without token" do - it "says it's not authorized" do - get "/phil/food/" - - last_response.status.must_equal 401 - last_response.body.must_equal "Unauthorized" - end - end - - describe "with wrong token" do - it "says it's not authorized" do - header "Authorization", "Bearer wrongtoken" - get "/phil/food/" - - last_response.status.must_equal 401 - last_response.body.must_equal "Unauthorized" - end - end - - end - - context "authorized" do - - before do - redis.sadd "authorizations:phil:amarillo", [":rw"] - header "Authorization", "Bearer amarillo" - - put_stub = OpenStruct.new(headers: { - etag: "0815etag", - last_modified: "Fri, 04 Mar 2016 12:20:18 GMT" - }) - - RestClient.stub :put, put_stub do - put "/phil/food/aguacate", "si" - put "/phil/food/camaron", "yummi" - put "/phil/food/desayunos/bolon", "wow" - end - end - - describe "documents" do - - it "returns the required response headers" do - get_stub = OpenStruct.new(body: "si", headers: { - etag: "0815etag", - last_modified: "Fri, 04 Mar 2016 12:20:18 GMT", - content_type: "text/plain; charset=utf-8", - content_length: 2 - }) - - RestClient.stub :get, get_stub do - get "/phil/food/aguacate" - end - - last_response.status.must_equal 200 - last_response.headers["ETag"].must_equal "\"0815etag\"" - last_response.headers["Cache-Control"].must_equal "no-cache" - last_response.headers["Content-Type"].must_equal "text/plain; charset=utf-8" - end - - it "returns a 404 when data doesn't exist" do - raises_exception = ->(url, headers) { raise RestClient::ResourceNotFound.new } - RestClient.stub :get, raises_exception do - get "/phil/food/steak" - end - - last_response.status.must_equal 404 - last_response.body.must_equal "Not Found" - end - - it "responds with 304 when IF_NONE_MATCH header contains the ETag" do - - header "If-None-Match", "\"0815etag\"" - - get "/phil/food/aguacate" - - last_response.status.must_equal 304 - last_response.headers["ETag"].must_equal "\"0815etag\"" - last_response.headers["Last-Modified"].must_equal "Fri, 04 Mar 2016 12:20:18 GMT" - end - - it "responds with 304 when IF_NONE_MATCH header contains weak ETAG matching the current ETag" do - header "If-None-Match", "W/\"0815etag\"" - - get "/phil/food/aguacate" - - last_response.status.must_equal 304 - last_response.headers["ETag"].must_equal "\"0815etag\"" - last_response.headers["Last-Modified"].must_equal "Fri, 04 Mar 2016 12:20:18 GMT" - end - - end - - describe "directory listings" do - - it "returns the correct ETag header" do - get "/phil/food/" - - last_response.status.must_equal 200 - last_response.headers["ETag"].must_equal "\"f9f85fbf5aa1fa378fd79ac8aa0a457d\"" - end - - it "returns a Cache-Control header with value 'no-cache'" do - get "/phil/food/" - - last_response.status.must_equal 200 - last_response.headers["Cache-Control"].must_equal "no-cache" - end - - it "responds with 304 when IF_NONE_MATCH header contains the ETag" do - header "If-None-Match", "\"f9f85fbf5aa1fa378fd79ac8aa0a457d\"" - get "/phil/food/" - - last_response.status.must_equal 304 - end - - it "responds with 304 when IF_NONE_MATCH header contains weak ETAG matching the ETag" do - header "If-None-Match", "W/\"f9f85fbf5aa1fa378fd79ac8aa0a457d\"" - get "/phil/food/" - - last_response.status.must_equal 304 - end - - it "contains all items in the directory" do - get "/phil/food/" - - last_response.status.must_equal 200 - last_response.content_type.must_equal "application/ld+json" - - content = JSON.parse(last_response.body) - content["@context"].must_equal "http://remotestorage.io/spec/folder-description" - content["items"]["aguacate"].wont_be_nil - content["items"]["aguacate"]["Content-Type"].must_equal "text/plain; charset=utf-8" - content["items"]["aguacate"]["Content-Length"].must_equal 2 - content["items"]["aguacate"]["ETag"].must_equal "0815etag" - content["items"]["camaron"].wont_be_nil - content["items"]["camaron"]["Content-Type"].must_equal "text/plain; charset=utf-8" - content["items"]["camaron"]["Content-Length"].must_equal 5 - content["items"]["camaron"]["ETag"].must_equal "0815etag" - content["items"]["desayunos/"].wont_be_nil - content["items"]["desayunos/"]["ETag"].must_equal "dd36e3cfe52b5f33421150b289a7d48d" - end - - it "contains all items in the root directory" do - get "phil/" - - last_response.status.must_equal 200 - last_response.content_type.must_equal "application/ld+json" - - content = JSON.parse(last_response.body) - content["items"]["food/"].wont_be_nil - content["items"]["food/"]["ETag"].must_equal "f9f85fbf5aa1fa378fd79ac8aa0a457d" - end - - it "responds with an empty directory liting when directory doesn't exist" do - get "phil/some-non-existing-dir/" - - last_response.status.must_equal 200 - last_response.content_type.must_equal "application/ld+json" - - content = JSON.parse(last_response.body) - content["items"].must_equal({}) - end - - end - end - - end - - describe "HEAD requests" do - - before do - purge_redis - end - - context "not authorized" do - - describe "without token" do - it "says it's not authorized" do - head "/phil/food/camarones" - - last_response.status.must_equal 401 - last_response.body.must_be_empty - end - end - - describe "with wrong token" do - it "says it's not authorized" do - header "Authorization", "Bearer wrongtoken" - head "/phil/food/camarones" - - last_response.status.must_equal 401 - last_response.body.must_be_empty - end - end - - end - - context "authorized" do - - before do - redis.sadd "authorizations:phil:amarillo", [":rw"] - header "Authorization", "Bearer amarillo" - - put_stub = OpenStruct.new(headers: { - etag: "0815etag", - last_modified: "Fri, 04 Mar 2016 12:20:18 GMT" - }) - - RestClient.stub :put, put_stub do - put "/phil/food/aguacate", "si" - put "/phil/food/camaron", "yummi" - put "/phil/food/desayunos/bolon", "wow" - end - end - - describe "directory listings" do - it "returns the correct header information" do - get "/phil/food/" - - last_response.status.must_equal 200 - last_response.content_type.must_equal "application/ld+json" - last_response.headers["ETag"].must_equal "\"f9f85fbf5aa1fa378fd79ac8aa0a457d\"" - end - end - - describe "documents" do - context "when the document doesn't exist" do - it "returns a 404" do - head "/phil/food/steak" - - last_response.status.must_equal 404 - last_response.body.must_be_empty - end - end - - context "when the document exists" do - it "returns the required response headers" do - head "/phil/food/aguacate" - - last_response.status.must_equal 200 - last_response.headers["ETag"].must_equal "\"0815etag\"" - last_response.headers["Cache-Control"].must_equal "no-cache" - end - - it "responds with 304 when IF_NONE_MATCH header contains the ETag" do - header "If-None-Match", "\"0815etag\"" - - head "/phil/food/aguacate" - - last_response.status.must_equal 304 - last_response.headers["ETag"].must_equal "\"0815etag\"" - last_response.headers["Last-Modified"].must_equal "Fri, 04 Mar 2016 12:20:18 GMT" - end - - it "responds with 304 when IF_NONE_MATCH header contains weak ETAG matching the current ETag" do - header "If-None-Match", "W/\"0815etag\"" - - head "/phil/food/aguacate" - - last_response.status.must_equal 304 - last_response.headers["ETag"].must_equal "\"0815etag\"" - last_response.headers["Last-Modified"].must_equal "Fri, 04 Mar 2016 12:20:18 GMT" - end - end - end - end - end + it_behaves_like 'a REST adapter' end - From b00fc5bee2a1e5df81d8018720bbe514bd549122 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Greg=20Kar=C3=A9kinian?= Date: Mon, 30 Apr 2018 15:10:55 +0200 Subject: [PATCH 11/29] Reverse the not found logic in the delete_data method to make it clearer --- lib/remote_storage/rest_provider.rb | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/lib/remote_storage/rest_provider.rb b/lib/remote_storage/rest_provider.rb index 0e95577..1b59601 100644 --- a/lib/remote_storage/rest_provider.rb +++ b/lib/remote_storage/rest_provider.rb @@ -165,7 +165,6 @@ module RemoteStorage def delete_data(user, directory, key) url = url_for_key(user, directory, key) - not_found = false existing_metadata = redis.hgetall "rs:m:#{user}:#{directory}/#{key}" @@ -175,17 +174,17 @@ module RemoteStorage end end - not_found = !try_to_delete(url) + found = try_to_delete(url) log_size_difference(user, existing_metadata["s"], 0) delete_metadata_objects(user, directory, key) delete_dir_objects(user, directory) - if not_found - server.halt 404, "Not Found" - else + if found server.headers["Etag"] = %Q("#{existing_metadata["e"]}") server.halt 200 + else + server.halt 404, "Not Found" end end From 3b72b8d0bb5d139301773c631adbb470ff5c9df7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Greg=20Kar=C3=A9kinian?= Date: Mon, 30 Apr 2018 15:12:53 +0200 Subject: [PATCH 12/29] Pass the headers to the set_response_headers directly, not the response --- lib/remote_storage/rest_provider.rb | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/lib/remote_storage/rest_provider.rb b/lib/remote_storage/rest_provider.rb index 1b59601..c394689 100644 --- a/lib/remote_storage/rest_provider.rb +++ b/lib/remote_storage/rest_provider.rb @@ -67,7 +67,7 @@ module RemoteStorage res = do_get_request(url) - set_response_headers(res) + set_response_headers(res.headers) return res.body rescue RestClient::ResourceNotFound @@ -208,11 +208,11 @@ module RemoteStorage raise NotImplementedError end - def set_response_headers(response) - server.headers["ETag"] = format_etag(response.headers[:etag]) - server.headers["Content-Type"] = response.headers[:content_type] - server.headers["Content-Length"] = response.headers[:content_length] - server.headers["Last-Modified"] = response.headers[:last_modified] + def set_response_headers(headers) + server.headers["ETag"] = format_etag(headers[:etag]) + server.headers["Content-Type"] = headers[:content_type] + server.headers["Content-Length"] = headers[:content_length] + server.headers["Last-Modified"] = headers[:last_modified] end def extract_category(directory) From 97cd5ec837ef7eed630a4f191e904a162c6a808d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Greg=20Kar=C3=A9kinian?= Date: Mon, 30 Apr 2018 15:15:04 +0200 Subject: [PATCH 13/29] Move a comment to the relevant line --- lib/remote_storage/s3_rest.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/remote_storage/s3_rest.rb b/lib/remote_storage/s3_rest.rb index 5f3db7d..48b6cb4 100644 --- a/lib/remote_storage/s3_rest.rb +++ b/lib/remote_storage/s3_rest.rb @@ -23,9 +23,9 @@ module RemoteStorage end end - # S3 does not return a Last-Modified response header on PUTs def do_put_request_and_return_etag_and_last_modified(url, data, content_type) res = do_put_request(url, data, content_type) + # S3 does not return a Last-Modified response header on PUTs head_res = do_head_request(url) return [res.headers[:etag].delete('"'), timestamp_for(head_res.headers[:last_modified])] From 1532a23d296f30fb66db4469e79c2eb48787f382 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Greg=20Kar=C3=A9kinian?= Date: Mon, 30 Apr 2018 15:30:16 +0200 Subject: [PATCH 14/29] Reorder the argument in authorization_headers_for Make content_type and md5 optional (set to nil by default) --- lib/remote_storage/s3_rest.rb | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/lib/remote_storage/s3_rest.rb b/lib/remote_storage/s3_rest.rb index 48b6cb4..fd9d62d 100644 --- a/lib/remote_storage/s3_rest.rb +++ b/lib/remote_storage/s3_rest.rb @@ -18,7 +18,7 @@ module RemoteStorage def do_put_request(url, data, content_type) deal_with_unauthorized_requests do md5 = Digest::MD5.base64digest(data) - authorization_headers = authorization_headers_for("PUT", md5, content_type, url) + authorization_headers = authorization_headers_for("PUT", url, md5, content_type) RestClient.put(url, data, authorization_headers.merge({ "Content-Type" => content_type, "Content-Md5" => md5})) end end @@ -33,21 +33,21 @@ module RemoteStorage def do_get_request(url, &block) deal_with_unauthorized_requests do - authorization_headers = authorization_headers_for("GET", "", "", url) + authorization_headers = authorization_headers_for("GET", url) RestClient.get(url, authorization_headers, &block) end end def do_head_request(url, &block) deal_with_unauthorized_requests do - authorization_headers = authorization_headers_for("HEAD", "", "", url) + authorization_headers = authorization_headers_for("HEAD", url) RestClient.head(url, authorization_headers, &block) end end def do_delete_request(url) deal_with_unauthorized_requests do - authorization_headers = authorization_headers_for("DELETE", "", "", url) + authorization_headers = authorization_headers_for("DELETE", url) RestClient.delete(url, authorization_headers) end end @@ -68,10 +68,10 @@ module RemoteStorage # This is using the S3 authorizations, not the newer AW V4 Signatures # (https://s3.amazonaws.com/doc/s3-developer-guide/RESTAuthentication.html) - def authorization_headers_for(http_verb, md5, content_type, url) + def authorization_headers_for(http_verb, url, md5 = nil, content_type = nil) url = File.join("/", url.gsub(base_url, "")) date = Time.now.httpdate - signed_data = signature(http_verb, md5, content_type, date, url) + signed_data = generate_s3_signature(http_verb, md5, content_type, date, url) { "Authorization" => "AWS #{credentials[:access_key_id]}:#{signed_data}", "Date" => date} end @@ -92,7 +92,7 @@ module RemoteStorage WEBrick::HTTPUtils.escape(s).gsub('%5B', '[').gsub('%5D', ']') end - def signature(http_verb, md5, content_type, date, url) + def generate_s3_signature(http_verb, md5, content_type, date, url) string_to_sign = [http_verb, md5, content_type, date, url].join "\n" signature = digest(credentials[:secret_key_id], string_to_sign) uri_escape(signature) From 454f02dfafba15686770d39de6978ecf618bb402 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Greg=20Kar=C3=A9kinian?= Date: Mon, 30 Apr 2018 15:44:28 +0200 Subject: [PATCH 15/29] Rename the spec root description --- spec/s3/app_spec.rb | 2 +- spec/swift/app_spec.rb | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/spec/s3/app_spec.rb b/spec/s3/app_spec.rb index 9874edc..eb0bd39 100644 --- a/spec/s3/app_spec.rb +++ b/spec/s3/app_spec.rb @@ -1,6 +1,6 @@ require_relative "../spec_helper" -describe "App" do +describe "S3 provider" do def container_url_for(user) "#{app.settings.s3["endpoint"]}#{app.settings.s3["bucket"]}/#{user}" end diff --git a/spec/swift/app_spec.rb b/spec/swift/app_spec.rb index a90c2d7..ef31cb9 100644 --- a/spec/swift/app_spec.rb +++ b/spec/swift/app_spec.rb @@ -1,6 +1,6 @@ require_relative "../spec_helper" -describe "App" do +describe "Swift provider" do def container_url_for(user) "#{app.settings.swift["host"]}/rs:documents:test/#{user}" end From 5da0d0b5c345e2a2b199b08494435e630806bdf0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Greg=20Kar=C3=A9kinian?= Date: Mon, 30 Apr 2018 15:44:49 +0200 Subject: [PATCH 16/29] Delete unused methods --- spec/shared_examples.rb | 4 ---- spec/swift/app_spec.rb | 4 ---- 2 files changed, 8 deletions(-) diff --git a/spec/shared_examples.rb b/spec/shared_examples.rb index ec74e15..baa6900 100644 --- a/spec/shared_examples.rb +++ b/spec/shared_examples.rb @@ -11,10 +11,6 @@ shared_examples_for 'a REST adapter' do raise NotImplementedError end - def config_file - raise NotImplementedError - end - it "returns 404 on non-existing routes" do get "/virginmargarita" last_response.status.must_equal 404 diff --git a/spec/swift/app_spec.rb b/spec/swift/app_spec.rb index ef31cb9..41884f3 100644 --- a/spec/swift/app_spec.rb +++ b/spec/swift/app_spec.rb @@ -9,10 +9,6 @@ describe "Swift provider" do RemoteStorage::Swift end - def config_file - "config.yml.example.swift" - end - before do stub_request(:put, "#{container_url_for("phil")}/food/aguacate"). to_return(status: 200, headers: { etag: "0815etag", last_modified: "Fri, 04 Mar 2016 12:20:18 GMT" }) From ca0127d6a67c0702a6dfa13dc1f7a0012c4e0483 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Greg=20Kar=C3=A9kinian?= Date: Mon, 30 Apr 2018 15:46:18 +0200 Subject: [PATCH 17/29] Rename the S3 provider to just S3 --- lib/remote_storage/{s3_rest.rb => s3.rb} | 2 +- liquor-cabinet.rb | 4 ++-- spec/s3/app_spec.rb | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) rename lib/remote_storage/{s3_rest.rb => s3.rb} (99%) diff --git a/lib/remote_storage/s3_rest.rb b/lib/remote_storage/s3.rb similarity index 99% rename from lib/remote_storage/s3_rest.rb rename to lib/remote_storage/s3.rb index fd9d62d..c195272 100644 --- a/lib/remote_storage/s3_rest.rb +++ b/lib/remote_storage/s3.rb @@ -5,7 +5,7 @@ require "openssl" require "webrick/httputils" module RemoteStorage - class S3Rest + class S3 include RestProvider private diff --git a/liquor-cabinet.rb b/liquor-cabinet.rb index 2d21522..6f6324f 100644 --- a/liquor-cabinet.rb +++ b/liquor-cabinet.rb @@ -5,7 +5,7 @@ require "sinatra/base" require 'sinatra/config_file' require "sinatra/reloader" require "remote_storage/swift" -require "remote_storage/s3_rest" +require "remote_storage/s3" class LiquorCabinet < Sinatra::Base @@ -132,7 +132,7 @@ class LiquorCabinet < Sinatra::Base if settings.respond_to? :swift RemoteStorage::Swift.new(settings, self) elsif settings.respond_to? :s3 - RemoteStorage::S3Rest.new(settings, self) + RemoteStorage::S3.new(settings, self) else puts <<-EOF You need to set one storage backend in your config.yml file. diff --git a/spec/s3/app_spec.rb b/spec/s3/app_spec.rb index eb0bd39..828c6f8 100644 --- a/spec/s3/app_spec.rb +++ b/spec/s3/app_spec.rb @@ -6,7 +6,7 @@ describe "S3 provider" do end def storage_class - RemoteStorage::S3Rest + RemoteStorage::S3 end before do From f14ef4dc21e729da3f8d04f44fee382716f1ef1b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Greg=20Kar=C3=A9kinian?= Date: Mon, 30 Apr 2018 16:39:12 +0200 Subject: [PATCH 18/29] Consider that the metadata has changed when the Last-Modified changes Also adds a spec for it --- lib/remote_storage/rest_provider.rb | 3 +-- spec/s3/app_spec.rb | 8 ++++++++ spec/shared_examples.rb | 11 +++++++++++ spec/swift/app_spec.rb | 6 ++++++ 4 files changed, 26 insertions(+), 2 deletions(-) diff --git a/lib/remote_storage/rest_provider.rb b/lib/remote_storage/rest_provider.rb index c394689..621d456 100644 --- a/lib/remote_storage/rest_provider.rb +++ b/lib/remote_storage/rest_provider.rb @@ -289,10 +289,9 @@ module RemoteStorage def metadata_changed?(old_metadata, new_metadata) # check metadata relevant to the directory listing - # ie. the timestamp (m) is not relevant, because it's not used in - # the listing return old_metadata["e"] != new_metadata[:e] || old_metadata["s"] != new_metadata[:s].to_s || + old_metadata["m"] != new_metadata[:m] || old_metadata["t"] != new_metadata[:t] end diff --git a/spec/s3/app_spec.rb b/spec/s3/app_spec.rb index 828c6f8..6c922b2 100644 --- a/spec/s3/app_spec.rb +++ b/spec/s3/app_spec.rb @@ -24,6 +24,14 @@ describe "S3 provider" do stub_request(:put, "#{container_url_for("phil")}/food/aguacate"). with(body: "muy deliciosa"). to_return(status: 200, headers: { etag: '"0815etag"' }) + stub_request(:put, "#{container_url_for("phil")}/food/banano"). + with(body: "si"). + to_return(status: 200, headers: { etag: '"0815etag"' }) + stub_request(:put, "#{container_url_for("phil")}/food/banano"). + with(body: "oh, no"). + to_return(status: 200, headers: { etag: '"0817etag"' }) + stub_request(:head, "#{container_url_for("phil")}/food/banano"). + to_return(status: 200, headers: { last_modified: "Fri, 04 Mar 2016 12:20:20 GMT" }) stub_request(:head, "#{container_url_for("phil")}/food/aguacate"). to_return(status: 200, headers: { last_modified: "Fri, 04 Mar 2016 12:20:18 GMT" }) stub_request(:delete, "#{container_url_for("phil")}/food/aguacate"). diff --git a/spec/shared_examples.rb b/spec/shared_examples.rb index baa6900..fd5f742 100644 --- a/spec/shared_examples.rb +++ b/spec/shared_examples.rb @@ -38,6 +38,17 @@ shared_examples_for 'a REST adapter' do metadata["m"].length.must_equal 13 end + it "updates the metadata object in redis when it changes" do + put "/phil/food/banano", "si" + put "/phil/food/banano", "oh, no" + + metadata = redis.hgetall "rs:m:phil:food/banano" + metadata["s"].must_equal "6" + metadata["t"].must_equal "text/plain; charset=utf-8" + metadata["e"].must_equal "0817etag" + metadata["m"].must_equal "1457094020000" + end + it "creates the directory objects metadata in redis" do put "/phil/food/aguacate", "si" put "/phil/food/camaron", "yummi" diff --git a/spec/swift/app_spec.rb b/spec/swift/app_spec.rb index 41884f3..2bc913d 100644 --- a/spec/swift/app_spec.rb +++ b/spec/swift/app_spec.rb @@ -24,6 +24,12 @@ describe "Swift provider" do stub_request(:put, "#{container_url_for("phil")}/food/aguacate"). with(body: "muy deliciosa"). to_return(status: 200, headers: { etag: "0815etag", last_modified: "Fri, 04 Mar 2016 12:20:18 GMT" }) + stub_request(:put, "#{container_url_for("phil")}/food/banano"). + with(body: "si"). + to_return(status: 200, headers: { etag: "0815etag", last_modified: "Fri, 04 Mar 2016 12:20:18 GMT" }) + stub_request(:put, "#{container_url_for("phil")}/food/banano"). + with(body: "oh, no"). + to_return(status: 200, headers: { etag: "0817etag", last_modified: "Fri, 04 Mar 2016 12:20:20 GMT" }) stub_request(:head, "#{container_url_for("phil")}/food/aguacate"). to_return(status: 200, headers: { last_modified: "Fri, 04 Mar 2016 12:20:18 GMT" }) stub_request(:delete, "#{container_url_for("phil")}/food/aguacate"). From a92286703d56253d7e7a31c6060c32dfc64de96a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Greg=20Kar=C3=A9kinian?= Date: Mon, 30 Apr 2018 16:46:28 +0200 Subject: [PATCH 19/29] Uncomment the development config in the examples --- config.yml.example.s3 | 21 ++++++++++----------- config.yml.example.swift | 14 +++++++------- 2 files changed, 17 insertions(+), 18 deletions(-) diff --git a/config.yml.example.s3 b/config.yml.example.s3 index 0acb9fd..1ae5766 100644 --- a/config.yml.example.s3 +++ b/config.yml.example.s3 @@ -1,16 +1,15 @@ development: &defaults maintenance: false - # # uncomment this section - # s3: - # endpoint: "https://some-endpoint" - # region: "region" - # access_key_id: "" - # secret_key_id: "" - # bucket: "test-bucket" - # # Redis is needed for the swift backend - # redis: - # host: localhost - # port: 6379 + s3: + endpoint: "https://some-endpoint" + region: "region" + access_key_id: "" + secret_key_id: "" + bucket: "test-bucket" + # Redis is needed for the swift backend + redis: + host: localhost + port: 6379 test: <<: *defaults diff --git a/config.yml.example.swift b/config.yml.example.swift index b44c579..e7cad01 100644 --- a/config.yml.example.swift +++ b/config.yml.example.swift @@ -1,12 +1,12 @@ development: &defaults maintenance: false - # # uncomment this section - # swift: &swift_defaults - # host: "https://swift.example.com" - # # Redis is needed for the swift backend - # redis: - # host: localhost - # port: 6379 + # uncomment this section + swift: &swift_defaults + host: "https://swift.example.com" + # Redis is needed for the swift backend + redis: + host: localhost + port: 6379 test: <<: *defaults From d0a28c711129bc7f4a562d576902254b82e81c55 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Greg=20Kar=C3=A9kinian?= Date: Wed, 9 May 2018 15:21:51 +0200 Subject: [PATCH 20/29] Remove useless comments and remove duplicate section in the example configs --- config.yml.example.s3 | 4 ---- config.yml.example.swift | 5 ----- 2 files changed, 9 deletions(-) diff --git a/config.yml.example.s3 b/config.yml.example.s3 index 1ae5766..a0e654e 100644 --- a/config.yml.example.s3 +++ b/config.yml.example.s3 @@ -6,7 +6,6 @@ development: &defaults access_key_id: "" secret_key_id: "" bucket: "test-bucket" - # Redis is needed for the swift backend redis: host: localhost port: 6379 @@ -19,9 +18,6 @@ test: access_key_id: "" secret_key_id: "" bucket: "test-bucket" - redis: - host: localhost - port: 6379 staging: <<: *defaults diff --git a/config.yml.example.swift b/config.yml.example.swift index e7cad01..40fa39f 100644 --- a/config.yml.example.swift +++ b/config.yml.example.swift @@ -1,9 +1,7 @@ development: &defaults maintenance: false - # uncomment this section swift: &swift_defaults host: "https://swift.example.com" - # Redis is needed for the swift backend redis: host: localhost port: 6379 @@ -12,9 +10,6 @@ test: <<: *defaults swift: host: "https://swift.example.com" - redis: - host: localhost - port: 6379 staging: <<: *defaults From 0ec76c8140bb6f28d5de8f018d5ad630a4a83de4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Greg=20Kar=C3=A9kinian?= Date: Wed, 9 May 2018 15:26:32 +0200 Subject: [PATCH 21/29] Fix coding style for a hash --- lib/remote_storage/s3.rb | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/lib/remote_storage/s3.rb b/lib/remote_storage/s3.rb index c195272..b3929a6 100644 --- a/lib/remote_storage/s3.rb +++ b/lib/remote_storage/s3.rb @@ -72,8 +72,10 @@ module RemoteStorage url = File.join("/", url.gsub(base_url, "")) date = Time.now.httpdate signed_data = generate_s3_signature(http_verb, md5, content_type, date, url) - { "Authorization" => "AWS #{credentials[:access_key_id]}:#{signed_data}", - "Date" => date} + { + "Authorization" => "AWS #{credentials[:access_key_id]}:#{signed_data}", + "Date" => date + } end def credentials From 709f63555dbcb77a0bf3ef4343a7c7ba399d3af5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Greg=20Kar=C3=A9kinian?= Date: Wed, 9 May 2018 15:55:00 +0200 Subject: [PATCH 22/29] Rewrite to avoid long lines --- lib/remote_storage/s3.rb | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/lib/remote_storage/s3.rb b/lib/remote_storage/s3.rb index b3929a6..f2346d4 100644 --- a/lib/remote_storage/s3.rb +++ b/lib/remote_storage/s3.rb @@ -18,8 +18,10 @@ module RemoteStorage def do_put_request(url, data, content_type) deal_with_unauthorized_requests do md5 = Digest::MD5.base64digest(data) - authorization_headers = authorization_headers_for("PUT", url, md5, content_type) - RestClient.put(url, data, authorization_headers.merge({ "Content-Type" => content_type, "Content-Md5" => md5})) + authorization_headers = authorization_headers_for( + "PUT", url, md5, content_type + ).merge({ "Content-Type" => content_type, "Content-Md5" => md5 }) + RestClient.put(url, data, authorization_headers) end end From 639c3724f02a06310f2e5c1507e193b4091a71fa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Greg=20Kar=C3=A9kinian?= Date: Wed, 9 May 2018 16:01:14 +0200 Subject: [PATCH 23/29] Refactor the put_request method to have a return value --- lib/remote_storage/rest_provider.rb | 15 +++++++-------- lib/remote_storage/s3.rb | 17 ++++++++--------- 2 files changed, 15 insertions(+), 17 deletions(-) diff --git a/lib/remote_storage/rest_provider.rb b/lib/remote_storage/rest_provider.rb index 621d456..3c5d38d 100644 --- a/lib/remote_storage/rest_provider.rb +++ b/lib/remote_storage/rest_provider.rb @@ -141,7 +141,7 @@ module RemoteStorage server.halt 412, "Precondition Failed" unless existing_metadata.empty? end - etag, timestamp = do_put_request_and_return_etag_and_last_modified(url, data, content_type) + etag, timestamp = do_put_request(url, data, content_type) metadata = { e: etag, @@ -392,16 +392,15 @@ module RemoteStorage def do_put_request(url, data, content_type) deal_with_unauthorized_requests do - RestClient.put(url, data, default_headers.merge({content_type: content_type})) + res = RestClient.put(url, data, default_headers.merge({content_type: content_type})) + + return [ + res.headers[:etag], + timestamp_for(res.headers[:last_modified]) + ] end end - def do_put_request_and_return_etag_and_last_modified(url, data, content_type) - res = do_put_request(url, data, content_type) - - return [res.headers[:etag], timestamp_for(res.headers[:last_modified])] - end - def do_get_request(url, &block) deal_with_unauthorized_requests do RestClient.get(url, default_headers, &block) diff --git a/lib/remote_storage/s3.rb b/lib/remote_storage/s3.rb index f2346d4..875f596 100644 --- a/lib/remote_storage/s3.rb +++ b/lib/remote_storage/s3.rb @@ -21,18 +21,17 @@ module RemoteStorage authorization_headers = authorization_headers_for( "PUT", url, md5, content_type ).merge({ "Content-Type" => content_type, "Content-Md5" => md5 }) - RestClient.put(url, data, authorization_headers) + res = RestClient.put(url, data, authorization_headers) + # S3 does not return a Last-Modified response header on PUTs + head_res = do_head_request(url) + + return [ + res.headers[:etag].delete('"'), + timestamp_for(head_res.headers[:last_modified]) + ] end end - def do_put_request_and_return_etag_and_last_modified(url, data, content_type) - res = do_put_request(url, data, content_type) - # S3 does not return a Last-Modified response header on PUTs - head_res = do_head_request(url) - - return [res.headers[:etag].delete('"'), timestamp_for(head_res.headers[:last_modified])] - end - def do_get_request(url, &block) deal_with_unauthorized_requests do authorization_headers = authorization_headers_for("GET", url) From c0d88f1da6496a72dd7e5c98065ce1e8dbc68df7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Greg=20Kar=C3=A9kinian?= Date: Wed, 9 May 2018 16:18:22 +0200 Subject: [PATCH 24/29] Run Travis builds on the Docker infrastructure --- .travis.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.travis.yml b/.travis.yml index 6c1356e..02acefa 100644 --- a/.travis.yml +++ b/.travis.yml @@ -22,3 +22,5 @@ notifications: env: - BACKEND=s3 - BACKEND=swift +# Run on Docker infrastructure +sudo: false From df65190df8fb2776ca5983342814b2f9c8db14a7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Greg=20Kar=C3=A9kinian?= Date: Wed, 9 May 2018 18:42:04 +0200 Subject: [PATCH 25/29] Simply the request stubs, add comments Remove the stubs that are not required, making everything easier to understand --- spec/s3/app_spec.rb | 28 ++++++++++++++-------------- spec/swift/app_spec.rb | 30 +++++++++++++++--------------- 2 files changed, 29 insertions(+), 29 deletions(-) diff --git a/spec/s3/app_spec.rb b/spec/s3/app_spec.rb index 6c922b2..8829ab9 100644 --- a/spec/s3/app_spec.rb +++ b/spec/s3/app_spec.rb @@ -12,18 +12,18 @@ describe "S3 provider" do before do stub_request(:put, "#{container_url_for("phil")}/food/aguacate"). to_return(status: 200, headers: { etag: '"0815etag"' }) - stub_request(:put, "#{container_url_for("phil")}/food/aguacate"). - with(body: "si"). - to_return(status: 200, headers: { etag: '"0815etag"' }) + # Write new content with an If-Match header (a new Etag is returned) stub_request(:put, "#{container_url_for("phil")}/food/aguacate"). with(body: "aye"). to_return(status: 200, headers: { etag: '"0915etag"' }) - stub_request(:put, "#{container_url_for("phil")}/food/aguacate"). - with(body: "deliciosa"). - to_return(status: 200, headers: { etag: '"0815etag"' }) - stub_request(:put, "#{container_url_for("phil")}/food/aguacate"). - with(body: "muy deliciosa"). + stub_request(:head, "#{container_url_for("phil")}/food/aguacate"). + to_return(status: 200, headers: { last_modified: "Fri, 04 Mar 2016 12:20:18 GMT" }) + stub_request(:get, "#{container_url_for("phil")}/food/aguacate"). + to_return(status: 200, body: "rootbody", headers: { etag: '"0817etag"', content_type: "text/plain; charset=utf-8" }) + stub_request(:delete, "#{container_url_for("phil")}/food/aguacate"). to_return(status: 200, headers: { etag: '"0815etag"' }) + + # PUT requests authorized updates the metadata object in redis when it changes stub_request(:put, "#{container_url_for("phil")}/food/banano"). with(body: "si"). to_return(status: 200, headers: { etag: '"0815etag"' }) @@ -32,28 +32,28 @@ describe "S3 provider" do to_return(status: 200, headers: { etag: '"0817etag"' }) stub_request(:head, "#{container_url_for("phil")}/food/banano"). to_return(status: 200, headers: { last_modified: "Fri, 04 Mar 2016 12:20:20 GMT" }) - stub_request(:head, "#{container_url_for("phil")}/food/aguacate"). - to_return(status: 200, headers: { last_modified: "Fri, 04 Mar 2016 12:20:18 GMT" }) - stub_request(:delete, "#{container_url_for("phil")}/food/aguacate"). - to_return(status: 200, headers: { etag: '"0815etag"' }) + stub_request(:put, "#{container_url_for("phil")}/food/camaron"). to_return(status: 200, headers: { etag: '"0816etag"' }) stub_request(:head, "#{container_url_for("phil")}/food/camaron"). to_return(status: 200, headers: { last_modified: "Fri, 04 Mar 2016 12:20:18 GMT" }) stub_request(:delete, "#{container_url_for("phil")}/food/camaron"). to_return(status: 200, headers: { etag: '"0816etag"' }) + stub_request(:put, "#{container_url_for("phil")}/food/desayunos/bolon"). to_return(status: 200, headers: { etag: '"0817etag"' }) stub_request(:head, "#{container_url_for("phil")}/food/desayunos/bolon"). to_return(status: 200, headers: { last_modified: "Fri, 04 Mar 2016 12:20:18 GMT" }) stub_request(:delete, "#{container_url_for("phil")}/food/desayunos/bolon"). to_return(status: 200, headers: { etag: '"0817etag"' }) - stub_request(:get, "#{container_url_for("phil")}/food/aguacate"). - to_return(status: 200, body: "rootbody", headers: { etag: '"0817etag"', content_type: "text/plain; charset=utf-8" }) + + # objects in root dir stub_request(:put, "#{container_url_for("phil")}/bamboo.txt"). to_return(status: 200, headers: { etag: '"0818etag"' }) stub_request(:head, "#{container_url_for("phil")}/bamboo.txt"). to_return(status: 200, headers: { last_modified: "Fri, 04 Mar 2016 12:20:18 GMT" }) + + # 404 stub_request(:head, "#{container_url_for("phil")}/food/steak"). to_return(status: 404) stub_request(:get, "#{container_url_for("phil")}/food/steak"). diff --git a/spec/swift/app_spec.rb b/spec/swift/app_spec.rb index 2bc913d..13e7145 100644 --- a/spec/swift/app_spec.rb +++ b/spec/swift/app_spec.rb @@ -12,40 +12,40 @@ describe "Swift provider" do before do stub_request(:put, "#{container_url_for("phil")}/food/aguacate"). to_return(status: 200, headers: { etag: "0815etag", last_modified: "Fri, 04 Mar 2016 12:20:18 GMT" }) - stub_request(:put, "#{container_url_for("phil")}/food/aguacate"). - with(body: "si"). - to_return(status: 200, headers: { etag: "0815etag", last_modified: "Fri, 04 Mar 2016 12:20:18 GMT" }) + # Write new content with an If-Match header (a new Etag is returned) stub_request(:put, "#{container_url_for("phil")}/food/aguacate"). with(body: "aye"). to_return(status: 200, headers: { etag: "0915etag", last_modified: "Fri, 04 Mar 2016 12:20:18 GMT" }) - stub_request(:put, "#{container_url_for("phil")}/food/aguacate"). - with(body: "deliciosa"). - to_return(status: 200, headers: { etag: "0815etag", last_modified: "Fri, 04 Mar 2016 12:20:18 GMT" }) - stub_request(:put, "#{container_url_for("phil")}/food/aguacate"). - with(body: "muy deliciosa"). - to_return(status: 200, headers: { etag: "0815etag", last_modified: "Fri, 04 Mar 2016 12:20:18 GMT" }) + stub_request(:head, "#{container_url_for("phil")}/food/aguacate"). + to_return(status: 200, headers: { last_modified: "Fri, 04 Mar 2016 12:20:18 GMT" }) + stub_request(:get, "#{container_url_for("phil")}/food/aguacate"). + to_return(status: 200, body: "rootbody", headers: { etag: "0817etag", content_type: "text/plain; charset=utf-8" }) + stub_request(:delete, "#{container_url_for("phil")}/food/aguacate"). + to_return(status: 200, headers: { etag: "0815etag" }) + + # PUT requests authorized updates the metadata object in redis when it changes stub_request(:put, "#{container_url_for("phil")}/food/banano"). with(body: "si"). to_return(status: 200, headers: { etag: "0815etag", last_modified: "Fri, 04 Mar 2016 12:20:18 GMT" }) stub_request(:put, "#{container_url_for("phil")}/food/banano"). with(body: "oh, no"). to_return(status: 200, headers: { etag: "0817etag", last_modified: "Fri, 04 Mar 2016 12:20:20 GMT" }) - stub_request(:head, "#{container_url_for("phil")}/food/aguacate"). - to_return(status: 200, headers: { last_modified: "Fri, 04 Mar 2016 12:20:18 GMT" }) - stub_request(:delete, "#{container_url_for("phil")}/food/aguacate"). - to_return(status: 200, headers: { etag: "0815etag" }) + stub_request(:put, "#{container_url_for("phil")}/food/camaron"). to_return(status: 200, headers: { etag: "0816etag", last_modified: "Fri, 04 Mar 2016 12:20:18 GMT" }) stub_request(:delete, "#{container_url_for("phil")}/food/camaron"). to_return(status: 200, headers: { etag: "0816etag" }) + stub_request(:put, "#{container_url_for("phil")}/food/desayunos/bolon"). to_return(status: 200, headers: { etag: "0817etag", last_modified: "Fri, 04 Mar 2016 12:20:18 GMT" }) stub_request(:delete, "#{container_url_for("phil")}/food/desayunos/bolon"). to_return(status: 200, headers: { etag: "0817etag" }) - stub_request(:get, "#{container_url_for("phil")}/food/aguacate"). - to_return(status: 200, body: "rootbody", headers: { etag: "0817etag", content_type: "text/plain; charset=utf-8" }) + + # objects in root dir stub_request(:put, "#{container_url_for("phil")}/bamboo.txt"). to_return(status: 200, headers: { etag: "0818etag", last_modified: "Fri, 04 Mar 2016 12:20:18 GMT" }) + + # 404 stub_request(:head, "#{container_url_for("phil")}/food/steak"). to_return(status: 404) stub_request(:get, "#{container_url_for("phil")}/food/steak"). From be33b0e195850d7ceed1481e524b0a37f7fe4cd5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Greg=20Kar=C3=A9kinian?= Date: Wed, 9 May 2018 19:08:32 +0200 Subject: [PATCH 26/29] Replace placeholder comment that I forgot to replace --- spec/s3/app_spec.rb | 2 +- spec/swift/app_spec.rb | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/spec/s3/app_spec.rb b/spec/s3/app_spec.rb index 8829ab9..6e60c41 100644 --- a/spec/s3/app_spec.rb +++ b/spec/s3/app_spec.rb @@ -23,7 +23,7 @@ describe "S3 provider" do stub_request(:delete, "#{container_url_for("phil")}/food/aguacate"). to_return(status: 200, headers: { etag: '"0815etag"' }) - # PUT requests authorized updates the metadata object in redis when it changes + # Write new content to check the metadata in Redis stub_request(:put, "#{container_url_for("phil")}/food/banano"). with(body: "si"). to_return(status: 200, headers: { etag: '"0815etag"' }) diff --git a/spec/swift/app_spec.rb b/spec/swift/app_spec.rb index 13e7145..eff8bd9 100644 --- a/spec/swift/app_spec.rb +++ b/spec/swift/app_spec.rb @@ -23,7 +23,7 @@ describe "Swift provider" do stub_request(:delete, "#{container_url_for("phil")}/food/aguacate"). to_return(status: 200, headers: { etag: "0815etag" }) - # PUT requests authorized updates the metadata object in redis when it changes + # Write new content to check the metadata in Redis stub_request(:put, "#{container_url_for("phil")}/food/banano"). with(body: "si"). to_return(status: 200, headers: { etag: "0815etag", last_modified: "Fri, 04 Mar 2016 12:20:18 GMT" }) From 0f91c8c27e6548f20054d190fdb1a1ad7f2a19fc Mon Sep 17 00:00:00 2001 From: Sebastian Kippe Date: Fri, 11 May 2018 15:35:33 +0200 Subject: [PATCH 27/29] Improve README Update for new backend, improve text in general. --- README.md | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/README.md b/README.md index 9f3af01..7c45437 100644 --- a/README.md +++ b/README.md @@ -3,16 +3,19 @@ # Liquor Cabinet Liquor Cabinet is where Frank stores all his stuff. It's a -remoteStorage-compatible storage provider API, based on Sinatra and currently -using Openstack Swift as backend. You can use it on its own, or e.g. mount it -from a Rails application. +[remoteStorage](https://remotestorage.io) HTTP API, based on Sinatra. The +metadata and OAuth tokens are stored in Redis, and documents can be stored in +anything that supports the storage API of either Openstack Swift or Amazon S3. -It's merely implementing the storage API, not including the Webfinger and OAuth -parts of remoteStorage. You have to set the authorization keys/values in the -database yourself. +Liquor Cabinet only implements the storage API part of the remoteStorage +protocol, but does not include the Webfinger and OAuth parts. It is meant to be +added to existing systems and user accounts, so you will have to add your own +OAuth dialog for remoteStorage authorizations and persist the tokens in Redis. -If you have any questions about this thing, drop by #remotestorage on Freenode, and -we'll happily answer them. +If you have any questions about this program, drop by #remotestorage on +Freenode, or [post to the RS +forums](https://community.remotestorage.io/c/server-development), and we'll +happily answer them. ## Contributing From 533c9b6346890d297e3146049d20b0434324c929 Mon Sep 17 00:00:00 2001 From: Garret Alfert Date: Tue, 22 May 2018 11:59:55 +0200 Subject: [PATCH 28/29] Set Content-type header on HEAD requests Fixes #121 --- lib/remote_storage/rest_provider.rb | 5 +++-- spec/shared_examples.rb | 2 ++ 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/lib/remote_storage/rest_provider.rb b/lib/remote_storage/rest_provider.rb index 3c5d38d..02ab904 100644 --- a/lib/remote_storage/rest_provider.rb +++ b/lib/remote_storage/rest_provider.rb @@ -44,8 +44,9 @@ module RemoteStorage server.halt 404 if metadata.empty? # Set the response headers for a 304 or 200 response - server.headers["ETag"] = %Q("#{metadata["e"]}") - server.headers["Last-Modified"] = Time.at(metadata["m"].to_i / 1000).httpdate + server.headers["ETag"] = %Q("#{metadata["e"]}") + server.headers["Last-Modified"] = Time.at(metadata["m"].to_i / 1000).httpdate + server.headers["Content-Type"] = metadata["t"] if none_match.include? %Q("#{metadata["e"]}") server.halt 304 diff --git a/spec/shared_examples.rb b/spec/shared_examples.rb index fd5f742..e95cbc4 100644 --- a/spec/shared_examples.rb +++ b/spec/shared_examples.rb @@ -614,6 +614,8 @@ shared_examples_for 'a REST adapter' do last_response.status.must_equal 200 last_response.headers["ETag"].must_equal "\"0815etag\"" last_response.headers["Cache-Control"].must_equal "no-cache" + last_response.headers["Last-Modified"].must_equal "Fri, 04 Mar 2016 12:20:18 GMT" + last_response.headers["Content-Type"].must_equal "text/plain; charset=utf-8" end it "responds with 304 when IF_NONE_MATCH header contains the ETag" do From b4ad4c47936c51cd3c7891269aa99c9377c8d198 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Greg=20Kar=C3=A9kinian?= Date: Tue, 22 May 2018 12:44:24 +0200 Subject: [PATCH 29/29] Set the Content-Length on a HEAD requests We get it as part of the Redis metadata RFC: https://www.w3.org/Protocols/rfc2616/rfc2616-sec14.html#sec14.13 --- lib/remote_storage/rest_provider.rb | 1 + spec/shared_examples.rb | 1 + 2 files changed, 2 insertions(+) diff --git a/lib/remote_storage/rest_provider.rb b/lib/remote_storage/rest_provider.rb index 02ab904..bddf75f 100644 --- a/lib/remote_storage/rest_provider.rb +++ b/lib/remote_storage/rest_provider.rb @@ -47,6 +47,7 @@ module RemoteStorage server.headers["ETag"] = %Q("#{metadata["e"]}") server.headers["Last-Modified"] = Time.at(metadata["m"].to_i / 1000).httpdate server.headers["Content-Type"] = metadata["t"] + server.headers["Content-Length"] = metadata["s"] if none_match.include? %Q("#{metadata["e"]}") server.halt 304 diff --git a/spec/shared_examples.rb b/spec/shared_examples.rb index e95cbc4..eea85f9 100644 --- a/spec/shared_examples.rb +++ b/spec/shared_examples.rb @@ -616,6 +616,7 @@ shared_examples_for 'a REST adapter' do last_response.headers["Cache-Control"].must_equal "no-cache" last_response.headers["Last-Modified"].must_equal "Fri, 04 Mar 2016 12:20:18 GMT" last_response.headers["Content-Type"].must_equal "text/plain; charset=utf-8" + last_response.headers["Content-Length"].must_equal "2" end it "responds with 304 when IF_NONE_MATCH header contains the ETag" do