From bf3919e721d673fd682167b55738498758755ceb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Greg=20Kar=C3=A9kinian?= Date: Wed, 20 Jan 2016 18:15:09 +0000 Subject: [PATCH 01/58] Update redis gem to 3.2.2 --- Gemfile.lock | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Gemfile.lock b/Gemfile.lock index 2c575e0..d234198 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -70,7 +70,7 @@ GEM unicorn (~> 4.8) raindrops (0.13.0) rake (10.4.2) - redis (3.2.1) + redis (3.2.2) rest-client (1.8.0) http-cookie (>= 1.0.2, < 2.0) mime-types (>= 1.16, < 3.0) @@ -120,4 +120,4 @@ DEPENDENCIES sinatra-contrib BUNDLED WITH - 1.10.6 + 1.11.2 From 1a8ba680ab8a7a08574b166af3300667ac6bf376 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Greg=20Kar=C3=A9kinian?= Date: Wed, 20 Jan 2016 18:36:44 +0000 Subject: [PATCH 02/58] Travis CI: Update Ruby to 2.2.4 and switch to Trusty infrastructure --- .travis.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 7e02b80..135619f 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,6 +1,8 @@ language: ruby rvm: - - 2.2 + - 2.2.4 +sudo: required +dist: trusty before_install: - sh .travis/install_riakcs.sh before_script: From aa0caac3c4f60243c5e59034dd11e6de966ce9ce Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Greg=20Kar=C3=A9kinian?= Date: Wed, 20 Jan 2016 18:57:34 +0000 Subject: [PATCH 03/58] Add bundler caching to Travis build --- .travis.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.travis.yml b/.travis.yml index 135619f..e13c4b9 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,4 +1,5 @@ language: ruby +cache: bundler rvm: - 2.2.4 sudo: required From b520e591c745671f1f5719d62a006dfa37e0a6c6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Greg=20Kar=C3=A9kinian?= Date: Wed, 20 Jan 2016 19:02:52 +0000 Subject: [PATCH 04/58] Manually install bundler in Travis build --- .travis.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.travis.yml b/.travis.yml index e13c4b9..f07d701 100644 --- a/.travis.yml +++ b/.travis.yml @@ -6,6 +6,7 @@ sudo: required dist: trusty before_install: - sh .travis/install_riakcs.sh + - gem install bundler before_script: - cp config.yml.example config.yml script: bundle exec rake test From d0dd4af3d942df347067cfaf0661beaadb402324 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Greg=20Kar=C3=A9kinian?= Date: Wed, 20 Jan 2016 20:21:25 +0000 Subject: [PATCH 05/58] Add riak service to Travis CI --- .travis.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.travis.yml b/.travis.yml index f07d701..a365d06 100644 --- a/.travis.yml +++ b/.travis.yml @@ -21,3 +21,5 @@ notifications: - http://hook-juggler.herokuapp.com/hooks/travis on_success: always on_failure: always +services: + - riak From 77bd54b0097ce97bc00373ecb586769ae8e59262 Mon Sep 17 00:00:00 2001 From: Garret Alfert Date: Wed, 20 Jan 2016 15:32:44 -0500 Subject: [PATCH 06/58] Move old specs into riak subdir --- spec/{ => riak}/app_spec.rb | 1 + spec/{ => riak}/directories_spec.rb | 0 spec/{ => riak}/permissions_spec.rb | 0 spec/{ => riak}/riak_spec.rb | 0 4 files changed, 1 insertion(+) rename spec/{ => riak}/app_spec.rb (99%) rename spec/{ => riak}/directories_spec.rb (100%) rename spec/{ => riak}/permissions_spec.rb (100%) rename spec/{ => riak}/riak_spec.rb (100%) diff --git a/spec/app_spec.rb b/spec/riak/app_spec.rb similarity index 99% rename from spec/app_spec.rb rename to spec/riak/app_spec.rb index ccdb8a1..1eb0ef1 100644 --- a/spec/app_spec.rb +++ b/spec/riak/app_spec.rb @@ -11,4 +11,5 @@ describe "App" do get "/virginmargarita" last_response.status.must_equal 404 end + end diff --git a/spec/directories_spec.rb b/spec/riak/directories_spec.rb similarity index 100% rename from spec/directories_spec.rb rename to spec/riak/directories_spec.rb diff --git a/spec/permissions_spec.rb b/spec/riak/permissions_spec.rb similarity index 100% rename from spec/permissions_spec.rb rename to spec/riak/permissions_spec.rb diff --git a/spec/riak_spec.rb b/spec/riak/riak_spec.rb similarity index 100% rename from spec/riak_spec.rb rename to spec/riak/riak_spec.rb From 990ea9cf288e07a0a0aa4e6c60f9c54a5c89c9b5 Mon Sep 17 00:00:00 2001 From: Garret Alfert Date: Wed, 20 Jan 2016 15:33:35 -0500 Subject: [PATCH 07/58] Save object metadata in redis --- Gemfile | 1 + Gemfile.lock | 2 ++ lib/remote_storage/swift.rb | 17 +++++++++++++++- spec/spec_helper.rb | 9 +++++++++ spec/swift/app_spec.rb | 40 +++++++++++++++++++++++++++++++++++++ 5 files changed, 68 insertions(+), 1 deletion(-) create mode 100644 spec/swift/app_spec.rb diff --git a/Gemfile b/Gemfile index f5ee5c7..694b773 100644 --- a/Gemfile +++ b/Gemfile @@ -13,6 +13,7 @@ group :test do gem 'rake' gem 'purdytest', :require => false gem 'm' + gem 'minitest-stub_any_instance' end group :staging, :production do diff --git a/Gemfile.lock b/Gemfile.lock index d234198..9d3cc8f 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -50,6 +50,7 @@ GEM method_source (0.8.2) mime-types (2.6.1) minitest (5.7.0) + minitest-stub_any_instance (1.0.1) multi_json (1.11.1) multipart-post (2.0.0) net-scp (1.0.4) @@ -109,6 +110,7 @@ DEPENDENCIES fog m mime-types (~> 2.6.1) + minitest-stub_any_instance purdytest rainbows rake diff --git a/lib/remote_storage/swift.rb b/lib/remote_storage/swift.rb index 5fabbab..c3f5c2f 100644 --- a/lib/remote_storage/swift.rb +++ b/lib/remote_storage/swift.rb @@ -125,7 +125,16 @@ module RemoteStorage res = do_put_request(url, data, content_type) - if update_dir_objects(user, directory) + # TODO get last modified from response and add to metadata + metadata = { + etag: res.headers[:etag], + size: data.length, + type: content_type + } + + if update_metadata_object(user, directory, key, metadata) && + # TODO provide the last modified to use for the dir objects as well + update_dir_objects(user, directory) server.headers["ETag"] = %Q("#{res.headers[:etag]}") server.halt 200 else @@ -255,7 +264,13 @@ module RemoteStorage parent_directories end + def update_metadata_object(user, directory, key, metadata) + key = "users:#{user}:data:#{directory}/#{key}" + redis.hmset(key, *metadata) + end + def update_dir_objects(user, directory) + # TODO use actual last modified time from the document put request timestamp = (Time.now.to_f * 1000).to_i parent_directories_for(directory).each do |dir| diff --git a/spec/spec_helper.rb b/spec/spec_helper.rb index ee4daac..585725a 100644 --- a/spec/spec_helper.rb +++ b/spec/spec_helper.rb @@ -9,6 +9,10 @@ require 'minitest/autorun' require 'rack/test' require 'purdytest' require 'riak' +require "redis" +require "rest_client" +require "minitest/stub_any_instance" +require "ostruct" def app LiquorCabinet @@ -28,6 +32,11 @@ def write_last_response_to_file(filename = "last_response.html") end alias context describe + +def redis + @redis ||= Redis.new(host: app.settings.redis["host"], port: app.settings.redis["port"]) +end + if app.settings.respond_to? :riak ::Riak.disable_list_keys_warnings = true diff --git a/spec/swift/app_spec.rb b/spec/swift/app_spec.rb new file mode 100644 index 0000000..bc3ec0a --- /dev/null +++ b/spec/swift/app_spec.rb @@ -0,0 +1,40 @@ +require_relative "../spec_helper" + +describe "App" do + include Rack::Test::Methods + + def app + LiquorCabinet + end + + it "returns 404 on non-existing routes" do + get "/virginmargarita" + last_response.status.must_equal 404 + end + + describe "PUT requests" do + context "authorized" do + before do + redis.sadd "authorizations:phil:amarillo", [":rw"] + header "Authorization", "Bearer amarillo" + + end + + it "creates the metadata object in redis" do + put_stub = OpenStruct.new(headers: {etag: "bla"}) + RemoteStorage::Swift.stub_any_instance :has_name_collision?, false do + RestClient.stub :put, put_stub do + put "/phil/food/aguacate", "si" + end + end + + metadata = redis.hgetall "users:phil:data:food/aguacate" + metadata["size"].must_equal "2" + metadata["type"].must_equal "text/plain; charset=utf-8" + metadata["etag"].must_equal "bla" + metadata["modified"].must_equal nil + end + end + end +end + From cd2c0865e8aa0279b3bbedd65fdc1ae5b12b563f Mon Sep 17 00:00:00 2001 From: Garret Alfert Date: Wed, 20 Jan 2016 15:53:22 -0500 Subject: [PATCH 08/58] Save directory metadata in redis (WIP) --- lib/remote_storage/swift.rb | 7 +++++-- spec/swift/app_spec.rb | 17 +++++++++++++++-- 2 files changed, 20 insertions(+), 4 deletions(-) diff --git a/lib/remote_storage/swift.rb b/lib/remote_storage/swift.rb index c3f5c2f..4c42e10 100644 --- a/lib/remote_storage/swift.rb +++ b/lib/remote_storage/swift.rb @@ -265,7 +265,7 @@ module RemoteStorage end def update_metadata_object(user, directory, key, metadata) - key = "users:#{user}:data:#{directory}/#{key}" + key = "rs_meta:#{user}:#{directory}/#{key}" redis.hmset(key, *metadata) end @@ -274,7 +274,10 @@ module RemoteStorage timestamp = (Time.now.to_f * 1000).to_i parent_directories_for(directory).each do |dir| - do_put_request("#{url_for_directory(user, dir)}/", timestamp.to_s, "text/plain") + res = do_put_request("#{url_for_directory(user, dir)}/", timestamp.to_s, "text/plain") + key = "rs_meta:#{user}:#{dir}/" + metadata = {etag: res.headers[:etag], modified: timestamp} + redis.hmset(key, *metadata) end true diff --git a/spec/swift/app_spec.rb b/spec/swift/app_spec.rb index bc3ec0a..fdbc39b 100644 --- a/spec/swift/app_spec.rb +++ b/spec/swift/app_spec.rb @@ -17,7 +17,6 @@ describe "App" do before do redis.sadd "authorizations:phil:amarillo", [":rw"] header "Authorization", "Bearer amarillo" - end it "creates the metadata object in redis" do @@ -28,12 +27,26 @@ describe "App" do end end - metadata = redis.hgetall "users:phil:data:food/aguacate" + metadata = redis.hgetall "rs_meta:phil:food/aguacate" metadata["size"].must_equal "2" metadata["type"].must_equal "text/plain; charset=utf-8" metadata["etag"].must_equal "bla" metadata["modified"].must_equal nil end + + it "creates the directory objects metadata in redis" do + put_stub = OpenStruct.new(headers: {etag: "bla"}) + RemoteStorage::Swift.stub_any_instance :has_name_collision?, false do + RestClient.stub :put, put_stub do + put "/phil/food/aguacate", "si" + end + end + + metadata = redis.hgetall "rs_meta:phil:food/" + metadata["etag"].must_equal "bla" + metadata["modified"].length.must_equal 13 + metadata = redis.hgetall "rs_meta:phil:food/" + end end end end From d8ba27cb6304957516d21515ddedb0a90b34b002 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Greg=20Kar=C3=A9kinian?= Date: Wed, 20 Jan 2016 21:00:42 +0000 Subject: [PATCH 09/58] Back to precise builds on Travis --- .travis.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index a365d06..c86a496 100644 --- a/.travis.yml +++ b/.travis.yml @@ -2,8 +2,6 @@ language: ruby cache: bundler rvm: - 2.2.4 -sudo: required -dist: trusty before_install: - sh .travis/install_riakcs.sh - gem install bundler From d1651799cf37bb91fe1eac39712dcbe353343fc0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Greg=20Kar=C3=A9kinian?= Date: Wed, 20 Jan 2016 21:40:28 +0000 Subject: [PATCH 10/58] Revert "Add riak service to Travis CI" This reverts commit d0dd4af3d942df347067cfaf0661beaadb402324. --- .travis.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index c86a496..4131003 100644 --- a/.travis.yml +++ b/.travis.yml @@ -19,5 +19,3 @@ notifications: - http://hook-juggler.herokuapp.com/hooks/travis on_success: always on_failure: always -services: - - riak From 189d04af250386c2a1cffa7709c8863637d8846b Mon Sep 17 00:00:00 2001 From: Garret Alfert Date: Wed, 20 Jan 2016 17:27:00 -0500 Subject: [PATCH 11/58] Save list of directory items in redis --- lib/remote_storage/swift.rb | 16 +++++++++++++--- spec/swift/app_spec.rb | 7 +++++++ 2 files changed, 20 insertions(+), 3 deletions(-) diff --git a/lib/remote_storage/swift.rb b/lib/remote_storage/swift.rb index 4c42e10..197d1f8 100644 --- a/lib/remote_storage/swift.rb +++ b/lib/remote_storage/swift.rb @@ -128,7 +128,7 @@ module RemoteStorage # TODO get last modified from response and add to metadata metadata = { etag: res.headers[:etag], - size: data.length, + size: data.size, type: content_type } @@ -264,9 +264,18 @@ module RemoteStorage parent_directories end + def parent_directory_for(directory) + if directory.match(/\//) + return directory[0..directory.rindex("/")-1] + elsif directory != "" + return "/" + end + end + def update_metadata_object(user, directory, key, metadata) - key = "rs_meta:#{user}:#{directory}/#{key}" - redis.hmset(key, *metadata) + redis_key = "rs_meta:#{user}:#{directory}/#{key}" + redis.hmset(redis_key, *metadata) + redis.sadd "rs_meta:#{user}:#{directory}/:items", key end def update_dir_objects(user, directory) @@ -278,6 +287,7 @@ module RemoteStorage key = "rs_meta:#{user}:#{dir}/" metadata = {etag: res.headers[:etag], modified: timestamp} redis.hmset(key, *metadata) + redis.sadd "rs_meta:#{user}:#{parent_directory_for(dir)}:items", "#{dir}/" end true diff --git a/spec/swift/app_spec.rb b/spec/swift/app_spec.rb index fdbc39b..84bb9ff 100644 --- a/spec/swift/app_spec.rb +++ b/spec/swift/app_spec.rb @@ -39,6 +39,7 @@ describe "App" do RemoteStorage::Swift.stub_any_instance :has_name_collision?, false do RestClient.stub :put, put_stub do put "/phil/food/aguacate", "si" + put "/phil/food/camaron", "yummi" end end @@ -46,6 +47,12 @@ describe "App" do metadata["etag"].must_equal "bla" metadata["modified"].length.must_equal 13 metadata = redis.hgetall "rs_meta:phil:food/" + + food_items = redis.smembers "rs_meta:phil:food/:items" + food_items.must_equal ["camaron", "aguacate"] + + root_items = redis.smembers "rs_meta:phil:/:items" + root_items.must_equal ["food/"] end end end From 972378e67f3d6a009fcc98fd41c903096a17762a Mon Sep 17 00:00:00 2001 From: Garret Alfert Date: Fri, 22 Jan 2016 14:46:41 -0500 Subject: [PATCH 12/58] Don't care for order of array when comparing --- spec/swift/app_spec.rb | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/spec/swift/app_spec.rb b/spec/swift/app_spec.rb index 84bb9ff..a7e25c8 100644 --- a/spec/swift/app_spec.rb +++ b/spec/swift/app_spec.rb @@ -49,7 +49,9 @@ describe "App" do metadata = redis.hgetall "rs_meta:phil:food/" food_items = redis.smembers "rs_meta:phil:food/:items" - food_items.must_equal ["camaron", "aguacate"] + food_items.each do |food_item| + ["camaron", "aguacate"].must_include food_item + end root_items = redis.smembers "rs_meta:phil:/:items" root_items.must_equal ["food/"] From 02e5d0b5ab693508fb01c20b9ad07503a7a2e7a5 Mon Sep 17 00:00:00 2001 From: Garret Alfert Date: Fri, 22 Jan 2016 14:49:57 -0500 Subject: [PATCH 13/58] Fix relative path in riak specs --- spec/riak/app_spec.rb | 2 +- spec/riak/directories_spec.rb | 2 +- spec/riak/permissions_spec.rb | 2 +- spec/riak/riak_spec.rb | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/spec/riak/app_spec.rb b/spec/riak/app_spec.rb index 1eb0ef1..411c6c4 100644 --- a/spec/riak/app_spec.rb +++ b/spec/riak/app_spec.rb @@ -1,4 +1,4 @@ -require_relative "spec_helper" +require_relative "../spec_helper" describe "App" do include Rack::Test::Methods diff --git a/spec/riak/directories_spec.rb b/spec/riak/directories_spec.rb index ea62080..6551a7f 100644 --- a/spec/riak/directories_spec.rb +++ b/spec/riak/directories_spec.rb @@ -1,4 +1,4 @@ -require_relative "spec_helper" +require_relative "../spec_helper" describe "Directories" do include Rack::Test::Methods diff --git a/spec/riak/permissions_spec.rb b/spec/riak/permissions_spec.rb index 6652c85..6250a2e 100644 --- a/spec/riak/permissions_spec.rb +++ b/spec/riak/permissions_spec.rb @@ -1,4 +1,4 @@ -require_relative "spec_helper" +require_relative "../spec_helper" describe "Permissions" do include Rack::Test::Methods diff --git a/spec/riak/riak_spec.rb b/spec/riak/riak_spec.rb index b3116f2..40a457a 100644 --- a/spec/riak/riak_spec.rb +++ b/spec/riak/riak_spec.rb @@ -1,4 +1,4 @@ -require_relative "spec_helper" +require_relative "../spec_helper" describe "App with Riak backend" do include Rack::Test::Methods From 25c170021e852a6c301d750ab17ceafc4b207cfd Mon Sep 17 00:00:00 2001 From: Garret Alfert Date: Fri, 22 Jan 2016 16:04:28 -0500 Subject: [PATCH 14/58] Delete metadata from Redis when deleting objects --- lib/remote_storage/swift.rb | 13 +++++- spec/swift/app_spec.rb | 82 ++++++++++++++++++++++++++++++++++++- 2 files changed, 93 insertions(+), 2 deletions(-) diff --git a/lib/remote_storage/swift.rb b/lib/remote_storage/swift.rb index 197d1f8..913f683 100644 --- a/lib/remote_storage/swift.rb +++ b/lib/remote_storage/swift.rb @@ -152,6 +152,7 @@ module RemoteStorage end do_delete_request(url) + delete_metadata_objects(user, directory, key) delete_dir_objects(user, directory) server.halt 200 @@ -299,13 +300,23 @@ module RemoteStorage false end + def delete_metadata_objects(user, directory, key) + redis_key = "rs_meta:#{user}:#{directory}/#{key}" + redis.del(redis_key) + redis.srem "rs_meta:#{user}:#{directory}/:items", key + end + def delete_dir_objects(user, directory) parent_directories_for(directory).each do |dir| if dir_empty?(user, dir) do_delete_request("#{url_for_directory(user, dir)}/") + redis.del "rs_meta:#{user}:#{directory}/" + redis.srem "rs_meta:#{user}:#{parent_directory_for(dir)}:items", "#{dir}/" else timestamp = (Time.now.to_f * 1000).to_i - do_put_request("#{url_for_directory(user, dir)}/", timestamp.to_s, "text/plain") + res = do_put_request("#{url_for_directory(user, dir)}/", timestamp.to_s, "text/plain") + metadata = {etag: res.headers[:etag], modified: timestamp} + redis.hmset("rs_meta:#{user}:#{dir}/", *metadata) end end end diff --git a/spec/swift/app_spec.rb b/spec/swift/app_spec.rb index a7e25c8..17d1555 100644 --- a/spec/swift/app_spec.rb +++ b/spec/swift/app_spec.rb @@ -46,7 +46,6 @@ describe "App" do metadata = redis.hgetall "rs_meta:phil:food/" metadata["etag"].must_equal "bla" metadata["modified"].length.must_equal 13 - metadata = redis.hgetall "rs_meta:phil:food/" food_items = redis.smembers "rs_meta:phil:food/:items" food_items.each do |food_item| @@ -58,5 +57,86 @@ describe "App" do end end end + + describe "DELETE requests" do + context "authorized" do + before do + redis.sadd "authorizations:phil:amarillo", [":rw"] + header "Authorization", "Bearer amarillo" + + put_stub = OpenStruct.new(headers: {etag: "bla"}) + RemoteStorage::Swift.stub_any_instance :has_name_collision?, false do + RestClient.stub :put, put_stub do + put "/phil/food/aguacate", "si" + put "/phil/food/camaron", "yummi" + end + end + end + + it "deletes the metadata object in redis" do + put_stub = OpenStruct.new(headers: {etag: "bla"}) + RemoteStorage::Swift.stub_any_instance :dir_empty?, false do + RestClient.stub :put, put_stub do + RestClient.stub :delete, "" do + delete "/phil/food/aguacate" + end + end + end + + metadata = redis.hgetall "rs_meta:phil:food/aguacate" + metadata.must_be_empty + end + + it "deletes the directory objects metadata in redis" do + old_metadata = redis.hgetall "rs_meta:phil:food/" + + put_stub = OpenStruct.new(headers: {etag: "newetag"}) + RemoteStorage::Swift.stub_any_instance :dir_empty?, false do + RestClient.stub :put, put_stub do + RestClient.stub :delete, "" do + delete "/phil/food/aguacate" + end + end + end + + metadata = redis.hgetall "rs_meta:phil:food/" + metadata["etag"].must_equal "newetag" + metadata["modified"].length.must_equal 13 + metadata["modified"].wont_equal old_metadata["modified"] + + food_items = redis.smembers "rs_meta:phil:food/:items" + food_items.must_equal ["camaron"] + + root_items = redis.smembers "rs_meta:phil:/:items" + root_items.must_equal ["food/"] + end + + it "deletes the parent directory objects metadata when deleting all items" do + put_stub = OpenStruct.new(headers: {etag: "bla"}) + RemoteStorage::Swift.stub_any_instance :dir_empty?, false do + RestClient.stub :put, put_stub do + RestClient.stub :delete, "" do + delete "/phil/food/aguacate" + end + end + end + + RemoteStorage::Swift.stub_any_instance :dir_empty?, true do + RestClient.stub :delete, "" do + delete "/phil/food/camaron" + end + end + + metadata = redis.hgetall "rs_meta:phil:food/" + metadata.must_be_empty + + food_items = redis.smembers "rs_meta:phil:food/:items" + food_items.must_be_empty + + root_items = redis.smembers "rs_meta:phil:/:items" + root_items.must_be_empty + end + end + end end From 9fe03498ea370d2c57a9e8df1ad714d1ce084992 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Greg=20Kar=C3=A9kinian?= Date: Fri, 22 Jan 2016 23:00:13 +0000 Subject: [PATCH 15/58] Force legacy Blue Box build for now on Travis --- .travis.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.travis.yml b/.travis.yml index 4131003..f5705a6 100644 --- a/.travis.yml +++ b/.travis.yml @@ -19,3 +19,6 @@ notifications: - http://hook-juggler.herokuapp.com/hooks/travis on_success: always on_failure: always +# Force legacy Blue Box build for now +- sudo: required + group: legacy From 7d5bd3a62468d0a5c47aced8cdfccd8c868df201 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Greg=20Kar=C3=A9kinian?= Date: Fri, 22 Jan 2016 23:01:33 +0000 Subject: [PATCH 16/58] Fix .travis.yml file --- .travis.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index f5705a6..3c8de3f 100644 --- a/.travis.yml +++ b/.travis.yml @@ -20,5 +20,5 @@ notifications: on_success: always on_failure: always # Force legacy Blue Box build for now -- sudo: required - group: legacy +sudo: required +group: legacy From f73c286ce9207e2fa85b12c594a620d5f2473ad3 Mon Sep 17 00:00:00 2001 From: Garret Alfert Date: Sat, 23 Jan 2016 12:34:08 -0500 Subject: [PATCH 17/58] Purge all keys from redis before every test run --- spec/spec_helper.rb | 6 ++++++ spec/swift/app_spec.rb | 10 ++++++++++ 2 files changed, 16 insertions(+) diff --git a/spec/spec_helper.rb b/spec/spec_helper.rb index 585725a..c92c47f 100644 --- a/spec/spec_helper.rb +++ b/spec/spec_helper.rb @@ -37,6 +37,12 @@ def redis @redis ||= Redis.new(host: app.settings.redis["host"], port: app.settings.redis["port"]) end +def purge_redis + redis.keys("*").each do |key| + redis.del key + end +end + if app.settings.respond_to? :riak ::Riak.disable_list_keys_warnings = true diff --git a/spec/swift/app_spec.rb b/spec/swift/app_spec.rb index 17d1555..9afbdf9 100644 --- a/spec/swift/app_spec.rb +++ b/spec/swift/app_spec.rb @@ -13,6 +13,11 @@ describe "App" do end describe "PUT requests" do + + before do + purge_redis + end + context "authorized" do before do redis.sadd "authorizations:phil:amarillo", [":rw"] @@ -59,6 +64,11 @@ describe "App" do end describe "DELETE requests" do + + before do + purge_redis + end + context "authorized" do before do redis.sadd "authorizations:phil:amarillo", [":rw"] From dfc8a5909642b21226ec898ce35c4d4ea73ec995 Mon Sep 17 00:00:00 2001 From: Garret Alfert Date: Sat, 23 Jan 2016 13:20:42 -0500 Subject: [PATCH 18/58] Fix redis keys and content for nested directories --- lib/remote_storage/swift.rb | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/lib/remote_storage/swift.rb b/lib/remote_storage/swift.rb index 913f683..58a6a5f 100644 --- a/lib/remote_storage/swift.rb +++ b/lib/remote_storage/swift.rb @@ -265,9 +265,17 @@ module RemoteStorage parent_directories end + def top_directory(directory) + if directory.match(/\//) + directory.split("/").last + elsif directory != "" + return directory + end + end + def parent_directory_for(directory) if directory.match(/\//) - return directory[0..directory.rindex("/")-1] + return directory[0..directory.rindex("/")] elsif directory != "" return "/" end @@ -277,6 +285,8 @@ module RemoteStorage redis_key = "rs_meta:#{user}:#{directory}/#{key}" redis.hmset(redis_key, *metadata) redis.sadd "rs_meta:#{user}:#{directory}/:items", key + + true end def update_dir_objects(user, directory) @@ -288,7 +298,7 @@ module RemoteStorage key = "rs_meta:#{user}:#{dir}/" metadata = {etag: res.headers[:etag], modified: timestamp} redis.hmset(key, *metadata) - redis.sadd "rs_meta:#{user}:#{parent_directory_for(dir)}:items", "#{dir}/" + redis.sadd "rs_meta:#{user}:#{parent_directory_for(dir)}:items", "#{top_directory(dir)}/" end true From 599865cf3f2dbed5f88148fd034f1b16047c3dd5 Mon Sep 17 00:00:00 2001 From: Garret Alfert Date: Sat, 23 Jan 2016 13:26:38 -0500 Subject: [PATCH 19/58] Generate directory listing from Redis metadata --- lib/remote_storage/swift.rb | 42 +++++++++++++++++++++++++++ spec/swift/app_spec.rb | 58 +++++++++++++++++++++++++++++++++++++ 2 files changed, 100 insertions(+) diff --git a/lib/remote_storage/swift.rb b/lib/remote_storage/swift.rb index 58a6a5f..94ccafc 100644 --- a/lib/remote_storage/swift.rb +++ b/lib/remote_storage/swift.rb @@ -75,6 +75,48 @@ module RemoteStorage end def get_directory_listing(user, directory) + # TODO add ETag header + # TODO check IF_NONE_MATCH header + + server.headers["Content-Type"] = "application/json" + + listing = { + "@context" => "http://remotestorage.io/spec/folder-description", + "items" => {} + } + + items = redis.smembers "rs_meta:#{user}:#{directory}/:items" + + items.sort.each do |name| + redis_key = if directory.empty? + "rs_meta:phil:#{name}" + else + "rs_meta:phil:#{directory}/#{name}" + end + metadata = redis.hgetall redis_key + + if name[-1] == "/" # It's a directory + listing["items"].merge!({ + name => { + "ETag" => metadata["etag"] + } + }) + else # It's a file + listing["items"].merge!({ + name => { + "ETag" => metadata["etag"], + "Content-Type" => metadata["type"], + "Content-Length" => metadata["size"].to_i + } + }); + end + + end + + listing.to_json + end + + def get_directory_listing_from_swift(user, directory) is_root_listing = directory.empty? server.headers["Content-Type"] = "application/json" diff --git a/spec/swift/app_spec.rb b/spec/swift/app_spec.rb index 9afbdf9..08bb344 100644 --- a/spec/swift/app_spec.rb +++ b/spec/swift/app_spec.rb @@ -148,5 +148,63 @@ describe "App" do end end end + + describe "GET requests" do + + before do + purge_redis + end + + context "authorized" do + before do + redis.sadd "authorizations:phil:amarillo", [":rw"] + header "Authorization", "Bearer amarillo" + + put_stub = OpenStruct.new(headers: {etag: "bla"}) + RemoteStorage::Swift.stub_any_instance :has_name_collision?, false do + RestClient.stub :put, put_stub do + put "/phil/food/aguacate", "si" + put "/phil/food/camaron", "yummi" + put "/phil/food/desunyos/bolon", "wow" + end + end + end + + describe "directory listings" do + + it "contains all items in the directory" do + get "/phil/food/" + + last_response.status.must_equal 200 + last_response.content_type.must_equal "application/json" + + content = JSON.parse(last_response.body) + content["@context"].must_equal "http://remotestorage.io/spec/folder-description" + content["items"]["aguacate"].wont_be_nil + content["items"]["aguacate"]["Content-Type"].must_equal "text/plain; charset=utf-8" + content["items"]["aguacate"]["Content-Length"].must_equal 2 + content["items"]["aguacate"]["ETag"].must_equal "bla" + content["items"]["camaron"].wont_be_nil + content["items"]["camaron"]["Content-Type"].must_equal "text/plain; charset=utf-8" + content["items"]["camaron"]["Content-Length"].must_equal 5 + content["items"]["camaron"]["ETag"].must_equal "bla" + content["items"]["desunyos/"].wont_be_nil + content["items"]["desunyos/"]["ETag"].must_equal "bla" + end + + it "contains all items in the root directory" do + get "phil/" + + last_response.status.must_equal 200 + last_response.content_type.must_equal "application/json" + + content = JSON.parse(last_response.body) + content["items"]["food/"].wont_be_nil + content["items"]["food/"]["ETag"].must_equal "bla" + end + + end + end + end end From c676c1a6a8fe0efc1e24db8b3ee0c26fa1c2d089 Mon Sep 17 00:00:00 2001 From: Garret Alfert Date: Sat, 23 Jan 2016 14:28:14 -0500 Subject: [PATCH 20/58] Add metadata entry for root directory --- lib/remote_storage/swift.rb | 6 ++++++ spec/swift/app_spec.rb | 4 ++++ 2 files changed, 10 insertions(+) diff --git a/lib/remote_storage/swift.rb b/lib/remote_storage/swift.rb index 94ccafc..ceb65a4 100644 --- a/lib/remote_storage/swift.rb +++ b/lib/remote_storage/swift.rb @@ -304,6 +304,8 @@ module RemoteStorage directories.pop end + parent_directories << "" # add empty string for the root directory + parent_directories end @@ -336,6 +338,7 @@ module RemoteStorage timestamp = (Time.now.to_f * 1000).to_i parent_directories_for(directory).each do |dir| + # TODO check if we can actually do a put request to the root dir res = do_put_request("#{url_for_directory(user, dir)}/", timestamp.to_s, "text/plain") key = "rs_meta:#{user}:#{dir}/" metadata = {etag: res.headers[:etag], modified: timestamp} @@ -346,6 +349,7 @@ module RemoteStorage true rescue parent_directories_for(directory).each do |dir| + # TODO check if we can actually do a delete request to the root dir do_delete_request("#{url_for_directory(user, dir)}/") rescue false end @@ -361,11 +365,13 @@ module RemoteStorage def delete_dir_objects(user, directory) parent_directories_for(directory).each do |dir| if dir_empty?(user, dir) + # TODO check if we can actually do a delete request to the root dir do_delete_request("#{url_for_directory(user, dir)}/") redis.del "rs_meta:#{user}:#{directory}/" redis.srem "rs_meta:#{user}:#{parent_directory_for(dir)}:items", "#{dir}/" else timestamp = (Time.now.to_f * 1000).to_i + # TODO check if we can actually do a put request to the root dir res = do_put_request("#{url_for_directory(user, dir)}/", timestamp.to_s, "text/plain") metadata = {etag: res.headers[:etag], modified: timestamp} redis.hmset("rs_meta:#{user}:#{dir}/", *metadata) diff --git a/spec/swift/app_spec.rb b/spec/swift/app_spec.rb index 08bb344..924a259 100644 --- a/spec/swift/app_spec.rb +++ b/spec/swift/app_spec.rb @@ -48,6 +48,10 @@ describe "App" do end end + metadata = redis.hgetall "rs_meta:phil:/" + metadata["etag"].must_equal "bla" + metadata["modified"].length.must_equal 13 + metadata = redis.hgetall "rs_meta:phil:food/" metadata["etag"].must_equal "bla" metadata["modified"].length.must_equal 13 From bec9f7a6cce21f597e24bfafbd3aaf3ae0a09dc0 Mon Sep 17 00:00:00 2001 From: Garret Alfert Date: Sat, 23 Jan 2016 14:29:18 -0500 Subject: [PATCH 21/58] Set ETag header for directory listings --- lib/remote_storage/swift.rb | 4 +++- spec/swift/app_spec.rb | 7 +++++++ 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/lib/remote_storage/swift.rb b/lib/remote_storage/swift.rb index ceb65a4..216b7ab 100644 --- a/lib/remote_storage/swift.rb +++ b/lib/remote_storage/swift.rb @@ -75,10 +75,12 @@ module RemoteStorage end def get_directory_listing(user, directory) - # TODO add ETag header # TODO check IF_NONE_MATCH header + etag = redis.hget "rs_meta:#{user}:#{directory}/", "etag" + server.headers["Content-Type"] = "application/json" + server.headers["ETag"] = %Q("#{etag}") listing = { "@context" => "http://remotestorage.io/spec/folder-description", diff --git a/spec/swift/app_spec.rb b/spec/swift/app_spec.rb index 924a259..877b0f7 100644 --- a/spec/swift/app_spec.rb +++ b/spec/swift/app_spec.rb @@ -176,6 +176,13 @@ describe "App" do describe "directory listings" do + it "has an ETag in the header" do + get "/phil/food/" + + last_response.status.must_equal 200 + last_response.headers["ETag"].must_equal "\"bla\"" + end + it "contains all items in the directory" do get "/phil/food/" From 15196ca4b9d9c8733e97142e9c4f2aaf58b5a3f8 Mon Sep 17 00:00:00 2001 From: Garret Alfert Date: Sat, 23 Jan 2016 14:29:37 -0500 Subject: [PATCH 22/58] Check IF_NONE_MATCH header for directory listings --- lib/remote_storage/swift.rb | 3 ++- spec/swift/app_spec.rb | 7 +++++++ 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/lib/remote_storage/swift.rb b/lib/remote_storage/swift.rb index 216b7ab..2581ab5 100644 --- a/lib/remote_storage/swift.rb +++ b/lib/remote_storage/swift.rb @@ -75,9 +75,10 @@ module RemoteStorage end def get_directory_listing(user, directory) - # TODO check IF_NONE_MATCH header etag = redis.hget "rs_meta:#{user}:#{directory}/", "etag" + none_match = (server.env["HTTP_IF_NONE_MATCH"] || "").split(",").map(&:strip) + server.halt 304 if none_match.include? etag server.headers["Content-Type"] = "application/json" server.headers["ETag"] = %Q("#{etag}") diff --git a/spec/swift/app_spec.rb b/spec/swift/app_spec.rb index 877b0f7..9022913 100644 --- a/spec/swift/app_spec.rb +++ b/spec/swift/app_spec.rb @@ -183,6 +183,13 @@ describe "App" do last_response.headers["ETag"].must_equal "\"bla\"" end + it "responds with 304 when IF_NONE_MATCH header contains the ETag" do + header "If-None-Match", "bla" + get "/phil/food/" + + last_response.status.must_equal 304 + end + it "contains all items in the directory" do get "/phil/food/" From 737be34e73947b959bfff1e48d7ecddb79d35d04 Mon Sep 17 00:00:00 2001 From: Garret Alfert Date: Tue, 26 Jan 2016 20:21:45 +0100 Subject: [PATCH 23/58] Configure the source of dir listing (redis or swift) --- config.yml.example | 1 + lib/remote_storage/swift.rb | 8 ++++++++ 2 files changed, 9 insertions(+) diff --git a/config.yml.example b/config.yml.example index f0ed36d..5f65092 100644 --- a/config.yml.example +++ b/config.yml.example @@ -20,6 +20,7 @@ development: &defaults # redis: # host: localhost # port: 6379 + # use_redis_dir_listing: true test: <<: *defaults diff --git a/lib/remote_storage/swift.rb b/lib/remote_storage/swift.rb index 2581ab5..80a8406 100644 --- a/lib/remote_storage/swift.rb +++ b/lib/remote_storage/swift.rb @@ -75,6 +75,14 @@ module RemoteStorage end def get_directory_listing(user, directory) + if settings.use_redis_dir_listing + get_directory_listing_from_redis(user, directory) + else + get_directory_listing_from_swift + end + end + + def get_directory_listing_from_redis(user, directory) etag = redis.hget "rs_meta:#{user}:#{directory}/", "etag" none_match = (server.env["HTTP_IF_NONE_MATCH"] || "").split(",").map(&:strip) From 0745354b8a4f789df6fdc8e1899a67f8955157ef Mon Sep 17 00:00:00 2001 From: Garret Alfert Date: Tue, 26 Jan 2016 20:32:29 +0100 Subject: [PATCH 24/58] Retrieve directory listing from Redis via Lua --- lib/remote_storage/swift.rb | 70 ++++++++++++++++++++++--------------- 1 file changed, 41 insertions(+), 29 deletions(-) diff --git a/lib/remote_storage/swift.rb b/lib/remote_storage/swift.rb index 80a8406..9738f36 100644 --- a/lib/remote_storage/swift.rb +++ b/lib/remote_storage/swift.rb @@ -82,6 +82,46 @@ module RemoteStorage end end + def get_directory_listing_from_redis_via_lua(user, directory) + lua_script = <<-EOF + local user = ARGV[1] + local directory = ARGV[2] + local items = redis.call("smembers", "rs_meta:"..user..":"..directory.."/:items") + local listing = {} + + for index, name in pairs(items) do + local redis_key = "rs_meta:"..user..":" + if directory == "" then + redis_key = redis_key..name + else + redis_key = redis_key..directory.."/"..name + end + + local metadata_values = redis.call("hgetall", redis_key) + local metadata = {} + + -- redis returns hashes as a single list of alternating keys and values + -- this collates it into a table + for idx = 1, #metadata_values, 2 do + metadata[metadata_values[idx]] = metadata_values[idx + 1] + end + + listing[name] = {} + if string.sub(name, -1) == "/" then + listing[name]["ETag"] = metadata["etag"] + else + listing[name]["ETag"] = metadata["etag"] + listing[name]["Content-Type"] = metadata["type"] + listing[name]["Content-Length"] = tonumber(metadata["size"]) + end + end + + return cjson.encode(listing) + EOF + + JSON.parse(redis.eval(lua_script, nil, [user, directory])) + end + def get_directory_listing_from_redis(user, directory) etag = redis.hget "rs_meta:#{user}:#{directory}/", "etag" @@ -93,37 +133,9 @@ module RemoteStorage listing = { "@context" => "http://remotestorage.io/spec/folder-description", - "items" => {} + "items" => get_directory_listing_from_redis_via_lua(user, directory) } - items = redis.smembers "rs_meta:#{user}:#{directory}/:items" - - items.sort.each do |name| - redis_key = if directory.empty? - "rs_meta:phil:#{name}" - else - "rs_meta:phil:#{directory}/#{name}" - end - metadata = redis.hgetall redis_key - - if name[-1] == "/" # It's a directory - listing["items"].merge!({ - name => { - "ETag" => metadata["etag"] - } - }) - else # It's a file - listing["items"].merge!({ - name => { - "ETag" => metadata["etag"], - "Content-Type" => metadata["type"], - "Content-Length" => metadata["size"].to_i - } - }); - end - - end - listing.to_json end From d973771192df9bbcd8431ae9b103bf79f2d479b1 Mon Sep 17 00:00:00 2001 From: Garret Alfert Date: Tue, 26 Jan 2016 20:33:43 +0100 Subject: [PATCH 25/58] Remove some duplication --- lib/remote_storage/swift.rb | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/lib/remote_storage/swift.rb b/lib/remote_storage/swift.rb index 9738f36..71d0572 100644 --- a/lib/remote_storage/swift.rb +++ b/lib/remote_storage/swift.rb @@ -106,11 +106,8 @@ module RemoteStorage metadata[metadata_values[idx]] = metadata_values[idx + 1] end - listing[name] = {} - if string.sub(name, -1) == "/" then - listing[name]["ETag"] = metadata["etag"] - else - listing[name]["ETag"] = metadata["etag"] + listing[name] = {["ETag"] = metadata["etag"]} + if string.sub(name, -1) ~= "/" then listing[name]["Content-Type"] = metadata["type"] listing[name]["Content-Length"] = tonumber(metadata["size"]) end From 4e7c8f68bbfbe53a5f1d9d2f471687933ba7c5e4 Mon Sep 17 00:00:00 2001 From: Garret Alfert Date: Thu, 28 Jan 2016 17:48:20 +0100 Subject: [PATCH 26/58] Add missing arguments to method call --- lib/remote_storage/swift.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/remote_storage/swift.rb b/lib/remote_storage/swift.rb index 71d0572..98eb1df 100644 --- a/lib/remote_storage/swift.rb +++ b/lib/remote_storage/swift.rb @@ -78,7 +78,7 @@ module RemoteStorage if settings.use_redis_dir_listing get_directory_listing_from_redis(user, directory) else - get_directory_listing_from_swift + get_directory_listing_from_swift(user, directory) end end From 4ca67c7ea9e5b7b81a593c6f0495f52186d45c7e Mon Sep 17 00:00:00 2001 From: Garret Alfert Date: Thu, 28 Jan 2016 19:06:24 +0100 Subject: [PATCH 27/58] Do collision detection via Redis metadata --- lib/remote_storage/swift.rb | 52 +++++++++++++++++++++++++++++++++++++ spec/swift/app_spec.rb | 40 ++++++++++++++++++++++++++++ 2 files changed, 92 insertions(+) diff --git a/lib/remote_storage/swift.rb b/lib/remote_storage/swift.rb index 98eb1df..6074492 100644 --- a/lib/remote_storage/swift.rb +++ b/lib/remote_storage/swift.rb @@ -289,6 +289,58 @@ module RemoteStorage end def has_name_collision?(user, directory, key) + if settings.use_redis_dir_listing + has_name_collision_via_redis?(user, directory, key) + else + has_name_collision_via_swift?(user, directory, key) + end + end + + def has_name_collision_via_redis?(user, directory, key) + lua_script = <<-EOF + local user = ARGV[1] + local directory = ARGV[2] + local key = ARGV[3] + + -- build table with parent directories from remaining arguments + local parent_dir_count = #ARGV - 3 + local parent_directories = {} + for i = 4, 4 + parent_dir_count do + table.insert(parent_directories, ARGV[i]) + end + + -- check for existing directory with the same name as the document + local redis_key = "rs_meta:"..user..":" + if directory == "" then + redis_key = redis_key..key.."/" + else + redis_key = redis_key..directory.."/"..key.."/" + end + if redis.call("hget", redis_key, "etag") then + return true + end + + for index, dir in pairs(parent_directories) do + if redis.call("hget", "rs_meta:"..user..":"..dir.."/", "etag") then + -- the directory already exists, no need to do further checks + return false + else + -- check for existing document with same name as directory + if redis.call("hget", "rs_meta:"..user..":"..dir, "etag") then + return true + end + end + end + + return false + EOF + + parent_directories = parent_directories_for(directory) + + redis.eval(lua_script, nil, [user, directory, key, *parent_directories]) + end + + def has_name_collision_via_swift?(user, directory, key) # check for existing directory with the same name as the document url = url_for_key(user, directory, key) do_head_request("#{url}/") do |res| diff --git a/spec/swift/app_spec.rb b/spec/swift/app_spec.rb index 9022913..9d20293 100644 --- a/spec/swift/app_spec.rb +++ b/spec/swift/app_spec.rb @@ -64,6 +64,46 @@ describe "App" do root_items = redis.smembers "rs_meta:phil:/:items" root_items.must_equal ["food/"] end + + describe "name collision checks" do + it "is successful when there is no name collision" do + put_stub = OpenStruct.new(headers: {etag: "bla"}) + RestClient.stub :put, put_stub do + put "/phil/food/aguacate", "si" + end + + last_response.status.must_equal 200 + + metadata = redis.hgetall "rs_meta:phil:food/aguacate" + metadata["size"].must_equal "2" + end + + it "conflicts when there is a directory with same name as document" do + put_stub = OpenStruct.new(headers: {etag: "bla"}) + RestClient.stub :put, put_stub do + put "/phil/food/aguacate", "si" + put "/phil/food", "wontwork" + end + + last_response.status.must_equal 409 + + metadata = redis.hgetall "rs_meta:phil:food" + metadata.must_be_empty + end + + it "conflicts when there is a document with same name as directory" do + put_stub = OpenStruct.new(headers: {etag: "bla"}) + RestClient.stub :put, put_stub do + put "/phil/food/aguacate", "si" + put "/phil/food/aguacate/empanado", "wontwork" + end + + last_response.status.must_equal 409 + + metadata = redis.hgetall "rs_meta:phil:food/aguacate/empanado" + metadata.must_be_empty + end + end end end From 16dcc56fbadf8d174df29eb175697d95e354dc0e Mon Sep 17 00:00:00 2001 From: Garret Alfert Date: Fri, 29 Jan 2016 15:16:24 +0100 Subject: [PATCH 28/58] No need to stub has_name_collision? anymore --- spec/swift/app_spec.rb | 32 ++++++++++++-------------------- 1 file changed, 12 insertions(+), 20 deletions(-) diff --git a/spec/swift/app_spec.rb b/spec/swift/app_spec.rb index 9d20293..6b2a876 100644 --- a/spec/swift/app_spec.rb +++ b/spec/swift/app_spec.rb @@ -26,10 +26,8 @@ describe "App" do it "creates the metadata object in redis" do put_stub = OpenStruct.new(headers: {etag: "bla"}) - RemoteStorage::Swift.stub_any_instance :has_name_collision?, false do - RestClient.stub :put, put_stub do - put "/phil/food/aguacate", "si" - end + RestClient.stub :put, put_stub do + put "/phil/food/aguacate", "si" end metadata = redis.hgetall "rs_meta:phil:food/aguacate" @@ -41,11 +39,9 @@ describe "App" do it "creates the directory objects metadata in redis" do put_stub = OpenStruct.new(headers: {etag: "bla"}) - RemoteStorage::Swift.stub_any_instance :has_name_collision?, false do - RestClient.stub :put, put_stub do - put "/phil/food/aguacate", "si" - put "/phil/food/camaron", "yummi" - end + RestClient.stub :put, put_stub do + put "/phil/food/aguacate", "si" + put "/phil/food/camaron", "yummi" end metadata = redis.hgetall "rs_meta:phil:/" @@ -119,11 +115,9 @@ describe "App" do header "Authorization", "Bearer amarillo" put_stub = OpenStruct.new(headers: {etag: "bla"}) - RemoteStorage::Swift.stub_any_instance :has_name_collision?, false do - RestClient.stub :put, put_stub do - put "/phil/food/aguacate", "si" - put "/phil/food/camaron", "yummi" - end + RestClient.stub :put, put_stub do + put "/phil/food/aguacate", "si" + put "/phil/food/camaron", "yummi" end end @@ -205,12 +199,10 @@ describe "App" do header "Authorization", "Bearer amarillo" put_stub = OpenStruct.new(headers: {etag: "bla"}) - RemoteStorage::Swift.stub_any_instance :has_name_collision?, false do - RestClient.stub :put, put_stub do - put "/phil/food/aguacate", "si" - put "/phil/food/camaron", "yummi" - put "/phil/food/desunyos/bolon", "wow" - end + RestClient.stub :put, put_stub do + put "/phil/food/aguacate", "si" + put "/phil/food/camaron", "yummi" + put "/phil/food/desunyos/bolon", "wow" end end From a4673e9661a744810e8756f08397c91edc8d7430 Mon Sep 17 00:00:00 2001 From: Garret Alfert Date: Fri, 29 Jan 2016 16:18:09 +0100 Subject: [PATCH 29/58] Use directory backend config from redis instead of config file --- config.yml.example | 1 - lib/remote_storage/swift.rb | 8 ++++++-- spec/spec_helper.rb | 2 +- spec/swift/app_spec.rb | 31 +++++++++++++++++++++++++++++++ 4 files changed, 38 insertions(+), 4 deletions(-) diff --git a/config.yml.example b/config.yml.example index 5f65092..f0ed36d 100644 --- a/config.yml.example +++ b/config.yml.example @@ -20,7 +20,6 @@ development: &defaults # redis: # host: localhost # port: 6379 - # use_redis_dir_listing: true test: <<: *defaults diff --git a/lib/remote_storage/swift.rb b/lib/remote_storage/swift.rb index 6074492..535947e 100644 --- a/lib/remote_storage/swift.rb +++ b/lib/remote_storage/swift.rb @@ -75,7 +75,7 @@ module RemoteStorage end def get_directory_listing(user, directory) - if settings.use_redis_dir_listing + if directory_backend(user).match /new/ get_directory_listing_from_redis(user, directory) else get_directory_listing_from_swift(user, directory) @@ -289,7 +289,7 @@ module RemoteStorage end def has_name_collision?(user, directory, key) - if settings.use_redis_dir_listing + if directory_backend(user).match /new/ has_name_collision_via_redis?(user, directory, key) else has_name_collision_via_swift?(user, directory, key) @@ -518,6 +518,10 @@ module RemoteStorage @redis ||= Redis.new(host: settings.redis["host"], port: settings.redis["port"]) end + def directory_backend(user) + @directory_backend ||= redis.get("rs_config:dir_backend:#{user}") || "legacy" + end + def etag_for(body) objects = JSON.parse(body) diff --git a/spec/spec_helper.rb b/spec/spec_helper.rb index c92c47f..8cec109 100644 --- a/spec/spec_helper.rb +++ b/spec/spec_helper.rb @@ -38,7 +38,7 @@ def redis end def purge_redis - redis.keys("*").each do |key| + redis.keys("rs_*").each do |key| redis.del key end end diff --git a/spec/swift/app_spec.rb b/spec/swift/app_spec.rb index 6b2a876..b4b9d38 100644 --- a/spec/swift/app_spec.rb +++ b/spec/swift/app_spec.rb @@ -16,6 +16,7 @@ describe "App" do before do purge_redis + redis.set "rs_config:dir_backend:phil", "new" end context "authorized" do @@ -107,6 +108,7 @@ describe "App" do before do purge_redis + redis.set "rs_config:dir_backend:phil", "new" end context "authorized" do @@ -191,9 +193,11 @@ describe "App" do before do purge_redis + redis.set "rs_config:dir_backend:phil", "new" end context "authorized" do + before do redis.sadd "authorizations:phil:amarillo", [":rw"] header "Authorization", "Bearer amarillo" @@ -255,6 +259,33 @@ describe "App" do end end + + context "with legacy directory backend" do + + before do + redis.sadd "authorizations:phil:amarillo", [":rw"] + header "Authorization", "Bearer amarillo" + + put_stub = OpenStruct.new(headers: {etag: "bla"}) + RestClient.stub :put, put_stub do + put "/phil/food/aguacate", "si" + put "/phil/food/camaron", "yummi" + end + + redis.set "rs_config:dir_backend:phil", "legacy" + end + + it "serves directory listing from Swift backend" do + RemoteStorage::Swift.stub_any_instance :get_directory_listing_from_swift, "directory listing" do + get "/phil/food/" + end + + last_response.status.must_equal 200 + last_response.body.must_equal "directory listing" + end + + end + end end From 573dadf3657de3e8bc9873893d236d65bf3b2250 Mon Sep 17 00:00:00 2001 From: Garret Alfert Date: Fri, 29 Jan 2016 16:18:49 +0100 Subject: [PATCH 30/58] Respond with 503 when directory backend is locked --- lib/remote_storage/swift.rb | 1 + spec/swift/app_spec.rb | 32 ++++++++++++++++++++++++++++++++ 2 files changed, 33 insertions(+) diff --git a/lib/remote_storage/swift.rb b/lib/remote_storage/swift.rb index 535947e..b815ca3 100644 --- a/lib/remote_storage/swift.rb +++ b/lib/remote_storage/swift.rb @@ -28,6 +28,7 @@ module RemoteStorage server.halt 401 unless permission if ["PUT", "DELETE"].include? request_method + server.halt 503 if directory_backend(user).match /locked/ server.halt 401 unless permission == "rw" end end diff --git a/spec/swift/app_spec.rb b/spec/swift/app_spec.rb index b4b9d38..ac0b41a 100644 --- a/spec/swift/app_spec.rb +++ b/spec/swift/app_spec.rb @@ -101,6 +101,38 @@ describe "App" do metadata.must_be_empty end end + + describe "directory backend configuration" do + context "locked new backed" do + before do + redis.set "rs_config:dir_backend:phil", "new-locked" + end + + it "responds with 503" do + put "/phil/food/aguacate", "si" + + last_response.status.must_equal 503 + + metadata = redis.hgetall "rs_meta:phil:food/aguacate" + metadata.must_be_empty + end + end + + context "locked legacy backend" do + before do + redis.set "rs_config:dir_backend:phil", "legacy-locked" + end + + it "responds with 503" do + put "/phil/food/aguacate", "si" + + last_response.status.must_equal 503 + + metadata = redis.hgetall "rs_meta:phil:food/aguacate" + metadata.must_be_empty + end + end + end end end From 7084a50d1b21093a8a2556f0e506a47ebbdb8d33 Mon Sep 17 00:00:00 2001 From: Garret Alfert Date: Fri, 29 Jan 2016 16:45:17 +0100 Subject: [PATCH 31/58] Make dir_empty? method use Redis metadata --- lib/remote_storage/swift.rb | 8 ++++++-- spec/swift/app_spec.rb | 27 ++++++++------------------- 2 files changed, 14 insertions(+), 21 deletions(-) diff --git a/lib/remote_storage/swift.rb b/lib/remote_storage/swift.rb index b815ca3..4a64d13 100644 --- a/lib/remote_storage/swift.rb +++ b/lib/remote_storage/swift.rb @@ -453,8 +453,12 @@ module RemoteStorage end def dir_empty?(user, dir) - do_get_request("#{container_url_for(user)}/?format=plain&limit=1&path=#{escape(dir)}/") do |res| - return res.headers[:content_length] == "0" + if directory_backend(user).match /new/ + redis.smembers("rs_meta:#{user}:#{dir}/:items").empty? + else + do_get_request("#{container_url_for(user)}/?format=plain&limit=1&path=#{escape(dir)}/") do |res| + return res.headers[:content_length] == "0" + end end end diff --git a/spec/swift/app_spec.rb b/spec/swift/app_spec.rb index ac0b41a..dc51ee4 100644 --- a/spec/swift/app_spec.rb +++ b/spec/swift/app_spec.rb @@ -157,11 +157,9 @@ describe "App" do it "deletes the metadata object in redis" do put_stub = OpenStruct.new(headers: {etag: "bla"}) - RemoteStorage::Swift.stub_any_instance :dir_empty?, false do - RestClient.stub :put, put_stub do - RestClient.stub :delete, "" do - delete "/phil/food/aguacate" - end + RestClient.stub :put, put_stub do + RestClient.stub :delete, "" do + delete "/phil/food/aguacate" end end @@ -173,11 +171,9 @@ describe "App" do old_metadata = redis.hgetall "rs_meta:phil:food/" put_stub = OpenStruct.new(headers: {etag: "newetag"}) - RemoteStorage::Swift.stub_any_instance :dir_empty?, false do - RestClient.stub :put, put_stub do - RestClient.stub :delete, "" do - delete "/phil/food/aguacate" - end + RestClient.stub :put, put_stub do + RestClient.stub :delete, "" do + delete "/phil/food/aguacate" end end @@ -195,16 +191,9 @@ describe "App" do it "deletes the parent directory objects metadata when deleting all items" do put_stub = OpenStruct.new(headers: {etag: "bla"}) - RemoteStorage::Swift.stub_any_instance :dir_empty?, false do - RestClient.stub :put, put_stub do - RestClient.stub :delete, "" do - delete "/phil/food/aguacate" - end - end - end - - RemoteStorage::Swift.stub_any_instance :dir_empty?, true do + RestClient.stub :put, put_stub do RestClient.stub :delete, "" do + delete "/phil/food/aguacate" delete "/phil/food/camaron" end end From 9ba85446efa1656c69ba7c61123e10d1d334bc63 Mon Sep 17 00:00:00 2001 From: Garret Alfert Date: Mon, 1 Feb 2016 14:58:19 +0100 Subject: [PATCH 32/58] Use parentheses to fix syntax warnings --- lib/remote_storage/swift.rb | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/lib/remote_storage/swift.rb b/lib/remote_storage/swift.rb index 4a64d13..a53ee2a 100644 --- a/lib/remote_storage/swift.rb +++ b/lib/remote_storage/swift.rb @@ -28,7 +28,7 @@ module RemoteStorage server.halt 401 unless permission if ["PUT", "DELETE"].include? request_method - server.halt 503 if directory_backend(user).match /locked/ + server.halt 503 if directory_backend(user).match(/locked/) server.halt 401 unless permission == "rw" end end @@ -76,7 +76,7 @@ module RemoteStorage end def get_directory_listing(user, directory) - if directory_backend(user).match /new/ + if directory_backend(user).match(/new/) get_directory_listing_from_redis(user, directory) else get_directory_listing_from_swift(user, directory) @@ -290,7 +290,7 @@ module RemoteStorage end def has_name_collision?(user, directory, key) - if directory_backend(user).match /new/ + if directory_backend(user).match(/new/) has_name_collision_via_redis?(user, directory, key) else has_name_collision_via_swift?(user, directory, key) @@ -453,7 +453,7 @@ module RemoteStorage end def dir_empty?(user, dir) - if directory_backend(user).match /new/ + if directory_backend(user).match(/new/) redis.smembers("rs_meta:#{user}:#{dir}/:items").empty? else do_get_request("#{container_url_for(user)}/?format=plain&limit=1&path=#{escape(dir)}/") do |res| From 409628c9e726fa837c7e4615d67463a10c0ecef6 Mon Sep 17 00:00:00 2001 From: Garret Alfert Date: Mon, 1 Feb 2016 18:04:35 +0100 Subject: [PATCH 33/58] Trying to fix specs when run on Travis --- config.yml.example | 5 +++++ spec/riak/directories_spec.rb | 4 ++-- spec/riak/riak_spec.rb | 10 +++++----- spec/spec_helper.rb | 14 ++++++++------ 4 files changed, 20 insertions(+), 13 deletions(-) diff --git a/config.yml.example b/config.yml.example index f0ed36d..b4e3ac3 100644 --- a/config.yml.example +++ b/config.yml.example @@ -32,6 +32,11 @@ test: cs_binaries: rs.binaries.test authorizations: rs_authorizations_test opslog: rs_opslog_test + swift: + host: "https://swift.example.com" + redis: + host: localhost + port: 6379 staging: <<: *defaults diff --git a/spec/riak/directories_spec.rb b/spec/riak/directories_spec.rb index 6551a7f..eea4f82 100644 --- a/spec/riak/directories_spec.rb +++ b/spec/riak/directories_spec.rb @@ -251,7 +251,7 @@ describe "Directories" do context "charset given in content-type header" do before do header "Content-Type", "image/jpeg; charset=binary" - filename = File.join(File.expand_path(File.dirname(__FILE__)), "fixtures", "rockrule.jpeg") + filename = File.join(File.expand_path(File.dirname(__FILE__)), "..", "fixtures", "rockrule.jpeg") @image = File.open(filename, "r").read put "/jimmy/tasks/jaypeg.jpg", @image end @@ -273,7 +273,7 @@ describe "Directories" do context "no charset in content-type header" do before do header "Content-Type", "image/jpeg" - filename = File.join(File.expand_path(File.dirname(__FILE__)), "fixtures", "rockrule.jpeg") + filename = File.join(File.expand_path(File.dirname(__FILE__)), "..", "fixtures", "rockrule.jpeg") @image = File.open(filename, "r").read put "/jimmy/tasks/jaypeg.jpg", @image end diff --git a/spec/riak/riak_spec.rb b/spec/riak/riak_spec.rb index 40a457a..19c2d69 100644 --- a/spec/riak/riak_spec.rb +++ b/spec/riak/riak_spec.rb @@ -446,7 +446,7 @@ describe "App with Riak backend" do context "binary charset in content-type header" do before do header "Content-Type", "image/jpeg; charset=binary" - filename = File.join(File.expand_path(File.dirname(__FILE__)), "fixtures", "rockrule.jpeg") + filename = File.join(File.expand_path(File.dirname(__FILE__)), "..", "fixtures", "rockrule.jpeg") @image = File.open(filename, "r").read put "/jimmy/documents/jaypeg", @image end @@ -502,7 +502,7 @@ describe "App with Riak backend" do context "overwriting existing file with same file" do before do header "Content-Type", "image/jpeg; charset=binary" - filename = File.join(File.expand_path(File.dirname(__FILE__)), "fixtures", "rockrule.jpeg") + filename = File.join(File.expand_path(File.dirname(__FILE__)), "..", "fixtures", "rockrule.jpeg") @image = File.open(filename, "r").read put "/jimmy/documents/jaypeg", @image end @@ -518,7 +518,7 @@ describe "App with Riak backend" do context "overwriting existing file with different file" do before do header "Content-Type", "image/jpeg; charset=binary" - filename = File.join(File.expand_path(File.dirname(__FILE__)), "fixtures", "rockrule.jpeg") + filename = File.join(File.expand_path(File.dirname(__FILE__)), "..", "fixtures", "rockrule.jpeg") @image = File.open(filename, "r").read put "/jimmy/documents/jaypeg", @image+"foo" end @@ -540,7 +540,7 @@ describe "App with Riak backend" do context "no binary charset in content-type header" do before do header "Content-Type", "image/jpeg" - filename = File.join(File.expand_path(File.dirname(__FILE__)), "fixtures", "rockrule.jpeg") + filename = File.join(File.expand_path(File.dirname(__FILE__)), "..", "fixtures", "rockrule.jpeg") @image = File.open(filename, "r").read put "/jimmy/documents/jaypeg", @image end @@ -705,7 +705,7 @@ describe "App with Riak backend" do context "binary data" do before do header "Content-Type", "image/jpeg; charset=binary" - filename = File.join(File.expand_path(File.dirname(__FILE__)), "fixtures", "rockrule.jpeg") + filename = File.join(File.expand_path(File.dirname(__FILE__)), "..", "fixtures", "rockrule.jpeg") @image = File.open(filename, "r").read put "/jimmy/documents/jaypeg", @image diff --git a/spec/spec_helper.rb b/spec/spec_helper.rb index 8cec109..4975f52 100644 --- a/spec/spec_helper.rb +++ b/spec/spec_helper.rb @@ -33,13 +33,15 @@ end alias context describe -def redis - @redis ||= Redis.new(host: app.settings.redis["host"], port: app.settings.redis["port"]) -end +if app.settings.respond_to? :redis + def redis + @redis ||= Redis.new(host: app.settings.redis["host"], port: app.settings.redis["port"]) + end -def purge_redis - redis.keys("rs_*").each do |key| - redis.del key + def purge_redis + redis.keys("rs_*").each do |key| + redis.del key + end end end From 536df233369537099f7564edb4cb3196b1da88cb Mon Sep 17 00:00:00 2001 From: Garret Alfert Date: Tue, 2 Feb 2016 00:10:47 +0100 Subject: [PATCH 34/58] Make Travis create a Swift token file --- .travis.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.travis.yml b/.travis.yml index 135619f..0f78e31 100644 --- a/.travis.yml +++ b/.travis.yml @@ -7,6 +7,7 @@ before_install: - sh .travis/install_riakcs.sh before_script: - cp config.yml.example config.yml + - mkdir -p tmp && echo "swifttoken" > tmp/swift_token.txt script: bundle exec rake test branches: only: From 954f046dd9d15f80ce9882f256065f573c908214 Mon Sep 17 00:00:00 2001 From: Garret Alfert Date: Tue, 2 Feb 2016 00:19:15 +0100 Subject: [PATCH 35/58] Make Travis install Redis --- .travis.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.travis.yml b/.travis.yml index 0f78e31..dbf71dc 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,6 +1,8 @@ language: ruby rvm: - 2.2.4 +services: + - redis-server sudo: required dist: trusty before_install: From 2516e9090ec4a79574dc9bb692f44b26849d8c44 Mon Sep 17 00:00:00 2001 From: Garret Alfert Date: Tue, 2 Feb 2016 00:57:53 +0100 Subject: [PATCH 36/58] Make Travis only run Swift specs for now --- .travis.yml | 2 +- config.yml.example | 18 +++++++++--------- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/.travis.yml b/.travis.yml index b883498..d3d7d89 100644 --- a/.travis.yml +++ b/.travis.yml @@ -10,7 +10,7 @@ before_install: before_script: - cp config.yml.example config.yml - mkdir -p tmp && echo "swifttoken" > tmp/swift_token.txt -script: bundle exec rake test +script: ruby spec/swift/* branches: only: - master diff --git a/config.yml.example b/config.yml.example index b4e3ac3..e2aac55 100644 --- a/config.yml.example +++ b/config.yml.example @@ -23,15 +23,15 @@ development: &defaults test: <<: *defaults - riak: - <<: *riak_defaults - buckets: - data: rs_data_test - directories: rs_directories_test - binaries: rs_binaries_test - cs_binaries: rs.binaries.test - authorizations: rs_authorizations_test - opslog: rs_opslog_test + # riak: + # <<: *riak_defaults + # buckets: + # data: rs_data_test + # directories: rs_directories_test + # binaries: rs_binaries_test + # cs_binaries: rs.binaries.test + # authorizations: rs_authorizations_test + # opslog: rs_opslog_test swift: host: "https://swift.example.com" redis: From d7a71d039f29fd67c16b0e60c16ea6f355d7b0f0 Mon Sep 17 00:00:00 2001 From: Garret Alfert Date: Tue, 2 Feb 2016 01:04:49 +0100 Subject: [PATCH 37/58] Comment out the Riak settings from example config --- config.yml.example | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/config.yml.example b/config.yml.example index e2aac55..6fb3307 100644 --- a/config.yml.example +++ b/config.yml.example @@ -1,18 +1,18 @@ development: &defaults maintenance: false - riak: &riak_defaults - host: localhost - http_port: 8098 - riak_cs: - credentials_file: "cs_credentials.json" - endpoint: "http://cs.example.com:8080" - buckets: - data: rs_data - directories: rs_directories - binaries: rs_binaries - cs_binaries: rs.binaries - authorizations: rs_authorizations - opslog: rs_opslog + # riak: &riak_defaults + # host: localhost + # http_port: 8098 + # riak_cs: + # credentials_file: "cs_credentials.json" + # endpoint: "http://cs.example.com:8080" + # buckets: + # data: rs_data + # directories: rs_directories + # binaries: rs_binaries + # cs_binaries: rs.binaries + # authorizations: rs_authorizations + # opslog: rs_opslog # # uncomment this section and comment the riak one # swift: &swift_defaults # host: "https://swift.example.com" From c730333143a2e27c46e994e99ee03bfd54f1b732 Mon Sep 17 00:00:00 2001 From: Garret Alfert Date: Tue, 2 Feb 2016 18:07:12 +0100 Subject: [PATCH 38/58] Don't try to do put or delete requests to root dir in order to update etag --- lib/remote_storage/swift.rb | 33 ++++++++++++++++++++--------- spec/swift/app_spec.rb | 41 +++++++++++++++++++++++++++++-------- 2 files changed, 56 insertions(+), 18 deletions(-) diff --git a/lib/remote_storage/swift.rb b/lib/remote_storage/swift.rb index a53ee2a..b35021d 100644 --- a/lib/remote_storage/swift.rb +++ b/lib/remote_storage/swift.rb @@ -411,10 +411,16 @@ module RemoteStorage timestamp = (Time.now.to_f * 1000).to_i parent_directories_for(directory).each do |dir| - # TODO check if we can actually do a put request to the root dir - res = do_put_request("#{url_for_directory(user, dir)}/", timestamp.to_s, "text/plain") + unless dir == "" + res = do_put_request("#{url_for_directory(user, dir)}/", timestamp.to_s, "text/plain") + etag = res.headers[:etag] + else + get_response = do_get_request("#{container_url_for(user)}/?format=json&path=") + etag = etag_for(get_response.body) + end + key = "rs_meta:#{user}:#{dir}/" - metadata = {etag: res.headers[:etag], modified: timestamp} + metadata = {etag: etag, modified: timestamp} redis.hmset(key, *metadata) redis.sadd "rs_meta:#{user}:#{parent_directory_for(dir)}:items", "#{top_directory(dir)}/" end @@ -422,8 +428,9 @@ module RemoteStorage true rescue parent_directories_for(directory).each do |dir| - # TODO check if we can actually do a delete request to the root dir - do_delete_request("#{url_for_directory(user, dir)}/") rescue false + unless dir == "" + do_delete_request("#{url_for_directory(user, dir)}/") rescue false + end end false @@ -438,15 +445,21 @@ module RemoteStorage def delete_dir_objects(user, directory) parent_directories_for(directory).each do |dir| if dir_empty?(user, dir) - # TODO check if we can actually do a delete request to the root dir - do_delete_request("#{url_for_directory(user, dir)}/") + unless dir == "" + do_delete_request("#{url_for_directory(user, dir)}/") + end redis.del "rs_meta:#{user}:#{directory}/" redis.srem "rs_meta:#{user}:#{parent_directory_for(dir)}:items", "#{dir}/" else timestamp = (Time.now.to_f * 1000).to_i - # TODO check if we can actually do a put request to the root dir - res = do_put_request("#{url_for_directory(user, dir)}/", timestamp.to_s, "text/plain") - metadata = {etag: res.headers[:etag], modified: timestamp} + unless dir == "" + res = do_put_request("#{url_for_directory(user, dir)}/", timestamp.to_s, "text/plain") + etag = res.headers[:etag] + else + get_response = do_get_request("#{container_url_for(user)}/?format=json&path=") + etag = etag_for(get_response.body) + end + metadata = {etag: etag, modified: timestamp} redis.hmset("rs_meta:#{user}:#{dir}/", *metadata) end end diff --git a/spec/swift/app_spec.rb b/spec/swift/app_spec.rb index dc51ee4..052b750 100644 --- a/spec/swift/app_spec.rb +++ b/spec/swift/app_spec.rb @@ -40,13 +40,18 @@ describe "App" do it "creates the directory objects metadata in redis" do put_stub = OpenStruct.new(headers: {etag: "bla"}) + get_stub = OpenStruct.new(body: "rootbody") RestClient.stub :put, put_stub do - put "/phil/food/aguacate", "si" - put "/phil/food/camaron", "yummi" + RestClient.stub :get, get_stub do + RemoteStorage::Swift.stub_any_instance :etag_for, "rootetag" do + put "/phil/food/aguacate", "si" + put "/phil/food/camaron", "yummi" + end + end end metadata = redis.hgetall "rs_meta:phil:/" - metadata["etag"].must_equal "bla" + metadata["etag"].must_equal "rootetag" metadata["modified"].length.must_equal 13 metadata = redis.hgetall "rs_meta:phil:food/" @@ -65,8 +70,13 @@ describe "App" do describe "name collision checks" do it "is successful when there is no name collision" do put_stub = OpenStruct.new(headers: {etag: "bla"}) + get_stub = OpenStruct.new(body: "rootbody") RestClient.stub :put, put_stub do - put "/phil/food/aguacate", "si" + RestClient.stub :get, get_stub do + RemoteStorage::Swift.stub_any_instance :etag_for, "rootetag" do + put "/phil/food/aguacate", "si" + end + end end last_response.status.must_equal 200 @@ -157,9 +167,14 @@ describe "App" do it "deletes the metadata object in redis" do put_stub = OpenStruct.new(headers: {etag: "bla"}) + get_stub = OpenStruct.new(body: "rootbody") RestClient.stub :put, put_stub do RestClient.stub :delete, "" do - delete "/phil/food/aguacate" + RestClient.stub :get, get_stub do + RemoteStorage::Swift.stub_any_instance :etag_for, "rootetag" do + delete "/phil/food/aguacate" + end + end end end @@ -171,9 +186,14 @@ describe "App" do old_metadata = redis.hgetall "rs_meta:phil:food/" put_stub = OpenStruct.new(headers: {etag: "newetag"}) + get_stub = OpenStruct.new(body: "rootbody") RestClient.stub :put, put_stub do RestClient.stub :delete, "" do - delete "/phil/food/aguacate" + RestClient.stub :get, get_stub do + RemoteStorage::Swift.stub_any_instance :etag_for, "rootetag" do + delete "/phil/food/aguacate" + end + end end end @@ -191,10 +211,15 @@ describe "App" do it "deletes the parent directory objects metadata when deleting all items" do put_stub = OpenStruct.new(headers: {etag: "bla"}) + get_stub = OpenStruct.new(body: "rootbody") RestClient.stub :put, put_stub do RestClient.stub :delete, "" do - delete "/phil/food/aguacate" - delete "/phil/food/camaron" + RestClient.stub :get, get_stub do + RemoteStorage::Swift.stub_any_instance :etag_for, "rootetag" do + delete "/phil/food/aguacate" + delete "/phil/food/camaron" + end + end end end From 0e10f3b6ff7e5aa18a1adb0ee5832ca32fed7b2b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Greg=20Kar=C3=A9kinian?= Date: Tue, 2 Feb 2016 22:20:38 +0000 Subject: [PATCH 39/58] Change the way to load the Redis config to allow clusters --- lib/remote_storage/swift.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/remote_storage/swift.rb b/lib/remote_storage/swift.rb index b35021d..1b2dfee 100644 --- a/lib/remote_storage/swift.rb +++ b/lib/remote_storage/swift.rb @@ -533,7 +533,7 @@ module RemoteStorage end def redis - @redis ||= Redis.new(host: settings.redis["host"], port: settings.redis["port"]) + @redis ||= Redis.new(settings.redis) end def directory_backend(user) From 5370df9c75a0dea5364177a9a2fb408d91bc2520 Mon Sep 17 00:00:00 2001 From: Garret Alfert Date: Mon, 15 Feb 2016 15:15:01 +0100 Subject: [PATCH 40/58] Basic migration script using actual RS dir listings --- migrate_metadata_to_redis.rb | 117 +++++++++++++++++++++++++++++++++++ 1 file changed, 117 insertions(+) create mode 100755 migrate_metadata_to_redis.rb diff --git a/migrate_metadata_to_redis.rb b/migrate_metadata_to_redis.rb new file mode 100755 index 0000000..de1e812 --- /dev/null +++ b/migrate_metadata_to_redis.rb @@ -0,0 +1,117 @@ +#!/usr/bin/env ruby + +require "rest_client" +require "redis" + +class Migrator + + attr_accessor :username, :token, :base_url + + def initialize(username, token) + @username = username + @token = token + @base_url = "https://storage.5apps.com" + end + + def configure_redis(redis_config) + @redis_config = redis_config + end + + def root_url + "#{@base_url}/#{@username}" + end + + def headers + {"authorization" => "Bearer #{@token}"} + end + + def is_dir?(name) + name[-1] == "/" + end + + def url_for(directory, parent_directory="") + # base_path = [root_url, parent_directory].join("/") + "#{root_url}#{parent_directory}#{directory}" + end + + def migrate + work_on_dir("", "/") + end + + def work_on_dir(directory, parent_directory) + url = url_for(directory, parent_directory) + + # puts "work on dir: #{url}" + + response = RestClient.get(url, headers) + listing = JSON.parse(response.body) + + timestamp = (Time.now.to_f * 1000).to_i + + if listing["items"].any? + items = listing["items"] + items.each do |item, data| + if is_dir? item + save_directory_data("#{parent_directory}#{directory}", item, data, timestamp) + + # get dir listing and repeat + work_on_dir(item, "#{parent_directory}#{directory}") + else + save_document_data("#{parent_directory}#{directory}", item, data, timestamp) + end + + add_item_to_parent_dir("#{parent_directory}#{directory}", item) + end + end + end + + def add_item_to_parent_dir(dir, item) + key = "rs_meta:#{username}:#{parent_directory_for(dir)}:items" + # puts "adding item #{item} to #{key}" + redis.sadd key, item + end + + def save_directory_data(dir, item, data, timestamp) + key = "rs_meta:#{username}:#{dir.gsub(/^\//, "")}#{item}" + metadata = {etag: data["ETag"], modified: timestamp} + + # puts "metadata for dir #{key}: #{metadata}" + redis.hmset(key, *metadata) + end + + def save_document_data(dir, item, data, timestamp) + key = "rs_meta:#{username}:#{dir.gsub(/^\//, "")}#{item}" + metadata = { + etag: data["ETag"], + size: data["Content-Length"], + type: data["Content-Type"], + modified: timestamp + } + # puts "metadata for document #{key}: #{metadata}" + redis.hmset(key, *metadata) + end + + def parent_directory_for(directory) + return directory if directory == "/" + + return directory[0..directory.rindex("/")].gsub(/^\//, "") + end + + def redis + @redis ||= Redis.new(@redis_config) + end + +end + +username = ARGV[0] +token = ARGV[1] + +migrator = Migrator.new username, token +migrator.configure_redis({host: "localhost", port: 6379}) + +migrator.migrate + + + + + From 54176b6928b85e425d585a04caf60359211f6a06 Mon Sep 17 00:00:00 2001 From: Garret Alfert Date: Mon, 22 Feb 2016 16:08:53 +0100 Subject: [PATCH 41/58] Use data directly from Swift for metadata migration --- migrate_metadata_to_redis.rb | 167 +++++++++++++++++++++++++++-------- 1 file changed, 132 insertions(+), 35 deletions(-) diff --git a/migrate_metadata_to_redis.rb b/migrate_metadata_to_redis.rb index de1e812..7458ff8 100755 --- a/migrate_metadata_to_redis.rb +++ b/migrate_metadata_to_redis.rb @@ -2,49 +2,45 @@ require "rest_client" require "redis" +require "yaml" class Migrator - attr_accessor :username, :token, :base_url + attr_accessor :username, :base_url, :swift_host, :swift_token, + :environment, :dry_run, :logging, :settings - def initialize(username, token) + def initialize(username) @username = username - @token = token - @base_url = "https://storage.5apps.com" - end - def configure_redis(redis_config) - @redis_config = redis_config + @environment = ENV["ENVIRONMENT"] || "staging" + @settings = YAML.load(File.read('config.yml'))[@environment] + @swift_host = @settings["swift"]["host"] + @swift_token = File.read("tmp/swift_token.txt") + + @dry_run = false # disables writing anything to Redis when true + @logging = true end def root_url "#{@base_url}/#{@username}" end - def headers - {"authorization" => "Bearer #{@token}"} - end - def is_dir?(name) name[-1] == "/" end def url_for(directory, parent_directory="") - # base_path = [root_url, parent_directory].join("/") "#{root_url}#{parent_directory}#{directory}" end def migrate - work_on_dir("", "/") + work_on_dir("", "") end def work_on_dir(directory, parent_directory) - url = url_for(directory, parent_directory) + puts "retrieving listing for '#{parent_directory}#{directory}'" if logging - # puts "work on dir: #{url}" - - response = RestClient.get(url, headers) - listing = JSON.parse(response.body) + listing = get_directory_listing_from_swift("#{parent_directory}#{directory}") timestamp = (Time.now.to_f * 1000).to_i @@ -67,16 +63,16 @@ class Migrator def add_item_to_parent_dir(dir, item) key = "rs_meta:#{username}:#{parent_directory_for(dir)}:items" - # puts "adding item #{item} to #{key}" - redis.sadd key, item + puts "adding item #{item} to #{key}" if logging + redis.sadd(key, item) unless dry_run end def save_directory_data(dir, item, data, timestamp) key = "rs_meta:#{username}:#{dir.gsub(/^\//, "")}#{item}" metadata = {etag: data["ETag"], modified: timestamp} - # puts "metadata for dir #{key}: #{metadata}" - redis.hmset(key, *metadata) + puts "metadata for dir #{key}: #{metadata}" if logging + redis.hmset(key, *metadata) unless dry_run end def save_document_data(dir, item, data, timestamp) @@ -87,31 +83,132 @@ class Migrator type: data["Content-Type"], modified: timestamp } - # puts "metadata for document #{key}: #{metadata}" - redis.hmset(key, *metadata) + puts "metadata for document #{key}: #{metadata}" if logging + redis.hmset(key, *metadata) unless dry_run end def parent_directory_for(directory) - return directory if directory == "/" - - return directory[0..directory.rindex("/")].gsub(/^\//, "") + if directory.match(/\//) + return directory[0..directory.rindex("/")] + else + return "/" + end end def redis - @redis ||= Redis.new(@redis_config) + @redis ||= Redis.new(@settings["redis"]) end + def get_directory_listing_from_swift(directory) + is_root_listing = directory.empty? + + get_response = nil + + do_head_request("#{url_for_directory(@username, directory)}") do |response| + return directory_listing([]) if response.code == 404 + + if is_root_listing + get_response = do_get_request("#{container_url_for(@username)}/?format=json&path=") + else + get_response = do_get_request("#{container_url_for(@username)}/?format=json&path=#{escape(directory)}/") + end + end + + if body = JSON.parse(get_response.body) + listing = directory_listing(body) + else + puts "listing not JSON" + end + + listing + end + + def directory_listing(res_body) + listing = { + "@context" => "http://remotestorage.io/spec/folder-description", + "items" => {} + } + + res_body.each do |entry| + name = entry["name"] + name.sub!("#{File.dirname(entry["name"])}/", '') + if name[-1] == "/" # It's a directory + listing["items"].merge!({ + name => { + "ETag" => entry["hash"], + } + }) + else # It's a file + listing["items"].merge!({ + name => { + "ETag" => entry["hash"], + "Content-Type" => entry["content_type"], + "Content-Length" => entry["bytes"] + } + }) + end + end + + listing + end + + def etag_for(body) + objects = JSON.parse(body) + + if objects.empty? + Digest::MD5.hexdigest "" + else + Digest::MD5.hexdigest objects.map { |o| o["hash"] }.join + end + end + + def do_head_request(url, &block) + RestClient.head(url, default_headers, &block) + end + + def do_get_request(url, &block) + RestClient.get(url, default_headers, &block) + end + + def default_headers + {"x-auth-token" => @swift_token} + end + + def url_for_directory(user, directory) + if directory.empty? + container_url_for(user) + else + "#{container_url_for(user)}/#{escape(directory)}" + end + end + + def container_url_for(user) + "#{base_url}/#{container_for(user)}" + end + + def base_url + @base_url ||= @swift_host + end + + def container_for(user) + "rs:#{environment.to_s.chars.first}:#{user}" + end + + def escape(url) + # We want spaces to turn into %20 and slashes to stay slashes + CGI::escape(url).gsub('+', '%20').gsub('%2F', '/') + end end username = ARGV[0] -token = ARGV[1] -migrator = Migrator.new username, token -migrator.configure_redis({host: "localhost", port: 6379}) +unless username + puts "No username given." + puts "Usage:" + puts "ENVIRONMENT=staging ./migrate_metadata_to_redis.rb " + exit 1 +end +migrator = Migrator.new username migrator.migrate - - - - From 570e3dcdb5d2c2c07a8cdc13964ad53e1b02c484 Mon Sep 17 00:00:00 2001 From: Garret Alfert Date: Wed, 24 Feb 2016 12:23:42 +0100 Subject: [PATCH 42/58] Set directory backend type during migration --- migrate_metadata_to_redis.rb | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/migrate_metadata_to_redis.rb b/migrate_metadata_to_redis.rb index 7458ff8..5a4e482 100755 --- a/migrate_metadata_to_redis.rb +++ b/migrate_metadata_to_redis.rb @@ -34,7 +34,20 @@ class Migrator end def migrate - work_on_dir("", "") + set_directory_backend("legacy_locked") + begin + work_on_dir("", "") + rescue Exception => ex + puts "Error migrating metadata for '#{username}': #{ex}" if logging + set_directory_backend("legacy") + # TODO write username to file for later reference + exit 1 + end + set_directory_backend("new") + end + + def set_directory_backend(backend) + redis.set("rs_config:dir_backend:#{username}", backend) unless dry_run end def work_on_dir(directory, parent_directory) From 3b391246ed920998fc28ab77c8f20cac90eea50b Mon Sep 17 00:00:00 2001 From: Garret Alfert Date: Wed, 24 Feb 2016 12:48:12 +0100 Subject: [PATCH 43/58] Use proper logger for migration script --- migrate_metadata_to_redis.rb | 22 +++++++++++++++------- 1 file changed, 15 insertions(+), 7 deletions(-) diff --git a/migrate_metadata_to_redis.rb b/migrate_metadata_to_redis.rb index 5a4e482..c850bac 100755 --- a/migrate_metadata_to_redis.rb +++ b/migrate_metadata_to_redis.rb @@ -3,22 +3,28 @@ require "rest_client" require "redis" require "yaml" +require "logger" class Migrator attr_accessor :username, :base_url, :swift_host, :swift_token, - :environment, :dry_run, :logging, :settings + :environment, :dry_run, :settings, :logger def initialize(username) @username = username @environment = ENV["ENVIRONMENT"] || "staging" @settings = YAML.load(File.read('config.yml'))[@environment] + @swift_host = @settings["swift"]["host"] @swift_token = File.read("tmp/swift_token.txt") @dry_run = false # disables writing anything to Redis when true - @logging = true + + @logger = Logger.new("log/migrate_metadata_to_redis.log") + log_level = ENV["LOGLEVEL"] || "INFO" + logger.level = Kernel.const_get "Logger::#{log_level}" + logger.progname = username end def root_url @@ -34,16 +40,18 @@ class Migrator end def migrate + logger.info "Starting migration for '#{username}'" set_directory_backend("legacy_locked") begin work_on_dir("", "") rescue Exception => ex - puts "Error migrating metadata for '#{username}': #{ex}" if logging + logger.error "Error migrating metadata for '#{username}': #{ex}" set_directory_backend("legacy") # TODO write username to file for later reference exit 1 end set_directory_backend("new") + logger.info "Finished migration for '#{username}'" end def set_directory_backend(backend) @@ -51,7 +59,7 @@ class Migrator end def work_on_dir(directory, parent_directory) - puts "retrieving listing for '#{parent_directory}#{directory}'" if logging + logger.debug "Retrieving listing for '#{parent_directory}#{directory}'" listing = get_directory_listing_from_swift("#{parent_directory}#{directory}") @@ -76,7 +84,7 @@ class Migrator def add_item_to_parent_dir(dir, item) key = "rs_meta:#{username}:#{parent_directory_for(dir)}:items" - puts "adding item #{item} to #{key}" if logging + logger.debug "Adding item #{item} to #{key}" redis.sadd(key, item) unless dry_run end @@ -84,7 +92,7 @@ class Migrator key = "rs_meta:#{username}:#{dir.gsub(/^\//, "")}#{item}" metadata = {etag: data["ETag"], modified: timestamp} - puts "metadata for dir #{key}: #{metadata}" if logging + logger.debug "Metadata for dir #{key}: #{metadata}" redis.hmset(key, *metadata) unless dry_run end @@ -96,7 +104,7 @@ class Migrator type: data["Content-Type"], modified: timestamp } - puts "metadata for document #{key}: #{metadata}" if logging + logger.debug "Metadata for document #{key}: #{metadata}" redis.hmset(key, *metadata) unless dry_run end From 33731f793ad0177aca88c612e624b422a7bec58e Mon Sep 17 00:00:00 2001 From: Garret Alfert Date: Wed, 24 Feb 2016 13:04:49 +0100 Subject: [PATCH 44/58] Set dry_run via environment variable Can be enabled via `DRYRUN=true` --- migrate_metadata_to_redis.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/migrate_metadata_to_redis.rb b/migrate_metadata_to_redis.rb index c850bac..abfd3bc 100755 --- a/migrate_metadata_to_redis.rb +++ b/migrate_metadata_to_redis.rb @@ -19,7 +19,7 @@ class Migrator @swift_host = @settings["swift"]["host"] @swift_token = File.read("tmp/swift_token.txt") - @dry_run = false # disables writing anything to Redis when true + @dry_run = ENV["DRYRUN"] || false # disables writing anything to Redis when true @logger = Logger.new("log/migrate_metadata_to_redis.log") log_level = ENV["LOGLEVEL"] || "INFO" From b396a6dfd1c1a6910f579796bdefb07a0882ec3e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Greg=20Kar=C3=A9kinian?= Date: Fri, 26 Feb 2016 13:47:34 +0100 Subject: [PATCH 45/58] Update Ruby gems --- Gemfile | 8 +++-- Gemfile.lock | 82 +++++++++++++++++++++++++++------------------------- 2 files changed, 48 insertions(+), 42 deletions(-) diff --git a/Gemfile b/Gemfile index 694b773..7157326 100644 --- a/Gemfile +++ b/Gemfile @@ -2,12 +2,14 @@ source "https://rubygems.org" gem "sinatra", '~> 1.4' gem "sinatra-contrib" -gem "activesupport", '~> 4.2' +gem "activesupport" gem "riak-client", :github => "5apps/riak-ruby-client", :branch => "invalid_uri_error" -gem "fog" +gem "fog-aws" gem "rest-client" gem "redis" -gem "mime-types", "~> 2.6.1", require: 'mime/types/columnar' +# Remove require when we can update to 3.0, which sets the new storage +# format to columnar by default. Increases performance +gem "mime-types", "~> 2.99", require: 'mime/types/columnar' group :test do gem 'rake' diff --git a/Gemfile.lock b/Gemfile.lock index 9d3cc8f..1601888 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -13,51 +13,56 @@ GIT GEM remote: https://rubygems.org/ specs: - activesupport (4.2.2) + activesupport (4.2.5.1) i18n (~> 0.7) json (~> 1.7, >= 1.7.7) minitest (~> 5.1) thread_safe (~> 0.3, >= 0.3.4) tzinfo (~> 1.1) - backports (3.6.4) + backports (3.6.8) beefcake (0.3.7) builder (3.2.2) - domain_name (0.5.24) + domain_name (0.5.20160216) unf (>= 0.0.5, < 1.0.0) - excon (0.16.10) - faraday (0.9.1) + excon (0.45.4) + faraday (0.9.2) multipart-post (>= 1.2, < 3) - fog (1.7.0) + fog-aws (0.8.1) + fog-core (~> 1.27) + fog-json (~> 1.0) + fog-xml (~> 0.1) + ipaddress (~> 0.8) + fog-core (1.36.0) builder - excon (~> 0.14) - formatador (~> 0.2.0) - mime-types - multi_json (~> 1.0) - net-scp (~> 1.0.4) - net-ssh (>= 2.1.3) - nokogiri (~> 1.5.0) - ruby-hmac + excon (~> 0.45) + formatador (~> 0.2) + fog-json (1.0.2) + fog-core (~> 1.0) + multi_json (~> 1.10) + fog-xml (0.1.2) + fog-core + nokogiri (~> 1.5, >= 1.5.11) formatador (0.2.5) http-cookie (1.0.2) domain_name (~> 0.5) i18n (0.7.0) innertube (1.0.2) + ipaddress (0.8.3) json (1.8.3) - kgio (2.9.3) - m (1.3.4) + kgio (2.10.0) + m (1.4.2) method_source (>= 0.6.7) rake (>= 0.9.2.2) method_source (0.8.2) - mime-types (2.6.1) - minitest (5.7.0) + mime-types (2.99.1) + mini_portile2 (2.0.0) + minitest (5.8.4) minitest-stub_any_instance (1.0.1) - multi_json (1.11.1) + multi_json (1.11.2) multipart-post (2.0.0) - net-scp (1.0.4) - net-ssh (>= 1.99.1) - net-ssh (2.6.7) - netrc (0.10.3) - nokogiri (1.5.11) + netrc (0.11.0) + nokogiri (1.6.7.2) + mini_portile2 (~> 2.0.0.rc2) purdytest (2.0.0) minitest (~> 5.5) rack (1.6.4) @@ -65,25 +70,24 @@ GEM rack rack-test (0.6.3) rack (>= 1.0) - rainbows (4.6.2) + rainbows (5.0.0) kgio (~> 2.5) rack (~> 1.1) - unicorn (~> 4.8) - raindrops (0.13.0) - rake (10.4.2) + unicorn (~> 5.0) + raindrops (0.15.0) + rake (10.5.0) redis (3.2.2) rest-client (1.8.0) http-cookie (>= 1.0.2, < 2.0) mime-types (>= 1.16, < 3.0) netrc (~> 0.7) - ruby-hmac (0.4.0) - sentry-raven (0.15.2) + sentry-raven (0.15.6) faraday (>= 0.7.6) - sinatra (1.4.6) - rack (~> 1.4) + sinatra (1.4.7) + rack (~> 1.5) rack-protection (~> 1.4) tilt (>= 1.3, < 3) - sinatra-contrib (1.4.4) + sinatra-contrib (1.4.6) backports (>= 2.0) multi_json rack-protection @@ -91,13 +95,13 @@ GEM sinatra (~> 1.4.0) tilt (>= 1.3, < 3) thread_safe (0.3.5) - tilt (2.0.1) + tilt (2.0.2) tzinfo (1.2.2) thread_safe (~> 0.1) unf (0.1.4) unf_ext - unf_ext (0.0.7.1) - unicorn (4.9.0) + unf_ext (0.0.7.2) + unicorn (5.0.1) kgio (~> 2.6) rack raindrops (~> 0.7) @@ -106,10 +110,10 @@ PLATFORMS ruby DEPENDENCIES - activesupport (~> 4.2) - fog + activesupport + fog-aws m - mime-types (~> 2.6.1) + mime-types (~> 2.99) minitest-stub_any_instance purdytest rainbows From 41074e35b35a59dd3b4f25209ede0c8a31af5072 Mon Sep 17 00:00:00 2001 From: Garret Alfert Date: Mon, 29 Feb 2016 15:35:37 +0100 Subject: [PATCH 46/58] Log usernames for failed migration to separate file This way they can be easily retried later on. --- migrate_metadata_to_redis.rb | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/migrate_metadata_to_redis.rb b/migrate_metadata_to_redis.rb index abfd3bc..3897ca0 100755 --- a/migrate_metadata_to_redis.rb +++ b/migrate_metadata_to_redis.rb @@ -47,7 +47,8 @@ class Migrator rescue Exception => ex logger.error "Error migrating metadata for '#{username}': #{ex}" set_directory_backend("legacy") - # TODO write username to file for later reference + # write username to file for later reference + File.open('log/failed_migration.log', 'a') { |f| f.puts username } exit 1 end set_directory_backend("new") From e7a35636fdda9460885f2e3135b9e4732d2b657e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Greg=20Kar=C3=A9kinian?= Date: Tue, 1 Mar 2016 15:02:56 +0100 Subject: [PATCH 47/58] Turn Redis parameters into symbols --- lib/remote_storage/swift.rb | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/lib/remote_storage/swift.rb b/lib/remote_storage/swift.rb index 1b2dfee..c929d5d 100644 --- a/lib/remote_storage/swift.rb +++ b/lib/remote_storage/swift.rb @@ -3,6 +3,7 @@ require "json" require "cgi" require "active_support/core_ext/time/conversions" require "active_support/core_ext/numeric/time" +require "active_support/core_ext/hash" require "redis" require "digest/md5" @@ -533,7 +534,7 @@ module RemoteStorage end def redis - @redis ||= Redis.new(settings.redis) + @redis ||= Redis.new(settings.redis.symbolize_keys) end def directory_backend(user) From 7107c77a3f752e6016add8c0bf599b9ccde6f5a4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Greg=20Kar=C3=A9kinian?= Date: Tue, 1 Mar 2016 15:06:23 +0100 Subject: [PATCH 48/58] Turn Redis parameters into symbols --- migrate_metadata_to_redis.rb | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/migrate_metadata_to_redis.rb b/migrate_metadata_to_redis.rb index 3897ca0..fa2a705 100755 --- a/migrate_metadata_to_redis.rb +++ b/migrate_metadata_to_redis.rb @@ -4,6 +4,7 @@ require "rest_client" require "redis" require "yaml" require "logger" +require "active_support/core_ext/hash" class Migrator @@ -118,7 +119,7 @@ class Migrator end def redis - @redis ||= Redis.new(@settings["redis"]) + @redis ||= Redis.new(@settings["redis"].symbolize_keys) end def get_directory_listing_from_swift(directory) From f1f450cc723f86319079ebb3627dac43bae41b8e Mon Sep 17 00:00:00 2001 From: Garret Alfert Date: Tue, 1 Mar 2016 22:57:54 +0100 Subject: [PATCH 49/58] Don't add double trailing slash to container URL The directory name already contains a trailing slash. --- migrate_metadata_to_redis.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/migrate_metadata_to_redis.rb b/migrate_metadata_to_redis.rb index fa2a705..0f37a49 100755 --- a/migrate_metadata_to_redis.rb +++ b/migrate_metadata_to_redis.rb @@ -133,7 +133,7 @@ class Migrator if is_root_listing get_response = do_get_request("#{container_url_for(@username)}/?format=json&path=") else - get_response = do_get_request("#{container_url_for(@username)}/?format=json&path=#{escape(directory)}/") + get_response = do_get_request("#{container_url_for(@username)}/?format=json&path=#{escape(directory)}") end end From d18de83893498ce18360289e0f9bbf050358beb8 Mon Sep 17 00:00:00 2001 From: Garret Alfert Date: Wed, 2 Mar 2016 13:59:44 +0100 Subject: [PATCH 50/58] Strip trailing whitespace from Swift token --- migrate_metadata_to_redis.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/migrate_metadata_to_redis.rb b/migrate_metadata_to_redis.rb index 0f37a49..9ecddd0 100755 --- a/migrate_metadata_to_redis.rb +++ b/migrate_metadata_to_redis.rb @@ -18,7 +18,7 @@ class Migrator @settings = YAML.load(File.read('config.yml'))[@environment] @swift_host = @settings["swift"]["host"] - @swift_token = File.read("tmp/swift_token.txt") + @swift_token = File.read("tmp/swift_token.txt").strip @dry_run = ENV["DRYRUN"] || false # disables writing anything to Redis when true From 19f6fcd0684c9e6250b3b2494912690b4b0d7d3f Mon Sep 17 00:00:00 2001 From: Garret Alfert Date: Wed, 2 Mar 2016 14:00:59 +0100 Subject: [PATCH 51/58] Use actual Last-Modified time we get from Swift --- migrate_metadata_to_redis.rb | 19 ++++++++++++++----- 1 file changed, 14 insertions(+), 5 deletions(-) diff --git a/migrate_metadata_to_redis.rb b/migrate_metadata_to_redis.rb index 9ecddd0..e7e66dd 100755 --- a/migrate_metadata_to_redis.rb +++ b/migrate_metadata_to_redis.rb @@ -76,7 +76,7 @@ class Migrator # get dir listing and repeat work_on_dir(item, "#{parent_directory}#{directory}") else - save_document_data("#{parent_directory}#{directory}", item, data, timestamp) + save_document_data("#{parent_directory}#{directory}", item, data) end add_item_to_parent_dir("#{parent_directory}#{directory}", item) @@ -92,19 +92,22 @@ class Migrator def save_directory_data(dir, item, data, timestamp) key = "rs_meta:#{username}:#{dir.gsub(/^\//, "")}#{item}" - metadata = {etag: data["ETag"], modified: timestamp} + metadata = { + etag: data["ETag"], + modified: timestamp_for(data["Last-Modified"]) + } logger.debug "Metadata for dir #{key}: #{metadata}" redis.hmset(key, *metadata) unless dry_run end - def save_document_data(dir, item, data, timestamp) + def save_document_data(dir, item, data) key = "rs_meta:#{username}:#{dir.gsub(/^\//, "")}#{item}" metadata = { etag: data["ETag"], size: data["Content-Length"], type: data["Content-Type"], - modified: timestamp + modified: timestamp_for(data["Last-Modified"]) } logger.debug "Metadata for document #{key}: #{metadata}" redis.hmset(key, *metadata) unless dry_run @@ -118,6 +121,10 @@ class Migrator end end + def timestamp_for(date) + return DateTime.parse(date).strftime("%Q").to_i + end + def redis @redis ||= Redis.new(@settings["redis"].symbolize_keys) end @@ -159,6 +166,7 @@ class Migrator listing["items"].merge!({ name => { "ETag" => entry["hash"], + "Last-Modified" => entry["last_modified"] } }) else # It's a file @@ -166,7 +174,8 @@ class Migrator name => { "ETag" => entry["hash"], "Content-Type" => entry["content_type"], - "Content-Length" => entry["bytes"] + "Content-Length" => entry["bytes"], + "Last-Modified" => entry["last_modified"] } }) end From ff25b4038546eb5ade90f44c3b9729d8ad19b921 Mon Sep 17 00:00:00 2001 From: Garret Alfert Date: Wed, 2 Mar 2016 14:05:53 +0100 Subject: [PATCH 52/58] Save last modified timestamp for objects as well --- lib/remote_storage/swift.rb | 18 +++++++++--------- spec/swift/app_spec.rb | 2 +- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/lib/remote_storage/swift.rb b/lib/remote_storage/swift.rb index 1b2dfee..8dde63b 100644 --- a/lib/remote_storage/swift.rb +++ b/lib/remote_storage/swift.rb @@ -188,16 +188,18 @@ module RemoteStorage res = do_put_request(url, data, content_type) - # TODO get last modified from response and add to metadata + # TODO use actual last modified time from the document put request + timestamp = (Time.now.to_f * 1000).to_i + metadata = { etag: res.headers[:etag], size: data.size, - type: content_type + type: content_type, + modified: timestamp } if update_metadata_object(user, directory, key, metadata) && - # TODO provide the last modified to use for the dir objects as well - update_dir_objects(user, directory) + update_dir_objects(user, directory, timestamp) server.headers["ETag"] = %Q("#{res.headers[:etag]}") server.halt 200 else @@ -406,10 +408,7 @@ module RemoteStorage true end - def update_dir_objects(user, directory) - # TODO use actual last modified time from the document put request - timestamp = (Time.now.to_f * 1000).to_i - + def update_dir_objects(user, directory, timestamp) parent_directories_for(directory).each do |dir| unless dir == "" res = do_put_request("#{url_for_directory(user, dir)}/", timestamp.to_s, "text/plain") @@ -443,6 +442,8 @@ module RemoteStorage end def delete_dir_objects(user, directory) + timestamp = (Time.now.to_f * 1000).to_i + parent_directories_for(directory).each do |dir| if dir_empty?(user, dir) unless dir == "" @@ -451,7 +452,6 @@ module RemoteStorage redis.del "rs_meta:#{user}:#{directory}/" redis.srem "rs_meta:#{user}:#{parent_directory_for(dir)}:items", "#{dir}/" else - timestamp = (Time.now.to_f * 1000).to_i unless dir == "" res = do_put_request("#{url_for_directory(user, dir)}/", timestamp.to_s, "text/plain") etag = res.headers[:etag] diff --git a/spec/swift/app_spec.rb b/spec/swift/app_spec.rb index 052b750..3323b26 100644 --- a/spec/swift/app_spec.rb +++ b/spec/swift/app_spec.rb @@ -35,7 +35,7 @@ describe "App" do metadata["size"].must_equal "2" metadata["type"].must_equal "text/plain; charset=utf-8" metadata["etag"].must_equal "bla" - metadata["modified"].must_equal nil + metadata["modified"].length.must_equal 13 end it "creates the directory objects metadata in redis" do From 067f4a155378ae0611e7c6970e66be5d466cf88a Mon Sep 17 00:00:00 2001 From: Garret Alfert Date: Wed, 2 Mar 2016 14:24:16 +0100 Subject: [PATCH 53/58] Use shorter Redis key for dir backend config rsc:db:username instead of rs_config:dir_backend:username --- lib/remote_storage/swift.rb | 2 +- migrate_metadata_to_redis.rb | 2 +- spec/spec_helper.rb | 2 +- spec/swift/app_spec.rb | 12 ++++++------ 4 files changed, 9 insertions(+), 9 deletions(-) diff --git a/lib/remote_storage/swift.rb b/lib/remote_storage/swift.rb index 8dde63b..1be255c 100644 --- a/lib/remote_storage/swift.rb +++ b/lib/remote_storage/swift.rb @@ -537,7 +537,7 @@ module RemoteStorage end def directory_backend(user) - @directory_backend ||= redis.get("rs_config:dir_backend:#{user}") || "legacy" + @directory_backend ||= redis.get("rsc:db:#{user}") || "legacy" end def etag_for(body) diff --git a/migrate_metadata_to_redis.rb b/migrate_metadata_to_redis.rb index e7e66dd..952aea4 100755 --- a/migrate_metadata_to_redis.rb +++ b/migrate_metadata_to_redis.rb @@ -57,7 +57,7 @@ class Migrator end def set_directory_backend(backend) - redis.set("rs_config:dir_backend:#{username}", backend) unless dry_run + redis.set("rsc:db:#{username}", backend) unless dry_run end def work_on_dir(directory, parent_directory) diff --git a/spec/spec_helper.rb b/spec/spec_helper.rb index 4975f52..dfc955f 100644 --- a/spec/spec_helper.rb +++ b/spec/spec_helper.rb @@ -39,7 +39,7 @@ if app.settings.respond_to? :redis end def purge_redis - redis.keys("rs_*").each do |key| + redis.keys("rs*").each do |key| redis.del key end end diff --git a/spec/swift/app_spec.rb b/spec/swift/app_spec.rb index 3323b26..b42a95a 100644 --- a/spec/swift/app_spec.rb +++ b/spec/swift/app_spec.rb @@ -16,7 +16,7 @@ describe "App" do before do purge_redis - redis.set "rs_config:dir_backend:phil", "new" + redis.set "rsc:db:phil", "new" end context "authorized" do @@ -115,7 +115,7 @@ describe "App" do describe "directory backend configuration" do context "locked new backed" do before do - redis.set "rs_config:dir_backend:phil", "new-locked" + redis.set "rsc:db:phil", "new-locked" end it "responds with 503" do @@ -130,7 +130,7 @@ describe "App" do context "locked legacy backend" do before do - redis.set "rs_config:dir_backend:phil", "legacy-locked" + redis.set "rsc:db:phil", "legacy-locked" end it "responds with 503" do @@ -150,7 +150,7 @@ describe "App" do before do purge_redis - redis.set "rs_config:dir_backend:phil", "new" + redis.set "rsc:db:phil", "new" end context "authorized" do @@ -239,7 +239,7 @@ describe "App" do before do purge_redis - redis.set "rs_config:dir_backend:phil", "new" + redis.set "rsc:db:phil", "new" end context "authorized" do @@ -318,7 +318,7 @@ describe "App" do put "/phil/food/camaron", "yummi" end - redis.set "rs_config:dir_backend:phil", "legacy" + redis.set "rsc:db:phil", "legacy" end it "serves directory listing from Swift backend" do From 650da91a1da71bd4a87d3e148591a15d72dc0dcb Mon Sep 17 00:00:00 2001 From: Garret Alfert Date: Wed, 2 Mar 2016 14:43:28 +0100 Subject: [PATCH 54/58] Shorten Redis metadata keys Use rsm:::i instead of rs_meta:::items --- lib/remote_storage/swift.rb | 32 ++++++++++++++++---------------- migrate_metadata_to_redis.rb | 6 +++--- spec/swift/app_spec.rb | 36 ++++++++++++++++++------------------ 3 files changed, 37 insertions(+), 37 deletions(-) diff --git a/lib/remote_storage/swift.rb b/lib/remote_storage/swift.rb index 1be255c..75526a5 100644 --- a/lib/remote_storage/swift.rb +++ b/lib/remote_storage/swift.rb @@ -87,11 +87,11 @@ module RemoteStorage lua_script = <<-EOF local user = ARGV[1] local directory = ARGV[2] - local items = redis.call("smembers", "rs_meta:"..user..":"..directory.."/:items") + local items = redis.call("smembers", "rsm:"..user..":"..directory.."/:i") local listing = {} for index, name in pairs(items) do - local redis_key = "rs_meta:"..user..":" + local redis_key = "rsm:"..user..":" if directory == "" then redis_key = redis_key..name else @@ -121,7 +121,7 @@ module RemoteStorage end def get_directory_listing_from_redis(user, directory) - etag = redis.hget "rs_meta:#{user}:#{directory}/", "etag" + etag = redis.hget "rsm:#{user}:#{directory}/", "etag" none_match = (server.env["HTTP_IF_NONE_MATCH"] || "").split(",").map(&:strip) server.halt 304 if none_match.include? etag @@ -313,7 +313,7 @@ module RemoteStorage end -- check for existing directory with the same name as the document - local redis_key = "rs_meta:"..user..":" + local redis_key = "rsm:"..user..":" if directory == "" then redis_key = redis_key..key.."/" else @@ -324,12 +324,12 @@ module RemoteStorage end for index, dir in pairs(parent_directories) do - if redis.call("hget", "rs_meta:"..user..":"..dir.."/", "etag") then + if redis.call("hget", "rsm:"..user..":"..dir.."/", "etag") then -- the directory already exists, no need to do further checks return false else -- check for existing document with same name as directory - if redis.call("hget", "rs_meta:"..user..":"..dir, "etag") then + if redis.call("hget", "rsm:"..user..":"..dir, "etag") then return true end end @@ -401,9 +401,9 @@ module RemoteStorage end def update_metadata_object(user, directory, key, metadata) - redis_key = "rs_meta:#{user}:#{directory}/#{key}" + redis_key = "rsm:#{user}:#{directory}/#{key}" redis.hmset(redis_key, *metadata) - redis.sadd "rs_meta:#{user}:#{directory}/:items", key + redis.sadd "rsm:#{user}:#{directory}/:i", key true end @@ -418,10 +418,10 @@ module RemoteStorage etag = etag_for(get_response.body) end - key = "rs_meta:#{user}:#{dir}/" + key = "rsm:#{user}:#{dir}/" metadata = {etag: etag, modified: timestamp} redis.hmset(key, *metadata) - redis.sadd "rs_meta:#{user}:#{parent_directory_for(dir)}:items", "#{top_directory(dir)}/" + redis.sadd "rsm:#{user}:#{parent_directory_for(dir)}:i", "#{top_directory(dir)}/" end true @@ -436,9 +436,9 @@ module RemoteStorage end def delete_metadata_objects(user, directory, key) - redis_key = "rs_meta:#{user}:#{directory}/#{key}" + redis_key = "rsm:#{user}:#{directory}/#{key}" redis.del(redis_key) - redis.srem "rs_meta:#{user}:#{directory}/:items", key + redis.srem "rsm:#{user}:#{directory}/:i", key end def delete_dir_objects(user, directory) @@ -449,8 +449,8 @@ module RemoteStorage unless dir == "" do_delete_request("#{url_for_directory(user, dir)}/") end - redis.del "rs_meta:#{user}:#{directory}/" - redis.srem "rs_meta:#{user}:#{parent_directory_for(dir)}:items", "#{dir}/" + redis.del "rsm:#{user}:#{directory}/" + redis.srem "rsm:#{user}:#{parent_directory_for(dir)}:i", "#{dir}/" else unless dir == "" res = do_put_request("#{url_for_directory(user, dir)}/", timestamp.to_s, "text/plain") @@ -460,14 +460,14 @@ module RemoteStorage etag = etag_for(get_response.body) end metadata = {etag: etag, modified: timestamp} - redis.hmset("rs_meta:#{user}:#{dir}/", *metadata) + redis.hmset("rsm:#{user}:#{dir}/", *metadata) end end end def dir_empty?(user, dir) if directory_backend(user).match(/new/) - redis.smembers("rs_meta:#{user}:#{dir}/:items").empty? + redis.smembers("rsm:#{user}:#{dir}/:i").empty? else do_get_request("#{container_url_for(user)}/?format=plain&limit=1&path=#{escape(dir)}/") do |res| return res.headers[:content_length] == "0" diff --git a/migrate_metadata_to_redis.rb b/migrate_metadata_to_redis.rb index 952aea4..d1211c8 100755 --- a/migrate_metadata_to_redis.rb +++ b/migrate_metadata_to_redis.rb @@ -85,13 +85,13 @@ class Migrator end def add_item_to_parent_dir(dir, item) - key = "rs_meta:#{username}:#{parent_directory_for(dir)}:items" + key = "rsm:#{username}:#{parent_directory_for(dir)}:i" logger.debug "Adding item #{item} to #{key}" redis.sadd(key, item) unless dry_run end def save_directory_data(dir, item, data, timestamp) - key = "rs_meta:#{username}:#{dir.gsub(/^\//, "")}#{item}" + key = "rsm:#{username}:#{dir.gsub(/^\//, "")}#{item}" metadata = { etag: data["ETag"], modified: timestamp_for(data["Last-Modified"]) @@ -102,7 +102,7 @@ class Migrator end def save_document_data(dir, item, data) - key = "rs_meta:#{username}:#{dir.gsub(/^\//, "")}#{item}" + key = "rsm:#{username}:#{dir.gsub(/^\//, "")}#{item}" metadata = { etag: data["ETag"], size: data["Content-Length"], diff --git a/spec/swift/app_spec.rb b/spec/swift/app_spec.rb index b42a95a..831c089 100644 --- a/spec/swift/app_spec.rb +++ b/spec/swift/app_spec.rb @@ -31,7 +31,7 @@ describe "App" do put "/phil/food/aguacate", "si" end - metadata = redis.hgetall "rs_meta:phil:food/aguacate" + metadata = redis.hgetall "rsm:phil:food/aguacate" metadata["size"].must_equal "2" metadata["type"].must_equal "text/plain; charset=utf-8" metadata["etag"].must_equal "bla" @@ -50,20 +50,20 @@ describe "App" do end end - metadata = redis.hgetall "rs_meta:phil:/" + metadata = redis.hgetall "rsm:phil:/" metadata["etag"].must_equal "rootetag" metadata["modified"].length.must_equal 13 - metadata = redis.hgetall "rs_meta:phil:food/" + metadata = redis.hgetall "rsm:phil:food/" metadata["etag"].must_equal "bla" metadata["modified"].length.must_equal 13 - food_items = redis.smembers "rs_meta:phil:food/:items" + food_items = redis.smembers "rsm:phil:food/:i" food_items.each do |food_item| ["camaron", "aguacate"].must_include food_item end - root_items = redis.smembers "rs_meta:phil:/:items" + root_items = redis.smembers "rsm:phil:/:i" root_items.must_equal ["food/"] end @@ -81,7 +81,7 @@ describe "App" do last_response.status.must_equal 200 - metadata = redis.hgetall "rs_meta:phil:food/aguacate" + metadata = redis.hgetall "rsm:phil:food/aguacate" metadata["size"].must_equal "2" end @@ -94,7 +94,7 @@ describe "App" do last_response.status.must_equal 409 - metadata = redis.hgetall "rs_meta:phil:food" + metadata = redis.hgetall "rsm:phil:food" metadata.must_be_empty end @@ -107,7 +107,7 @@ describe "App" do last_response.status.must_equal 409 - metadata = redis.hgetall "rs_meta:phil:food/aguacate/empanado" + metadata = redis.hgetall "rsm:phil:food/aguacate/empanado" metadata.must_be_empty end end @@ -123,7 +123,7 @@ describe "App" do last_response.status.must_equal 503 - metadata = redis.hgetall "rs_meta:phil:food/aguacate" + metadata = redis.hgetall "rsm:phil:food/aguacate" metadata.must_be_empty end end @@ -138,7 +138,7 @@ describe "App" do last_response.status.must_equal 503 - metadata = redis.hgetall "rs_meta:phil:food/aguacate" + metadata = redis.hgetall "rsm:phil:food/aguacate" metadata.must_be_empty end end @@ -178,12 +178,12 @@ describe "App" do end end - metadata = redis.hgetall "rs_meta:phil:food/aguacate" + metadata = redis.hgetall "rsm:phil:food/aguacate" metadata.must_be_empty end it "deletes the directory objects metadata in redis" do - old_metadata = redis.hgetall "rs_meta:phil:food/" + old_metadata = redis.hgetall "rsm:phil:food/" put_stub = OpenStruct.new(headers: {etag: "newetag"}) get_stub = OpenStruct.new(body: "rootbody") @@ -197,15 +197,15 @@ describe "App" do end end - metadata = redis.hgetall "rs_meta:phil:food/" + metadata = redis.hgetall "rsm:phil:food/" metadata["etag"].must_equal "newetag" metadata["modified"].length.must_equal 13 metadata["modified"].wont_equal old_metadata["modified"] - food_items = redis.smembers "rs_meta:phil:food/:items" + food_items = redis.smembers "rsm:phil:food/:i" food_items.must_equal ["camaron"] - root_items = redis.smembers "rs_meta:phil:/:items" + root_items = redis.smembers "rsm:phil:/:i" root_items.must_equal ["food/"] end @@ -223,13 +223,13 @@ describe "App" do end end - metadata = redis.hgetall "rs_meta:phil:food/" + metadata = redis.hgetall "rsm:phil:food/" metadata.must_be_empty - food_items = redis.smembers "rs_meta:phil:food/:items" + food_items = redis.smembers "rsm:phil:food/:i" food_items.must_be_empty - root_items = redis.smembers "rs_meta:phil:/:items" + root_items = redis.smembers "rsm:phil:/:i" root_items.must_be_empty end end From 88f2dceb2775a27a9a5aec74ff1f0270c28b2010 Mon Sep 17 00:00:00 2001 From: Garret Alfert Date: Wed, 2 Mar 2016 15:00:33 +0100 Subject: [PATCH 55/58] Use single letters for Redis metadata keys Saves 16 characters per document object and 10 characters per directory object --- lib/remote_storage/swift.rb | 26 +++++++++++++------------- migrate_metadata_to_redis.rb | 12 ++++++------ spec/swift/app_spec.rb | 24 ++++++++++++------------ 3 files changed, 31 insertions(+), 31 deletions(-) diff --git a/lib/remote_storage/swift.rb b/lib/remote_storage/swift.rb index 75526a5..1d0b4f5 100644 --- a/lib/remote_storage/swift.rb +++ b/lib/remote_storage/swift.rb @@ -107,10 +107,10 @@ module RemoteStorage metadata[metadata_values[idx]] = metadata_values[idx + 1] end - listing[name] = {["ETag"] = metadata["etag"]} + listing[name] = {["ETag"] = metadata["e"]} if string.sub(name, -1) ~= "/" then - listing[name]["Content-Type"] = metadata["type"] - listing[name]["Content-Length"] = tonumber(metadata["size"]) + listing[name]["Content-Type"] = metadata["t"] + listing[name]["Content-Length"] = tonumber(metadata["s"]) end end @@ -121,7 +121,7 @@ module RemoteStorage end def get_directory_listing_from_redis(user, directory) - etag = redis.hget "rsm:#{user}:#{directory}/", "etag" + etag = redis.hget "rsm:#{user}:#{directory}/", "e" none_match = (server.env["HTTP_IF_NONE_MATCH"] || "").split(",").map(&:strip) server.halt 304 if none_match.include? etag @@ -192,10 +192,10 @@ module RemoteStorage timestamp = (Time.now.to_f * 1000).to_i metadata = { - etag: res.headers[:etag], - size: data.size, - type: content_type, - modified: timestamp + e: res.headers[:etag], + s: data.size, + t: content_type, + m: timestamp } if update_metadata_object(user, directory, key, metadata) && @@ -319,17 +319,17 @@ module RemoteStorage else redis_key = redis_key..directory.."/"..key.."/" end - if redis.call("hget", redis_key, "etag") then + if redis.call("hget", redis_key, "e") then return true end for index, dir in pairs(parent_directories) do - if redis.call("hget", "rsm:"..user..":"..dir.."/", "etag") then + if redis.call("hget", "rsm:"..user..":"..dir.."/", "e") then -- the directory already exists, no need to do further checks return false else -- check for existing document with same name as directory - if redis.call("hget", "rsm:"..user..":"..dir, "etag") then + if redis.call("hget", "rsm:"..user..":"..dir, "e") then return true end end @@ -419,7 +419,7 @@ module RemoteStorage end key = "rsm:#{user}:#{dir}/" - metadata = {etag: etag, modified: timestamp} + metadata = {e: etag, m: timestamp} redis.hmset(key, *metadata) redis.sadd "rsm:#{user}:#{parent_directory_for(dir)}:i", "#{top_directory(dir)}/" end @@ -459,7 +459,7 @@ module RemoteStorage get_response = do_get_request("#{container_url_for(user)}/?format=json&path=") etag = etag_for(get_response.body) end - metadata = {etag: etag, modified: timestamp} + metadata = {e: etag, m: timestamp} redis.hmset("rsm:#{user}:#{dir}/", *metadata) end end diff --git a/migrate_metadata_to_redis.rb b/migrate_metadata_to_redis.rb index d1211c8..a421d27 100755 --- a/migrate_metadata_to_redis.rb +++ b/migrate_metadata_to_redis.rb @@ -93,8 +93,8 @@ class Migrator def save_directory_data(dir, item, data, timestamp) key = "rsm:#{username}:#{dir.gsub(/^\//, "")}#{item}" metadata = { - etag: data["ETag"], - modified: timestamp_for(data["Last-Modified"]) + e: data["ETag"], + m: timestamp_for(data["Last-Modified"]) } logger.debug "Metadata for dir #{key}: #{metadata}" @@ -104,10 +104,10 @@ class Migrator def save_document_data(dir, item, data) key = "rsm:#{username}:#{dir.gsub(/^\//, "")}#{item}" metadata = { - etag: data["ETag"], - size: data["Content-Length"], - type: data["Content-Type"], - modified: timestamp_for(data["Last-Modified"]) + e: data["ETag"], + s: data["Content-Length"], + t: data["Content-Type"], + m: timestamp_for(data["Last-Modified"]) } logger.debug "Metadata for document #{key}: #{metadata}" redis.hmset(key, *metadata) unless dry_run diff --git a/spec/swift/app_spec.rb b/spec/swift/app_spec.rb index 831c089..61204e3 100644 --- a/spec/swift/app_spec.rb +++ b/spec/swift/app_spec.rb @@ -32,10 +32,10 @@ describe "App" do end metadata = redis.hgetall "rsm:phil:food/aguacate" - metadata["size"].must_equal "2" - metadata["type"].must_equal "text/plain; charset=utf-8" - metadata["etag"].must_equal "bla" - metadata["modified"].length.must_equal 13 + metadata["s"].must_equal "2" + metadata["t"].must_equal "text/plain; charset=utf-8" + metadata["e"].must_equal "bla" + metadata["m"].length.must_equal 13 end it "creates the directory objects metadata in redis" do @@ -51,12 +51,12 @@ describe "App" do end metadata = redis.hgetall "rsm:phil:/" - metadata["etag"].must_equal "rootetag" - metadata["modified"].length.must_equal 13 + metadata["e"].must_equal "rootetag" + metadata["m"].length.must_equal 13 metadata = redis.hgetall "rsm:phil:food/" - metadata["etag"].must_equal "bla" - metadata["modified"].length.must_equal 13 + metadata["e"].must_equal "bla" + metadata["m"].length.must_equal 13 food_items = redis.smembers "rsm:phil:food/:i" food_items.each do |food_item| @@ -82,7 +82,7 @@ describe "App" do last_response.status.must_equal 200 metadata = redis.hgetall "rsm:phil:food/aguacate" - metadata["size"].must_equal "2" + metadata["s"].must_equal "2" end it "conflicts when there is a directory with same name as document" do @@ -198,9 +198,9 @@ describe "App" do end metadata = redis.hgetall "rsm:phil:food/" - metadata["etag"].must_equal "newetag" - metadata["modified"].length.must_equal 13 - metadata["modified"].wont_equal old_metadata["modified"] + metadata["e"].must_equal "newetag" + metadata["m"].length.must_equal 13 + metadata["m"].wont_equal old_metadata["m"] food_items = redis.smembers "rsm:phil:food/:i" food_items.must_equal ["camaron"] From c2fd9bc6d0f1245797673900fc765cd8bda48b36 Mon Sep 17 00:00:00 2001 From: Garret Alfert Date: Wed, 2 Mar 2016 18:04:38 +0100 Subject: [PATCH 56/58] Change Redis metadata key prefix to rs:m --- lib/remote_storage/swift.rb | 32 ++++++++++++++++---------------- migrate_metadata_to_redis.rb | 6 +++--- spec/swift/app_spec.rb | 36 ++++++++++++++++++------------------ 3 files changed, 37 insertions(+), 37 deletions(-) diff --git a/lib/remote_storage/swift.rb b/lib/remote_storage/swift.rb index 1d0b4f5..1cf7a34 100644 --- a/lib/remote_storage/swift.rb +++ b/lib/remote_storage/swift.rb @@ -87,11 +87,11 @@ module RemoteStorage lua_script = <<-EOF local user = ARGV[1] local directory = ARGV[2] - local items = redis.call("smembers", "rsm:"..user..":"..directory.."/:i") + local items = redis.call("smembers", "rs:m:"..user..":"..directory.."/:i") local listing = {} for index, name in pairs(items) do - local redis_key = "rsm:"..user..":" + local redis_key = "rs:m:"..user..":" if directory == "" then redis_key = redis_key..name else @@ -121,7 +121,7 @@ module RemoteStorage end def get_directory_listing_from_redis(user, directory) - etag = redis.hget "rsm:#{user}:#{directory}/", "e" + etag = redis.hget "rs:m:#{user}:#{directory}/", "e" none_match = (server.env["HTTP_IF_NONE_MATCH"] || "").split(",").map(&:strip) server.halt 304 if none_match.include? etag @@ -313,7 +313,7 @@ module RemoteStorage end -- check for existing directory with the same name as the document - local redis_key = "rsm:"..user..":" + local redis_key = "rs:m:"..user..":" if directory == "" then redis_key = redis_key..key.."/" else @@ -324,12 +324,12 @@ module RemoteStorage end for index, dir in pairs(parent_directories) do - if redis.call("hget", "rsm:"..user..":"..dir.."/", "e") then + if redis.call("hget", "rs:m:"..user..":"..dir.."/", "e") then -- the directory already exists, no need to do further checks return false else -- check for existing document with same name as directory - if redis.call("hget", "rsm:"..user..":"..dir, "e") then + if redis.call("hget", "rs:m:"..user..":"..dir, "e") then return true end end @@ -401,9 +401,9 @@ module RemoteStorage end def update_metadata_object(user, directory, key, metadata) - redis_key = "rsm:#{user}:#{directory}/#{key}" + redis_key = "rs:m:#{user}:#{directory}/#{key}" redis.hmset(redis_key, *metadata) - redis.sadd "rsm:#{user}:#{directory}/:i", key + redis.sadd "rs:m:#{user}:#{directory}/:i", key true end @@ -418,10 +418,10 @@ module RemoteStorage etag = etag_for(get_response.body) end - key = "rsm:#{user}:#{dir}/" + key = "rs:m:#{user}:#{dir}/" metadata = {e: etag, m: timestamp} redis.hmset(key, *metadata) - redis.sadd "rsm:#{user}:#{parent_directory_for(dir)}:i", "#{top_directory(dir)}/" + redis.sadd "rs:m:#{user}:#{parent_directory_for(dir)}:i", "#{top_directory(dir)}/" end true @@ -436,9 +436,9 @@ module RemoteStorage end def delete_metadata_objects(user, directory, key) - redis_key = "rsm:#{user}:#{directory}/#{key}" + redis_key = "rs:m:#{user}:#{directory}/#{key}" redis.del(redis_key) - redis.srem "rsm:#{user}:#{directory}/:i", key + redis.srem "rs:m:#{user}:#{directory}/:i", key end def delete_dir_objects(user, directory) @@ -449,8 +449,8 @@ module RemoteStorage unless dir == "" do_delete_request("#{url_for_directory(user, dir)}/") end - redis.del "rsm:#{user}:#{directory}/" - redis.srem "rsm:#{user}:#{parent_directory_for(dir)}:i", "#{dir}/" + redis.del "rs:m:#{user}:#{directory}/" + redis.srem "rs:m:#{user}:#{parent_directory_for(dir)}:i", "#{dir}/" else unless dir == "" res = do_put_request("#{url_for_directory(user, dir)}/", timestamp.to_s, "text/plain") @@ -460,14 +460,14 @@ module RemoteStorage etag = etag_for(get_response.body) end metadata = {e: etag, m: timestamp} - redis.hmset("rsm:#{user}:#{dir}/", *metadata) + redis.hmset("rs:m:#{user}:#{dir}/", *metadata) end end end def dir_empty?(user, dir) if directory_backend(user).match(/new/) - redis.smembers("rsm:#{user}:#{dir}/:i").empty? + redis.smembers("rs:m:#{user}:#{dir}/:i").empty? else do_get_request("#{container_url_for(user)}/?format=plain&limit=1&path=#{escape(dir)}/") do |res| return res.headers[:content_length] == "0" diff --git a/migrate_metadata_to_redis.rb b/migrate_metadata_to_redis.rb index a421d27..336f498 100755 --- a/migrate_metadata_to_redis.rb +++ b/migrate_metadata_to_redis.rb @@ -85,13 +85,13 @@ class Migrator end def add_item_to_parent_dir(dir, item) - key = "rsm:#{username}:#{parent_directory_for(dir)}:i" + key = "rs:m:#{username}:#{parent_directory_for(dir)}:i" logger.debug "Adding item #{item} to #{key}" redis.sadd(key, item) unless dry_run end def save_directory_data(dir, item, data, timestamp) - key = "rsm:#{username}:#{dir.gsub(/^\//, "")}#{item}" + key = "rs:m:#{username}:#{dir.gsub(/^\//, "")}#{item}" metadata = { e: data["ETag"], m: timestamp_for(data["Last-Modified"]) @@ -102,7 +102,7 @@ class Migrator end def save_document_data(dir, item, data) - key = "rsm:#{username}:#{dir.gsub(/^\//, "")}#{item}" + key = "rs:m:#{username}:#{dir.gsub(/^\//, "")}#{item}" metadata = { e: data["ETag"], s: data["Content-Length"], diff --git a/spec/swift/app_spec.rb b/spec/swift/app_spec.rb index 61204e3..29504f7 100644 --- a/spec/swift/app_spec.rb +++ b/spec/swift/app_spec.rb @@ -31,7 +31,7 @@ describe "App" do put "/phil/food/aguacate", "si" end - metadata = redis.hgetall "rsm:phil:food/aguacate" + metadata = redis.hgetall "rs:m:phil:food/aguacate" metadata["s"].must_equal "2" metadata["t"].must_equal "text/plain; charset=utf-8" metadata["e"].must_equal "bla" @@ -50,20 +50,20 @@ describe "App" do end end - metadata = redis.hgetall "rsm:phil:/" + metadata = redis.hgetall "rs:m:phil:/" metadata["e"].must_equal "rootetag" metadata["m"].length.must_equal 13 - metadata = redis.hgetall "rsm:phil:food/" + metadata = redis.hgetall "rs:m:phil:food/" metadata["e"].must_equal "bla" metadata["m"].length.must_equal 13 - food_items = redis.smembers "rsm:phil:food/:i" + food_items = redis.smembers "rs:m:phil:food/:i" food_items.each do |food_item| ["camaron", "aguacate"].must_include food_item end - root_items = redis.smembers "rsm:phil:/:i" + root_items = redis.smembers "rs:m:phil:/:i" root_items.must_equal ["food/"] end @@ -81,7 +81,7 @@ describe "App" do last_response.status.must_equal 200 - metadata = redis.hgetall "rsm:phil:food/aguacate" + metadata = redis.hgetall "rs:m:phil:food/aguacate" metadata["s"].must_equal "2" end @@ -94,7 +94,7 @@ describe "App" do last_response.status.must_equal 409 - metadata = redis.hgetall "rsm:phil:food" + metadata = redis.hgetall "rs:m:phil:food" metadata.must_be_empty end @@ -107,7 +107,7 @@ describe "App" do last_response.status.must_equal 409 - metadata = redis.hgetall "rsm:phil:food/aguacate/empanado" + metadata = redis.hgetall "rs:m:phil:food/aguacate/empanado" metadata.must_be_empty end end @@ -123,7 +123,7 @@ describe "App" do last_response.status.must_equal 503 - metadata = redis.hgetall "rsm:phil:food/aguacate" + metadata = redis.hgetall "rs:m:phil:food/aguacate" metadata.must_be_empty end end @@ -138,7 +138,7 @@ describe "App" do last_response.status.must_equal 503 - metadata = redis.hgetall "rsm:phil:food/aguacate" + metadata = redis.hgetall "rs:m:phil:food/aguacate" metadata.must_be_empty end end @@ -178,12 +178,12 @@ describe "App" do end end - metadata = redis.hgetall "rsm:phil:food/aguacate" + metadata = redis.hgetall "rs:m:phil:food/aguacate" metadata.must_be_empty end it "deletes the directory objects metadata in redis" do - old_metadata = redis.hgetall "rsm:phil:food/" + old_metadata = redis.hgetall "rs:m:phil:food/" put_stub = OpenStruct.new(headers: {etag: "newetag"}) get_stub = OpenStruct.new(body: "rootbody") @@ -197,15 +197,15 @@ describe "App" do end end - metadata = redis.hgetall "rsm:phil:food/" + metadata = redis.hgetall "rs:m:phil:food/" metadata["e"].must_equal "newetag" metadata["m"].length.must_equal 13 metadata["m"].wont_equal old_metadata["m"] - food_items = redis.smembers "rsm:phil:food/:i" + food_items = redis.smembers "rs:m:phil:food/:i" food_items.must_equal ["camaron"] - root_items = redis.smembers "rsm:phil:/:i" + root_items = redis.smembers "rs:m:phil:/:i" root_items.must_equal ["food/"] end @@ -223,13 +223,13 @@ describe "App" do end end - metadata = redis.hgetall "rsm:phil:food/" + metadata = redis.hgetall "rs:m:phil:food/" metadata.must_be_empty - food_items = redis.smembers "rsm:phil:food/:i" + food_items = redis.smembers "rs:m:phil:food/:i" food_items.must_be_empty - root_items = redis.smembers "rsm:phil:/:i" + root_items = redis.smembers "rs:m:phil:/:i" root_items.must_be_empty end end From 9b7b224f79b97115875b862269057af4da43e3cc Mon Sep 17 00:00:00 2001 From: Garret Alfert Date: Wed, 2 Mar 2016 18:06:42 +0100 Subject: [PATCH 57/58] Use more explicit key name for items --- lib/remote_storage/swift.rb | 12 ++++++------ migrate_metadata_to_redis.rb | 2 +- spec/swift/app_spec.rb | 12 ++++++------ 3 files changed, 13 insertions(+), 13 deletions(-) diff --git a/lib/remote_storage/swift.rb b/lib/remote_storage/swift.rb index 1cf7a34..818f6e2 100644 --- a/lib/remote_storage/swift.rb +++ b/lib/remote_storage/swift.rb @@ -87,7 +87,7 @@ module RemoteStorage lua_script = <<-EOF local user = ARGV[1] local directory = ARGV[2] - local items = redis.call("smembers", "rs:m:"..user..":"..directory.."/:i") + local items = redis.call("smembers", "rs:m:"..user..":"..directory.."/:items") local listing = {} for index, name in pairs(items) do @@ -403,7 +403,7 @@ module RemoteStorage def update_metadata_object(user, directory, key, metadata) redis_key = "rs:m:#{user}:#{directory}/#{key}" redis.hmset(redis_key, *metadata) - redis.sadd "rs:m:#{user}:#{directory}/:i", key + redis.sadd "rs:m:#{user}:#{directory}/:items", key true end @@ -421,7 +421,7 @@ module RemoteStorage key = "rs:m:#{user}:#{dir}/" metadata = {e: etag, m: timestamp} redis.hmset(key, *metadata) - redis.sadd "rs:m:#{user}:#{parent_directory_for(dir)}:i", "#{top_directory(dir)}/" + redis.sadd "rs:m:#{user}:#{parent_directory_for(dir)}:items", "#{top_directory(dir)}/" end true @@ -438,7 +438,7 @@ module RemoteStorage def delete_metadata_objects(user, directory, key) redis_key = "rs:m:#{user}:#{directory}/#{key}" redis.del(redis_key) - redis.srem "rs:m:#{user}:#{directory}/:i", key + redis.srem "rs:m:#{user}:#{directory}/:items", key end def delete_dir_objects(user, directory) @@ -450,7 +450,7 @@ module RemoteStorage do_delete_request("#{url_for_directory(user, dir)}/") end redis.del "rs:m:#{user}:#{directory}/" - redis.srem "rs:m:#{user}:#{parent_directory_for(dir)}:i", "#{dir}/" + redis.srem "rs:m:#{user}:#{parent_directory_for(dir)}:items", "#{dir}/" else unless dir == "" res = do_put_request("#{url_for_directory(user, dir)}/", timestamp.to_s, "text/plain") @@ -467,7 +467,7 @@ module RemoteStorage def dir_empty?(user, dir) if directory_backend(user).match(/new/) - redis.smembers("rs:m:#{user}:#{dir}/:i").empty? + redis.smembers("rs:m:#{user}:#{dir}/:items").empty? else do_get_request("#{container_url_for(user)}/?format=plain&limit=1&path=#{escape(dir)}/") do |res| return res.headers[:content_length] == "0" diff --git a/migrate_metadata_to_redis.rb b/migrate_metadata_to_redis.rb index 336f498..bd28e75 100755 --- a/migrate_metadata_to_redis.rb +++ b/migrate_metadata_to_redis.rb @@ -85,7 +85,7 @@ class Migrator end def add_item_to_parent_dir(dir, item) - key = "rs:m:#{username}:#{parent_directory_for(dir)}:i" + key = "rs:m:#{username}:#{parent_directory_for(dir)}:items" logger.debug "Adding item #{item} to #{key}" redis.sadd(key, item) unless dry_run end diff --git a/spec/swift/app_spec.rb b/spec/swift/app_spec.rb index 29504f7..56d03a1 100644 --- a/spec/swift/app_spec.rb +++ b/spec/swift/app_spec.rb @@ -58,12 +58,12 @@ describe "App" do metadata["e"].must_equal "bla" metadata["m"].length.must_equal 13 - food_items = redis.smembers "rs:m:phil:food/:i" + food_items = redis.smembers "rs:m:phil:food/:items" food_items.each do |food_item| ["camaron", "aguacate"].must_include food_item end - root_items = redis.smembers "rs:m:phil:/:i" + root_items = redis.smembers "rs:m:phil:/:items" root_items.must_equal ["food/"] end @@ -202,10 +202,10 @@ describe "App" do metadata["m"].length.must_equal 13 metadata["m"].wont_equal old_metadata["m"] - food_items = redis.smembers "rs:m:phil:food/:i" + food_items = redis.smembers "rs:m:phil:food/:items" food_items.must_equal ["camaron"] - root_items = redis.smembers "rs:m:phil:/:i" + root_items = redis.smembers "rs:m:phil:/:items" root_items.must_equal ["food/"] end @@ -226,10 +226,10 @@ describe "App" do metadata = redis.hgetall "rs:m:phil:food/" metadata.must_be_empty - food_items = redis.smembers "rs:m:phil:food/:i" + food_items = redis.smembers "rs:m:phil:food/:items" food_items.must_be_empty - root_items = redis.smembers "rs:m:phil:/:i" + root_items = redis.smembers "rs:m:phil:/:items" root_items.must_be_empty end end From 14a522f09f8374fe681ef7b70d0d15df000d1e89 Mon Sep 17 00:00:00 2001 From: Garret Alfert Date: Thu, 3 Mar 2016 00:39:35 +0100 Subject: [PATCH 58/58] Load bundled environment to access all libraries --- migrate_metadata_to_redis.rb | 2 ++ 1 file changed, 2 insertions(+) diff --git a/migrate_metadata_to_redis.rb b/migrate_metadata_to_redis.rb index bd28e75..df3705f 100755 --- a/migrate_metadata_to_redis.rb +++ b/migrate_metadata_to_redis.rb @@ -1,5 +1,7 @@ #!/usr/bin/env ruby +require "rubygems" +require "bundler/setup" require "rest_client" require "redis" require "yaml"