From 995ba5288604b57262de94b5990881f23d0d8eee Mon Sep 17 00:00:00 2001 From: Garret Alfert Date: Mon, 13 Jun 2016 13:18:10 +0200 Subject: [PATCH 1/5] Log user's storage size in Redis --- lib/remote_storage/swift.rb | 7 +++++++ spec/swift/app_spec.rb | 41 +++++++++++++++++++++++++++++++++++++ 2 files changed, 48 insertions(+) diff --git a/lib/remote_storage/swift.rb b/lib/remote_storage/swift.rb index 679d8b7..0a41055 100644 --- a/lib/remote_storage/swift.rb +++ b/lib/remote_storage/swift.rb @@ -159,6 +159,7 @@ module RemoteStorage if update_metadata_object(user, directory, key, metadata) if metadata_changed?(existing_metadata, metadata) update_dir_objects(user, directory, timestamp, checksum_for(data)) + log_size_difference(user, existing_metadata["s"], metadata[:s]) end server.headers["ETag"] = %Q("#{res.headers[:etag]}") @@ -168,6 +169,11 @@ module RemoteStorage end end + def log_size_difference(user, old_size, new_size) + delta = new_size.to_i - old_size.to_i + redis.incrby "rs:s:#{user}", delta + end + def checksum_for(data) Digest::MD5.hexdigest(data) end @@ -182,6 +188,7 @@ module RemoteStorage end do_delete_request(url) + log_size_difference(user, existing_metadata["s"], 0) delete_metadata_objects(user, directory, key) delete_dir_objects(user, directory) diff --git a/spec/swift/app_spec.rb b/spec/swift/app_spec.rb index a30c52e..eadbc68 100644 --- a/spec/swift/app_spec.rb +++ b/spec/swift/app_spec.rb @@ -74,6 +74,36 @@ describe "App" do root_items.must_equal ["food/"] end + context "logging usage size" do + before do + @put_stub = OpenStruct.new(headers: { + etag: "bla", + last_modified: "Fri, 04 Mar 2016 12:20:18 GMT" + }) + + end + + it "logs the complete size when creating new objects" do + RestClient.stub :put, @put_stub do + put "/phil/food/aguacate", "1234567890" + end + + size_log = redis.get "rs:s:phil" + size_log.must_equal "10" + end + + it "logs the size difference when updating existing objects" do + RestClient.stub :put, @put_stub do + put "/phil/food/camaron", "1234567890" + put "/phil/food/aguacate", "1234567890" + put "/phil/food/aguacate", "123" + end + + size_log = redis.get "rs:s:phil" + size_log.must_equal "13" + end + end + describe "objects in root dir" do before do put_stub = OpenStruct.new(headers: { @@ -289,6 +319,17 @@ describe "App" do end end + it "decreases the size log by size of deleted object" do + RestClient.stub :delete, "" do + RemoteStorage::Swift.stub_any_instance :etag_for, "rootetag" do + delete "/phil/food/aguacate" + end + end + + size_log = redis.get "rs:s:phil" + size_log.must_equal "8" + end + it "deletes the metadata object in redis" do RestClient.stub :delete, "" do RemoteStorage::Swift.stub_any_instance :etag_for, "rootetag" do From ad0ea12059c072e4e232378388ae97d69682e154 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Greg=20Kar=C3=A9kinian?= Date: Wed, 20 Jul 2016 17:27:44 +0200 Subject: [PATCH 2/5] Return 401 when getting an empty bearer token For example: Authorization: Bearer The cause of the empty bearer also needs to be investigated --- lib/remote_storage/swift.rb | 2 +- spec/swift/app_spec.rb | 11 ++++++++++- 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/lib/remote_storage/swift.rb b/lib/remote_storage/swift.rb index 0a41055..4efa8ed 100644 --- a/lib/remote_storage/swift.rb +++ b/lib/remote_storage/swift.rb @@ -24,7 +24,7 @@ module RemoteStorage return true if ["GET", "HEAD"].include?(request_method) && !listing end - server.halt 401, "Unauthorized" if token.empty? + server.halt 401, "Unauthorized" if token.nil? || token.empty? authorizations = redis.smembers("authorizations:#{user}:#{token}") permission = directory_permission(authorizations, directory) diff --git a/spec/swift/app_spec.rb b/spec/swift/app_spec.rb index eadbc68..5c8d24f 100644 --- a/spec/swift/app_spec.rb +++ b/spec/swift/app_spec.rb @@ -280,7 +280,6 @@ describe "App" do end context "not authorized" do - describe "with no token" do it "says it's not authorized" do delete "/phil/food/aguacate" @@ -290,6 +289,16 @@ describe "App" do end end + describe "with empty token" do + it "says it's not authorized" do + header "Authorization", "Bearer " + delete "/phil/food/aguacate" + + last_response.status.must_equal 401 + last_response.body.must_equal "Unauthorized" + end + end + describe "with wrong token" do it "says it's not authorized" do header "Authorization", "Bearer wrongtoken" From 9710bb320c1ef06a49112d835a19036975b8cf4e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Greg=20Kar=C3=A9kinian?= Date: Thu, 21 Jul 2016 11:06:53 +0200 Subject: [PATCH 3/5] Update nokogiri --- Gemfile.lock | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/Gemfile.lock b/Gemfile.lock index 1601888..98c9bb3 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -55,14 +55,16 @@ GEM rake (>= 0.9.2.2) method_source (0.8.2) mime-types (2.99.1) - mini_portile2 (2.0.0) + mini_portile2 (2.1.0) minitest (5.8.4) minitest-stub_any_instance (1.0.1) multi_json (1.11.2) multipart-post (2.0.0) netrc (0.11.0) - nokogiri (1.6.7.2) - mini_portile2 (~> 2.0.0.rc2) + nokogiri (1.6.8) + mini_portile2 (~> 2.1.0) + pkg-config (~> 1.1.7) + pkg-config (1.1.7) purdytest (2.0.0) minitest (~> 5.5) rack (1.6.4) From 56474bc08f14fbe40fa743a59ad48c16ff66ddbc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Greg=20Kar=C3=A9kinian?= Date: Thu, 21 Jul 2016 11:26:21 +0200 Subject: [PATCH 4/5] Bump Ruby version on Travis --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index d3d7d89..f2759c3 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,7 +1,7 @@ language: ruby cache: bundler rvm: - - 2.2.4 + - 2.2.5 services: - redis-server before_install: From 8caff843d6eebf44bcbc9c3876026af412eb1c5c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Greg=20Kar=C3=A9kinian?= Date: Thu, 21 Jul 2016 11:26:32 +0200 Subject: [PATCH 5/5] Update gem dependencies --- Gemfile | 2 +- Gemfile.lock | 54 ++++++++++++++++++++++++++-------------------------- 2 files changed, 28 insertions(+), 28 deletions(-) diff --git a/Gemfile b/Gemfile index 7157326..49a4894 100644 --- a/Gemfile +++ b/Gemfile @@ -9,7 +9,7 @@ gem "rest-client" gem "redis" # Remove require when we can update to 3.0, which sets the new storage # format to columnar by default. Increases performance -gem "mime-types", "~> 2.99", require: 'mime/types/columnar' +gem "mime-types" group :test do gem 'rake' diff --git a/Gemfile.lock b/Gemfile.lock index 98c9bb3..3e86df4 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -13,28 +13,28 @@ GIT GEM remote: https://rubygems.org/ specs: - activesupport (4.2.5.1) + activesupport (5.0.0) + concurrent-ruby (~> 1.0, >= 1.0.2) i18n (~> 0.7) - json (~> 1.7, >= 1.7.7) minitest (~> 5.1) - thread_safe (~> 0.3, >= 0.3.4) tzinfo (~> 1.1) backports (3.6.8) beefcake (0.3.7) builder (3.2.2) - domain_name (0.5.20160216) + concurrent-ruby (1.0.2) + domain_name (0.5.20160615) unf (>= 0.0.5, < 1.0.0) - excon (0.45.4) + excon (0.51.0) faraday (0.9.2) multipart-post (>= 1.2, < 3) - fog-aws (0.8.1) - fog-core (~> 1.27) + fog-aws (0.10.0) + fog-core (~> 1.38) fog-json (~> 1.0) fog-xml (~> 0.1) ipaddress (~> 0.8) - fog-core (1.36.0) + fog-core (1.42.0) builder - excon (~> 0.45) + excon (~> 0.49) formatador (~> 0.2) fog-json (1.0.2) fog-core (~> 1.0) @@ -48,17 +48,18 @@ GEM i18n (0.7.0) innertube (1.0.2) ipaddress (0.8.3) - json (1.8.3) kgio (2.10.0) - m (1.4.2) + m (1.5.0) method_source (>= 0.6.7) rake (>= 0.9.2.2) method_source (0.8.2) - mime-types (2.99.1) + mime-types (3.1) + mime-types-data (~> 3.2015) + mime-types-data (3.2016.0521) mini_portile2 (2.1.0) - minitest (5.8.4) + minitest (5.9.0) minitest-stub_any_instance (1.0.1) - multi_json (1.11.2) + multi_json (1.12.1) multipart-post (2.0.0) netrc (0.11.0) nokogiri (1.6.8) @@ -76,20 +77,20 @@ GEM kgio (~> 2.5) rack (~> 1.1) unicorn (~> 5.0) - raindrops (0.15.0) - rake (10.5.0) - redis (3.2.2) - rest-client (1.8.0) + raindrops (0.16.0) + rake (11.2.2) + redis (3.3.1) + rest-client (2.0.0) http-cookie (>= 1.0.2, < 2.0) - mime-types (>= 1.16, < 3.0) - netrc (~> 0.7) - sentry-raven (0.15.6) - faraday (>= 0.7.6) + mime-types (>= 1.16, < 4.0) + netrc (~> 0.8) + sentry-raven (1.2.0) + faraday (>= 0.7.6, < 0.10.x) sinatra (1.4.7) rack (~> 1.5) rack-protection (~> 1.4) tilt (>= 1.3, < 3) - sinatra-contrib (1.4.6) + sinatra-contrib (1.4.7) backports (>= 2.0) multi_json rack-protection @@ -97,15 +98,14 @@ GEM sinatra (~> 1.4.0) tilt (>= 1.3, < 3) thread_safe (0.3.5) - tilt (2.0.2) + tilt (2.0.5) tzinfo (1.2.2) thread_safe (~> 0.1) unf (0.1.4) unf_ext unf_ext (0.0.7.2) - unicorn (5.0.1) + unicorn (5.1.0) kgio (~> 2.6) - rack raindrops (~> 0.7) PLATFORMS @@ -115,7 +115,7 @@ DEPENDENCIES activesupport fog-aws m - mime-types (~> 2.99) + mime-types minitest-stub_any_instance purdytest rainbows