From 5370df9c75a0dea5364177a9a2fb408d91bc2520 Mon Sep 17 00:00:00 2001
From: Garret Alfert <garret@5apps.com>
Date: Mon, 15 Feb 2016 15:15:01 +0100
Subject: [PATCH 01/16] Basic migration script using actual RS dir listings

---
 migrate_metadata_to_redis.rb | 117 +++++++++++++++++++++++++++++++++++
 1 file changed, 117 insertions(+)
 create mode 100755 migrate_metadata_to_redis.rb

diff --git a/migrate_metadata_to_redis.rb b/migrate_metadata_to_redis.rb
new file mode 100755
index 0000000..de1e812
--- /dev/null
+++ b/migrate_metadata_to_redis.rb
@@ -0,0 +1,117 @@
+#!/usr/bin/env ruby
+
+require "rest_client"
+require "redis"
+
+class Migrator
+
+  attr_accessor :username, :token, :base_url
+
+  def initialize(username, token)
+    @username = username
+    @token = token
+    @base_url = "https://storage.5apps.com"
+  end
+
+  def configure_redis(redis_config)
+    @redis_config = redis_config
+  end
+
+  def root_url
+    "#{@base_url}/#{@username}"
+  end
+
+  def headers
+    {"authorization" => "Bearer #{@token}"}
+  end
+
+  def is_dir?(name)
+    name[-1] == "/"
+  end
+
+  def url_for(directory, parent_directory="")
+    # base_path = [root_url, parent_directory].join("/")
+    "#{root_url}#{parent_directory}#{directory}"
+  end
+
+  def migrate
+    work_on_dir("", "/")
+  end
+
+  def work_on_dir(directory, parent_directory)
+    url = url_for(directory, parent_directory)
+
+    # puts "work on dir: #{url}"
+
+    response = RestClient.get(url, headers)
+    listing = JSON.parse(response.body)
+
+    timestamp = (Time.now.to_f * 1000).to_i
+
+    if listing["items"].any?
+      items = listing["items"]
+      items.each do |item, data|
+        if is_dir? item
+          save_directory_data("#{parent_directory}#{directory}", item, data, timestamp)
+
+          # get dir listing and repeat
+          work_on_dir(item, "#{parent_directory}#{directory}")
+        else
+          save_document_data("#{parent_directory}#{directory}", item, data, timestamp)
+        end
+
+        add_item_to_parent_dir("#{parent_directory}#{directory}", item)
+      end
+    end
+  end
+
+  def add_item_to_parent_dir(dir, item)
+    key = "rs_meta:#{username}:#{parent_directory_for(dir)}:items"
+    # puts "adding item #{item} to #{key}"
+    redis.sadd key, item
+  end
+
+  def save_directory_data(dir, item, data, timestamp)
+    key = "rs_meta:#{username}:#{dir.gsub(/^\//, "")}#{item}"
+    metadata = {etag: data["ETag"], modified: timestamp}
+
+    # puts "metadata for dir #{key}: #{metadata}"
+    redis.hmset(key, *metadata)
+  end
+
+  def save_document_data(dir, item, data, timestamp)
+    key = "rs_meta:#{username}:#{dir.gsub(/^\//, "")}#{item}"
+    metadata = {
+      etag: data["ETag"],
+      size: data["Content-Length"],
+      type: data["Content-Type"],
+      modified: timestamp
+    }
+    # puts "metadata for document #{key}: #{metadata}"
+    redis.hmset(key, *metadata)
+  end
+
+  def parent_directory_for(directory)
+    return directory if directory == "/"
+
+    return directory[0..directory.rindex("/")].gsub(/^\//, "")
+  end
+
+  def redis
+    @redis ||= Redis.new(@redis_config)
+  end
+
+end
+
+username = ARGV[0]
+token = ARGV[1]
+
+migrator = Migrator.new username, token
+migrator.configure_redis({host: "localhost", port: 6379})
+
+migrator.migrate
+
+
+
+
+

From 54176b6928b85e425d585a04caf60359211f6a06 Mon Sep 17 00:00:00 2001
From: Garret Alfert <garret@5apps.com>
Date: Mon, 22 Feb 2016 16:08:53 +0100
Subject: [PATCH 02/16] Use data directly from Swift for metadata migration

---
 migrate_metadata_to_redis.rb | 167 +++++++++++++++++++++++++++--------
 1 file changed, 132 insertions(+), 35 deletions(-)

diff --git a/migrate_metadata_to_redis.rb b/migrate_metadata_to_redis.rb
index de1e812..7458ff8 100755
--- a/migrate_metadata_to_redis.rb
+++ b/migrate_metadata_to_redis.rb
@@ -2,49 +2,45 @@
 
 require "rest_client"
 require "redis"
+require "yaml"
 
 class Migrator
 
-  attr_accessor :username, :token, :base_url
+  attr_accessor :username, :base_url, :swift_host, :swift_token,
+                :environment, :dry_run, :logging, :settings
 
-  def initialize(username, token)
+  def initialize(username)
     @username = username
-    @token = token
-    @base_url = "https://storage.5apps.com"
-  end
 
-  def configure_redis(redis_config)
-    @redis_config = redis_config
+    @environment = ENV["ENVIRONMENT"] || "staging"
+    @settings = YAML.load(File.read('config.yml'))[@environment]
+    @swift_host = @settings["swift"]["host"]
+    @swift_token = File.read("tmp/swift_token.txt")
+
+    @dry_run = false # disables writing anything to Redis when true
+    @logging = true
   end
 
   def root_url
     "#{@base_url}/#{@username}"
   end
 
-  def headers
-    {"authorization" => "Bearer #{@token}"}
-  end
-
   def is_dir?(name)
     name[-1] == "/"
   end
 
   def url_for(directory, parent_directory="")
-    # base_path = [root_url, parent_directory].join("/")
     "#{root_url}#{parent_directory}#{directory}"
   end
 
   def migrate
-    work_on_dir("", "/")
+    work_on_dir("", "")
   end
 
   def work_on_dir(directory, parent_directory)
-    url = url_for(directory, parent_directory)
+    puts "retrieving listing for '#{parent_directory}#{directory}'" if logging
 
-    # puts "work on dir: #{url}"
-
-    response = RestClient.get(url, headers)
-    listing = JSON.parse(response.body)
+    listing = get_directory_listing_from_swift("#{parent_directory}#{directory}")
 
     timestamp = (Time.now.to_f * 1000).to_i
 
@@ -67,16 +63,16 @@ class Migrator
 
   def add_item_to_parent_dir(dir, item)
     key = "rs_meta:#{username}:#{parent_directory_for(dir)}:items"
-    # puts "adding item #{item} to #{key}"
-    redis.sadd key, item
+    puts "adding item #{item} to #{key}" if logging
+    redis.sadd(key, item) unless dry_run
   end
 
   def save_directory_data(dir, item, data, timestamp)
     key = "rs_meta:#{username}:#{dir.gsub(/^\//, "")}#{item}"
     metadata = {etag: data["ETag"], modified: timestamp}
 
-    # puts "metadata for dir #{key}: #{metadata}"
-    redis.hmset(key, *metadata)
+    puts "metadata for dir #{key}: #{metadata}" if logging
+    redis.hmset(key, *metadata) unless dry_run
   end
 
   def save_document_data(dir, item, data, timestamp)
@@ -87,31 +83,132 @@ class Migrator
       type: data["Content-Type"],
       modified: timestamp
     }
-    # puts "metadata for document #{key}: #{metadata}"
-    redis.hmset(key, *metadata)
+    puts "metadata for document #{key}: #{metadata}" if logging
+    redis.hmset(key, *metadata) unless dry_run
   end
 
   def parent_directory_for(directory)
-    return directory if directory == "/"
-
-    return directory[0..directory.rindex("/")].gsub(/^\//, "")
+    if directory.match(/\//)
+      return directory[0..directory.rindex("/")]
+    else
+      return "/"
+    end
   end
 
   def redis
-    @redis ||= Redis.new(@redis_config)
+    @redis ||= Redis.new(@settings["redis"])
   end
 
+  def get_directory_listing_from_swift(directory)
+    is_root_listing = directory.empty?
+
+    get_response = nil
+
+    do_head_request("#{url_for_directory(@username, directory)}") do |response|
+      return directory_listing([]) if response.code == 404
+
+      if is_root_listing
+        get_response = do_get_request("#{container_url_for(@username)}/?format=json&path=")
+      else
+        get_response = do_get_request("#{container_url_for(@username)}/?format=json&path=#{escape(directory)}/")
+      end
+    end
+
+    if body = JSON.parse(get_response.body)
+      listing = directory_listing(body)
+    else
+      puts "listing not JSON"
+    end
+
+    listing
+  end
+
+  def directory_listing(res_body)
+    listing = {
+      "@context" => "http://remotestorage.io/spec/folder-description",
+      "items"    => {}
+    }
+
+    res_body.each do |entry|
+      name = entry["name"]
+      name.sub!("#{File.dirname(entry["name"])}/", '')
+      if name[-1] == "/" # It's a directory
+        listing["items"].merge!({
+          name => {
+            "ETag"           => entry["hash"],
+          }
+        })
+      else # It's a file
+        listing["items"].merge!({
+          name => {
+            "ETag"           => entry["hash"],
+            "Content-Type"   => entry["content_type"],
+            "Content-Length" => entry["bytes"]
+          }
+        })
+      end
+    end
+
+    listing
+  end
+
+  def etag_for(body)
+    objects = JSON.parse(body)
+
+    if objects.empty?
+      Digest::MD5.hexdigest ""
+    else
+      Digest::MD5.hexdigest objects.map { |o| o["hash"] }.join
+    end
+  end
+
+  def do_head_request(url, &block)
+    RestClient.head(url, default_headers, &block)
+  end
+
+  def do_get_request(url, &block)
+    RestClient.get(url, default_headers, &block)
+  end
+
+  def default_headers
+    {"x-auth-token" => @swift_token}
+  end
+
+  def url_for_directory(user, directory)
+    if directory.empty?
+      container_url_for(user)
+    else
+      "#{container_url_for(user)}/#{escape(directory)}"
+    end
+  end
+
+  def container_url_for(user)
+    "#{base_url}/#{container_for(user)}"
+  end
+
+  def base_url
+    @base_url ||= @swift_host
+  end
+
+  def container_for(user)
+    "rs:#{environment.to_s.chars.first}:#{user}"
+  end
+
+  def escape(url)
+    # We want spaces to turn into %20 and slashes to stay slashes
+    CGI::escape(url).gsub('+', '%20').gsub('%2F', '/')
+  end
 end
 
 username = ARGV[0]
-token = ARGV[1]
 
-migrator = Migrator.new username, token
-migrator.configure_redis({host: "localhost", port: 6379})
+unless username
+  puts "No username given."
+  puts "Usage:"
+  puts "ENVIRONMENT=staging ./migrate_metadata_to_redis.rb <username>"
+  exit 1
+end
 
+migrator = Migrator.new username
 migrator.migrate
 
-
-
-
-

From 570e3dcdb5d2c2c07a8cdc13964ad53e1b02c484 Mon Sep 17 00:00:00 2001
From: Garret Alfert <garret@5apps.com>
Date: Wed, 24 Feb 2016 12:23:42 +0100
Subject: [PATCH 03/16] Set directory backend type during migration

---
 migrate_metadata_to_redis.rb | 15 ++++++++++++++-
 1 file changed, 14 insertions(+), 1 deletion(-)

diff --git a/migrate_metadata_to_redis.rb b/migrate_metadata_to_redis.rb
index 7458ff8..5a4e482 100755
--- a/migrate_metadata_to_redis.rb
+++ b/migrate_metadata_to_redis.rb
@@ -34,7 +34,20 @@ class Migrator
   end
 
   def migrate
-    work_on_dir("", "")
+    set_directory_backend("legacy_locked")
+    begin
+      work_on_dir("", "")
+    rescue Exception => ex
+      puts "Error migrating metadata for '#{username}': #{ex}" if logging
+      set_directory_backend("legacy")
+      # TODO write username to file for later reference
+      exit 1
+    end
+    set_directory_backend("new")
+  end
+
+  def set_directory_backend(backend)
+    redis.set("rs_config:dir_backend:#{username}", backend) unless dry_run
   end
 
   def work_on_dir(directory, parent_directory)

From 3b391246ed920998fc28ab77c8f20cac90eea50b Mon Sep 17 00:00:00 2001
From: Garret Alfert <garret@5apps.com>
Date: Wed, 24 Feb 2016 12:48:12 +0100
Subject: [PATCH 04/16] Use proper logger for migration script

---
 migrate_metadata_to_redis.rb | 22 +++++++++++++++-------
 1 file changed, 15 insertions(+), 7 deletions(-)

diff --git a/migrate_metadata_to_redis.rb b/migrate_metadata_to_redis.rb
index 5a4e482..c850bac 100755
--- a/migrate_metadata_to_redis.rb
+++ b/migrate_metadata_to_redis.rb
@@ -3,22 +3,28 @@
 require "rest_client"
 require "redis"
 require "yaml"
+require "logger"
 
 class Migrator
 
   attr_accessor :username, :base_url, :swift_host, :swift_token,
-                :environment, :dry_run, :logging, :settings
+                :environment, :dry_run, :settings, :logger
 
   def initialize(username)
     @username = username
 
     @environment = ENV["ENVIRONMENT"] || "staging"
     @settings = YAML.load(File.read('config.yml'))[@environment]
+
     @swift_host = @settings["swift"]["host"]
     @swift_token = File.read("tmp/swift_token.txt")
 
     @dry_run = false # disables writing anything to Redis when true
-    @logging = true
+
+    @logger = Logger.new("log/migrate_metadata_to_redis.log")
+    log_level = ENV["LOGLEVEL"] || "INFO"
+    logger.level = Kernel.const_get "Logger::#{log_level}"
+    logger.progname = username
   end
 
   def root_url
@@ -34,16 +40,18 @@ class Migrator
   end
 
   def migrate
+    logger.info "Starting migration for '#{username}'"
     set_directory_backend("legacy_locked")
     begin
       work_on_dir("", "")
     rescue Exception => ex
-      puts "Error migrating metadata for '#{username}': #{ex}" if logging
+      logger.error "Error migrating metadata for '#{username}': #{ex}"
       set_directory_backend("legacy")
       # TODO write username to file for later reference
       exit 1
     end
     set_directory_backend("new")
+    logger.info "Finished migration for '#{username}'"
   end
 
   def set_directory_backend(backend)
@@ -51,7 +59,7 @@ class Migrator
   end
 
   def work_on_dir(directory, parent_directory)
-    puts "retrieving listing for '#{parent_directory}#{directory}'" if logging
+    logger.debug "Retrieving listing for '#{parent_directory}#{directory}'"
 
     listing = get_directory_listing_from_swift("#{parent_directory}#{directory}")
 
@@ -76,7 +84,7 @@ class Migrator
 
   def add_item_to_parent_dir(dir, item)
     key = "rs_meta:#{username}:#{parent_directory_for(dir)}:items"
-    puts "adding item #{item} to #{key}" if logging
+    logger.debug "Adding item #{item} to #{key}"
     redis.sadd(key, item) unless dry_run
   end
 
@@ -84,7 +92,7 @@ class Migrator
     key = "rs_meta:#{username}:#{dir.gsub(/^\//, "")}#{item}"
     metadata = {etag: data["ETag"], modified: timestamp}
 
-    puts "metadata for dir #{key}: #{metadata}" if logging
+    logger.debug "Metadata for dir #{key}: #{metadata}"
     redis.hmset(key, *metadata) unless dry_run
   end
 
@@ -96,7 +104,7 @@ class Migrator
       type: data["Content-Type"],
       modified: timestamp
     }
-    puts "metadata for document #{key}: #{metadata}" if logging
+    logger.debug "Metadata for document #{key}: #{metadata}"
     redis.hmset(key, *metadata) unless dry_run
   end
 

From 33731f793ad0177aca88c612e624b422a7bec58e Mon Sep 17 00:00:00 2001
From: Garret Alfert <garret@5apps.com>
Date: Wed, 24 Feb 2016 13:04:49 +0100
Subject: [PATCH 05/16] Set dry_run via environment variable

Can be enabled via `DRYRUN=true`
---
 migrate_metadata_to_redis.rb | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/migrate_metadata_to_redis.rb b/migrate_metadata_to_redis.rb
index c850bac..abfd3bc 100755
--- a/migrate_metadata_to_redis.rb
+++ b/migrate_metadata_to_redis.rb
@@ -19,7 +19,7 @@ class Migrator
     @swift_host = @settings["swift"]["host"]
     @swift_token = File.read("tmp/swift_token.txt")
 
-    @dry_run = false # disables writing anything to Redis when true
+    @dry_run = ENV["DRYRUN"] || false # disables writing anything to Redis when true
 
     @logger = Logger.new("log/migrate_metadata_to_redis.log")
     log_level = ENV["LOGLEVEL"] || "INFO"

From 41074e35b35a59dd3b4f25209ede0c8a31af5072 Mon Sep 17 00:00:00 2001
From: Garret Alfert <garret@5apps.com>
Date: Mon, 29 Feb 2016 15:35:37 +0100
Subject: [PATCH 06/16] Log usernames for failed migration to separate file

This way they can be easily retried later on.
---
 migrate_metadata_to_redis.rb | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/migrate_metadata_to_redis.rb b/migrate_metadata_to_redis.rb
index abfd3bc..3897ca0 100755
--- a/migrate_metadata_to_redis.rb
+++ b/migrate_metadata_to_redis.rb
@@ -47,7 +47,8 @@ class Migrator
     rescue Exception => ex
       logger.error "Error migrating metadata for '#{username}': #{ex}"
       set_directory_backend("legacy")
-      # TODO write username to file for later reference
+      # write username to file for later reference
+      File.open('log/failed_migration.log', 'a') { |f| f.puts username }
       exit 1
     end
     set_directory_backend("new")

From 7107c77a3f752e6016add8c0bf599b9ccde6f5a4 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Greg=20Kar=C3=A9kinian?= <greg@karekinian.com>
Date: Tue, 1 Mar 2016 15:06:23 +0100
Subject: [PATCH 07/16] Turn Redis parameters into symbols

---
 migrate_metadata_to_redis.rb | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/migrate_metadata_to_redis.rb b/migrate_metadata_to_redis.rb
index 3897ca0..fa2a705 100755
--- a/migrate_metadata_to_redis.rb
+++ b/migrate_metadata_to_redis.rb
@@ -4,6 +4,7 @@ require "rest_client"
 require "redis"
 require "yaml"
 require "logger"
+require "active_support/core_ext/hash"
 
 class Migrator
 
@@ -118,7 +119,7 @@ class Migrator
   end
 
   def redis
-    @redis ||= Redis.new(@settings["redis"])
+    @redis ||= Redis.new(@settings["redis"].symbolize_keys)
   end
 
   def get_directory_listing_from_swift(directory)

From f1f450cc723f86319079ebb3627dac43bae41b8e Mon Sep 17 00:00:00 2001
From: Garret Alfert <garret@5apps.com>
Date: Tue, 1 Mar 2016 22:57:54 +0100
Subject: [PATCH 08/16] Don't add double trailing slash to container URL

The directory name already contains a trailing slash.
---
 migrate_metadata_to_redis.rb | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/migrate_metadata_to_redis.rb b/migrate_metadata_to_redis.rb
index fa2a705..0f37a49 100755
--- a/migrate_metadata_to_redis.rb
+++ b/migrate_metadata_to_redis.rb
@@ -133,7 +133,7 @@ class Migrator
       if is_root_listing
         get_response = do_get_request("#{container_url_for(@username)}/?format=json&path=")
       else
-        get_response = do_get_request("#{container_url_for(@username)}/?format=json&path=#{escape(directory)}/")
+        get_response = do_get_request("#{container_url_for(@username)}/?format=json&path=#{escape(directory)}")
       end
     end
 

From d18de83893498ce18360289e0f9bbf050358beb8 Mon Sep 17 00:00:00 2001
From: Garret Alfert <garret@5apps.com>
Date: Wed, 2 Mar 2016 13:59:44 +0100
Subject: [PATCH 09/16] Strip trailing whitespace from Swift token

---
 migrate_metadata_to_redis.rb | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/migrate_metadata_to_redis.rb b/migrate_metadata_to_redis.rb
index 0f37a49..9ecddd0 100755
--- a/migrate_metadata_to_redis.rb
+++ b/migrate_metadata_to_redis.rb
@@ -18,7 +18,7 @@ class Migrator
     @settings = YAML.load(File.read('config.yml'))[@environment]
 
     @swift_host = @settings["swift"]["host"]
-    @swift_token = File.read("tmp/swift_token.txt")
+    @swift_token = File.read("tmp/swift_token.txt").strip
 
     @dry_run = ENV["DRYRUN"] || false # disables writing anything to Redis when true
 

From 19f6fcd0684c9e6250b3b2494912690b4b0d7d3f Mon Sep 17 00:00:00 2001
From: Garret Alfert <garret@5apps.com>
Date: Wed, 2 Mar 2016 14:00:59 +0100
Subject: [PATCH 10/16] Use actual Last-Modified time we get from Swift

---
 migrate_metadata_to_redis.rb | 19 ++++++++++++++-----
 1 file changed, 14 insertions(+), 5 deletions(-)

diff --git a/migrate_metadata_to_redis.rb b/migrate_metadata_to_redis.rb
index 9ecddd0..e7e66dd 100755
--- a/migrate_metadata_to_redis.rb
+++ b/migrate_metadata_to_redis.rb
@@ -76,7 +76,7 @@ class Migrator
           # get dir listing and repeat
           work_on_dir(item, "#{parent_directory}#{directory}")
         else
-          save_document_data("#{parent_directory}#{directory}", item, data, timestamp)
+          save_document_data("#{parent_directory}#{directory}", item, data)
         end
 
         add_item_to_parent_dir("#{parent_directory}#{directory}", item)
@@ -92,19 +92,22 @@ class Migrator
 
   def save_directory_data(dir, item, data, timestamp)
     key = "rs_meta:#{username}:#{dir.gsub(/^\//, "")}#{item}"
-    metadata = {etag: data["ETag"], modified: timestamp}
+    metadata = {
+      etag: data["ETag"],
+      modified: timestamp_for(data["Last-Modified"])
+    }
 
     logger.debug "Metadata for dir #{key}: #{metadata}"
     redis.hmset(key, *metadata) unless dry_run
   end
 
-  def save_document_data(dir, item, data, timestamp)
+  def save_document_data(dir, item, data)
     key = "rs_meta:#{username}:#{dir.gsub(/^\//, "")}#{item}"
     metadata = {
       etag: data["ETag"],
       size: data["Content-Length"],
       type: data["Content-Type"],
-      modified: timestamp
+      modified: timestamp_for(data["Last-Modified"])
     }
     logger.debug "Metadata for document #{key}: #{metadata}"
     redis.hmset(key, *metadata) unless dry_run
@@ -118,6 +121,10 @@ class Migrator
     end
   end
 
+  def timestamp_for(date)
+    return DateTime.parse(date).strftime("%Q").to_i
+  end
+
   def redis
     @redis ||= Redis.new(@settings["redis"].symbolize_keys)
   end
@@ -159,6 +166,7 @@ class Migrator
         listing["items"].merge!({
           name => {
             "ETag"           => entry["hash"],
+            "Last-Modified"  => entry["last_modified"]
           }
         })
       else # It's a file
@@ -166,7 +174,8 @@ class Migrator
           name => {
             "ETag"           => entry["hash"],
             "Content-Type"   => entry["content_type"],
-            "Content-Length" => entry["bytes"]
+            "Content-Length" => entry["bytes"],
+            "Last-Modified"  => entry["last_modified"]
           }
         })
       end

From ff25b4038546eb5ade90f44c3b9729d8ad19b921 Mon Sep 17 00:00:00 2001
From: Garret Alfert <garret@5apps.com>
Date: Wed, 2 Mar 2016 14:05:53 +0100
Subject: [PATCH 11/16] Save last modified timestamp for objects as well

---
 lib/remote_storage/swift.rb | 18 +++++++++---------
 spec/swift/app_spec.rb      |  2 +-
 2 files changed, 10 insertions(+), 10 deletions(-)

diff --git a/lib/remote_storage/swift.rb b/lib/remote_storage/swift.rb
index 1b2dfee..8dde63b 100644
--- a/lib/remote_storage/swift.rb
+++ b/lib/remote_storage/swift.rb
@@ -188,16 +188,18 @@ module RemoteStorage
 
       res = do_put_request(url, data, content_type)
 
-      # TODO get last modified from response and add to metadata
+      # TODO use actual last modified time from the document put request
+      timestamp = (Time.now.to_f * 1000).to_i
+
       metadata = {
         etag: res.headers[:etag],
         size: data.size,
-        type: content_type
+        type: content_type,
+        modified: timestamp
       }
 
       if update_metadata_object(user, directory, key, metadata) &&
-         # TODO provide the last modified to use for the dir objects as well
-         update_dir_objects(user, directory)
+         update_dir_objects(user, directory, timestamp)
         server.headers["ETag"] = %Q("#{res.headers[:etag]}")
         server.halt 200
       else
@@ -406,10 +408,7 @@ module RemoteStorage
       true
     end
 
-    def update_dir_objects(user, directory)
-      # TODO use actual last modified time from the document put request
-      timestamp = (Time.now.to_f * 1000).to_i
-
+    def update_dir_objects(user, directory, timestamp)
       parent_directories_for(directory).each do |dir|
         unless dir == ""
           res = do_put_request("#{url_for_directory(user, dir)}/", timestamp.to_s, "text/plain")
@@ -443,6 +442,8 @@ module RemoteStorage
     end
 
     def delete_dir_objects(user, directory)
+      timestamp = (Time.now.to_f * 1000).to_i
+
       parent_directories_for(directory).each do |dir|
         if dir_empty?(user, dir)
           unless dir == ""
@@ -451,7 +452,6 @@ module RemoteStorage
           redis.del "rs_meta:#{user}:#{directory}/"
           redis.srem "rs_meta:#{user}:#{parent_directory_for(dir)}:items", "#{dir}/"
         else
-          timestamp = (Time.now.to_f * 1000).to_i
           unless dir == ""
             res = do_put_request("#{url_for_directory(user, dir)}/", timestamp.to_s, "text/plain")
             etag = res.headers[:etag]
diff --git a/spec/swift/app_spec.rb b/spec/swift/app_spec.rb
index 052b750..3323b26 100644
--- a/spec/swift/app_spec.rb
+++ b/spec/swift/app_spec.rb
@@ -35,7 +35,7 @@ describe "App" do
         metadata["size"].must_equal "2"
         metadata["type"].must_equal "text/plain; charset=utf-8"
         metadata["etag"].must_equal "bla"
-        metadata["modified"].must_equal nil
+        metadata["modified"].length.must_equal 13
       end
 
       it "creates the directory objects metadata in redis" do

From 067f4a155378ae0611e7c6970e66be5d466cf88a Mon Sep 17 00:00:00 2001
From: Garret Alfert <garret@5apps.com>
Date: Wed, 2 Mar 2016 14:24:16 +0100
Subject: [PATCH 12/16] Use shorter Redis key for dir backend config

rsc:db:username instead of rs_config:dir_backend:username
---
 lib/remote_storage/swift.rb  |  2 +-
 migrate_metadata_to_redis.rb |  2 +-
 spec/spec_helper.rb          |  2 +-
 spec/swift/app_spec.rb       | 12 ++++++------
 4 files changed, 9 insertions(+), 9 deletions(-)

diff --git a/lib/remote_storage/swift.rb b/lib/remote_storage/swift.rb
index 8dde63b..1be255c 100644
--- a/lib/remote_storage/swift.rb
+++ b/lib/remote_storage/swift.rb
@@ -537,7 +537,7 @@ module RemoteStorage
     end
 
     def directory_backend(user)
-      @directory_backend ||= redis.get("rs_config:dir_backend:#{user}") || "legacy"
+      @directory_backend ||= redis.get("rsc:db:#{user}") || "legacy"
     end
 
     def etag_for(body)
diff --git a/migrate_metadata_to_redis.rb b/migrate_metadata_to_redis.rb
index e7e66dd..952aea4 100755
--- a/migrate_metadata_to_redis.rb
+++ b/migrate_metadata_to_redis.rb
@@ -57,7 +57,7 @@ class Migrator
   end
 
   def set_directory_backend(backend)
-    redis.set("rs_config:dir_backend:#{username}", backend) unless dry_run
+    redis.set("rsc:db:#{username}", backend) unless dry_run
   end
 
   def work_on_dir(directory, parent_directory)
diff --git a/spec/spec_helper.rb b/spec/spec_helper.rb
index 4975f52..dfc955f 100644
--- a/spec/spec_helper.rb
+++ b/spec/spec_helper.rb
@@ -39,7 +39,7 @@ if app.settings.respond_to? :redis
   end
 
   def purge_redis
-    redis.keys("rs_*").each do |key|
+    redis.keys("rs*").each do |key|
       redis.del key
     end
   end
diff --git a/spec/swift/app_spec.rb b/spec/swift/app_spec.rb
index 3323b26..b42a95a 100644
--- a/spec/swift/app_spec.rb
+++ b/spec/swift/app_spec.rb
@@ -16,7 +16,7 @@ describe "App" do
 
     before do
       purge_redis
-      redis.set "rs_config:dir_backend:phil", "new"
+      redis.set "rsc:db:phil", "new"
     end
 
     context "authorized" do
@@ -115,7 +115,7 @@ describe "App" do
       describe "directory backend configuration" do
         context "locked new backed" do
           before do
-            redis.set "rs_config:dir_backend:phil", "new-locked"
+            redis.set "rsc:db:phil", "new-locked"
           end
 
           it "responds with 503" do
@@ -130,7 +130,7 @@ describe "App" do
 
         context "locked legacy backend" do
           before do
-            redis.set "rs_config:dir_backend:phil", "legacy-locked"
+            redis.set "rsc:db:phil", "legacy-locked"
           end
 
           it "responds with 503" do
@@ -150,7 +150,7 @@ describe "App" do
 
     before do
       purge_redis
-      redis.set "rs_config:dir_backend:phil", "new"
+      redis.set "rsc:db:phil", "new"
     end
 
     context "authorized" do
@@ -239,7 +239,7 @@ describe "App" do
 
     before do
       purge_redis
-      redis.set "rs_config:dir_backend:phil", "new"
+      redis.set "rsc:db:phil", "new"
     end
 
     context "authorized" do
@@ -318,7 +318,7 @@ describe "App" do
           put "/phil/food/camaron", "yummi"
         end
 
-        redis.set "rs_config:dir_backend:phil", "legacy"
+        redis.set "rsc:db:phil", "legacy"
       end
 
       it "serves directory listing from Swift backend" do

From 650da91a1da71bd4a87d3e148591a15d72dc0dcb Mon Sep 17 00:00:00 2001
From: Garret Alfert <garret@5apps.com>
Date: Wed, 2 Mar 2016 14:43:28 +0100
Subject: [PATCH 13/16] Shorten Redis metadata keys

Use rsm:<username>:<dir>:i instead of
rs_meta:<username>:<dir>:items
---
 lib/remote_storage/swift.rb  | 32 ++++++++++++++++----------------
 migrate_metadata_to_redis.rb |  6 +++---
 spec/swift/app_spec.rb       | 36 ++++++++++++++++++------------------
 3 files changed, 37 insertions(+), 37 deletions(-)

diff --git a/lib/remote_storage/swift.rb b/lib/remote_storage/swift.rb
index 1be255c..75526a5 100644
--- a/lib/remote_storage/swift.rb
+++ b/lib/remote_storage/swift.rb
@@ -87,11 +87,11 @@ module RemoteStorage
       lua_script = <<-EOF
         local user = ARGV[1]
         local directory = ARGV[2]
-        local items = redis.call("smembers", "rs_meta:"..user..":"..directory.."/:items")
+        local items = redis.call("smembers", "rsm:"..user..":"..directory.."/:i")
         local listing = {}
 
         for index, name in pairs(items) do
-          local redis_key = "rs_meta:"..user..":"
+          local redis_key = "rsm:"..user..":"
           if directory == "" then
             redis_key = redis_key..name
           else
@@ -121,7 +121,7 @@ module RemoteStorage
     end
 
     def get_directory_listing_from_redis(user, directory)
-      etag = redis.hget "rs_meta:#{user}:#{directory}/", "etag"
+      etag = redis.hget "rsm:#{user}:#{directory}/", "etag"
 
       none_match = (server.env["HTTP_IF_NONE_MATCH"] || "").split(",").map(&:strip)
       server.halt 304 if none_match.include? etag
@@ -313,7 +313,7 @@ module RemoteStorage
         end
 
         -- check for existing directory with the same name as the document
-        local redis_key = "rs_meta:"..user..":"
+        local redis_key = "rsm:"..user..":"
         if directory == "" then
           redis_key = redis_key..key.."/"
         else
@@ -324,12 +324,12 @@ module RemoteStorage
         end
 
         for index, dir in pairs(parent_directories) do
-          if redis.call("hget", "rs_meta:"..user..":"..dir.."/", "etag") then
+          if redis.call("hget", "rsm:"..user..":"..dir.."/", "etag") then
             -- the directory already exists, no need to do further checks
             return false
           else
             -- check for existing document with same name as directory
-            if redis.call("hget", "rs_meta:"..user..":"..dir, "etag") then
+            if redis.call("hget", "rsm:"..user..":"..dir, "etag") then
               return true
             end
           end
@@ -401,9 +401,9 @@ module RemoteStorage
     end
 
     def update_metadata_object(user, directory, key, metadata)
-      redis_key = "rs_meta:#{user}:#{directory}/#{key}"
+      redis_key = "rsm:#{user}:#{directory}/#{key}"
       redis.hmset(redis_key, *metadata)
-      redis.sadd "rs_meta:#{user}:#{directory}/:items", key
+      redis.sadd "rsm:#{user}:#{directory}/:i", key
 
       true
     end
@@ -418,10 +418,10 @@ module RemoteStorage
           etag = etag_for(get_response.body)
         end
 
-        key = "rs_meta:#{user}:#{dir}/"
+        key = "rsm:#{user}:#{dir}/"
         metadata = {etag: etag, modified: timestamp}
         redis.hmset(key, *metadata)
-        redis.sadd "rs_meta:#{user}:#{parent_directory_for(dir)}:items", "#{top_directory(dir)}/"
+        redis.sadd "rsm:#{user}:#{parent_directory_for(dir)}:i", "#{top_directory(dir)}/"
       end
 
       true
@@ -436,9 +436,9 @@ module RemoteStorage
     end
 
     def delete_metadata_objects(user, directory, key)
-      redis_key = "rs_meta:#{user}:#{directory}/#{key}"
+      redis_key = "rsm:#{user}:#{directory}/#{key}"
       redis.del(redis_key)
-      redis.srem "rs_meta:#{user}:#{directory}/:items", key
+      redis.srem "rsm:#{user}:#{directory}/:i", key
     end
 
     def delete_dir_objects(user, directory)
@@ -449,8 +449,8 @@ module RemoteStorage
           unless dir == ""
             do_delete_request("#{url_for_directory(user, dir)}/")
           end
-          redis.del "rs_meta:#{user}:#{directory}/"
-          redis.srem "rs_meta:#{user}:#{parent_directory_for(dir)}:items", "#{dir}/"
+          redis.del "rsm:#{user}:#{directory}/"
+          redis.srem "rsm:#{user}:#{parent_directory_for(dir)}:i", "#{dir}/"
         else
           unless dir == ""
             res = do_put_request("#{url_for_directory(user, dir)}/", timestamp.to_s, "text/plain")
@@ -460,14 +460,14 @@ module RemoteStorage
             etag = etag_for(get_response.body)
           end
           metadata = {etag: etag, modified: timestamp}
-          redis.hmset("rs_meta:#{user}:#{dir}/", *metadata)
+          redis.hmset("rsm:#{user}:#{dir}/", *metadata)
         end
       end
     end
 
     def dir_empty?(user, dir)
       if directory_backend(user).match(/new/)
-        redis.smembers("rs_meta:#{user}:#{dir}/:items").empty?
+        redis.smembers("rsm:#{user}:#{dir}/:i").empty?
       else
         do_get_request("#{container_url_for(user)}/?format=plain&limit=1&path=#{escape(dir)}/") do |res|
           return res.headers[:content_length] == "0"
diff --git a/migrate_metadata_to_redis.rb b/migrate_metadata_to_redis.rb
index 952aea4..d1211c8 100755
--- a/migrate_metadata_to_redis.rb
+++ b/migrate_metadata_to_redis.rb
@@ -85,13 +85,13 @@ class Migrator
   end
 
   def add_item_to_parent_dir(dir, item)
-    key = "rs_meta:#{username}:#{parent_directory_for(dir)}:items"
+    key = "rsm:#{username}:#{parent_directory_for(dir)}:i"
     logger.debug "Adding item #{item} to #{key}"
     redis.sadd(key, item) unless dry_run
   end
 
   def save_directory_data(dir, item, data, timestamp)
-    key = "rs_meta:#{username}:#{dir.gsub(/^\//, "")}#{item}"
+    key = "rsm:#{username}:#{dir.gsub(/^\//, "")}#{item}"
     metadata = {
       etag: data["ETag"],
       modified: timestamp_for(data["Last-Modified"])
@@ -102,7 +102,7 @@ class Migrator
   end
 
   def save_document_data(dir, item, data)
-    key = "rs_meta:#{username}:#{dir.gsub(/^\//, "")}#{item}"
+    key = "rsm:#{username}:#{dir.gsub(/^\//, "")}#{item}"
     metadata = {
       etag: data["ETag"],
       size: data["Content-Length"],
diff --git a/spec/swift/app_spec.rb b/spec/swift/app_spec.rb
index b42a95a..831c089 100644
--- a/spec/swift/app_spec.rb
+++ b/spec/swift/app_spec.rb
@@ -31,7 +31,7 @@ describe "App" do
           put "/phil/food/aguacate", "si"
         end
 
-        metadata = redis.hgetall "rs_meta:phil:food/aguacate"
+        metadata = redis.hgetall "rsm:phil:food/aguacate"
         metadata["size"].must_equal "2"
         metadata["type"].must_equal "text/plain; charset=utf-8"
         metadata["etag"].must_equal "bla"
@@ -50,20 +50,20 @@ describe "App" do
           end
         end
 
-        metadata = redis.hgetall "rs_meta:phil:/"
+        metadata = redis.hgetall "rsm:phil:/"
         metadata["etag"].must_equal "rootetag"
         metadata["modified"].length.must_equal 13
 
-        metadata = redis.hgetall "rs_meta:phil:food/"
+        metadata = redis.hgetall "rsm:phil:food/"
         metadata["etag"].must_equal "bla"
         metadata["modified"].length.must_equal 13
 
-        food_items = redis.smembers "rs_meta:phil:food/:items"
+        food_items = redis.smembers "rsm:phil:food/:i"
         food_items.each do |food_item|
           ["camaron", "aguacate"].must_include food_item
         end
 
-        root_items = redis.smembers "rs_meta:phil:/:items"
+        root_items = redis.smembers "rsm:phil:/:i"
         root_items.must_equal ["food/"]
       end
 
@@ -81,7 +81,7 @@ describe "App" do
 
           last_response.status.must_equal 200
 
-          metadata = redis.hgetall "rs_meta:phil:food/aguacate"
+          metadata = redis.hgetall "rsm:phil:food/aguacate"
           metadata["size"].must_equal "2"
         end
 
@@ -94,7 +94,7 @@ describe "App" do
 
           last_response.status.must_equal 409
 
-          metadata = redis.hgetall "rs_meta:phil:food"
+          metadata = redis.hgetall "rsm:phil:food"
           metadata.must_be_empty
         end
 
@@ -107,7 +107,7 @@ describe "App" do
 
           last_response.status.must_equal 409
 
-          metadata = redis.hgetall "rs_meta:phil:food/aguacate/empanado"
+          metadata = redis.hgetall "rsm:phil:food/aguacate/empanado"
           metadata.must_be_empty
         end
       end
@@ -123,7 +123,7 @@ describe "App" do
 
             last_response.status.must_equal 503
 
-            metadata = redis.hgetall "rs_meta:phil:food/aguacate"
+            metadata = redis.hgetall "rsm:phil:food/aguacate"
             metadata.must_be_empty
           end
         end
@@ -138,7 +138,7 @@ describe "App" do
 
             last_response.status.must_equal 503
 
-            metadata = redis.hgetall "rs_meta:phil:food/aguacate"
+            metadata = redis.hgetall "rsm:phil:food/aguacate"
             metadata.must_be_empty
           end
         end
@@ -178,12 +178,12 @@ describe "App" do
           end
         end
 
-        metadata = redis.hgetall "rs_meta:phil:food/aguacate"
+        metadata = redis.hgetall "rsm:phil:food/aguacate"
         metadata.must_be_empty
       end
 
       it "deletes the directory objects metadata in redis" do
-        old_metadata = redis.hgetall "rs_meta:phil:food/"
+        old_metadata = redis.hgetall "rsm:phil:food/"
 
         put_stub = OpenStruct.new(headers: {etag: "newetag"})
         get_stub = OpenStruct.new(body: "rootbody")
@@ -197,15 +197,15 @@ describe "App" do
           end
         end
 
-        metadata = redis.hgetall "rs_meta:phil:food/"
+        metadata = redis.hgetall "rsm:phil:food/"
         metadata["etag"].must_equal "newetag"
         metadata["modified"].length.must_equal 13
         metadata["modified"].wont_equal old_metadata["modified"]
 
-        food_items = redis.smembers "rs_meta:phil:food/:items"
+        food_items = redis.smembers "rsm:phil:food/:i"
         food_items.must_equal ["camaron"]
 
-        root_items = redis.smembers "rs_meta:phil:/:items"
+        root_items = redis.smembers "rsm:phil:/:i"
         root_items.must_equal ["food/"]
       end
 
@@ -223,13 +223,13 @@ describe "App" do
           end
         end
 
-        metadata = redis.hgetall "rs_meta:phil:food/"
+        metadata = redis.hgetall "rsm:phil:food/"
         metadata.must_be_empty
 
-        food_items = redis.smembers "rs_meta:phil:food/:items"
+        food_items = redis.smembers "rsm:phil:food/:i"
         food_items.must_be_empty
 
-        root_items = redis.smembers "rs_meta:phil:/:items"
+        root_items = redis.smembers "rsm:phil:/:i"
         root_items.must_be_empty
       end
     end

From 88f2dceb2775a27a9a5aec74ff1f0270c28b2010 Mon Sep 17 00:00:00 2001
From: Garret Alfert <garret@5apps.com>
Date: Wed, 2 Mar 2016 15:00:33 +0100
Subject: [PATCH 14/16] Use single letters for Redis metadata keys

Saves 16 characters per document object and 10
characters per directory object
---
 lib/remote_storage/swift.rb  | 26 +++++++++++++-------------
 migrate_metadata_to_redis.rb | 12 ++++++------
 spec/swift/app_spec.rb       | 24 ++++++++++++------------
 3 files changed, 31 insertions(+), 31 deletions(-)

diff --git a/lib/remote_storage/swift.rb b/lib/remote_storage/swift.rb
index 75526a5..1d0b4f5 100644
--- a/lib/remote_storage/swift.rb
+++ b/lib/remote_storage/swift.rb
@@ -107,10 +107,10 @@ module RemoteStorage
             metadata[metadata_values[idx]] = metadata_values[idx + 1]
           end
 
-          listing[name] = {["ETag"] = metadata["etag"]}
+          listing[name] = {["ETag"] = metadata["e"]}
           if string.sub(name, -1) ~= "/" then
-            listing[name]["Content-Type"]   = metadata["type"]
-            listing[name]["Content-Length"] = tonumber(metadata["size"])
+            listing[name]["Content-Type"]   = metadata["t"]
+            listing[name]["Content-Length"] = tonumber(metadata["s"])
           end
         end
 
@@ -121,7 +121,7 @@ module RemoteStorage
     end
 
     def get_directory_listing_from_redis(user, directory)
-      etag = redis.hget "rsm:#{user}:#{directory}/", "etag"
+      etag = redis.hget "rsm:#{user}:#{directory}/", "e"
 
       none_match = (server.env["HTTP_IF_NONE_MATCH"] || "").split(",").map(&:strip)
       server.halt 304 if none_match.include? etag
@@ -192,10 +192,10 @@ module RemoteStorage
       timestamp = (Time.now.to_f * 1000).to_i
 
       metadata = {
-        etag: res.headers[:etag],
-        size: data.size,
-        type: content_type,
-        modified: timestamp
+        e: res.headers[:etag],
+        s: data.size,
+        t: content_type,
+        m: timestamp
       }
 
       if update_metadata_object(user, directory, key, metadata) &&
@@ -319,17 +319,17 @@ module RemoteStorage
         else
           redis_key = redis_key..directory.."/"..key.."/"
         end
-        if redis.call("hget", redis_key, "etag") then
+        if redis.call("hget", redis_key, "e") then
           return true
         end
 
         for index, dir in pairs(parent_directories) do
-          if redis.call("hget", "rsm:"..user..":"..dir.."/", "etag") then
+          if redis.call("hget", "rsm:"..user..":"..dir.."/", "e") then
             -- the directory already exists, no need to do further checks
             return false
           else
             -- check for existing document with same name as directory
-            if redis.call("hget", "rsm:"..user..":"..dir, "etag") then
+            if redis.call("hget", "rsm:"..user..":"..dir, "e") then
               return true
             end
           end
@@ -419,7 +419,7 @@ module RemoteStorage
         end
 
         key = "rsm:#{user}:#{dir}/"
-        metadata = {etag: etag, modified: timestamp}
+        metadata = {e: etag, m: timestamp}
         redis.hmset(key, *metadata)
         redis.sadd "rsm:#{user}:#{parent_directory_for(dir)}:i", "#{top_directory(dir)}/"
       end
@@ -459,7 +459,7 @@ module RemoteStorage
             get_response = do_get_request("#{container_url_for(user)}/?format=json&path=")
             etag = etag_for(get_response.body)
           end
-          metadata = {etag: etag, modified: timestamp}
+          metadata = {e: etag, m: timestamp}
           redis.hmset("rsm:#{user}:#{dir}/", *metadata)
         end
       end
diff --git a/migrate_metadata_to_redis.rb b/migrate_metadata_to_redis.rb
index d1211c8..a421d27 100755
--- a/migrate_metadata_to_redis.rb
+++ b/migrate_metadata_to_redis.rb
@@ -93,8 +93,8 @@ class Migrator
   def save_directory_data(dir, item, data, timestamp)
     key = "rsm:#{username}:#{dir.gsub(/^\//, "")}#{item}"
     metadata = {
-      etag: data["ETag"],
-      modified: timestamp_for(data["Last-Modified"])
+      e: data["ETag"],
+      m: timestamp_for(data["Last-Modified"])
     }
 
     logger.debug "Metadata for dir #{key}: #{metadata}"
@@ -104,10 +104,10 @@ class Migrator
   def save_document_data(dir, item, data)
     key = "rsm:#{username}:#{dir.gsub(/^\//, "")}#{item}"
     metadata = {
-      etag: data["ETag"],
-      size: data["Content-Length"],
-      type: data["Content-Type"],
-      modified: timestamp_for(data["Last-Modified"])
+      e: data["ETag"],
+      s: data["Content-Length"],
+      t: data["Content-Type"],
+      m: timestamp_for(data["Last-Modified"])
     }
     logger.debug "Metadata for document #{key}: #{metadata}"
     redis.hmset(key, *metadata) unless dry_run
diff --git a/spec/swift/app_spec.rb b/spec/swift/app_spec.rb
index 831c089..61204e3 100644
--- a/spec/swift/app_spec.rb
+++ b/spec/swift/app_spec.rb
@@ -32,10 +32,10 @@ describe "App" do
         end
 
         metadata = redis.hgetall "rsm:phil:food/aguacate"
-        metadata["size"].must_equal "2"
-        metadata["type"].must_equal "text/plain; charset=utf-8"
-        metadata["etag"].must_equal "bla"
-        metadata["modified"].length.must_equal 13
+        metadata["s"].must_equal "2"
+        metadata["t"].must_equal "text/plain; charset=utf-8"
+        metadata["e"].must_equal "bla"
+        metadata["m"].length.must_equal 13
       end
 
       it "creates the directory objects metadata in redis" do
@@ -51,12 +51,12 @@ describe "App" do
         end
 
         metadata = redis.hgetall "rsm:phil:/"
-        metadata["etag"].must_equal "rootetag"
-        metadata["modified"].length.must_equal 13
+        metadata["e"].must_equal "rootetag"
+        metadata["m"].length.must_equal 13
 
         metadata = redis.hgetall "rsm:phil:food/"
-        metadata["etag"].must_equal "bla"
-        metadata["modified"].length.must_equal 13
+        metadata["e"].must_equal "bla"
+        metadata["m"].length.must_equal 13
 
         food_items = redis.smembers "rsm:phil:food/:i"
         food_items.each do |food_item|
@@ -82,7 +82,7 @@ describe "App" do
           last_response.status.must_equal 200
 
           metadata = redis.hgetall "rsm:phil:food/aguacate"
-          metadata["size"].must_equal "2"
+          metadata["s"].must_equal "2"
         end
 
         it "conflicts when there is a directory with same name as document" do
@@ -198,9 +198,9 @@ describe "App" do
         end
 
         metadata = redis.hgetall "rsm:phil:food/"
-        metadata["etag"].must_equal "newetag"
-        metadata["modified"].length.must_equal 13
-        metadata["modified"].wont_equal old_metadata["modified"]
+        metadata["e"].must_equal "newetag"
+        metadata["m"].length.must_equal 13
+        metadata["m"].wont_equal old_metadata["m"]
 
         food_items = redis.smembers "rsm:phil:food/:i"
         food_items.must_equal ["camaron"]

From c2fd9bc6d0f1245797673900fc765cd8bda48b36 Mon Sep 17 00:00:00 2001
From: Garret Alfert <garret@5apps.com>
Date: Wed, 2 Mar 2016 18:04:38 +0100
Subject: [PATCH 15/16] Change Redis metadata key prefix to rs:m

---
 lib/remote_storage/swift.rb  | 32 ++++++++++++++++----------------
 migrate_metadata_to_redis.rb |  6 +++---
 spec/swift/app_spec.rb       | 36 ++++++++++++++++++------------------
 3 files changed, 37 insertions(+), 37 deletions(-)

diff --git a/lib/remote_storage/swift.rb b/lib/remote_storage/swift.rb
index 1d0b4f5..1cf7a34 100644
--- a/lib/remote_storage/swift.rb
+++ b/lib/remote_storage/swift.rb
@@ -87,11 +87,11 @@ module RemoteStorage
       lua_script = <<-EOF
         local user = ARGV[1]
         local directory = ARGV[2]
-        local items = redis.call("smembers", "rsm:"..user..":"..directory.."/:i")
+        local items = redis.call("smembers", "rs:m:"..user..":"..directory.."/:i")
         local listing = {}
 
         for index, name in pairs(items) do
-          local redis_key = "rsm:"..user..":"
+          local redis_key = "rs:m:"..user..":"
           if directory == "" then
             redis_key = redis_key..name
           else
@@ -121,7 +121,7 @@ module RemoteStorage
     end
 
     def get_directory_listing_from_redis(user, directory)
-      etag = redis.hget "rsm:#{user}:#{directory}/", "e"
+      etag = redis.hget "rs:m:#{user}:#{directory}/", "e"
 
       none_match = (server.env["HTTP_IF_NONE_MATCH"] || "").split(",").map(&:strip)
       server.halt 304 if none_match.include? etag
@@ -313,7 +313,7 @@ module RemoteStorage
         end
 
         -- check for existing directory with the same name as the document
-        local redis_key = "rsm:"..user..":"
+        local redis_key = "rs:m:"..user..":"
         if directory == "" then
           redis_key = redis_key..key.."/"
         else
@@ -324,12 +324,12 @@ module RemoteStorage
         end
 
         for index, dir in pairs(parent_directories) do
-          if redis.call("hget", "rsm:"..user..":"..dir.."/", "e") then
+          if redis.call("hget", "rs:m:"..user..":"..dir.."/", "e") then
             -- the directory already exists, no need to do further checks
             return false
           else
             -- check for existing document with same name as directory
-            if redis.call("hget", "rsm:"..user..":"..dir, "e") then
+            if redis.call("hget", "rs:m:"..user..":"..dir, "e") then
               return true
             end
           end
@@ -401,9 +401,9 @@ module RemoteStorage
     end
 
     def update_metadata_object(user, directory, key, metadata)
-      redis_key = "rsm:#{user}:#{directory}/#{key}"
+      redis_key = "rs:m:#{user}:#{directory}/#{key}"
       redis.hmset(redis_key, *metadata)
-      redis.sadd "rsm:#{user}:#{directory}/:i", key
+      redis.sadd "rs:m:#{user}:#{directory}/:i", key
 
       true
     end
@@ -418,10 +418,10 @@ module RemoteStorage
           etag = etag_for(get_response.body)
         end
 
-        key = "rsm:#{user}:#{dir}/"
+        key = "rs:m:#{user}:#{dir}/"
         metadata = {e: etag, m: timestamp}
         redis.hmset(key, *metadata)
-        redis.sadd "rsm:#{user}:#{parent_directory_for(dir)}:i", "#{top_directory(dir)}/"
+        redis.sadd "rs:m:#{user}:#{parent_directory_for(dir)}:i", "#{top_directory(dir)}/"
       end
 
       true
@@ -436,9 +436,9 @@ module RemoteStorage
     end
 
     def delete_metadata_objects(user, directory, key)
-      redis_key = "rsm:#{user}:#{directory}/#{key}"
+      redis_key = "rs:m:#{user}:#{directory}/#{key}"
       redis.del(redis_key)
-      redis.srem "rsm:#{user}:#{directory}/:i", key
+      redis.srem "rs:m:#{user}:#{directory}/:i", key
     end
 
     def delete_dir_objects(user, directory)
@@ -449,8 +449,8 @@ module RemoteStorage
           unless dir == ""
             do_delete_request("#{url_for_directory(user, dir)}/")
           end
-          redis.del "rsm:#{user}:#{directory}/"
-          redis.srem "rsm:#{user}:#{parent_directory_for(dir)}:i", "#{dir}/"
+          redis.del "rs:m:#{user}:#{directory}/"
+          redis.srem "rs:m:#{user}:#{parent_directory_for(dir)}:i", "#{dir}/"
         else
           unless dir == ""
             res = do_put_request("#{url_for_directory(user, dir)}/", timestamp.to_s, "text/plain")
@@ -460,14 +460,14 @@ module RemoteStorage
             etag = etag_for(get_response.body)
           end
           metadata = {e: etag, m: timestamp}
-          redis.hmset("rsm:#{user}:#{dir}/", *metadata)
+          redis.hmset("rs:m:#{user}:#{dir}/", *metadata)
         end
       end
     end
 
     def dir_empty?(user, dir)
       if directory_backend(user).match(/new/)
-        redis.smembers("rsm:#{user}:#{dir}/:i").empty?
+        redis.smembers("rs:m:#{user}:#{dir}/:i").empty?
       else
         do_get_request("#{container_url_for(user)}/?format=plain&limit=1&path=#{escape(dir)}/") do |res|
           return res.headers[:content_length] == "0"
diff --git a/migrate_metadata_to_redis.rb b/migrate_metadata_to_redis.rb
index a421d27..336f498 100755
--- a/migrate_metadata_to_redis.rb
+++ b/migrate_metadata_to_redis.rb
@@ -85,13 +85,13 @@ class Migrator
   end
 
   def add_item_to_parent_dir(dir, item)
-    key = "rsm:#{username}:#{parent_directory_for(dir)}:i"
+    key = "rs:m:#{username}:#{parent_directory_for(dir)}:i"
     logger.debug "Adding item #{item} to #{key}"
     redis.sadd(key, item) unless dry_run
   end
 
   def save_directory_data(dir, item, data, timestamp)
-    key = "rsm:#{username}:#{dir.gsub(/^\//, "")}#{item}"
+    key = "rs:m:#{username}:#{dir.gsub(/^\//, "")}#{item}"
     metadata = {
       e: data["ETag"],
       m: timestamp_for(data["Last-Modified"])
@@ -102,7 +102,7 @@ class Migrator
   end
 
   def save_document_data(dir, item, data)
-    key = "rsm:#{username}:#{dir.gsub(/^\//, "")}#{item}"
+    key = "rs:m:#{username}:#{dir.gsub(/^\//, "")}#{item}"
     metadata = {
       e: data["ETag"],
       s: data["Content-Length"],
diff --git a/spec/swift/app_spec.rb b/spec/swift/app_spec.rb
index 61204e3..29504f7 100644
--- a/spec/swift/app_spec.rb
+++ b/spec/swift/app_spec.rb
@@ -31,7 +31,7 @@ describe "App" do
           put "/phil/food/aguacate", "si"
         end
 
-        metadata = redis.hgetall "rsm:phil:food/aguacate"
+        metadata = redis.hgetall "rs:m:phil:food/aguacate"
         metadata["s"].must_equal "2"
         metadata["t"].must_equal "text/plain; charset=utf-8"
         metadata["e"].must_equal "bla"
@@ -50,20 +50,20 @@ describe "App" do
           end
         end
 
-        metadata = redis.hgetall "rsm:phil:/"
+        metadata = redis.hgetall "rs:m:phil:/"
         metadata["e"].must_equal "rootetag"
         metadata["m"].length.must_equal 13
 
-        metadata = redis.hgetall "rsm:phil:food/"
+        metadata = redis.hgetall "rs:m:phil:food/"
         metadata["e"].must_equal "bla"
         metadata["m"].length.must_equal 13
 
-        food_items = redis.smembers "rsm:phil:food/:i"
+        food_items = redis.smembers "rs:m:phil:food/:i"
         food_items.each do |food_item|
           ["camaron", "aguacate"].must_include food_item
         end
 
-        root_items = redis.smembers "rsm:phil:/:i"
+        root_items = redis.smembers "rs:m:phil:/:i"
         root_items.must_equal ["food/"]
       end
 
@@ -81,7 +81,7 @@ describe "App" do
 
           last_response.status.must_equal 200
 
-          metadata = redis.hgetall "rsm:phil:food/aguacate"
+          metadata = redis.hgetall "rs:m:phil:food/aguacate"
           metadata["s"].must_equal "2"
         end
 
@@ -94,7 +94,7 @@ describe "App" do
 
           last_response.status.must_equal 409
 
-          metadata = redis.hgetall "rsm:phil:food"
+          metadata = redis.hgetall "rs:m:phil:food"
           metadata.must_be_empty
         end
 
@@ -107,7 +107,7 @@ describe "App" do
 
           last_response.status.must_equal 409
 
-          metadata = redis.hgetall "rsm:phil:food/aguacate/empanado"
+          metadata = redis.hgetall "rs:m:phil:food/aguacate/empanado"
           metadata.must_be_empty
         end
       end
@@ -123,7 +123,7 @@ describe "App" do
 
             last_response.status.must_equal 503
 
-            metadata = redis.hgetall "rsm:phil:food/aguacate"
+            metadata = redis.hgetall "rs:m:phil:food/aguacate"
             metadata.must_be_empty
           end
         end
@@ -138,7 +138,7 @@ describe "App" do
 
             last_response.status.must_equal 503
 
-            metadata = redis.hgetall "rsm:phil:food/aguacate"
+            metadata = redis.hgetall "rs:m:phil:food/aguacate"
             metadata.must_be_empty
           end
         end
@@ -178,12 +178,12 @@ describe "App" do
           end
         end
 
-        metadata = redis.hgetall "rsm:phil:food/aguacate"
+        metadata = redis.hgetall "rs:m:phil:food/aguacate"
         metadata.must_be_empty
       end
 
       it "deletes the directory objects metadata in redis" do
-        old_metadata = redis.hgetall "rsm:phil:food/"
+        old_metadata = redis.hgetall "rs:m:phil:food/"
 
         put_stub = OpenStruct.new(headers: {etag: "newetag"})
         get_stub = OpenStruct.new(body: "rootbody")
@@ -197,15 +197,15 @@ describe "App" do
           end
         end
 
-        metadata = redis.hgetall "rsm:phil:food/"
+        metadata = redis.hgetall "rs:m:phil:food/"
         metadata["e"].must_equal "newetag"
         metadata["m"].length.must_equal 13
         metadata["m"].wont_equal old_metadata["m"]
 
-        food_items = redis.smembers "rsm:phil:food/:i"
+        food_items = redis.smembers "rs:m:phil:food/:i"
         food_items.must_equal ["camaron"]
 
-        root_items = redis.smembers "rsm:phil:/:i"
+        root_items = redis.smembers "rs:m:phil:/:i"
         root_items.must_equal ["food/"]
       end
 
@@ -223,13 +223,13 @@ describe "App" do
           end
         end
 
-        metadata = redis.hgetall "rsm:phil:food/"
+        metadata = redis.hgetall "rs:m:phil:food/"
         metadata.must_be_empty
 
-        food_items = redis.smembers "rsm:phil:food/:i"
+        food_items = redis.smembers "rs:m:phil:food/:i"
         food_items.must_be_empty
 
-        root_items = redis.smembers "rsm:phil:/:i"
+        root_items = redis.smembers "rs:m:phil:/:i"
         root_items.must_be_empty
       end
     end

From 9b7b224f79b97115875b862269057af4da43e3cc Mon Sep 17 00:00:00 2001
From: Garret Alfert <garret@5apps.com>
Date: Wed, 2 Mar 2016 18:06:42 +0100
Subject: [PATCH 16/16] Use more explicit key name for items

---
 lib/remote_storage/swift.rb  | 12 ++++++------
 migrate_metadata_to_redis.rb |  2 +-
 spec/swift/app_spec.rb       | 12 ++++++------
 3 files changed, 13 insertions(+), 13 deletions(-)

diff --git a/lib/remote_storage/swift.rb b/lib/remote_storage/swift.rb
index 1cf7a34..818f6e2 100644
--- a/lib/remote_storage/swift.rb
+++ b/lib/remote_storage/swift.rb
@@ -87,7 +87,7 @@ module RemoteStorage
       lua_script = <<-EOF
         local user = ARGV[1]
         local directory = ARGV[2]
-        local items = redis.call("smembers", "rs:m:"..user..":"..directory.."/:i")
+        local items = redis.call("smembers", "rs:m:"..user..":"..directory.."/:items")
         local listing = {}
 
         for index, name in pairs(items) do
@@ -403,7 +403,7 @@ module RemoteStorage
     def update_metadata_object(user, directory, key, metadata)
       redis_key = "rs:m:#{user}:#{directory}/#{key}"
       redis.hmset(redis_key, *metadata)
-      redis.sadd "rs:m:#{user}:#{directory}/:i", key
+      redis.sadd "rs:m:#{user}:#{directory}/:items", key
 
       true
     end
@@ -421,7 +421,7 @@ module RemoteStorage
         key = "rs:m:#{user}:#{dir}/"
         metadata = {e: etag, m: timestamp}
         redis.hmset(key, *metadata)
-        redis.sadd "rs:m:#{user}:#{parent_directory_for(dir)}:i", "#{top_directory(dir)}/"
+        redis.sadd "rs:m:#{user}:#{parent_directory_for(dir)}:items", "#{top_directory(dir)}/"
       end
 
       true
@@ -438,7 +438,7 @@ module RemoteStorage
     def delete_metadata_objects(user, directory, key)
       redis_key = "rs:m:#{user}:#{directory}/#{key}"
       redis.del(redis_key)
-      redis.srem "rs:m:#{user}:#{directory}/:i", key
+      redis.srem "rs:m:#{user}:#{directory}/:items", key
     end
 
     def delete_dir_objects(user, directory)
@@ -450,7 +450,7 @@ module RemoteStorage
             do_delete_request("#{url_for_directory(user, dir)}/")
           end
           redis.del "rs:m:#{user}:#{directory}/"
-          redis.srem "rs:m:#{user}:#{parent_directory_for(dir)}:i", "#{dir}/"
+          redis.srem "rs:m:#{user}:#{parent_directory_for(dir)}:items", "#{dir}/"
         else
           unless dir == ""
             res = do_put_request("#{url_for_directory(user, dir)}/", timestamp.to_s, "text/plain")
@@ -467,7 +467,7 @@ module RemoteStorage
 
     def dir_empty?(user, dir)
       if directory_backend(user).match(/new/)
-        redis.smembers("rs:m:#{user}:#{dir}/:i").empty?
+        redis.smembers("rs:m:#{user}:#{dir}/:items").empty?
       else
         do_get_request("#{container_url_for(user)}/?format=plain&limit=1&path=#{escape(dir)}/") do |res|
           return res.headers[:content_length] == "0"
diff --git a/migrate_metadata_to_redis.rb b/migrate_metadata_to_redis.rb
index 336f498..bd28e75 100755
--- a/migrate_metadata_to_redis.rb
+++ b/migrate_metadata_to_redis.rb
@@ -85,7 +85,7 @@ class Migrator
   end
 
   def add_item_to_parent_dir(dir, item)
-    key = "rs:m:#{username}:#{parent_directory_for(dir)}:i"
+    key = "rs:m:#{username}:#{parent_directory_for(dir)}:items"
     logger.debug "Adding item #{item} to #{key}"
     redis.sadd(key, item) unless dry_run
   end
diff --git a/spec/swift/app_spec.rb b/spec/swift/app_spec.rb
index 29504f7..56d03a1 100644
--- a/spec/swift/app_spec.rb
+++ b/spec/swift/app_spec.rb
@@ -58,12 +58,12 @@ describe "App" do
         metadata["e"].must_equal "bla"
         metadata["m"].length.must_equal 13
 
-        food_items = redis.smembers "rs:m:phil:food/:i"
+        food_items = redis.smembers "rs:m:phil:food/:items"
         food_items.each do |food_item|
           ["camaron", "aguacate"].must_include food_item
         end
 
-        root_items = redis.smembers "rs:m:phil:/:i"
+        root_items = redis.smembers "rs:m:phil:/:items"
         root_items.must_equal ["food/"]
       end
 
@@ -202,10 +202,10 @@ describe "App" do
         metadata["m"].length.must_equal 13
         metadata["m"].wont_equal old_metadata["m"]
 
-        food_items = redis.smembers "rs:m:phil:food/:i"
+        food_items = redis.smembers "rs:m:phil:food/:items"
         food_items.must_equal ["camaron"]
 
-        root_items = redis.smembers "rs:m:phil:/:i"
+        root_items = redis.smembers "rs:m:phil:/:items"
         root_items.must_equal ["food/"]
       end
 
@@ -226,10 +226,10 @@ describe "App" do
         metadata = redis.hgetall "rs:m:phil:food/"
         metadata.must_be_empty
 
-        food_items = redis.smembers "rs:m:phil:food/:i"
+        food_items = redis.smembers "rs:m:phil:food/:items"
         food_items.must_be_empty
 
-        root_items = redis.smembers "rs:m:phil:/:i"
+        root_items = redis.smembers "rs:m:phil:/:items"
         root_items.must_be_empty
       end
     end