Merge branch 'master' into stable

lib/remote_storage/swift.rb

@@ -433,7 +433,10 @@ module RemoteStorage
 
     def reload_swift_token
       server.logger.debug "Reloading swift token. Old token: #{settings.swift_token}"
-      settings.swift_token           = File.read(swift_token_path)
+      # Remove the line break from the token file. The line break that the
+      # token script is adding to the file was causing Sentry to reject the
+      # token field
+      settings.swift_token           = File.read(swift_token_path).rstrip
       settings.swift_token_loaded_at = Time.now
       server.logger.debug "Reloaded swift token. New token: #{settings.swift_token}"
     end
@@ -454,7 +457,7 @@ module RemoteStorage
       rescue RestClient::Unauthorized => ex
         Raven.capture_exception(
           ex,
-          tags: { swift_token:           settings.swift_token,
+          tags: { swift_token:           settings.swift_token[0..19], # send the first 20 characters
                   swift_token_loaded_at: settings.swift_token_loaded_at }
         )
         server.halt 500

migrate_storage_size_from_metadata.rb

@@ -0,0 +1,134 @@
+#!/usr/bin/env ruby
+
+require "rubygems"
+require "bundler/setup"
+require "rest_client"
+require "redis"
+require "yaml"
+require "logger"
+require "active_support/core_ext/hash"
+
+class Migrator
+
+  attr_accessor :username, :base_url, :environment, :settings, :logger
+
+  def initialize(username)
+    @username = username
+
+    @environment = ENV["ENVIRONMENT"] || "staging"
+    @settings = YAML.load(File.read('config.yml'))[@environment]
+
+    @logger = Logger.new("log/migrate_storage_size_from_metadata.log")
+    log_level = ENV["LOGLEVEL"] || "INFO"
+    logger.level = Kernel.const_get "Logger::#{log_level}"
+    logger.progname = username
+  end
+
+  def migrate
+    logger.info "Starting migration for '#{username}'"
+    set_container_migration_state("in_progress")
+    begin
+      write_storage_size_from_redis_metadata(username)
+    rescue Exception => ex
+      logger.error "Error setting storage size from metadata for '#{username}': #{ex}"
+      set_container_migration_state("not_started")
+      # write username to file for later reference
+      File.open('log/failed_migration.log', 'a') { |f| f.puts username }
+      exit 1
+    end
+    delete_container_migration_state
+    logger.info "Finished migration for '#{username}'"
+  end
+
+  def redis
+    @redis ||= Redis.new(@settings["redis"].symbolize_keys)
+  end
+
+  def write_storage_size_from_redis_metadata(user)
+    lua_script = <<-EOF
+      local user = ARGV[1]
+      local total_size = 0
+      local size_key = KEYS[1]
+
+      local function get_size_from_items(parent, directory)
+        local path
+        if parent == "/" then
+          path = directory
+        else
+          path = parent..directory
+        end
+        local items = redis.call("smembers", "rs:m:"..user..":"..path..":items")
+        for index, name in pairs(items) do
+          local redis_key = "rs:m:"..user..":"
+
+          redis_key = redis_key..path..name
+
+          -- if it's a directory, get the items inside of it
+          if string.match(name, "/$") then
+            get_size_from_items(path, name)
+          -- if it's a file, get its size
+          else
+            local file_size = redis.call("hget", redis_key, "s")
+            total_size = total_size + file_size
+          end
+        end
+      end
+
+      get_size_from_items("", "") -- Start from the root
+
+      redis.call("set", size_key, total_size)
+    EOF
+
+    redis.eval(lua_script, ["rs:s:#{user}"], [user])
+  end
+
+  def set_container_migration_state(type)
+    redis.hset("rs:size_migration", username, type)
+  end
+
+  def delete_container_migration_state
+    redis.hdel("rs:size_migration", username)
+  end
+
+end
+
+class MigrationRunner
+  attr_accessor :environment, :settings
+
+  def initialize
+    @environment = ENV["ENVIRONMENT"] || "staging"
+    @settings = YAML.load(File.read('config.yml'))[@environment]
+  end
+
+  def migrate
+    while username = pick_unmigrated_user
+      migrator = Migrator.new username
+      migrator.migrate
+    end
+  end
+
+  def unmigrated_users
+    redis.hgetall("rs:size_migration").select { |_, value|
+      value == "not_started"
+    }.keys
+  end
+
+  def pick_unmigrated_user
+    unmigrated_users.sample # pick a random user from list
+  end
+
+  def redis
+    @redis ||= Redis.new(@settings["redis"].symbolize_keys)
+  end
+
+end
+
+username = ARGV[0]
+
+if username
+  migrator = Migrator.new username
+  migrator.migrate
+else
+  runner = MigrationRunner.new
+  runner.migrate
+end