diff --git a/lib/remote_storage/swift.rb b/lib/remote_storage/swift.rb index ec7544c..2fb5fa4 100644 --- a/lib/remote_storage/swift.rb +++ b/lib/remote_storage/swift.rb @@ -433,7 +433,10 @@ module RemoteStorage def reload_swift_token server.logger.debug "Reloading swift token. Old token: #{settings.swift_token}" - settings.swift_token = File.read(swift_token_path) + # Remove the line break from the token file. The line break that the + # token script is adding to the file was causing Sentry to reject the + # token field + settings.swift_token = File.read(swift_token_path).rstrip settings.swift_token_loaded_at = Time.now server.logger.debug "Reloaded swift token. New token: #{settings.swift_token}" end @@ -454,7 +457,7 @@ module RemoteStorage rescue RestClient::Unauthorized => ex Raven.capture_exception( ex, - tags: { swift_token: settings.swift_token, + tags: { swift_token: settings.swift_token[0..19], # send the first 20 characters swift_token_loaded_at: settings.swift_token_loaded_at } ) server.halt 500 diff --git a/migrate_storage_size_from_metadata.rb b/migrate_storage_size_from_metadata.rb new file mode 100755 index 0000000..bd04bd3 --- /dev/null +++ b/migrate_storage_size_from_metadata.rb @@ -0,0 +1,134 @@ +#!/usr/bin/env ruby + +require "rubygems" +require "bundler/setup" +require "rest_client" +require "redis" +require "yaml" +require "logger" +require "active_support/core_ext/hash" + +class Migrator + + attr_accessor :username, :base_url, :environment, :settings, :logger + + def initialize(username) + @username = username + + @environment = ENV["ENVIRONMENT"] || "staging" + @settings = YAML.load(File.read('config.yml'))[@environment] + + @logger = Logger.new("log/migrate_storage_size_from_metadata.log") + log_level = ENV["LOGLEVEL"] || "INFO" + logger.level = Kernel.const_get "Logger::#{log_level}" + logger.progname = username + end + + def migrate + logger.info "Starting migration for '#{username}'" + set_container_migration_state("in_progress") + begin + write_storage_size_from_redis_metadata(username) + rescue Exception => ex + logger.error "Error setting storage size from metadata for '#{username}': #{ex}" + set_container_migration_state("not_started") + # write username to file for later reference + File.open('log/failed_migration.log', 'a') { |f| f.puts username } + exit 1 + end + delete_container_migration_state + logger.info "Finished migration for '#{username}'" + end + + def redis + @redis ||= Redis.new(@settings["redis"].symbolize_keys) + end + + def write_storage_size_from_redis_metadata(user) + lua_script = <<-EOF + local user = ARGV[1] + local total_size = 0 + local size_key = KEYS[1] + + local function get_size_from_items(parent, directory) + local path + if parent == "/" then + path = directory + else + path = parent..directory + end + local items = redis.call("smembers", "rs:m:"..user..":"..path..":items") + for index, name in pairs(items) do + local redis_key = "rs:m:"..user..":" + + redis_key = redis_key..path..name + + -- if it's a directory, get the items inside of it + if string.match(name, "/$") then + get_size_from_items(path, name) + -- if it's a file, get its size + else + local file_size = redis.call("hget", redis_key, "s") + total_size = total_size + file_size + end + end + end + + get_size_from_items("", "") -- Start from the root + + redis.call("set", size_key, total_size) + EOF + + redis.eval(lua_script, ["rs:s:#{user}"], [user]) + end + + def set_container_migration_state(type) + redis.hset("rs:size_migration", username, type) + end + + def delete_container_migration_state + redis.hdel("rs:size_migration", username) + end + +end + +class MigrationRunner + attr_accessor :environment, :settings + + def initialize + @environment = ENV["ENVIRONMENT"] || "staging" + @settings = YAML.load(File.read('config.yml'))[@environment] + end + + def migrate + while username = pick_unmigrated_user + migrator = Migrator.new username + migrator.migrate + end + end + + def unmigrated_users + redis.hgetall("rs:size_migration").select { |_, value| + value == "not_started" + }.keys + end + + def pick_unmigrated_user + unmigrated_users.sample # pick a random user from list + end + + def redis + @redis ||= Redis.new(@settings["redis"].symbolize_keys) + end + +end + +username = ARGV[0] + +if username + migrator = Migrator.new username + migrator.migrate +else + runner = MigrationRunner.new + runner.migrate +end