From be43a7474a8f9916eaa68077e42c8c2f88d8e13a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Greg=20Kar=C3=A9kinian?= Date: Tue, 18 Oct 2016 18:03:02 +0200 Subject: [PATCH 1/5] Remove line break from the token file when loading it --- lib/remote_storage/swift.rb | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/lib/remote_storage/swift.rb b/lib/remote_storage/swift.rb index ec7544c..ed527bb 100644 --- a/lib/remote_storage/swift.rb +++ b/lib/remote_storage/swift.rb @@ -433,7 +433,10 @@ module RemoteStorage def reload_swift_token server.logger.debug "Reloading swift token. Old token: #{settings.swift_token}" - settings.swift_token = File.read(swift_token_path) + # Remove the line break from the token file. The line break that the + # token script is adding to the file was causing Sentry to reject the + # token field + settings.swift_token = File.read(swift_token_path).rstrip settings.swift_token_loaded_at = Time.now server.logger.debug "Reloaded swift token. New token: #{settings.swift_token}" end From c1ed996b1dce056982229c1d3105ff3956a668c5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Greg=20Kar=C3=A9kinian?= Date: Wed, 19 Oct 2016 15:05:30 +0200 Subject: [PATCH 2/5] Only send the first 20 characters of the token --- lib/remote_storage/swift.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/remote_storage/swift.rb b/lib/remote_storage/swift.rb index ed527bb..2fb5fa4 100644 --- a/lib/remote_storage/swift.rb +++ b/lib/remote_storage/swift.rb @@ -457,7 +457,7 @@ module RemoteStorage rescue RestClient::Unauthorized => ex Raven.capture_exception( ex, - tags: { swift_token: settings.swift_token, + tags: { swift_token: settings.swift_token[0..19], # send the first 20 characters swift_token_loaded_at: settings.swift_token_loaded_at } ) server.halt 500 From a7c58085936011d2795b6deacd01075cb4326e78 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Greg=20Kar=C3=A9kinian?= Date: Thu, 15 Dec 2016 13:19:17 +0100 Subject: [PATCH 3/5] Add a script to calculate a user's storage size from the metadata... ... And write it to Redis Usage: ENVIRONMENT=development ./migrate_storage_size_from_metadata.rb username --- migrate_storage_size_from_metadata.rb | 95 +++++++++++++++++++++++++++ 1 file changed, 95 insertions(+) create mode 100755 migrate_storage_size_from_metadata.rb diff --git a/migrate_storage_size_from_metadata.rb b/migrate_storage_size_from_metadata.rb new file mode 100755 index 0000000..4059d9d --- /dev/null +++ b/migrate_storage_size_from_metadata.rb @@ -0,0 +1,95 @@ +#!/usr/bin/env ruby + +require "rubygems" +require "bundler/setup" +require "rest_client" +require "redis" +require "yaml" +require "logger" +require "active_support/core_ext/hash" + +class Migrator + + attr_accessor :username, :base_url, :environment, :settings, :logger + + def initialize(username) + @username = username + + @environment = ENV["ENVIRONMENT"] || "staging" + @settings = YAML.load(File.read('config.yml'))[@environment] + + @logger = Logger.new("log/migrate_storage_size_from_metadata.log") + log_level = ENV["LOGLEVEL"] || "INFO" + logger.level = Kernel.const_get "Logger::#{log_level}" + logger.progname = username + end + + def migrate + logger.info "Starting migration for '#{username}'" + begin + write_storage_size_from_redis_metadata(username) + rescue Exception => ex + logger.error "Error setting storage size from metadata for '#{username}': #{ex}" + # write username to file for later reference + File.open('log/failed_migration.log', 'a') { |f| f.puts username } + exit 1 + end + logger.info "Finished migration for '#{username}'" + end + + def redis + @redis ||= Redis.new(@settings["redis"].symbolize_keys) + end + + def write_storage_size_from_redis_metadata(user) + lua_script = <<-EOF + local user = ARGV[1] + local total_size = 0 + local size_key = KEYS[1] + + local function get_size_from_items(parent, directory) + local path + if parent == "/" then + path = directory + else + path = parent..directory + end + local items = redis.call("smembers", "rs:m:"..user..":"..path..":items") + for index, name in pairs(items) do + local redis_key = "rs:m:"..user..":" + + redis_key = redis_key..path..name + + -- if it's a directory, get the items inside of it + if string.match(name, "/$") then + get_size_from_items(path, name) + -- if it's a file, get its size + else + local file_size = redis.call("hget", redis_key, "s") + total_size = total_size + file_size + end + end + end + + get_size_from_items("", "") -- Start from the root + + redis.call("set", size_key, total_size) + EOF + + redis.eval(lua_script, ["rs:s:#{user}", "test"], [user]) + end + +end + +username = ARGV[0] + +unless username + puts "No username given." + puts "Usage:" + puts "ENVIRONMENT=staging ./migrate_storage_size_from_metadata.rb " + exit 1 +end + +migrator = Migrator.new username +migrator.migrate + From 97a77e2a2cff6fc0ff023c2a3ea1adee8747a851 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Greg=20Kar=C3=A9kinian?= Date: Thu, 15 Dec 2016 15:34:51 +0100 Subject: [PATCH 4/5] Remove unused test key passed to the eval command --- migrate_storage_size_from_metadata.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/migrate_storage_size_from_metadata.rb b/migrate_storage_size_from_metadata.rb index 4059d9d..6d854fe 100755 --- a/migrate_storage_size_from_metadata.rb +++ b/migrate_storage_size_from_metadata.rb @@ -76,7 +76,7 @@ class Migrator redis.call("set", size_key, total_size) EOF - redis.eval(lua_script, ["rs:s:#{user}", "test"], [user]) + redis.eval(lua_script, ["rs:s:#{user}"], [user]) end end From 938666661083d88672090acbb1c7bca1195ef3be Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Greg=20Kar=C3=A9kinian?= Date: Thu, 15 Dec 2016 16:28:19 +0100 Subject: [PATCH 5/5] Add a migration runner to be able to run the script on all users... ... on as many servers as we want Set the migration key in platform for all users: User.storage_customer.each do |u| FiveAppsStore::Application.redis_remotestorage.hset( "rs:size_migration", u.username, "not_started" ) end --- migrate_storage_size_from_metadata.rb | 57 ++++++++++++++++++++++----- 1 file changed, 48 insertions(+), 9 deletions(-) diff --git a/migrate_storage_size_from_metadata.rb b/migrate_storage_size_from_metadata.rb index 6d854fe..bd04bd3 100755 --- a/migrate_storage_size_from_metadata.rb +++ b/migrate_storage_size_from_metadata.rb @@ -26,14 +26,17 @@ class Migrator def migrate logger.info "Starting migration for '#{username}'" + set_container_migration_state("in_progress") begin write_storage_size_from_redis_metadata(username) rescue Exception => ex logger.error "Error setting storage size from metadata for '#{username}': #{ex}" + set_container_migration_state("not_started") # write username to file for later reference File.open('log/failed_migration.log', 'a') { |f| f.puts username } exit 1 end + delete_container_migration_state logger.info "Finished migration for '#{username}'" end @@ -79,17 +82,53 @@ class Migrator redis.eval(lua_script, ["rs:s:#{user}"], [user]) end + def set_container_migration_state(type) + redis.hset("rs:size_migration", username, type) + end + + def delete_container_migration_state + redis.hdel("rs:size_migration", username) + end + +end + +class MigrationRunner + attr_accessor :environment, :settings + + def initialize + @environment = ENV["ENVIRONMENT"] || "staging" + @settings = YAML.load(File.read('config.yml'))[@environment] + end + + def migrate + while username = pick_unmigrated_user + migrator = Migrator.new username + migrator.migrate + end + end + + def unmigrated_users + redis.hgetall("rs:size_migration").select { |_, value| + value == "not_started" + }.keys + end + + def pick_unmigrated_user + unmigrated_users.sample # pick a random user from list + end + + def redis + @redis ||= Redis.new(@settings["redis"].symbolize_keys) + end + end username = ARGV[0] -unless username - puts "No username given." - puts "Usage:" - puts "ENVIRONMENT=staging ./migrate_storage_size_from_metadata.rb " - exit 1 +if username + migrator = Migrator.new username + migrator.migrate +else + runner = MigrationRunner.new + runner.migrate end - -migrator = Migrator.new username -migrator.migrate -