diff --git a/lib/remote_storage/swift.rb b/lib/remote_storage/swift.rb index 4c9ed0e..ec7544c 100644 --- a/lib/remote_storage/swift.rb +++ b/lib/remote_storage/swift.rb @@ -20,10 +20,6 @@ module RemoteStorage def authorize_request(user, directory, token, listing=false) request_method = server.env["REQUEST_METHOD"] - if request_method.match(/PUT|DELETE/) && container_migration(user) == "in_progress" - server.halt 503, "Down for maintenance. Back soon!" - end - if directory.split("/").first == "public" return true if ["GET", "HEAD"].include?(request_method) && !listing end @@ -383,11 +379,7 @@ module RemoteStorage end def container_url_for(user) - if container_migration(user) - "#{base_url}/rs:#{settings.environment.to_s.chars.first}:#{user}" - else - "#{base_url}/rs:documents:#{settings.environment.to_s}/#{user}" - end + "#{base_url}/rs:documents:#{settings.environment.to_s}/#{user}" end def url_for_key(user, directory, key) @@ -398,10 +390,6 @@ module RemoteStorage @base_url ||= settings.swift["host"] end - def container_migration(user) - redis.hget("rs:container_migration", user) - end - def default_headers {"x-auth-token" => swift_token} end diff --git a/migrate_metadata_to_redis.rb b/migrate_metadata_to_redis.rb deleted file mode 100755 index df3705f..0000000 --- a/migrate_metadata_to_redis.rb +++ /dev/null @@ -1,248 +0,0 @@ -#!/usr/bin/env ruby - -require "rubygems" -require "bundler/setup" -require "rest_client" -require "redis" -require "yaml" -require "logger" -require "active_support/core_ext/hash" - -class Migrator - - attr_accessor :username, :base_url, :swift_host, :swift_token, - :environment, :dry_run, :settings, :logger - - def initialize(username) - @username = username - - @environment = ENV["ENVIRONMENT"] || "staging" - @settings = YAML.load(File.read('config.yml'))[@environment] - - @swift_host = @settings["swift"]["host"] - @swift_token = File.read("tmp/swift_token.txt").strip - - @dry_run = ENV["DRYRUN"] || false # disables writing anything to Redis when true - - @logger = Logger.new("log/migrate_metadata_to_redis.log") - log_level = ENV["LOGLEVEL"] || "INFO" - logger.level = Kernel.const_get "Logger::#{log_level}" - logger.progname = username - end - - def root_url - "#{@base_url}/#{@username}" - end - - def is_dir?(name) - name[-1] == "/" - end - - def url_for(directory, parent_directory="") - "#{root_url}#{parent_directory}#{directory}" - end - - def migrate - logger.info "Starting migration for '#{username}'" - set_directory_backend("legacy_locked") - begin - work_on_dir("", "") - rescue Exception => ex - logger.error "Error migrating metadata for '#{username}': #{ex}" - set_directory_backend("legacy") - # write username to file for later reference - File.open('log/failed_migration.log', 'a') { |f| f.puts username } - exit 1 - end - set_directory_backend("new") - logger.info "Finished migration for '#{username}'" - end - - def set_directory_backend(backend) - redis.set("rsc:db:#{username}", backend) unless dry_run - end - - def work_on_dir(directory, parent_directory) - logger.debug "Retrieving listing for '#{parent_directory}#{directory}'" - - listing = get_directory_listing_from_swift("#{parent_directory}#{directory}") - - timestamp = (Time.now.to_f * 1000).to_i - - if listing["items"].any? - items = listing["items"] - items.each do |item, data| - if is_dir? item - save_directory_data("#{parent_directory}#{directory}", item, data, timestamp) - - # get dir listing and repeat - work_on_dir(item, "#{parent_directory}#{directory}") - else - save_document_data("#{parent_directory}#{directory}", item, data) - end - - add_item_to_parent_dir("#{parent_directory}#{directory}", item) - end - end - end - - def add_item_to_parent_dir(dir, item) - key = "rs:m:#{username}:#{parent_directory_for(dir)}:items" - logger.debug "Adding item #{item} to #{key}" - redis.sadd(key, item) unless dry_run - end - - def save_directory_data(dir, item, data, timestamp) - key = "rs:m:#{username}:#{dir.gsub(/^\//, "")}#{item}" - metadata = { - e: data["ETag"], - m: timestamp_for(data["Last-Modified"]) - } - - logger.debug "Metadata for dir #{key}: #{metadata}" - redis.hmset(key, *metadata) unless dry_run - end - - def save_document_data(dir, item, data) - key = "rs:m:#{username}:#{dir.gsub(/^\//, "")}#{item}" - metadata = { - e: data["ETag"], - s: data["Content-Length"], - t: data["Content-Type"], - m: timestamp_for(data["Last-Modified"]) - } - logger.debug "Metadata for document #{key}: #{metadata}" - redis.hmset(key, *metadata) unless dry_run - end - - def parent_directory_for(directory) - if directory.match(/\//) - return directory[0..directory.rindex("/")] - else - return "/" - end - end - - def timestamp_for(date) - return DateTime.parse(date).strftime("%Q").to_i - end - - def redis - @redis ||= Redis.new(@settings["redis"].symbolize_keys) - end - - def get_directory_listing_from_swift(directory) - is_root_listing = directory.empty? - - get_response = nil - - do_head_request("#{url_for_directory(@username, directory)}") do |response| - return directory_listing([]) if response.code == 404 - - if is_root_listing - get_response = do_get_request("#{container_url_for(@username)}/?format=json&path=") - else - get_response = do_get_request("#{container_url_for(@username)}/?format=json&path=#{escape(directory)}") - end - end - - if body = JSON.parse(get_response.body) - listing = directory_listing(body) - else - puts "listing not JSON" - end - - listing - end - - def directory_listing(res_body) - listing = { - "@context" => "http://remotestorage.io/spec/folder-description", - "items" => {} - } - - res_body.each do |entry| - name = entry["name"] - name.sub!("#{File.dirname(entry["name"])}/", '') - if name[-1] == "/" # It's a directory - listing["items"].merge!({ - name => { - "ETag" => entry["hash"], - "Last-Modified" => entry["last_modified"] - } - }) - else # It's a file - listing["items"].merge!({ - name => { - "ETag" => entry["hash"], - "Content-Type" => entry["content_type"], - "Content-Length" => entry["bytes"], - "Last-Modified" => entry["last_modified"] - } - }) - end - end - - listing - end - - def etag_for(body) - objects = JSON.parse(body) - - if objects.empty? - Digest::MD5.hexdigest "" - else - Digest::MD5.hexdigest objects.map { |o| o["hash"] }.join - end - end - - def do_head_request(url, &block) - RestClient.head(url, default_headers, &block) - end - - def do_get_request(url, &block) - RestClient.get(url, default_headers, &block) - end - - def default_headers - {"x-auth-token" => @swift_token} - end - - def url_for_directory(user, directory) - if directory.empty? - container_url_for(user) - else - "#{container_url_for(user)}/#{escape(directory)}" - end - end - - def container_url_for(user) - "#{base_url}/#{container_for(user)}" - end - - def base_url - @base_url ||= @swift_host - end - - def container_for(user) - "rs:#{environment.to_s.chars.first}:#{user}" - end - - def escape(url) - # We want spaces to turn into %20 and slashes to stay slashes - CGI::escape(url).gsub('+', '%20').gsub('%2F', '/') - end -end - -username = ARGV[0] - -unless username - puts "No username given." - puts "Usage:" - puts "ENVIRONMENT=staging ./migrate_metadata_to_redis.rb " - exit 1 -end - -migrator = Migrator.new username -migrator.migrate - diff --git a/migrate_root_metadata_to_redis.rb b/migrate_root_metadata_to_redis.rb deleted file mode 100755 index 98b3a27..0000000 --- a/migrate_root_metadata_to_redis.rb +++ /dev/null @@ -1,110 +0,0 @@ -#!/usr/bin/env ruby - -require "rubygems" -require "bundler/setup" -require "rest_client" -require "redis" -require "yaml" -require "logger" -require "active_support/core_ext/hash" - -class Migrator - - attr_accessor :username, :base_url, :swift_host, :swift_token, - :environment, :dry_run, :settings, :logger - - def initialize(username) - @username = username - - @environment = ENV["ENVIRONMENT"] || "staging" - @settings = YAML.load(File.read('config.yml'))[@environment] - - @swift_host = @settings["swift"]["host"] - @swift_token = File.read("tmp/swift_token.txt").strip - - @dry_run = ENV["DRYRUN"] || false # disables writing anything to Redis when true - - @logger = Logger.new("log/migrate_root_metadata_to_redis.log") - log_level = ENV["LOGLEVEL"] || "INFO" - logger.level = Kernel.const_get "Logger::#{log_level}" - logger.progname = username - end - - def root_url - "#{@base_url}/#{@username}" - end - - def migrate - logger.info "Starting migration for '#{username}'" - begin - work_on_dir("") - rescue Exception => ex - logger.error "Error migrating metadata for '#{username}': #{ex}" - # write username to file for later reference - File.open('log/failed_root_migration.log', 'a') { |f| f.puts username } - exit 1 - end - logger.info "Finished migration for '#{username}'" - end - - def work_on_dir(directory) - logger.debug "Retrieving root metadata for '#{username}'" - - response = do_head_request("#{container_url_for(@username)}") - - save_root_directory_data(response) - end - - def save_root_directory_data(response) - key = "rs:m:#{username}:/" - - metadata = { - e: etag_for(response.headers[:x_timestamp], response.headers[:x_trans_id]), - m: (response.headers[:x_timestamp].to_f * 1000).to_i - } - - logger.debug "Metadata for dir #{key}: #{metadata}" - redis.hmset(key, *metadata) unless dry_run - end - - def etag_for(*args) - Digest::MD5.hexdigest args.join(":") - end - - def redis - @redis ||= Redis.new(@settings["redis"].symbolize_keys) - end - - def do_head_request(url, &block) - RestClient.head(url, default_headers, &block) - end - - def default_headers - {"x-auth-token" => @swift_token} - end - - def container_url_for(user) - "#{base_url}/#{container_for(user)}" - end - - def base_url - @base_url ||= @swift_host - end - - def container_for(user) - "rs:#{environment.to_s.chars.first}:#{user}" - end -end - -username = ARGV[0] - -unless username - puts "No username given." - puts "Usage:" - puts "ENVIRONMENT=staging ./migrate_root_metadata_to_redis.rb " - exit 1 -end - -migrator = Migrator.new username -migrator.migrate - diff --git a/migrate_to_single_container.rb b/migrate_to_single_container.rb deleted file mode 100755 index fa76a18..0000000 --- a/migrate_to_single_container.rb +++ /dev/null @@ -1,181 +0,0 @@ -#!/usr/bin/env ruby - -require "rubygems" -require "bundler/setup" -require "rest_client" -require "redis" -require "yaml" -require "logger" -require "json" -require "active_support/core_ext/hash" - -class Migrator - - attr_accessor :username, :base_url, :swift_host, :swift_token, - :environment, :dry_run, :settings, :logger - - def initialize(username) - @username = username - - @environment = ENV["ENVIRONMENT"] || "staging" - @settings = YAML.load(File.read('config.yml'))[@environment] - - @swift_host = @settings["swift"]["host"] - @swift_token = File.read("tmp/swift_token.txt").strip - - @dry_run = ENV["DRYRUN"] || false # disables writing anything when true - - @logger = Logger.new("log/migrate_to_single_container.log") - log_level = ENV["LOGLEVEL"] || "INFO" - logger.level = Kernel.const_get "Logger::#{log_level}" - logger.progname = username - end - - def migrate - logger.info "Starting migration for '#{username}'" - set_container_migration_state("in_progress") - begin - copy_all_documents - rescue Exception => ex - logger.error "Error migrating documents for '#{username}': #{ex}" - set_container_migration_state("not_started") - # write username to file for later reference - File.open('log/failed_migration.log', 'a') { |f| f.puts username } - exit 1 - end - delete_container_migration_state - File.open('log/finished_migration.log', 'a') { |f| f.puts username } - logger.info "Finished migration for '#{username}'" - end - - def is_document?(name) - name[-1] != "/" - end - - def set_container_migration_state(type) - redis.hset("rs:container_migration", username, type) unless dry_run - end - - def delete_container_migration_state - redis.hdel("rs:container_migration", username) unless dry_run - end - - def copy_all_documents - logger.debug "Retrieving object listing" - - listing = get_directory_listing_from_swift - - logger.debug "Full listing: #{listing}" - - if listing - - # skip user when there are more files than we can list - if listing.split("\n").size > 9999 - File.open('log/10k_users.log', 'a') { |f| f.puts username } - raise "User has too many files" - end - - listing.split("\n").each do |item| - if is_document? item - copy_document(item) - end - end - - end - end - - def copy_document(document_path) - old_document_url = "#{container_url_for(@username)}/#{escape(document_path)}" - - new_document_path = "rs:documents:#{environment.to_s.downcase}/#{@username}/#{escape(document_path)}" - - logger.debug "Copying document from #{old_document_url} to #{new_document_path}" - do_copy_request(old_document_url, new_document_path) unless dry_run - end - - def redis - @redis ||= Redis.new(@settings["redis"].symbolize_keys) - end - - def get_directory_listing_from_swift - get_response = do_get_request("#{container_url_for(@username)}/?prefix=") - - get_response.body - end - - def do_get_request(url, &block) - RestClient.get(url, default_headers, &block) - end - - def do_copy_request(url, destination_path) - RestClient::Request.execute( - method: :copy, - url: url, - headers: default_headers.merge({destination: destination_path}) - ) - end - - def default_headers - {"x-auth-token" => @swift_token} - end - - def container_url_for(user) - "#{base_url}/#{container_for(user)}" - end - - def base_url - @base_url ||= @swift_host - end - - def container_for(user) - "rs:#{environment.to_s.chars.first}:#{user}" - end - - def escape(url) - # We want spaces to turn into %20 and slashes to stay slashes - CGI::escape(url).gsub('+', '%20').gsub('%2F', '/') - end -end - -class MigrationRunner - - attr_accessor :environment, :settings - - def initialize - @environment = ENV["ENVIRONMENT"] || "staging" - @settings = YAML.load(File.read('config.yml'))[@environment] - end - - def migrate - while username = pick_unmigrated_user - migrator = Migrator.new username - migrator.migrate - end - end - - def unmigrated_users - redis.hgetall("rs:container_migration").select { |_, value| - value == "not_started" - }.keys - end - - def pick_unmigrated_user - unmigrated_users.sample # pick a random user from list - end - - def redis - @redis ||= Redis.new(@settings["redis"].symbolize_keys) - end - -end - -username = ARGV[0] - -if username - migrator = Migrator.new username - migrator.migrate -else - runner = MigrationRunner.new - runner.migrate -end -