5apps/liquor-cabinet
5apps/liquor-cabinet
4
0
Fork 0
Code Issues 1 Pull Requests Packages Releases Activity

Merge branch 'master' into stable

Browse Source
This commit is contained in:
Greg Karékinian 2016-09-16 11:25:50 +02:00
commit 28a6d1700b
4 changed files with 1 additions and 552 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

14
lib/remote_storage/swift.rb
View File

@ -20,10 +20,6 @@ module RemoteStorage
def authorize_request(user, directory, token, listing=false) def authorize_request(user, directory, token, listing=false)
request_method = server.env["REQUEST_METHOD"] request_method = server.env["REQUEST_METHOD"]
if request_method.match(/PUT|DELETE/) && container_migration(user) == "in_progress"
server.halt 503, "Down for maintenance. Back soon!"
end
if directory.split("/").first == "public" if directory.split("/").first == "public"
return true if ["GET", "HEAD"].include?(request_method) && !listing return true if ["GET", "HEAD"].include?(request_method) && !listing
end end
@ -383,11 +379,7 @@ module RemoteStorage
end end
def container_url_for(user) def container_url_for(user)
if container_migration(user) "#{base_url}/rs:documents:#{settings.environment.to_s}/#{user}"
"#{base_url}/rs:#{settings.environment.to_s.chars.first}:#{user}"
else
"#{base_url}/rs:documents:#{settings.environment.to_s}/#{user}"
end
end end
def url_for_key(user, directory, key) def url_for_key(user, directory, key)
@ -398,10 +390,6 @@ module RemoteStorage
@base_url ||= settings.swift["host"] @base_url ||= settings.swift["host"]
end end
def container_migration(user)
redis.hget("rs:container_migration", user)
end
def default_headers def default_headers
{"x-auth-token" => swift_token} {"x-auth-token" => swift_token}
end end

248
migrate_metadata_to_redis.rb
View File

@ -1,248 +0,0 @@
#!/usr/bin/env ruby
require "rubygems"
require "bundler/setup"
require "rest_client"
require "redis"
require "yaml"
require "logger"
require "active_support/core_ext/hash"
class Migrator
attr_accessor :username, :base_url, :swift_host, :swift_token,
:environment, :dry_run, :settings, :logger
def initialize(username)
@username = username
@environment = ENV["ENVIRONMENT"] || "staging"
@settings = YAML.load(File.read('config.yml'))[@environment]
@swift_host = @settings["swift"]["host"]
@swift_token = File.read("tmp/swift_token.txt").strip
@dry_run = ENV["DRYRUN"] || false # disables writing anything to Redis when true
@logger = Logger.new("log/migrate_metadata_to_redis.log")
log_level = ENV["LOGLEVEL"] || "INFO"
logger.level = Kernel.const_get "Logger::#{log_level}"
logger.progname = username
end
def root_url
"#{@base_url}/#{@username}"
end
def is_dir?(name)
name[-1] == "/"
end
def url_for(directory, parent_directory="")
"#{root_url}#{parent_directory}#{directory}"
end
def migrate
logger.info "Starting migration for '#{username}'"
set_directory_backend("legacy_locked")
begin
work_on_dir("", "")
rescue Exception => ex
logger.error "Error migrating metadata for '#{username}': #{ex}"
set_directory_backend("legacy")
# write username to file for later reference
File.open('log/failed_migration.log', 'a') { |f| f.puts username }
exit 1
end
set_directory_backend("new")
logger.info "Finished migration for '#{username}'"
end
def set_directory_backend(backend)
redis.set("rsc:db:#{username}", backend) unless dry_run
end
def work_on_dir(directory, parent_directory)
logger.debug "Retrieving listing for '#{parent_directory}#{directory}'"
listing = get_directory_listing_from_swift("#{parent_directory}#{directory}")
timestamp = (Time.now.to_f * 1000).to_i
if listing["items"].any?
items = listing["items"]
items.each do |item, data|
if is_dir? item
save_directory_data("#{parent_directory}#{directory}", item, data, timestamp)
# get dir listing and repeat
work_on_dir(item, "#{parent_directory}#{directory}")
else
save_document_data("#{parent_directory}#{directory}", item, data)
end
add_item_to_parent_dir("#{parent_directory}#{directory}", item)
end
end
end
def add_item_to_parent_dir(dir, item)
key = "rs:m:#{username}:#{parent_directory_for(dir)}:items"
logger.debug "Adding item #{item} to #{key}"
redis.sadd(key, item) unless dry_run
end
def save_directory_data(dir, item, data, timestamp)
key = "rs:m:#{username}:#{dir.gsub(/^\//, "")}#{item}"
metadata = {
e: data["ETag"],
m: timestamp_for(data["Last-Modified"])
}
logger.debug "Metadata for dir #{key}: #{metadata}"
redis.hmset(key, *metadata) unless dry_run
end
def save_document_data(dir, item, data)
key = "rs:m:#{username}:#{dir.gsub(/^\//, "")}#{item}"
metadata = {
e: data["ETag"],
s: data["Content-Length"],
t: data["Content-Type"],
m: timestamp_for(data["Last-Modified"])
}
logger.debug "Metadata for document #{key}: #{metadata}"
redis.hmset(key, *metadata) unless dry_run
end
def parent_directory_for(directory)
if directory.match(/\//)
return directory[0..directory.rindex("/")]
else
return "/"
end
end
def timestamp_for(date)
return DateTime.parse(date).strftime("%Q").to_i
end
def redis
@redis ||= Redis.new(@settings["redis"].symbolize_keys)
end
def get_directory_listing_from_swift(directory)
is_root_listing = directory.empty?
get_response = nil
do_head_request("#{url_for_directory(@username, directory)}") do |response|
return directory_listing([]) if response.code == 404
if is_root_listing
get_response = do_get_request("#{container_url_for(@username)}/?format=json&path=")
else
get_response = do_get_request("#{container_url_for(@username)}/?format=json&path=#{escape(directory)}")
end
end
if body = JSON.parse(get_response.body)
listing = directory_listing(body)
else
puts "listing not JSON"
end
listing
end
def directory_listing(res_body)
listing = {
"@context" => "http://remotestorage.io/spec/folder-description",
"items" => {}
}
res_body.each do |entry|
name = entry["name"]
name.sub!("#{File.dirname(entry["name"])}/", '')
if name[-1] == "/" # It's a directory
listing["items"].merge!({
name => {
"ETag" => entry["hash"],
"Last-Modified" => entry["last_modified"]
}
})
else # It's a file
listing["items"].merge!({
name => {
"ETag" => entry["hash"],
"Content-Type" => entry["content_type"],
"Content-Length" => entry["bytes"],
"Last-Modified" => entry["last_modified"]
}
})
end
end
listing
end
def etag_for(body)
objects = JSON.parse(body)
if objects.empty?
Digest::MD5.hexdigest ""
else
Digest::MD5.hexdigest objects.map { |o| o["hash"] }.join
end
end
def do_head_request(url, &block)
RestClient.head(url, default_headers, &block)
end
def do_get_request(url, &block)
RestClient.get(url, default_headers, &block)
end
def default_headers
{"x-auth-token" => @swift_token}
end
def url_for_directory(user, directory)
if directory.empty?
container_url_for(user)
else
"#{container_url_for(user)}/#{escape(directory)}"
end
end
def container_url_for(user)
"#{base_url}/#{container_for(user)}"
end
def base_url
@base_url ||= @swift_host
end
def container_for(user)
"rs:#{environment.to_s.chars.first}:#{user}"
end
def escape(url)
# We want spaces to turn into %20 and slashes to stay slashes
CGI::escape(url).gsub('+', '%20').gsub('%2F', '/')
end
end
username = ARGV[0]
unless username
puts "No username given."
puts "Usage:"
puts "ENVIRONMENT=staging ./migrate_metadata_to_redis.rb <username>"
exit 1
end
migrator = Migrator.new username
migrator.migrate

110
migrate_root_metadata_to_redis.rb
View File

@ -1,110 +0,0 @@
#!/usr/bin/env ruby
require "rubygems"
require "bundler/setup"
require "rest_client"
require "redis"
require "yaml"
require "logger"
require "active_support/core_ext/hash"
class Migrator
attr_accessor :username, :base_url, :swift_host, :swift_token,
:environment, :dry_run, :settings, :logger
def initialize(username)
@username = username
@environment = ENV["ENVIRONMENT"] || "staging"
@settings = YAML.load(File.read('config.yml'))[@environment]
@swift_host = @settings["swift"]["host"]
@swift_token = File.read("tmp/swift_token.txt").strip
@dry_run = ENV["DRYRUN"] || false # disables writing anything to Redis when true
@logger = Logger.new("log/migrate_root_metadata_to_redis.log")
log_level = ENV["LOGLEVEL"] || "INFO"
logger.level = Kernel.const_get "Logger::#{log_level}"
logger.progname = username
end
def root_url
"#{@base_url}/#{@username}"
end
def migrate
logger.info "Starting migration for '#{username}'"
begin
work_on_dir("")
rescue Exception => ex
logger.error "Error migrating metadata for '#{username}': #{ex}"
# write username to file for later reference
File.open('log/failed_root_migration.log', 'a') { |f| f.puts username }
exit 1
end
logger.info "Finished migration for '#{username}'"
end
def work_on_dir(directory)
logger.debug "Retrieving root metadata for '#{username}'"
response = do_head_request("#{container_url_for(@username)}")
save_root_directory_data(response)
end
def save_root_directory_data(response)
key = "rs:m:#{username}:/"
metadata = {
e: etag_for(response.headers[:x_timestamp], response.headers[:x_trans_id]),
m: (response.headers[:x_timestamp].to_f * 1000).to_i
}
logger.debug "Metadata for dir #{key}: #{metadata}"
redis.hmset(key, *metadata) unless dry_run
end
def etag_for(*args)
Digest::MD5.hexdigest args.join(":")
end
def redis
@redis ||= Redis.new(@settings["redis"].symbolize_keys)
end
def do_head_request(url, &block)
RestClient.head(url, default_headers, &block)
end
def default_headers
{"x-auth-token" => @swift_token}
end
def container_url_for(user)
"#{base_url}/#{container_for(user)}"
end
def base_url
@base_url ||= @swift_host
end
def container_for(user)
"rs:#{environment.to_s.chars.first}:#{user}"
end
end
username = ARGV[0]
unless username
puts "No username given."
puts "Usage:"
puts "ENVIRONMENT=staging ./migrate_root_metadata_to_redis.rb <username>"
exit 1
end
migrator = Migrator.new username
migrator.migrate

181
migrate_to_single_container.rb
View File

@ -1,181 +0,0 @@
#!/usr/bin/env ruby
require "rubygems"
require "bundler/setup"
require "rest_client"
require "redis"
require "yaml"
require "logger"
require "json"
require "active_support/core_ext/hash"
class Migrator
attr_accessor :username, :base_url, :swift_host, :swift_token,
:environment, :dry_run, :settings, :logger
def initialize(username)
@username = username
@environment = ENV["ENVIRONMENT"] || "staging"
@settings = YAML.load(File.read('config.yml'))[@environment]
@swift_host = @settings["swift"]["host"]
@swift_token = File.read("tmp/swift_token.txt").strip
@dry_run = ENV["DRYRUN"] || false # disables writing anything when true
@logger = Logger.new("log/migrate_to_single_container.log")
log_level = ENV["LOGLEVEL"] || "INFO"
logger.level = Kernel.const_get "Logger::#{log_level}"
logger.progname = username
end
def migrate
logger.info "Starting migration for '#{username}'"
set_container_migration_state("in_progress")
begin
copy_all_documents
rescue Exception => ex
logger.error "Error migrating documents for '#{username}': #{ex}"
set_container_migration_state("not_started")
# write username to file for later reference
File.open('log/failed_migration.log', 'a') { |f| f.puts username }
exit 1
end
delete_container_migration_state
File.open('log/finished_migration.log', 'a') { |f| f.puts username }
logger.info "Finished migration for '#{username}'"
end
def is_document?(name)
name[-1] != "/"
end
def set_container_migration_state(type)
redis.hset("rs:container_migration", username, type) unless dry_run
end
def delete_container_migration_state
redis.hdel("rs:container_migration", username) unless dry_run
end
def copy_all_documents
logger.debug "Retrieving object listing"
listing = get_directory_listing_from_swift
logger.debug "Full listing: #{listing}"
if listing
# skip user when there are more files than we can list
if listing.split("\n").size > 9999
File.open('log/10k_users.log', 'a') { |f| f.puts username }
raise "User has too many files"
end
listing.split("\n").each do |item|
if is_document? item
copy_document(item)
end
end
end
end
def copy_document(document_path)
old_document_url = "#{container_url_for(@username)}/#{escape(document_path)}"
new_document_path = "rs:documents:#{environment.to_s.downcase}/#{@username}/#{escape(document_path)}"
logger.debug "Copying document from #{old_document_url} to #{new_document_path}"
do_copy_request(old_document_url, new_document_path) unless dry_run
end
def redis
@redis ||= Redis.new(@settings["redis"].symbolize_keys)
end
def get_directory_listing_from_swift
get_response = do_get_request("#{container_url_for(@username)}/?prefix=")
get_response.body
end
def do_get_request(url, &block)
RestClient.get(url, default_headers, &block)
end
def do_copy_request(url, destination_path)
RestClient::Request.execute(
method: :copy,
url: url,
headers: default_headers.merge({destination: destination_path})
)
end
def default_headers
{"x-auth-token" => @swift_token}
end
def container_url_for(user)
"#{base_url}/#{container_for(user)}"
end
def base_url
@base_url ||= @swift_host
end
def container_for(user)
"rs:#{environment.to_s.chars.first}:#{user}"
end
def escape(url)
# We want spaces to turn into %20 and slashes to stay slashes
CGI::escape(url).gsub('+', '%20').gsub('%2F', '/')
end
end
class MigrationRunner
attr_accessor :environment, :settings
def initialize
@environment = ENV["ENVIRONMENT"] || "staging"
@settings = YAML.load(File.read('config.yml'))[@environment]
end
def migrate
while username = pick_unmigrated_user
migrator = Migrator.new username
migrator.migrate
end
end
def unmigrated_users
redis.hgetall("rs:container_migration").select { |_, value|
value == "not_started"
}.keys
end
def pick_unmigrated_user
unmigrated_users.sample # pick a random user from list
end
def redis
@redis ||= Redis.new(@settings["redis"].symbolize_keys)
end
end
username = ARGV[0]
if username
migrator = Migrator.new username
migrator.migrate
else
runner = MigrationRunner.new
runner.migrate
end