5apps/liquor-cabinet
5apps/liquor-cabinet
4
0
Fork 0
Code Issues 1 Pull Requests Packages Releases Activity

Use full dir listing instead of per subdir

Browse Source
This commit is contained in:
Garret Alfert 2016-09-07 17:59:14 +02:00
parent 685c82d068
commit 710657748b
1 changed files with 14 additions and 43 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

57
migrate_to_single_container.rb
View File

@ -35,7 +35,7 @@ class Migrator
logger.info "Starting migration for '#{username}'"
set_container_migration_state("in_progress")
begin
work_on_dir("", "")
copy_all_documents
rescue Exception => ex
logger.error "Error migrating documents for '#{username}': #{ex}"
set_container_migration_state("not_started")
@ -48,8 +48,8 @@ class Migrator
logger.info "Finished migration for '#{username}'"
end
def is_dir?(name)
name[-1] == "/"
def is_document?(name)
name[-1] != "/"
end
def set_container_migration_state(type)
@ -60,31 +60,26 @@ class Migrator
redis.hdel("rs:container_migration", username) unless dry_run
end
def work_on_dir(directory, parent_directory)
full_directory = "#{parent_directory}#{directory}"
logger.debug "Retrieving listing for '#{full_directory}'"
def copy_all_documents
logger.debug "Retrieving object listing"
listing = get_directory_listing_from_swift("#{full_directory}")
listing = get_directory_listing_from_swift
logger.debug "Listing for '#{full_directory}': #{listing}"
logger.debug "Full listing: #{listing}"
if listing
listing.split("\n").each do |item|
if is_dir? item
# get dir listing and repeat
work_on_dir(item, "#{parent_directory}") unless item == full_directory
else
copy_document("#{parent_directory}", item)
if is_document? item
copy_document(item)
end
end
end
end
def copy_document(directory, document)
old_document_url = "#{url_for_directory(@username, directory)}/#{escape(document)}"
def copy_document(document_path)
old_document_url = "#{container_url_for(@username)}/#{escape(document_path)}"
full_path = [directory, document].reject(&:empty?).join('/')
new_document_path = "rs:documents:#{environment.to_s.downcase}/#{@username}/#{escape(full_path)}"
new_document_path = "rs:documents:#{environment.to_s.downcase}/#{@username}/#{escape(document_path)}"
logger.debug "Copying document from #{old_document_url} to #{new_document_path}"
do_copy_request(old_document_url, new_document_path) unless dry_run
@ -94,32 +89,16 @@ class Migrator
@redis ||= Redis.new(@settings["redis"].symbolize_keys)
end
def get_directory_listing_from_swift(directory)
is_root_listing = directory.empty?
get_response = nil
if is_root_listing
get_response = do_get_request("#{container_url_for(@username)}/?delimiter=/&prefix=")
else
get_response = do_get_request("#{container_url_for(@username)}/?delimiter=/&prefix=#{escape(directory)}")
end
def get_directory_listing_from_swift
get_response = do_get_request("#{container_url_for(@username)}/?prefix=")
get_response.body
end
def do_head_request(url, &block)
RestClient.head(url, default_headers, &block)
end
def do_get_request(url, &block)
RestClient.get(url, default_headers, &block)
end
def do_put_request(url, data, content_type)
RestClient.put(url, data, default_headers.merge({content_type: content_type}))
end
def do_copy_request(url, destination_path)
RestClient::Request.execute(
method: :copy,
@ -132,14 +111,6 @@ class Migrator
{"x-auth-token" => @swift_token}
end
def url_for_directory(user, directory)
if directory.empty?
container_url_for(user)
else
"#{container_url_for(user)}/#{escape(directory)}"
end
end
def container_url_for(user)
"#{base_url}/#{container_for(user)}"
end