Greg Karékinian
commit
c29e681a40
3
Gemfile
3
Gemfile
@ -5,6 +5,8 @@ gem "sinatra-contrib"
|
||||
gem "activesupport", '~> 3.2'
|
||||
gem "riak-client", :github => "5apps/riak-ruby-client", :branch => "invalid_uri_error"
|
||||
gem "fog"
|
||||
gem "rest-client"
|
||||
gem "redis"
|
||||
|
||||
group :test do
|
||||
gem 'rake'
|
||||
@ -14,4 +16,5 @@ end
|
||||
|
||||
group :staging, :production do
|
||||
gem "rainbows"
|
||||
gem "sentry-raven"
|
||||
end
|
||||
|
||||
11
Gemfile.lock
11
Gemfile.lock
@ -21,6 +21,8 @@ GEM
|
||||
builder (3.2.2)
|
||||
eventmachine (1.0.3)
|
||||
excon (0.16.10)
|
||||
faraday (0.9.1)
|
||||
multipart-post (>= 1.2, < 3)
|
||||
fog (1.7.0)
|
||||
builder
|
||||
excon (~> 0.14)
|
||||
@ -42,6 +44,7 @@ GEM
|
||||
mime-types (1.23)
|
||||
minitest (2.10.0)
|
||||
multi_json (1.10.0)
|
||||
multipart-post (2.0.0)
|
||||
net-scp (1.0.4)
|
||||
net-ssh (>= 1.99.1)
|
||||
net-ssh (2.6.7)
|
||||
@ -59,7 +62,12 @@ GEM
|
||||
unicorn (~> 4.8)
|
||||
raindrops (0.13.0)
|
||||
rake (0.9.2.2)
|
||||
redis (3.0.7)
|
||||
rest-client (1.6.7)
|
||||
mime-types (>= 1.16)
|
||||
ruby-hmac (0.4.0)
|
||||
sentry-raven (0.13.1)
|
||||
faraday (>= 0.7.6)
|
||||
sinatra (1.4.5)
|
||||
rack (~> 1.4)
|
||||
rack-protection (~> 1.4)
|
||||
@ -87,6 +95,9 @@ DEPENDENCIES
|
||||
purdytest
|
||||
rainbows
|
||||
rake
|
||||
redis
|
||||
rest-client
|
||||
riak-client!
|
||||
sentry-raven
|
||||
sinatra (~> 1.4)
|
||||
sinatra-contrib
|
||||
|
||||
@ -13,6 +13,13 @@ development: &defaults
|
||||
cs_binaries: rs.binaries
|
||||
authorizations: rs_authorizations
|
||||
opslog: rs_opslog
|
||||
# # uncomment this section and comment the riak one
|
||||
# swift: &swift_defaults
|
||||
# host: "https://swift.example.com"
|
||||
# # Redis is needed for the swift backend
|
||||
# redis:
|
||||
# host: localhost
|
||||
# port: 6379
|
||||
|
||||
test:
|
||||
<<: *defaults
|
||||
|
||||
@ -15,7 +15,7 @@ module RemoteStorage
|
||||
self.settings = settings
|
||||
self.server = server
|
||||
|
||||
credentials = File.read(settings['riak_cs']['credentials_file'])
|
||||
credentials = File.read(settings.riak['riak_cs']['credentials_file'])
|
||||
self.cs_credentials = JSON.parse(credentials)
|
||||
end
|
||||
|
||||
@ -229,8 +229,7 @@ module RemoteStorage
|
||||
|
||||
# A URI object that can be used with HTTP backend methods
|
||||
def riak_uri(bucket, key)
|
||||
rc = settings.symbolize_keys
|
||||
URI.parse "http://#{rc[:host]}:#{rc[:http_port]}/riak/#{bucket}/#{key}"
|
||||
URI.parse "http://#{settings.riak["host"]}:#{settings.riak["http_port"]}/riak/#{bucket}/#{key}"
|
||||
end
|
||||
|
||||
def serializer_for(content_type)
|
||||
@ -471,13 +470,13 @@ module RemoteStorage
|
||||
end
|
||||
|
||||
def client
|
||||
@client ||= ::Riak::Client.new(:host => settings['host'],
|
||||
:http_port => settings['http_port'])
|
||||
@client ||= ::Riak::Client.new(:host => settings.riak['host'],
|
||||
:http_port => settings.riak['http_port'])
|
||||
end
|
||||
|
||||
def data_bucket
|
||||
@data_bucket ||= begin
|
||||
bucket = client.bucket(settings['buckets']['data'])
|
||||
bucket = client.bucket(settings.riak['buckets']['data'])
|
||||
bucket.allow_mult = false
|
||||
bucket
|
||||
end
|
||||
@ -485,7 +484,7 @@ module RemoteStorage
|
||||
|
||||
def directory_bucket
|
||||
@directory_bucket ||= begin
|
||||
bucket = client.bucket(settings['buckets']['directories'])
|
||||
bucket = client.bucket(settings.riak['buckets']['directories'])
|
||||
bucket.allow_mult = false
|
||||
bucket
|
||||
end
|
||||
@ -493,7 +492,7 @@ module RemoteStorage
|
||||
|
||||
def auth_bucket
|
||||
@auth_bucket ||= begin
|
||||
bucket = client.bucket(settings['buckets']['authorizations'])
|
||||
bucket = client.bucket(settings.riak['buckets']['authorizations'])
|
||||
bucket.allow_mult = false
|
||||
bucket
|
||||
end
|
||||
@ -501,7 +500,7 @@ module RemoteStorage
|
||||
|
||||
def binary_bucket
|
||||
@binary_bucket ||= begin
|
||||
bucket = client.bucket(settings['buckets']['binaries'])
|
||||
bucket = client.bucket(settings.riak['buckets']['binaries'])
|
||||
bucket.allow_mult = false
|
||||
bucket
|
||||
end
|
||||
@ -509,7 +508,7 @@ module RemoteStorage
|
||||
|
||||
def opslog_bucket
|
||||
@opslog_bucket ||= begin
|
||||
bucket = client.bucket(settings['buckets']['opslog'])
|
||||
bucket = client.bucket(settings.riak['buckets']['opslog'])
|
||||
bucket.allow_mult = false
|
||||
bucket
|
||||
end
|
||||
@ -520,12 +519,12 @@ module RemoteStorage
|
||||
:provider => 'AWS',
|
||||
:aws_access_key_id => cs_credentials['key_id'],
|
||||
:aws_secret_access_key => cs_credentials['key_secret'],
|
||||
:endpoint => settings['riak_cs']['endpoint']
|
||||
:endpoint => settings.riak['riak_cs']['endpoint']
|
||||
})
|
||||
end
|
||||
|
||||
def cs_binary_bucket
|
||||
@cs_binary_bucket ||= cs_client.directories.create(:key => settings['buckets']['cs_binaries'])
|
||||
@cs_binary_bucket ||= cs_client.directories.create(:key => settings.riak['buckets']['cs_binaries'])
|
||||
end
|
||||
|
||||
end
|
||||
|
||||
311
lib/remote_storage/swift.rb
Normal file
311
lib/remote_storage/swift.rb
Normal file
@ -0,0 +1,311 @@
|
||||
require "rest_client"
|
||||
require "json"
|
||||
require "cgi"
|
||||
require "active_support/core_ext/time/conversions"
|
||||
require "active_support/core_ext/numeric/time"
|
||||
require "redis"
|
||||
|
||||
module RemoteStorage
|
||||
class Swift
|
||||
|
||||
attr_accessor :settings, :server
|
||||
|
||||
def initialize(settings, server)
|
||||
self.settings = settings
|
||||
self.server = server
|
||||
end
|
||||
|
||||
def authorize_request(user, directory, token, listing=false)
|
||||
request_method = server.env["REQUEST_METHOD"]
|
||||
|
||||
if directory.split("/").first == "public"
|
||||
return true if ["GET", "HEAD"].include?(request_method) && !listing
|
||||
end
|
||||
|
||||
authorizations = redis.smembers("authorizations:#{user}:#{token}")
|
||||
permission = directory_permission(authorizations, directory)
|
||||
|
||||
server.halt 401 unless permission
|
||||
if ["PUT", "DELETE"].include? request_method
|
||||
server.halt 401 unless permission == "rw"
|
||||
end
|
||||
end
|
||||
|
||||
def get_head(user, directory, key)
|
||||
url = url_for(user, directory, key)
|
||||
|
||||
res = do_head_request(url)
|
||||
|
||||
set_response_headers(res)
|
||||
rescue RestClient::ResourceNotFound
|
||||
server.halt 404
|
||||
end
|
||||
|
||||
def get_data(user, directory, key)
|
||||
url = url_for(user, directory, key)
|
||||
|
||||
res = do_get_request(url)
|
||||
|
||||
set_response_headers(res)
|
||||
|
||||
none_match = (server.env["HTTP_IF_NONE_MATCH"] || "").split(",").map(&:strip)
|
||||
server.halt 304 if none_match.include? %Q("#{res.headers[:etag]}")
|
||||
|
||||
return res.body
|
||||
rescue RestClient::ResourceNotFound
|
||||
server.halt 404
|
||||
end
|
||||
|
||||
def get_head_directory_listing(user, directory)
|
||||
res = do_head_request("#{container_url_for(user)}/#{directory}/")
|
||||
|
||||
server.headers["Content-Type"] = "application/json"
|
||||
server.headers["ETag"] = %Q("#{res.headers[:etag]}")
|
||||
rescue RestClient::ResourceNotFound
|
||||
server.halt 404
|
||||
end
|
||||
|
||||
def get_directory_listing(user, directory)
|
||||
server.headers["Content-Type"] = "application/json"
|
||||
|
||||
do_head_request("#{container_url_for(user)}/#{directory}/") do |response|
|
||||
if response.code == 404
|
||||
return directory_listing([]).to_json
|
||||
else
|
||||
server.headers["ETag"] = %Q("#{response.headers[:etag]}")
|
||||
none_match = (server.env["HTTP_IF_NONE_MATCH"] || "").split(",").map(&:strip)
|
||||
server.halt 304 if none_match.include? %Q("#{response.headers[:etag]}")
|
||||
end
|
||||
end
|
||||
|
||||
res = do_get_request("#{container_url_for(user)}/?format=json&path=#{directory}/")
|
||||
|
||||
if body = JSON.parse(res.body)
|
||||
listing = directory_listing(body)
|
||||
else
|
||||
puts "listing not JSON"
|
||||
end
|
||||
|
||||
listing.to_json
|
||||
end
|
||||
|
||||
def put_data(user, directory, key, data, content_type)
|
||||
server.halt 409 if has_name_collision?(user, directory, key)
|
||||
|
||||
url = url_for(user, directory, key)
|
||||
|
||||
if required_match = server.env["HTTP_IF_MATCH"]
|
||||
do_head_request(url) do |response|
|
||||
server.halt 412 unless required_match == %Q("#{response.headers[:etag]}")
|
||||
end
|
||||
end
|
||||
if server.env["HTTP_IF_NONE_MATCH"] == "*"
|
||||
do_head_request(url) do |response|
|
||||
server.halt 412 unless response.code == 404
|
||||
end
|
||||
end
|
||||
|
||||
res = do_put_request(url, data, content_type)
|
||||
|
||||
if update_dir_objects(user, directory)
|
||||
server.headers["ETag"] = %Q("#{res.headers[:etag]}")
|
||||
server.halt 200
|
||||
else
|
||||
server.halt 500
|
||||
end
|
||||
end
|
||||
|
||||
def delete_data(user, directory, key)
|
||||
url = url_for(user, directory, key)
|
||||
|
||||
if required_match = server.env["HTTP_IF_MATCH"]
|
||||
do_head_request(url) do |response|
|
||||
server.halt 412 unless required_match == %Q("#{response.headers[:etag]}")
|
||||
end
|
||||
end
|
||||
|
||||
do_delete_request(url)
|
||||
delete_dir_objects(user, directory)
|
||||
|
||||
server.halt 200
|
||||
rescue RestClient::ResourceNotFound
|
||||
server.halt 404
|
||||
end
|
||||
|
||||
private
|
||||
|
||||
def set_response_headers(response)
|
||||
server.headers["ETag"] = %Q("#{response.headers[:etag]}")
|
||||
server.headers["Content-Type"] = response.headers[:content_type]
|
||||
server.headers["Content-Length"] = response.headers[:content_length]
|
||||
server.headers["Last-Modified"] = response.headers[:last_modified]
|
||||
end
|
||||
|
||||
def extract_category(directory)
|
||||
if directory.match(/^public\//)
|
||||
"public/#{directory.split('/')[1]}"
|
||||
else
|
||||
directory.split('/').first
|
||||
end
|
||||
end
|
||||
|
||||
def directory_permission(authorizations, directory)
|
||||
authorizations = authorizations.map do |auth|
|
||||
auth.index(":") ? auth.split(":") : [auth, "rw"]
|
||||
end
|
||||
authorizations = Hash[*authorizations.flatten]
|
||||
|
||||
permission = authorizations[""]
|
||||
|
||||
authorizations.each do |key, value|
|
||||
if directory.match(/^(public\/)?#{key}(\/|$)/)
|
||||
if permission.nil? || permission == "r"
|
||||
permission = value
|
||||
end
|
||||
return permission if permission == "rw"
|
||||
end
|
||||
end
|
||||
|
||||
permission
|
||||
end
|
||||
|
||||
def directory_listing(res_body)
|
||||
listing = {
|
||||
"@context" => "http://remotestorage.io/spec/folder-description",
|
||||
"items" => {}
|
||||
}
|
||||
|
||||
res_body.each do |entry|
|
||||
name = entry["name"].gsub("#{File.dirname(entry["name"])}/", '')
|
||||
if name[-1] == "/"
|
||||
listing["items"].merge!({
|
||||
name => {
|
||||
"ETag" => entry["hash"],
|
||||
}
|
||||
})
|
||||
else
|
||||
listing["items"].merge!({
|
||||
name => {
|
||||
"ETag" => entry["hash"],
|
||||
"Content-Type" => entry["content_type"],
|
||||
"Content-Length" => entry["bytes"]
|
||||
}
|
||||
})
|
||||
end
|
||||
end
|
||||
|
||||
listing
|
||||
end
|
||||
|
||||
def has_name_collision?(user, directory, key)
|
||||
# check for existing directory with the same name as the document
|
||||
do_head_request("#{container_url_for(user)}/#{directory}/#{key}/") do |res|
|
||||
return true if res.code == 200
|
||||
end
|
||||
|
||||
# check for existing documents with the same name as one of the parent directories
|
||||
parent_directories_for(directory).each do |dir|
|
||||
do_head_request("#{container_url_for(user)}/#{dir}/") do |res_dir|
|
||||
if res_dir.code == 200
|
||||
return false
|
||||
else
|
||||
do_head_request("#{container_url_for(user)}/#{dir}") do |res_key|
|
||||
if res_key.code == 200
|
||||
return true
|
||||
else
|
||||
next
|
||||
end
|
||||
end
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
false
|
||||
end
|
||||
|
||||
def parent_directories_for(directory)
|
||||
directories = directory.split("/")
|
||||
parent_directories = []
|
||||
|
||||
while directories.any?
|
||||
parent_directories << directories.join("/")
|
||||
directories.pop
|
||||
end
|
||||
|
||||
parent_directories
|
||||
end
|
||||
|
||||
def update_dir_objects(user, directory)
|
||||
timestamp = (Time.now.to_f * 1000).to_i
|
||||
|
||||
parent_directories_for(directory).each do |dir|
|
||||
do_put_request("#{container_url_for(user)}/#{dir}/", timestamp.to_s, "text/plain")
|
||||
end
|
||||
|
||||
true
|
||||
rescue
|
||||
parent_directories_for(directory).each do |dir|
|
||||
do_delete_request("#{container_url_for(user)}/#{dir}/") rescue false
|
||||
end
|
||||
|
||||
false
|
||||
end
|
||||
|
||||
def delete_dir_objects(user, directory)
|
||||
parent_directories_for(directory).each do |dir|
|
||||
if dir_empty?(user, dir)
|
||||
do_delete_request("#{container_url_for(user)}/#{dir}/")
|
||||
else
|
||||
timestamp = (Time.now.to_f * 1000).to_i
|
||||
do_put_request("#{container_url_for(user)}/#{dir}/", timestamp.to_s, "text/plain")
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
def dir_empty?(user, dir)
|
||||
do_get_request("#{container_url_for(user)}/?format=plain&limit=1&path=#{dir}/") do |res|
|
||||
return res.headers[:content_length] == "0"
|
||||
end
|
||||
end
|
||||
|
||||
def container_url_for(user)
|
||||
"#{base_url}/#{container_for(user)}"
|
||||
end
|
||||
|
||||
def url_for(user, directory, key)
|
||||
"#{container_url_for(user)}/#{directory}/#{key}"
|
||||
end
|
||||
|
||||
def base_url
|
||||
@base_url ||= settings.swift["host"]
|
||||
end
|
||||
|
||||
def container_for(user)
|
||||
"rs:#{settings.environment.to_s.chars.first}:#{user}"
|
||||
end
|
||||
|
||||
def default_headers
|
||||
@default_headers ||= {"x-auth-token" => settings.swift["token"]}
|
||||
end
|
||||
|
||||
def do_put_request(url, data, content_type)
|
||||
RestClient.put(url, data, default_headers.merge({content_type: content_type}))
|
||||
end
|
||||
|
||||
def do_get_request(url, &block)
|
||||
RestClient.get(url, default_headers, &block)
|
||||
end
|
||||
|
||||
def do_head_request(url, &block)
|
||||
RestClient.head(url, default_headers, &block)
|
||||
end
|
||||
|
||||
def do_delete_request(url)
|
||||
RestClient.delete(url, default_headers)
|
||||
end
|
||||
|
||||
def redis
|
||||
@redis ||= Redis.new(host: settings.redis["host"], port: settings.redis["port"])
|
||||
end
|
||||
end
|
||||
end
|
||||
@ -5,6 +5,7 @@ require "sinatra/base"
|
||||
require 'sinatra/config_file'
|
||||
require "sinatra/reloader"
|
||||
require "remote_storage/riak"
|
||||
require "remote_storage/swift"
|
||||
|
||||
class LiquorCabinet < Sinatra::Base
|
||||
|
||||
@ -23,11 +24,22 @@ class LiquorCabinet < Sinatra::Base
|
||||
|
||||
configure :development do
|
||||
register Sinatra::Reloader
|
||||
also_reload "lib/remote_storage/*.rb"
|
||||
enable :logging
|
||||
end
|
||||
|
||||
configure :production, :staging do
|
||||
require "rack/common_logger"
|
||||
if ENV['SENTRY_DSN']
|
||||
require "raven"
|
||||
|
||||
Raven.configure do |config|
|
||||
config.dsn = ENV['SENTRY_DSN']
|
||||
config.tags = { environment: settings.environment.to_s }
|
||||
end
|
||||
|
||||
use Raven::Rack
|
||||
end
|
||||
end
|
||||
|
||||
#
|
||||
@ -62,17 +74,19 @@ class LiquorCabinet < Sinatra::Base
|
||||
end
|
||||
|
||||
["/:user/*/:key", "/:user/:key"].each do |path|
|
||||
get path do
|
||||
storage.get_data(@user, @directory, @key)
|
||||
end
|
||||
|
||||
head path do
|
||||
storage.get_head(@user, @directory, @key)
|
||||
end
|
||||
|
||||
get path do
|
||||
storage.get_data(@user, @directory, @key)
|
||||
end
|
||||
|
||||
put path do
|
||||
data = request.body.read
|
||||
|
||||
halt 422 unless env['CONTENT_TYPE']
|
||||
|
||||
if env['CONTENT_TYPE'] == "application/x-www-form-urlencoded"
|
||||
content_type = "text/plain; charset=utf-8"
|
||||
else
|
||||
@ -88,23 +102,29 @@ class LiquorCabinet < Sinatra::Base
|
||||
end
|
||||
|
||||
["/:user/*/", "/:user/"].each do |path|
|
||||
get path do
|
||||
storage.get_directory_listing(@user, @directory)
|
||||
end
|
||||
|
||||
head path do
|
||||
storage.get_head_directory_listing(@user, @directory)
|
||||
end
|
||||
|
||||
get path do
|
||||
storage.get_directory_listing(@user, @directory)
|
||||
end
|
||||
end
|
||||
|
||||
private
|
||||
|
||||
def storage
|
||||
@storage ||= begin
|
||||
if settings.riak
|
||||
RemoteStorage::Riak.new(settings.riak, self)
|
||||
# elsif settings.redis
|
||||
# include RemoteStorage::Redis
|
||||
if settings.respond_to? :riak
|
||||
RemoteStorage::Riak.new(settings, self)
|
||||
elsif settings.respond_to? :swift
|
||||
settings.swift["token"] = File.read("tmp/swift_token.txt")
|
||||
RemoteStorage::Swift.new(settings, self)
|
||||
else
|
||||
puts <<-EOF
|
||||
You need to set one storage backend in your config.yml file.
|
||||
Riak and Swift are currently supported. See config.yml.example.
|
||||
EOF
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
1288
spec/riak_spec.rb
1288
spec/riak_spec.rb
File diff suppressed because it is too large
Load Diff
@ -28,8 +28,7 @@ def write_last_response_to_file(filename = "last_response.html")
|
||||
end
|
||||
|
||||
alias context describe
|
||||
|
||||
if app.settings.riak
|
||||
if app.settings.respond_to? :riak
|
||||
::Riak.disable_list_keys_warnings = true
|
||||
|
||||
def client
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user