5apps/liquor-cabinet
4
0
Fork 0
Code Issues 1 Pull Requests Packages Releases Activity

Merge pull request #30 from 5apps/riak-cs

Browse Source 
Save binary data in Riak CS
This commit is contained in:
galfert
2013-10-15 20:19:23 -07:00
parent 21d06a4970 0fd252208a
commit d7925f5b12
17 changed files with 522 additions and 124 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
.gitignore vendored
View File

@@ -1,2 +1,3 @@
config.yml config.yml
cs_credentials.json
pids pids

4
.travis.yml
View File

@@ -2,11 +2,13 @@ language: ruby
rvm: rvm:
- 1.9.2 - 1.9.2
- 1.9.3 - 1.9.3
services: riak before_install:
- sh .travis/install_riakcs.sh
before_script: before_script:
- cp config.yml.example config.yml - cp config.yml.example config.yml
script: rake test script: rake test
notifications: notifications:
email: false
webhooks: webhooks:
urls: urls:
- http://hook-juggler.heroku.com/hooks/travis - http://hook-juggler.heroku.com/hooks/travis

32
.travis/install_riakcs.sh Normal file
View File

@@ -0,0 +1,32 @@
#!/bin/bash
curl http://apt.basho.com/gpg/basho.apt.key | sudo apt-key add -
sudo bash -c "echo deb http://apt.basho.com $(lsb_release -sc) main > /etc/apt/sources.list.d/basho.list"
sudo apt-get update
sudo apt-get install -o Dpkg::Options::="--force-confnew" -q -y riak=1.4.2-1
sudo apt-get install -q -y riak-cs=1.4.1-1
sudo apt-get install -q -y stanchion=1.4.1-1
sudo bash -c "echo '127.0.0.1 cs.example.com' >> /etc/hosts"
sudo cp .travis/riak.app.config /etc/riak/app.config
sudo cp .travis/riak.vm.args /etc/riak/vm.args
sudo cp .travis/riakcs.app.config /etc/riak-cs/app.config
sudo cp .travis/riakcs.vm.args /etc/riak-cs/vm.args
sudo cp .travis/stanchion.app.config /etc/stanchion/app.config
sudo cp .travis/stanchion.vm.args /etc/stanchion/vm.args
sudo service riak start
sudo service riak-cs start
sudo service stanchion start
sleep 4
curl -H 'Content-Type: application/json' -X POST http://localhost:8080/riak-cs/user --data '{"email":"admin@5apps.com", "name":"admin"}' -o cs_admin_credentials.json
cat cs_admin_credentials.json
curl -H 'Content-Type: application/json' -X POST http://localhost:8080/riak-cs/user --data '{"email":"liquorcabinet@5apps.com", "name":"liquor cabinet"}' -o cs_credentials.json
cat cs_credentials.json
echo "\nFinished"

135
.travis/riak.app.config Normal file
View File

@@ -0,0 +1,135 @@
[
{kernel, [
{inet_dist_listen_max, 7999},
{inet_dist_listen_min, 6000}
]},
{lager, [
{crash_log, "/var/log/riak/crash.log"},
{crash_log_count, 5},
{crash_log_date, "$D0"},
{crash_log_msg_size, 65536},
{crash_log_size, 10485760},
{error_logger_hwm, 100},
{error_logger_redirect, true},
{handlers, [
{lager_file_backend, [
{"/var/log/riak/error.log", error, 10485760, "$D0", 5},
{"/var/log/riak/console.log", info, 10485760, "$D0", 5}
]}
]}
]},
{merge_index, [
{buffer_rollover_size, 1048576},
{data_root, "/var/lib/riak/merge_index"},
{max_compact_segments, 20}
]},
{riak_api, [
{pb, [
{"127.0.0.1", 8087}
]},
{pb_backlog, 128}
]},
{riak_control, [
{admin, true},
{auth, userlist},
{enabled, false},
{userlist, [
{"user", "pass"}
]}
]},
{riak_core, [
{cluster_mgr, {"127.0.0.1", 9085}},
{default_bucket_props, [
{allow_mult, true}
]},
{dtrace_support, false},
{handoff_port, 8099},
{http, [
{"127.0.0.1", 8098}
]},
{platform_bin_dir, "/usr/sbin"},
{platform_data_dir, "/var/lib/riak"},
{platform_etc_dir, "/etc/riak"},
{platform_lib_dir, "/usr/lib/riak"},
{platform_log_dir, "/var/log/riak"},
{ring_creation_size, 64},
{ring_state_dir, "/var/lib/riak/ring"}
]},
{riak_jmx, [
{enabled, false}
]},
{riak_kv, [
{add_paths, [
"/usr/lib/riak-cs/lib/riak_cs-1.4.1/ebin"
]},
{anti_entropy, {on, [
]}},
{anti_entropy_build_limit, {1, 3600000}},
{anti_entropy_concurrency, 2},
{anti_entropy_data_dir, "/var/lib/riak/anti_entropy"},
{anti_entropy_expire, 604800000},
{anti_entropy_leveldb_opts, [
{write_buffer_size, 4194304},
{max_open_files, 20}
]},
{anti_entropy_tick, 15000},
{fsm_limit, 50000},
{hook_js_vm_count, 2},
{http_url_encoding, on},
{js_max_vm_mem, 8},
{js_thread_stack, 16},
{listkeys_backpressure, true},
{map_js_vm_count, 8},
{mapred_2i_pipe, true},
{mapred_name, "mapred"},
{multi_backend, [
{be_default, riak_kv_eleveldb_backend, [
{cache_size, 4194304},
{data_root, "/var/lib/riak/leveldb"},
{max_open_files, 50}
]},
{be_blocks, riak_kv_bitcask_backend, [
{data_root, "/var/lib/riak/bitcask"}
]}
]},
{multi_backend_default, be_default},
{multi_backend_prefix_list, [
{<<"0b:">>, be_blocks}
]},
{object_format, v1},
{reduce_js_vm_count, 6},
{storage_backend, riak_cs_kv_multi_backend},
{vnode_vclocks, true}
]},
{riak_repl, [
{data_root, "/var/lib/riak/riak_repl"}
]},
{riak_search, [
{enabled, false}
]},
{riak_sysmon, [
{busy_dist_port, true},
{busy_port, true},
{gc_ms_limit, 0},
{heap_word_limit, 40111000},
{port_limit, 2},
{process_limit, 30}
]},
{sasl, [
{sasl_error_logger, false}
]},
{snmp, [
{agent, [
{config, [
{dir, "/etc/riak/snmp/agent/conf"},
{force_load, true}
]},
{db_dir, "/var/lib/riak/snmp/agent/db"},
{net_if, [
{options, [
{bind_to, true}
]}
]}
]}
]}
].

10
.travis/riak.vm.args Normal file
View File

@@ -0,0 +1,10 @@
+A 64
+K true
+W w
-env ERL_CRASH_DUMP /var/log/riak/erl_crash.dump
-env ERL_FULLSWEEP_AFTER 0
-env ERL_MAX_ETS_TABLES 256000
-env ERL_MAX_PORTS 64000
-name riak@127.0.0.1
-setcookie riak
-smp enable

65
.travis/riakcs.app.config Normal file
View File

@@ -0,0 +1,65 @@
[
{lager, [
{crash_log, "/var/log/riak-cs/crash.log"},
{crash_log_count, 5},
{crash_log_date, "$D0"},
{crash_log_msg_size, 65536},
{crash_log_size, 10485760},
{error_logger_redirect, true},
{handlers, [
{lager_file_backend, [
{"/var/log/riak-cs/error.log", error, 10485760, "$D0", 5},
{"/var/log/riak-cs/console.log", info, 10485760, "$D0", 5}
]}
]}
]},
{riak_cs, [
{access_archive_period, 3600},
{access_archiver_max_backlog, 2},
{access_log_flush_factor, 1},
{access_log_flush_size, 1000000},
{admin_key, "secret"},
{admin_secret, "secret"},
{anonymous_user_creation, true},
{auth_module, riak_cs_s3_auth},
{connection_pools, [
{bucket_list_pool, {5, 0}},
{request_pool, {128, 0}}
]},
{cs_ip, "127.0.0.1"},
{cs_port, 8080},
{cs_root_host, "cs.example.com"},
{cs_version, 10300},
{dtrace_support, false},
{fold_objects_for_list_keys, false},
{gc_interval, 900},
{gc_retry_interval, 21600},
{leeway_seconds, 86400},
{n_val_1_get_requests, true},
{rewrite_module, riak_cs_s3_rewrite},
{riak_ip, "127.0.0.1"},
{riak_pb_port, 8087},
{stanchion_ip, "127.0.0.1"},
{stanchion_port, 8085},
{stanchion_ssl, false},
{storage_archive_period, 86400},
{storage_schedule, [
]},
{trust_x_forwarded_for, false},
{usage_request_limit, 744}
]},
{sasl, [
{sasl_error_logger, false},
{utc_log, true}
]},
{webmachine, [
{log_handlers, [
{riak_cs_access_log_handler, [
]},
{webmachine_log_handler, [
"/var/log/riak-cs"
]}
]},
{server_name, "Riak CS"}
]}
].

8
.travis/riakcs.vm.args Normal file
View File

@@ -0,0 +1,8 @@
+A 64
+K true
+W w
-env ERL_CRASH_DUMP /var/log/riak-cs/erl_crash.dump
-env ERL_FULLSWEEP_AFTER 0
-env ERL_MAX_PORTS 4096
-name riak-cs@127.0.0.1
-setcookie riak-cs

29
.travis/stanchion.app.config Normal file
View File

@@ -0,0 +1,29 @@
[
{lager, [
{crash_log, "/var/log/stanchion/crash.log"},
{crash_log_count, 5},
{crash_log_date, "$D0"},
{crash_log_msg_size, 65536},
{crash_log_size, 10485760},
{error_logger_redirect, true},
{handlers, [
{lager_file_backend, [
{"/var/log/stanchion/error.log", error, 10485760, "$D0", 5},
{"/var/log/stanchion/console.log", info, 10485760, "$D0", 5}
]}
]}
]},
{sasl, [
{sasl_error_logger, false},
{utc_log, true}
]},
{stanchion, [
{admin_key, "secret"},
{admin_secret, "secret"},
{auth_bypass, false},
{riak_ip, "127.0.0.1"},
{riak_pb_port, 8087},
{stanchion_ip, "127.0.0.1"},
{stanchion_port, 8085}
]}
].

8
.travis/stanchion.vm.args Normal file
View File

@@ -0,0 +1,8 @@
+A 64
+K true
+W w
-env ERL_CRASH_DUMP /var/log/stanchion/erl_crash.dump
-env ERL_FULLSWEEP_AFTER 0
-env ERL_MAX_PORTS 4096
-name stanchion@127.0.0.1
-setcookie stanchion

1
Gemfile
View File

@@ -4,6 +4,7 @@ gem "sinatra"
gem "sinatra-contrib" gem "sinatra-contrib"
gem "activesupport" gem "activesupport"
gem "riak-client" gem "riak-client"
gem "fog"
group :test do group :test do
gem 'rake' gem 'rake'

27
Gemfile.lock
View File

@@ -6,16 +6,34 @@ GEM
multi_json (~> 1.0) multi_json (~> 1.0)
backports (3.3.0) backports (3.3.0)
beefcake (0.3.7) beefcake (0.3.7)
builder (3.1.3) builder (3.2.2)
eventmachine (1.0.3) eventmachine (1.0.3)
excon (0.16.10)
fog (1.7.0)
builder
excon (~> 0.14)
formatador (~> 0.2.0)
mime-types
multi_json (~> 1.0)
net-scp (~> 1.0.4)
net-ssh (>= 2.1.3)
nokogiri (~> 1.5.0)
ruby-hmac
formatador (0.2.4)
i18n (0.6.1) i18n (0.6.1)
innertube (1.0.2)
kgio (2.7.4) kgio (2.7.4)
m (1.2.1) m (1.2.1)
method_source (>= 0.6.7) method_source (>= 0.6.7)
rake (>= 0.9.2.2, < 1.0.0) rake (>= 0.9.2.2, < 1.0.0)
method_source (0.8) method_source (0.8)
mime-types (1.23)
minitest (2.10.0) minitest (2.10.0)
multi_json (1.3.6) multi_json (1.8.0)
net-scp (1.0.4)
net-ssh (>= 1.99.1)
net-ssh (2.6.7)
nokogiri (1.5.10)
purdytest (1.0.0) purdytest (1.0.0)
minitest (~> 2.2) minitest (~> 2.2)
rack (1.5.2) rack (1.5.2)
@@ -25,11 +43,13 @@ GEM
rack (>= 1.0) rack (>= 1.0)
raindrops (0.10.0) raindrops (0.10.0)
rake (0.9.2.2) rake (0.9.2.2)
riak-client (1.0.5) riak-client (1.4.1)
beefcake (~> 0.3.7) beefcake (~> 0.3.7)
builder (>= 2.1.2) builder (>= 2.1.2)
i18n (>= 0.4.0) i18n (>= 0.4.0)
innertube (~> 1.0.2)
multi_json (~> 1.0) multi_json (~> 1.0)
ruby-hmac (0.4.0)
sinatra (1.4.3) sinatra (1.4.3)
rack (~> 1.4) rack (~> 1.4)
rack-protection (~> 1.4) rack-protection (~> 1.4)
@@ -52,6 +72,7 @@ PLATFORMS
DEPENDENCIES DEPENDENCIES
activesupport activesupport
fog
m m
purdytest purdytest
rake rake

6
config.yml.example
View File

@@ -3,20 +3,26 @@ development: &defaults
riak: &riak_defaults riak: &riak_defaults
host: localhost host: localhost
http_port: 8098 http_port: 8098
riak_cs:
credentials_file: "cs_credentials.json"
endpoint: "http://cs.example.com:8080"
buckets: buckets:
data: rs_data data: rs_data
directories: rs_directories directories: rs_directories
binaries: rs_binaries binaries: rs_binaries
cs_binaries: rs.binaries
authorizations: rs_authorizations authorizations: rs_authorizations
opslog: rs_opslog opslog: rs_opslog
test: test:
<<: *defaults
riak: riak:
<<: *riak_defaults <<: *riak_defaults
buckets: buckets:
data: rs_data_test data: rs_data_test
directories: rs_directories_test directories: rs_directories_test
binaries: rs_binaries_test binaries: rs_binaries_test
cs_binaries: rs.binaries.test
authorizations: rs_authorizations_test authorizations: rs_authorizations_test
opslog: rs_opslog_test opslog: rs_opslog_test

9
lib/rack/common_logger.rb Normal file
View File

@@ -0,0 +1,9 @@
# Disable Rack logger completely
module Rack
class CommonLogger
def call(env)
# do nothing
@app.call(env)
end
end
end

168
lib/remote_storage/riak.rb
View File

@@ -5,37 +5,22 @@ require "active_support/core_ext/time/conversions"
require "active_support/core_ext/numeric/time" require "active_support/core_ext/numeric/time"
module RemoteStorage module RemoteStorage
module Riak class Riak
::Riak.url_decoding = true ::Riak.url_decoding = true
def client attr_accessor :settings, :server, :cs_credentials
@client ||= ::Riak::Client.new(:host => settings.riak['host'],
:http_port => settings.riak['http_port'])
end
def data_bucket def initialize(settings, server)
@data_bucket ||= client.bucket(settings.riak['buckets']['data']) self.settings = settings
end self.server = server
def directory_bucket credentials = File.read(settings['riak_cs']['credentials_file'])
@directory_bucket ||= client.bucket(settings.riak['buckets']['directories']) self.cs_credentials = JSON.parse(credentials)
end
def auth_bucket
@auth_bucket ||= client.bucket(settings.riak['buckets']['authorizations'])
end
def binary_bucket
@binary_bucket ||= client.bucket(settings.riak['buckets']['binaries'])
end
def opslog_bucket
@opslog_bucket ||= client.bucket(settings.riak['buckets']['opslog'])
end end
def authorize_request(user, directory, token, listing=false) def authorize_request(user, directory, token, listing=false)
request_method = env["REQUEST_METHOD"] request_method = server.env["REQUEST_METHOD"]
if directory.split("/").first == "public" if directory.split("/").first == "public"
return true if request_method == "GET" && !listing return true if request_method == "GET" && !listing
@@ -44,22 +29,29 @@ module RemoteStorage
authorizations = auth_bucket.get("#{user}:#{token}").data authorizations = auth_bucket.get("#{user}:#{token}").data
permission = directory_permission(authorizations, directory) permission = directory_permission(authorizations, directory)
halt 403 unless permission server.halt 403 unless permission
if ["PUT", "DELETE"].include? request_method if ["PUT", "DELETE"].include? request_method
halt 403 unless permission == "rw" server.halt 403 unless permission == "rw"
end end
rescue ::Riak::HTTPFailedRequest rescue ::Riak::HTTPFailedRequest
halt 403 server.halt 403
end end
def get_data(user, directory, key) def get_data(user, directory, key)
object = data_bucket.get("#{user}:#{directory}:#{key}") object = data_bucket.get("#{user}:#{directory}:#{key}")
headers["Content-Type"] = object.content_type server.headers["Content-Type"] = object.content_type
headers["Last-Modified"] = last_modified_date_for(object) server.headers["Last-Modified"] = last_modified_date_for(object)
if binary_link = object.links.select {|l| l.tag == "binary"}.first if binary_key = object.meta["binary_key"]
object = client[binary_link.bucket].get(binary_link.key) object = cs_binary_bucket.files.get(binary_key[0])
case object.content_type[/^[^;\s]+/]
when "application/json"
return object.body.to_json
else
return object.body
end
end end
case object.content_type[/^[^;\s]+/] case object.content_type[/^[^;\s]+/]
@@ -69,21 +61,21 @@ module RemoteStorage
return serializer_for(object.content_type) ? object.data : object.raw_data return serializer_for(object.content_type) ? object.data : object.raw_data
end end
rescue ::Riak::HTTPFailedRequest rescue ::Riak::HTTPFailedRequest
halt 404 server.halt 404
end end
def get_directory_listing(user, directory) def get_directory_listing(user, directory)
directory_object = directory_bucket.get("#{user}:#{directory}") directory_object = directory_bucket.get("#{user}:#{directory}")
timestamp = directory_object.data.to_i timestamp = directory_object.data.to_i
timestamp /= 1000 if timestamp.to_s.length == 13 timestamp /= 1000 if timestamp.to_s.length == 13
headers["Content-Type"] = "application/json" server.headers["Content-Type"] = "application/json"
headers["Last-Modified"] = Time.at(timestamp).to_s(:rfc822) server.headers["Last-Modified"] = Time.at(timestamp).to_s(:rfc822)
listing = directory_listing(user, directory) listing = directory_listing(user, directory)
return listing.to_json return listing.to_json
rescue ::Riak::HTTPFailedRequest rescue ::Riak::HTTPFailedRequest
headers["Content-Type"] = "application/json" server.headers["Content-Type"] = "application/json"
return "{}" return "{}"
end end
@@ -97,31 +89,32 @@ module RemoteStorage
object.meta["timestamp"] = timestamp object.meta["timestamp"] = timestamp
if binary_data?(object.content_type, data) if binary_data?(object.content_type, data)
save_binary_data(object, data) or halt 422 save_binary_data(object, data) or server.halt 422
new_object_size = data.size new_object_size = data.size
else else
set_object_data(object, data) or halt 422 set_object_data(object, data) or server.halt 422
new_object_size = object.raw_data.size new_object_size = object.raw_data.size
end end
object.store response = object.store
log_count = object_exists ? 0 : 1 log_count = object_exists ? 0 : 1
log_operation(user, directory, log_count, new_object_size, existing_object_size) log_operation(user, directory, log_count, new_object_size, existing_object_size)
update_all_directory_objects(user, directory, timestamp) update_all_directory_objects(user, directory, timestamp)
halt 200 server.halt 200
rescue ::Riak::HTTPFailedRequest rescue ::Riak::HTTPFailedRequest
halt 422 server.halt 422
end end
def delete_data(user, directory, key) def delete_data(user, directory, key)
object = data_bucket.get("#{user}:#{directory}:#{key}") object = data_bucket.get("#{user}:#{directory}:#{key}")
existing_object_size = object_size(object) existing_object_size = object_size(object)
if binary_link = object.links.select {|l| l.tag == "binary"}.first if binary_key = object.meta["binary_key"]
client[binary_link.bucket].delete(binary_link.key) object = cs_binary_bucket.files.get(binary_key[0])
object.destroy
end end
riak_response = data_bucket.delete("#{user}:#{directory}:#{key}") riak_response = data_bucket.delete("#{user}:#{directory}:#{key}")
@@ -133,12 +126,11 @@ module RemoteStorage
timestamp = (Time.now.to_f * 1000).to_i timestamp = (Time.now.to_f * 1000).to_i
delete_or_update_directory_objects(user, directory, timestamp) delete_or_update_directory_objects(user, directory, timestamp)
halt riak_response[:code] server.halt riak_response[:code]
rescue ::Riak::HTTPFailedRequest rescue ::Riak::HTTPFailedRequest
halt 404 server.halt 404
end end
private private
def extract_category(directory) def extract_category(directory)
@@ -174,9 +166,9 @@ module RemoteStorage
end end
def object_size(object) def object_size(object)
if binary_link = object.links.select {|l| l.tag == "binary"}.first if binary_key = object.meta["binary_key"]
response = head(settings.riak['buckets']['binaries'], escape(binary_link.key)) response = cs_client.head_object cs_binary_bucket.key, binary_key[0]
response[:headers]["content-length"].first.to_i response.headers["Content-Length"].to_i
else else
object.raw_data.nil? ? 0 : object.raw_data.size object.raw_data.nil? ? 0 : object.raw_data.size
end end
@@ -186,17 +178,9 @@ module RemoteStorage
::Riak.escaper.escape(string).gsub("+", "%20").gsub('/', "%2F") ::Riak.escaper.escape(string).gsub("+", "%20").gsub('/', "%2F")
end end
# Perform a HEAD request via the backend method
def head(bucket, key)
client.http do |h|
url = riak_uri(bucket, key)
h.head [200], url
end
end
# A URI object that can be used with HTTP backend methods # A URI object that can be used with HTTP backend methods
def riak_uri(bucket, key) def riak_uri(bucket, key)
rc = settings.riak.symbolize_keys rc = settings.symbolize_keys
URI.parse "http://#{rc[:host]}:#{rc[:http_port]}/riak/#{bucket}/#{key}" URI.parse "http://#{rc[:host]}:#{rc[:http_port]}/riak/#{bucket}/#{key}"
end end
@@ -373,14 +357,13 @@ module RemoteStorage
end end
def save_binary_data(object, data) def save_binary_data(object, data)
binary_object = binary_bucket.new(object.key) cs_binary_object = cs_binary_bucket.files.create(
binary_object.content_type = object.content_type :key => object.key,
binary_object.raw_data = data :body => data,
binary_object.indexes = object.indexes :content_type => object.content_type
binary_object.store )
link = ::Riak::Link.new(binary_bucket.name, binary_object.key, "binary") object.meta["binary_key"] = cs_binary_object.key
object.links << link
object.raw_data = "" object.raw_data = ""
end end
@@ -406,5 +389,64 @@ module RemoteStorage
parent_directories << "" parent_directories << ""
end end
def client
@client ||= ::Riak::Client.new(:host => settings['host'],
:http_port => settings['http_port'])
end
def data_bucket
@data_bucket ||= begin
bucket = client.bucket(settings['buckets']['data'])
bucket.allow_mult = false
bucket
end
end
def directory_bucket
@directory_bucket ||= begin
bucket = client.bucket(settings['buckets']['directories'])
bucket.allow_mult = false
bucket
end
end
def auth_bucket
@auth_bucket ||= begin
bucket = client.bucket(settings['buckets']['authorizations'])
bucket.allow_mult = false
bucket
end
end
def binary_bucket
@binary_bucket ||= begin
bucket = client.bucket(settings['buckets']['binaries'])
bucket.allow_mult = false
bucket
end
end
def opslog_bucket
@opslog_bucket ||= begin
bucket = client.bucket(settings['buckets']['opslog'])
bucket.allow_mult = false
bucket
end
end
def cs_client
@cs_client ||= Fog::Storage.new({
:provider => 'AWS',
:aws_access_key_id => cs_credentials['key_id'],
:aws_secret_access_key => cs_credentials['key_secret'],
:endpoint => settings['riak_cs']['endpoint']
})
end
def cs_binary_bucket
@cs_binary_bucket ||= cs_client.directories.create(:key => settings['buckets']['cs_binaries'])
end
end end
end end

57
liquor-cabinet.rb
View File

@@ -6,16 +6,6 @@ require 'sinatra/config_file'
require "sinatra/reloader" require "sinatra/reloader"
require "remote_storage/riak" require "remote_storage/riak"
# Disable Rack logger completely
module Rack
class CommonLogger
def call(env)
# do nothing
@app.call(env)
end
end
end
class LiquorCabinet < Sinatra::Base class LiquorCabinet < Sinatra::Base
# #
@@ -36,11 +26,8 @@ class LiquorCabinet < Sinatra::Base
enable :logging enable :logging
end end
if settings.riak configure :production, :staging do
include RemoteStorage::Riak require "rack/common_logger"
# elsif settings.redis
# include RemoteStorage::Redis
# end
end end
# #
@@ -64,23 +51,19 @@ class LiquorCabinet < Sinatra::Base
token = env["HTTP_AUTHORIZATION"] ? env["HTTP_AUTHORIZATION"].split(" ")[1] : "" token = env["HTTP_AUTHORIZATION"] ? env["HTTP_AUTHORIZATION"].split(" ")[1] : ""
authorize_request(@user, @directory, token, @key.blank?) unless request.options? storage.authorize_request(@user, @directory, token, @key.blank?) unless request.options?
end
options path do
halt 200
end end
end end
["/:user/*/:key", "/:user/:key"].each do |path| ["/:user/*/:key", "/:user/:key"].each do |path|
get path do get path do
get_data(@user, @directory, @key) storage.get_data(@user, @directory, @key)
end end
end
["/:user/*/", "/:user/"].each do |path|
get path do
get_directory_listing(@user, @directory)
end
end
["/:user/*/:key", "/:user/:key"].each do |path|
put path do put path do
data = request.body.read data = request.body.read
@@ -90,19 +73,29 @@ class LiquorCabinet < Sinatra::Base
content_type = env['CONTENT_TYPE'] content_type = env['CONTENT_TYPE']
end end
put_data(@user, @directory, @key, data, content_type) storage.put_data(@user, @directory, @key, data, content_type)
end end
end
["/:user/*/:key", "/:user/:key"].each do |path|
delete path do delete path do
delete_data(@user, @directory, @key) storage.delete_data(@user, @directory, @key)
end end
end end
["/:user/*/:key", "/:user/:key", "/:user/*/", "/:user/"].each do |path| ["/:user/*/", "/:user/"].each do |path|
options path do get path do
halt 200 storage.get_directory_listing(@user, @directory)
end
end
private
def storage
@storage ||= begin
if settings.riak
RemoteStorage::Riak.new(settings.riak, self)
# elsif settings.redis
# include RemoteStorage::Redis
end
end end
end end

30
spec/riak_spec.rb
View File

@@ -282,13 +282,13 @@ describe "App with Riak backend" do
last_response.body.must_equal @image last_response.body.must_equal @image
end end
it "indexes the binary set" do # it "indexes the binary set" do
indexes = binary_bucket.get("jimmy:documents:jaypeg").indexes # indexes = binary_bucket.get("jimmy:documents:jaypeg").indexes
indexes["user_id_bin"].must_be_kind_of Set # indexes["user_id_bin"].must_be_kind_of Set
indexes["user_id_bin"].must_include "jimmy" # indexes["user_id_bin"].must_include "jimmy"
indexes["directory_bin"].must_include "documents" # indexes["directory_bin"].must_include "documents"
end # end
it "logs the operation" do it "logs the operation" do
objects = [] objects = []
@@ -323,13 +323,13 @@ describe "App with Riak backend" do
last_response.body.must_equal @image last_response.body.must_equal @image
end end
it "indexes the binary set" do # it "indexes the binary set" do
indexes = binary_bucket.get("jimmy:documents:jaypeg").indexes # indexes = binary_bucket.get("jimmy:documents:jaypeg").indexes
indexes["user_id_bin"].must_be_kind_of Set # indexes["user_id_bin"].must_be_kind_of Set
indexes["user_id_bin"].must_include "jimmy" # indexes["user_id_bin"].must_include "jimmy"
indexes["directory_bin"].must_include "documents" # indexes["directory_bin"].must_include "documents"
end # end
end end
end end
@@ -427,9 +427,9 @@ describe "App with Riak backend" do
it "removes the binary object" do it "removes the binary object" do
last_response.status.must_equal 204 last_response.status.must_equal 204
lambda {
binary_bucket.get("jimmy:documents:jaypeg") binary = cs_binary_bucket.files.get("jimmy:documents:jaypeg")
}.must_raise Riak::HTTPFailedRequest binary.must_be_nil
end end
it "logs the operation" do it "logs the operation" do

56
spec/spec_helper.rb
View File

@@ -38,28 +38,64 @@ if app.settings.riak
end end
def data_bucket def data_bucket
@data_bucket ||= client.bucket(app.settings.riak['buckets']['data']) @data_bucket ||= begin
end bucket = client.bucket(app.settings.riak['buckets']['data'])
bucket.allow_mult = false
def auth_bucket bucket
@auth_bucket ||= client.bucket(app.settings.riak['buckets']['authorizations']) end
end end
def directory_bucket def directory_bucket
@directory_bucket ||= client.bucket(app.settings.riak['buckets']['directories']) @directory_bucket ||= begin
bucket = client.bucket(app.settings.riak['buckets']['directories'])
bucket.allow_mult = false
bucket
end
end end
def binary_bucket def auth_bucket
@binary_bucket ||= client.bucket(app.settings.riak['buckets']['binaries']) @auth_bucket ||= begin
bucket = client.bucket(app.settings.riak['buckets']['authorizations'])
bucket.allow_mult = false
bucket
end
end end
def opslog_bucket def opslog_bucket
@opslog_bucket ||= client.bucket(app.settings.riak['buckets']['opslog']) @opslog_bucket ||= begin
bucket = client.bucket(app.settings.riak['buckets']['opslog'])
bucket.allow_mult = false
bucket
end
end
def cs_credentials
@cs_credentials ||= begin
credentials = File.read(app.settings.riak['riak_cs']['credentials_file'])
JSON.parse(credentials)
end
end
def cs_client
@cs_client ||= Fog::Storage.new({
:provider => 'AWS',
:aws_access_key_id => cs_credentials['key_id'],
:aws_secret_access_key => cs_credentials['key_secret'],
:endpoint => app.settings.riak['riak_cs']['endpoint']
})
end
def cs_binary_bucket
@cs_binary_bucket ||= cs_client.directories.create(:key => app.settings.riak['buckets']['cs_binaries'])
end end
def purge_all_buckets def purge_all_buckets
[data_bucket, directory_bucket, auth_bucket, binary_bucket, opslog_bucket].each do |bucket| [data_bucket, directory_bucket, auth_bucket, opslog_bucket].each do |bucket|
bucket.keys.each {|key| bucket.delete key} bucket.keys.each {|key| bucket.delete key}
end end
cs_binary_bucket.files.each do |file|
file.destroy
end
end end
end end