Switch to AWS v4 signatures #5

Merged
basti merged 9 commits from feature/aws_v4_signatures into master 2023-11-21 14:19:38 +00:00
7 changed files with 47 additions and 50 deletions
+2 -3
View File
@@ -1,8 +1,7 @@
# FROM ruby:3.1.4 FROM ruby:3.1.4
FROM ruby:2.7.8
WORKDIR /liquorcabinet WORKDIR /liquorcabinet
ENV RACK_ENV=staging ENV RACK_ENV=production
basti marked this conversation as resolved
Review

Is this change intentional?

Is this change intentional?
Review

Yes, I think default should be production.

Yes, I think default should be production.
COPY Gemfile Gemfile.lock /liquorcabinet/ COPY Gemfile Gemfile.lock /liquorcabinet/
RUN bundle install RUN bundle install
+2 -1
View File
@@ -3,8 +3,9 @@ source "https://rubygems.org"
gem "sinatra", "~> 2.2.0" gem "sinatra", "~> 2.2.0"
gem "sinatra-contrib", "~> 2.2.0" gem "sinatra-contrib", "~> 2.2.0"
gem "activesupport", "~> 6.1.0" gem "activesupport", "~> 6.1.0"
gem "rest-client", "~> 2.1.0"
gem "redis", "~> 4.6.0" gem "redis", "~> 4.6.0"
gem "rest-client", "~> 2.1.0"
gem "aws-sigv4", "~> 1.0.0"
# Remove require when we can update to 3.0, which sets the new storage # Remove require when we can update to 3.0, which sets the new storage
# format to columnar by default. Increases performance # format to columnar by default. Increases performance
gem "mime-types" gem "mime-types"
+2
View File
@@ -9,6 +9,7 @@ GEM
zeitwerk (~> 2.3) zeitwerk (~> 2.3)
addressable (2.8.5) addressable (2.8.5)
public_suffix (>= 2.0.2, < 6.0) public_suffix (>= 2.0.2, < 6.0)
aws-sigv4 (1.0.3)
base64 (0.1.1) base64 (0.1.1)
concurrent-ruby (1.2.2) concurrent-ruby (1.2.2)
crack (0.4.5) crack (0.4.5)
@@ -93,6 +94,7 @@ PLATFORMS
DEPENDENCIES DEPENDENCIES
activesupport (~> 6.1.0) activesupport (~> 6.1.0)
aws-sigv4 (~> 1.0.0)
m m
mime-types mime-types
minitest minitest
+3 -3
View File
@@ -2,11 +2,11 @@ development: &defaults
maintenance: false maintenance: false
redis: redis:
host: <%= ENV["REDIS_HOST"] || "localhost" %> host: <%= ENV["REDIS_HOST"] || "localhost" %>
port: <%= ENV["REDIS_PORT"] || "6379" %> port: <%= ENV["REDIS_PORT"] || 6379 %>
db: <%= ENV["REDIS_DB"] || 0 %> db: <%= ENV["REDIS_DB"] || 1 %>
s3: &s3_defaults s3: &s3_defaults
endpoint: <%= ENV["S3_ENDPOINT"] || "http://127.0.0.1:9000" %> endpoint: <%= ENV["S3_ENDPOINT"] || "http://127.0.0.1:9000" %>
region: <%= ENV["S3_REGION"] %> region: <%= ENV["S3_REGION"] || "us-east-1" %>
access_key_id: <%= ENV["S3_ACCESS_KEY"] || "minioadmin" %> access_key_id: <%= ENV["S3_ACCESS_KEY"] || "minioadmin" %>
secret_key_id: <%= ENV["S3_SECRET_KEY"] || "minioadmin" %> secret_key_id: <%= ENV["S3_SECRET_KEY"] || "minioadmin" %>
bucket: <%= ENV["S3_BUCKET"] || "rs-development" %> bucket: <%= ENV["S3_BUCKET"] || "rs-development" %>
+26 -42
View File
@@ -10,18 +10,26 @@ module RemoteStorage
private private
# S3 already wraps the ETag around quotes def s3_signer
signer ||= Aws::Sigv4::Signer.new(
service: 's3',
region: settings.s3["region"],
access_key_id: settings.s3["access_key_id"].to_s,
secret_access_key: settings.s3["secret_key_id"].to_s
)
end
# S3 already wraps the ETag with quotes
def format_etag(etag) def format_etag(etag)
etag etag
end end
def do_put_request(url, data, content_type) def do_put_request(url, data, content_type)
deal_with_unauthorized_requests do deal_with_unauthorized_requests do
md5 = Digest::MD5.base64digest(data) headers = { "Content-Type" => content_type }
authorization_headers = authorization_headers_for( auth_headers = auth_headers_for("PUT", url, headers, data)
"PUT", url, md5, content_type
).merge({ "Content-Type" => content_type, "Content-Md5" => md5 }) res = RestClient.put(url, data, headers.merge(auth_headers))
res = RestClient.put(url, data, authorization_headers)
return [ return [
res.headers[:etag].delete('"'), res.headers[:etag].delete('"'),
@@ -32,24 +40,24 @@ module RemoteStorage
def do_get_request(url, &block) def do_get_request(url, &block)
deal_with_unauthorized_requests do deal_with_unauthorized_requests do
headers = { } headers = {}
headers["Range"] = server.env["HTTP_RANGE"] if server.env["HTTP_RANGE"] headers["Range"] = server.env["HTTP_RANGE"] if server.env["HTTP_RANGE"]
authorization_headers = authorization_headers_for("GET", url) auth_headers = auth_headers_for("GET", url, headers)
RestClient.get(url, authorization_headers.merge(headers), &block) RestClient.get(url, headers.merge(auth_headers), &block)
end end
end end
def do_head_request(url, &block) def do_head_request(url, &block)
deal_with_unauthorized_requests do deal_with_unauthorized_requests do
authorization_headers = authorization_headers_for("HEAD", url) auth_headers = auth_headers_for("HEAD", url)
RestClient.head(url, authorization_headers, &block) RestClient.head(url, auth_headers, &block)
end end
end end
def do_delete_request(url) def do_delete_request(url)
deal_with_unauthorized_requests do deal_with_unauthorized_requests do
authorization_headers = authorization_headers_for("DELETE", url) auth_headers = auth_headers_for("DELETE", url)
RestClient.delete(url, authorization_headers) RestClient.delete(url, auth_headers)
end end
end end
@@ -67,40 +75,17 @@ module RemoteStorage
return found return found
end end
# This is using the S3 authorizations, not the newer AW V4 Signatures def auth_headers_for(http_method, url, headers = {}, data = nil)
basti marked this conversation as resolved Outdated
Outdated
Review

The headers param could be made optional as well, then you wouldn't need to add an empty hash for the calls in do_head_request and do_delete_request.

auth_headers_for(http_method, url, headers = {}, data = nil)
The `headers` param could be made optional as well, then you wouldn't need to add an empty hash for the calls in `do_head_request` and `do_delete_request`. ```rb auth_headers_for(http_method, url, headers = {}, data = nil) ```
# (https://s3.amazonaws.com/doc/s3-developer-guide/RESTAuthentication.html) signature = s3_signer.sign_request(
def authorization_headers_for(http_verb, url, md5 = nil, content_type = nil) http_method: http_method, url: url, headers: headers, body: data
url = File.join("/", url.gsub(base_url, "")) )
date = Time.now.httpdate signature.headers
signed_data = generate_s3_signature(http_verb, md5, content_type, date, url)
{
"Authorization" => "AWS #{credentials[:access_key_id]}:#{signed_data}",
"Date" => date
}
end
def credentials
@credentials ||= { access_key_id: settings.s3["access_key_id"], secret_key_id: settings.s3["secret_key_id"] }
end
def digest(secret, string_to_sign)
Base64.encode64(hmac(secret, string_to_sign)).strip
end
def hmac(key, value)
OpenSSL::HMAC.digest(OpenSSL::Digest.new('sha1'), key, value)
end end
def uri_escape(s) def uri_escape(s)
CGI.escape(s).gsub('%5B', '[').gsub('%5D', ']') CGI.escape(s).gsub('%5B', '[').gsub('%5D', ']')
end end
def generate_s3_signature(http_verb, md5, content_type, date, url)
string_to_sign = [http_verb, md5, content_type, date, url].join "\n"
signature = digest(credentials[:secret_key_id], string_to_sign)
uri_escape(signature)
end
def base_url def base_url
@base_url ||= settings.s3["endpoint"] @base_url ||= settings.s3["endpoint"]
end end
@@ -109,5 +94,4 @@ module RemoteStorage
"#{base_url}/#{settings.s3["bucket"]}/#{user}" "#{base_url}/#{settings.s3["bucket"]}/#{user}"
end end
end end
end end
+1 -1
View File
@@ -64,7 +64,7 @@ class LiquorCabinet < Sinatra::Base
headers 'Access-Control-Allow-Origin' => '*', headers 'Access-Control-Allow-Origin' => '*',
'Access-Control-Allow-Methods' => 'GET, PUT, DELETE', 'Access-Control-Allow-Methods' => 'GET, PUT, DELETE',
'Access-Control-Allow-Headers' => 'Authorization, Content-Type, Origin, If-Match, If-None-Match, Range', 'Access-Control-Allow-Headers' => 'Authorization, Content-Type, Origin, If-Match, If-None-Match, Range',
'Access-Control-Expose-Headers' => 'ETag, Content-Length, Content-Range', 'Access-Control-Expose-Headers' => 'ETag, Content-Length, Content-Range, Content-Type',
'Accept-Ranges' => 'bytes' 'Accept-Ranges' => 'bytes'
headers['Access-Control-Allow-Origin'] = env["HTTP_ORIGIN"] if env["HTTP_ORIGIN"] headers['Access-Control-Allow-Origin'] = env["HTTP_ORIGIN"] if env["HTTP_ORIGIN"]
headers['Cache-Control'] = 'no-cache' headers['Cache-Control'] = 'no-cache'
Executable
+11
View File
@@ -0,0 +1,11 @@
#!/bin/bash
RACK_ENV=development \
REDIS_HOST=localhost \
REDIS_PORT=6379 \
REDIS_DB=1 \
S3_ENDPOINT='http://localhost:9000' \
S3_ACCESS_KEY='dev-key' \
S3_SECRET_KEY='123456789' \
S3_BUCKET=remotestorage \
bundle exec rackup -p 4567