From 189c304a0831166ddf437448e697ef5fe6d15365 Mon Sep 17 00:00:00 2001 From: Sebastian Kippe Date: Wed, 15 Nov 2023 19:03:33 +0100 Subject: [PATCH 1/9] Add example script for running server in dev --- run-dev.sh | 11 +++++++++++ 1 file changed, 11 insertions(+) create mode 100755 run-dev.sh diff --git a/run-dev.sh b/run-dev.sh new file mode 100755 index 0000000..e1b5e28 --- /dev/null +++ b/run-dev.sh @@ -0,0 +1,11 @@ +#!/bin/bash + +RACK_ENV=staging \ +REDIS_HOST=localhost \ +REDIS_PORT=6379 \ +REDIS_DB=1 \ +S3_ENDPOINT='http://localhost:9000' \ +S3_ACCESS_KEY='dev-key' \ +S3_SECRET_KEY='123456789' \ +S3_BUCKET=remotestorage \ +bundle exec rainbows --listen localhost:4567 -- 2.25.1 From 02820203f0a4c2588e8d0cc97436d7add09ab8a9 Mon Sep 17 00:00:00 2001 From: Sebastian Kippe Date: Wed, 15 Nov 2023 19:05:21 +0100 Subject: [PATCH 2/9] Switch to AWS V4 signatures --- Gemfile | 3 +- Gemfile.lock | 2 ++ lib/remote_storage/s3.rb | 72 +++++++++++++++++----------------------- 3 files changed, 35 insertions(+), 42 deletions(-) diff --git a/Gemfile b/Gemfile index b65ae92..d17ab0e 100644 --- a/Gemfile +++ b/Gemfile @@ -3,8 +3,9 @@ source "https://rubygems.org" gem "sinatra", "~> 2.2.0" gem "sinatra-contrib", "~> 2.2.0" gem "activesupport", "~> 6.1.0" -gem "rest-client", "~> 2.1.0" gem "redis", "~> 4.6.0" +gem "rest-client", "~> 2.1.0" +gem "aws-sigv4", "~> 1.0.0" # Remove require when we can update to 3.0, which sets the new storage # format to columnar by default. Increases performance gem "mime-types" diff --git a/Gemfile.lock b/Gemfile.lock index c9676ac..5f987a6 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -9,6 +9,7 @@ GEM zeitwerk (~> 2.3) addressable (2.8.5) public_suffix (>= 2.0.2, < 6.0) + aws-sigv4 (1.0.3) base64 (0.1.1) concurrent-ruby (1.2.2) crack (0.4.5) @@ -93,6 +94,7 @@ PLATFORMS DEPENDENCIES activesupport (~> 6.1.0) + aws-sigv4 (~> 1.0.0) m mime-types minitest diff --git a/lib/remote_storage/s3.rb b/lib/remote_storage/s3.rb index e6c2a2a..2a35a82 100644 --- a/lib/remote_storage/s3.rb +++ b/lib/remote_storage/s3.rb @@ -10,46 +10,58 @@ module RemoteStorage private - # S3 already wraps the ETag around quotes + def s3_signer + signer ||= Aws::Sigv4::Signer.new( + service: 's3', + region: settings.s3["region"], + access_key_id: settings.s3["access_key_id"].to_s, + secret_access_key: settings.s3["secret_key_id"].to_s + ) + end + + # S3 already wraps the ETag with quotes def format_etag(etag) etag end def do_put_request(url, data, content_type) deal_with_unauthorized_requests do - md5 = Digest::MD5.base64digest(data) - authorization_headers = authorization_headers_for( - "PUT", url, md5, content_type - ).merge({ "Content-Type" => content_type, "Content-Md5" => md5 }) - res = RestClient.put(url, data, authorization_headers) + headers = { "Content-Type" => content_type } + auth_headers = auth_headers_for("PUT", url, headers, data) + + res = RestClient.put(url, data, headers.merge(auth_headers)) return [ res.headers[:etag].delete('"'), timestamp_for(res.headers[:date]) # S3 does not return a Last-Modified response header on PUTs ] end + + rescue RestClient::Forbidden => ex + puts ex.response.to_s + raise ex end def do_get_request(url, &block) deal_with_unauthorized_requests do - headers = { } + headers = {} headers["Range"] = server.env["HTTP_RANGE"] if server.env["HTTP_RANGE"] - authorization_headers = authorization_headers_for("GET", url) - RestClient.get(url, authorization_headers.merge(headers), &block) + auth_headers = auth_headers_for("GET", url, headers) + RestClient.get(url, headers.merge(auth_headers), &block) end end def do_head_request(url, &block) deal_with_unauthorized_requests do - authorization_headers = authorization_headers_for("HEAD", url) - RestClient.head(url, authorization_headers, &block) + auth_headers = auth_headers_for("HEAD", url, {}) + RestClient.head(url, auth_headers, &block) end end def do_delete_request(url) deal_with_unauthorized_requests do - authorization_headers = authorization_headers_for("DELETE", url) - RestClient.delete(url, authorization_headers) + auth_headers = auth_headers_for("DELETE", url, {}) + RestClient.delete(url, auth_headers) end end @@ -67,40 +79,19 @@ module RemoteStorage return found end - # This is using the S3 authorizations, not the newer AW V4 Signatures - # (https://s3.amazonaws.com/doc/s3-developer-guide/RESTAuthentication.html) - def authorization_headers_for(http_verb, url, md5 = nil, content_type = nil) - url = File.join("/", url.gsub(base_url, "")) - date = Time.now.httpdate - signed_data = generate_s3_signature(http_verb, md5, content_type, date, url) - { - "Authorization" => "AWS #{credentials[:access_key_id]}:#{signed_data}", - "Date" => date - } - end + def auth_headers_for(http_method, url, headers, data = nil) + signature = s3_signer.sign_request( + http_method: http_method, url: url, headers: headers, body: data + ) - def credentials - @credentials ||= { access_key_id: settings.s3["access_key_id"], secret_key_id: settings.s3["secret_key_id"] } - end - - def digest(secret, string_to_sign) - Base64.encode64(hmac(secret, string_to_sign)).strip - end - - def hmac(key, value) - OpenSSL::HMAC.digest(OpenSSL::Digest.new('sha1'), key, value) + puts signature.headers.inspect + signature.headers end def uri_escape(s) CGI.escape(s).gsub('%5B', '[').gsub('%5D', ']') end - def generate_s3_signature(http_verb, md5, content_type, date, url) - string_to_sign = [http_verb, md5, content_type, date, url].join "\n" - signature = digest(credentials[:secret_key_id], string_to_sign) - uri_escape(signature) - end - def base_url @base_url ||= settings.s3["endpoint"] end @@ -109,5 +100,4 @@ module RemoteStorage "#{base_url}/#{settings.s3["bucket"]}/#{user}" end end - end -- 2.25.1 From b7ff3e7d42e8d54ab221a806ad1fe99d31058fe8 Mon Sep 17 00:00:00 2001 From: Sebastian Kippe Date: Thu, 16 Nov 2023 10:27:58 +0100 Subject: [PATCH 3/9] Remove debug output, rescue --- lib/remote_storage/s3.rb | 6 ------ 1 file changed, 6 deletions(-) diff --git a/lib/remote_storage/s3.rb b/lib/remote_storage/s3.rb index 2a35a82..7d2ab86 100644 --- a/lib/remote_storage/s3.rb +++ b/lib/remote_storage/s3.rb @@ -36,10 +36,6 @@ module RemoteStorage timestamp_for(res.headers[:date]) # S3 does not return a Last-Modified response header on PUTs ] end - - rescue RestClient::Forbidden => ex - puts ex.response.to_s - raise ex end def do_get_request(url, &block) @@ -83,8 +79,6 @@ module RemoteStorage signature = s3_signer.sign_request( http_method: http_method, url: url, headers: headers, body: data ) - - puts signature.headers.inspect signature.headers end -- 2.25.1 From 96417e3884c6bf275ce353a3005be44c86ac662c Mon Sep 17 00:00:00 2001 From: Sebastian Kippe Date: Thu, 16 Nov 2023 11:27:15 +0100 Subject: [PATCH 4/9] Use development env and Webrick in dev run script --- run-dev.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/run-dev.sh b/run-dev.sh index e1b5e28..fdb9a2b 100755 --- a/run-dev.sh +++ b/run-dev.sh @@ -1,6 +1,6 @@ #!/bin/bash -RACK_ENV=staging \ +RACK_ENV=development \ REDIS_HOST=localhost \ REDIS_PORT=6379 \ REDIS_DB=1 \ @@ -8,4 +8,4 @@ S3_ENDPOINT='http://localhost:9000' \ S3_ACCESS_KEY='dev-key' \ S3_SECRET_KEY='123456789' \ S3_BUCKET=remotestorage \ -bundle exec rainbows --listen localhost:4567 +bundle exec rackup -p 4567 -- 2.25.1 From 024f516a9d2a03ee966df7d4ec0be639d956a2c3 Mon Sep 17 00:00:00 2001 From: Sebastian Kippe Date: Thu, 16 Nov 2023 11:39:33 +0100 Subject: [PATCH 5/9] Expose Content-Type header --- liquor-cabinet.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/liquor-cabinet.rb b/liquor-cabinet.rb index f823375..ad25fdb 100644 --- a/liquor-cabinet.rb +++ b/liquor-cabinet.rb @@ -64,7 +64,7 @@ class LiquorCabinet < Sinatra::Base headers 'Access-Control-Allow-Origin' => '*', 'Access-Control-Allow-Methods' => 'GET, PUT, DELETE', 'Access-Control-Allow-Headers' => 'Authorization, Content-Type, Origin, If-Match, If-None-Match, Range', - 'Access-Control-Expose-Headers' => 'ETag, Content-Length, Content-Range', + 'Access-Control-Expose-Headers' => 'ETag, Content-Length, Content-Range, Content-Type', 'Accept-Ranges' => 'bytes' headers['Access-Control-Allow-Origin'] = env["HTTP_ORIGIN"] if env["HTTP_ORIGIN"] headers['Cache-Control'] = 'no-cache' -- 2.25.1 From d09c6e7a39d480ee16cfc4d4b5571d563e941bb2 Mon Sep 17 00:00:00 2001 From: Sebastian Kippe Date: Thu, 16 Nov 2023 11:41:57 +0100 Subject: [PATCH 6/9] Use production env for Docker image --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index c509fc7..16f7de7 100644 --- a/Dockerfile +++ b/Dockerfile @@ -2,7 +2,7 @@ FROM ruby:2.7.8 WORKDIR /liquorcabinet -ENV RACK_ENV=staging +ENV RACK_ENV=production COPY Gemfile Gemfile.lock /liquorcabinet/ RUN bundle install -- 2.25.1 From 91cadbf2283f1be64bf9d1c0240d6b9f2b747d95 Mon Sep 17 00:00:00 2001 From: Sebastian Kippe Date: Thu, 16 Nov 2023 11:45:07 +0100 Subject: [PATCH 7/9] Switch Docker image back to Ruby 3.1 --- Dockerfile | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index 16f7de7..6593f04 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,5 +1,4 @@ -# FROM ruby:3.1.4 -FROM ruby:2.7.8 +FROM ruby:3.1.4 WORKDIR /liquorcabinet ENV RACK_ENV=production -- 2.25.1 From 589dcf7fa2765f3fafaa6b7c85341e512960e62d Mon Sep 17 00:00:00 2001 From: Sebastian Kippe Date: Thu, 16 Nov 2023 12:39:46 +0100 Subject: [PATCH 8/9] Update example config --- config.yml.erb.example | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/config.yml.erb.example b/config.yml.erb.example index da19177..2ca3f82 100644 --- a/config.yml.erb.example +++ b/config.yml.erb.example @@ -2,11 +2,11 @@ development: &defaults maintenance: false redis: host: <%= ENV["REDIS_HOST"] || "localhost" %> - port: <%= ENV["REDIS_PORT"] || "6379" %> - db: <%= ENV["REDIS_DB"] || 0 %> + port: <%= ENV["REDIS_PORT"] || 6379 %> + db: <%= ENV["REDIS_DB"] || 1 %> s3: &s3_defaults endpoint: <%= ENV["S3_ENDPOINT"] || "http://127.0.0.1:9000" %> - region: <%= ENV["S3_REGION"] %> + region: <%= ENV["S3_REGION"] || "us-east-1" %> access_key_id: <%= ENV["S3_ACCESS_KEY"] || "minioadmin" %> secret_key_id: <%= ENV["S3_SECRET_KEY"] || "minioadmin" %> bucket: <%= ENV["S3_BUCKET"] || "rs-development" %> -- 2.25.1 From 209fcca5ea4c4b56baa2606872276227da3b4d80 Mon Sep 17 00:00:00 2001 From: Sebastian Kippe Date: Tue, 21 Nov 2023 15:18:13 +0100 Subject: [PATCH 9/9] Make headers argument optional --- lib/remote_storage/s3.rb | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/lib/remote_storage/s3.rb b/lib/remote_storage/s3.rb index 7d2ab86..4fd910b 100644 --- a/lib/remote_storage/s3.rb +++ b/lib/remote_storage/s3.rb @@ -49,14 +49,14 @@ module RemoteStorage def do_head_request(url, &block) deal_with_unauthorized_requests do - auth_headers = auth_headers_for("HEAD", url, {}) + auth_headers = auth_headers_for("HEAD", url) RestClient.head(url, auth_headers, &block) end end def do_delete_request(url) deal_with_unauthorized_requests do - auth_headers = auth_headers_for("DELETE", url, {}) + auth_headers = auth_headers_for("DELETE", url) RestClient.delete(url, auth_headers) end end @@ -75,7 +75,7 @@ module RemoteStorage return found end - def auth_headers_for(http_method, url, headers, data = nil) + def auth_headers_for(http_method, url, headers = {}, data = nil) signature = s3_signer.sign_request( http_method: http_method, url: url, headers: headers, body: data ) -- 2.25.1