Merge branch 'master' into stable

It was flattened by accident

Refs #62

.drone.yml

@@ -1,9 +0,0 @@
-kind: pipeline
-name: default
-
-steps:
-- name: test
-  image: ruby
-  commands:
-  - bundle install --jobs=3 --retry=3
-  - rake test

.gitea/workflows/ruby.yml

@@ -1,33 +0,0 @@
-name: Tests
-
-on:
-  push:
-    branches: [ master ]
-  pull_request:
-    branches: [ master ]
-
-permissions:
-  contents: read
-
-jobs:
-  test:
-    runs-on: ubuntu-latest
-    strategy:
-      matrix:
-        ruby-version: ['2.7', '3.0', '3.1']
-        redis-version: [6, 7]
-    steps:
-    - uses: actions/checkout@v3
-    - name: Set up Ruby
-      uses: ruby/setup-ruby@v1
-      with:
-        ruby-version: ${{ matrix.ruby-version }}
-        bundler-cache: true # runs 'bundle install' and caches installed gems automatically
-    - name: Start Redis
-      uses: supercharge/redis-github-action@1.4.0
-      with:
-        redis-version: ${{ matrix.redis-version }}
-    - name: Configure
-      run: cp config.yml.erb.example config.yml.erb
-    - name: Run tests
-      run: bundle exec rake test

.github/workflows/ruby.yml

@@ -11,11 +11,13 @@ permissions:
 
 jobs:
   test:
+
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        ruby-version: ['2.7', '3.0', '3.1']
-        redis-version: [6, 7]
+        ruby-version: ['2.5', '2.6', '2.7']
+        redis-version: [4, 5, 6]
+
     steps:
     - uses: actions/checkout@v3
     - name: Set up Ruby
@@ -28,6 +30,6 @@ jobs:
       with:
         redis-version: ${{ matrix.redis-version }}
     - name: Configure
-      run: cp config.yml.erb.example config.yml.erb
+      run: cp config.yml.example.s3 config.yml
     - name: Run tests
       run: bundle exec rake test

.gitignore

@@ -1,5 +1,4 @@
 config.yml
-config.yml.erb
 cs_credentials.json
 pids
 .bundle

Dockerfile

@@ -1,13 +0,0 @@
-FROM ruby:3.1.4
-
-WORKDIR /liquorcabinet
-ENV RACK_ENV=production
-
-COPY Gemfile Gemfile.lock /liquorcabinet/
-RUN bundle install
-COPY . /liquorcabinet
-COPY ./config.yml.erb.example /liquorcabinet/config.yml.erb
-
-EXPOSE 4567
-
-CMD ["bundle", "exec", "rainbows", "--listen", "0.0.0.0:4567"]

Gemfile

@@ -2,10 +2,9 @@ source "https://rubygems.org"
 
 gem "sinatra", "~> 2.2.0"
 gem "sinatra-contrib", "~> 2.2.0"
-gem "activesupport", "~> 6.1.0"
+gem "activesupport", "~> 6.0.5"
+gem "rest-client", "~> 2.1.0" # Fixes a memory leak in Ruby 2.4
 gem "redis", "~> 4.6.0"
-gem "rest-client", "~> 2.1.0"
-gem "aws-sigv4", "~> 1.0.0"
 # Remove require when we can update to 3.0, which sets the new storage
 # format to columnar by default. Increases performance
 gem "mime-types"
@@ -13,8 +12,8 @@ gem "mime-types"
 group :test do
   gem 'rake'
   gem 'rack-test'
+  gem 'purdytest', :require => false
   gem 'm'
-  gem 'minitest'
   gem 'minitest-stub_any_instance'
   gem 'webmock'
 end

Gemfile.lock

@@ -1,104 +1,102 @@
 GEM
   remote: https://rubygems.org/
   specs:
-    activesupport (6.1.7.6)
+    activesupport (6.0.5)
       concurrent-ruby (~> 1.0, >= 1.0.2)
-      i18n (>= 1.6, < 2)
-      minitest (>= 5.1)
-      tzinfo (~> 2.0)
-      zeitwerk (~> 2.3)
-    addressable (2.8.5)
-      public_suffix (>= 2.0.2, < 6.0)
-    aws-sigv4 (1.0.3)
-    base64 (0.1.1)
-    concurrent-ruby (1.2.2)
-    crack (0.4.5)
-      rexml
+      i18n (>= 0.7, < 2)
+      minitest (~> 5.1)
+      tzinfo (~> 1.1)
+      zeitwerk (~> 2.2, >= 2.2.2)
+    addressable (2.8.0)
+      public_suffix (>= 2.0.2, < 5.0)
+    concurrent-ruby (1.1.10)
+    crack (0.4.3)
+      safe_yaml (~> 1.0.0)
     domain_name (0.5.20190701)
       unf (>= 0.0.5, < 1.0.0)
-    faraday (2.7.11)
-      base64
-      faraday-net_http (>= 2.0, < 3.1)
-      ruby2_keywords (>= 0.0.4)
-    faraday-net_http (3.0.2)
-    hashdiff (1.0.1)
+    faraday (0.17.0)
+      multipart-post (>= 1.2, < 3)
+    hashdiff (1.0.0)
     http-accept (1.7.0)
-    http-cookie (1.0.5)
+    http-cookie (1.0.3)
       domain_name (~> 0.5)
-    i18n (1.14.1)
+    i18n (1.10.0)
       concurrent-ruby (~> 1.0)
-    kgio (2.11.4)
-    m (1.6.2)
+    kgio (2.11.2)
+    m (1.5.1)
       method_source (>= 0.6.7)
       rake (>= 0.9.2.2)
-    method_source (1.0.0)
-    mime-types (3.5.1)
+    method_source (0.9.2)
+    mime-types (3.3)
       mime-types-data (~> 3.2015)
-    mime-types-data (3.2023.1003)
-    minitest (5.20.0)
-    minitest-stub_any_instance (1.0.3)
+    mime-types-data (3.2019.1009)
+    minitest (5.13.0)
+    minitest-stub_any_instance (1.0.2)
     multi_json (1.15.0)
-    mustermann (2.0.2)
+    multipart-post (2.1.1)
+    mustermann (1.1.1)
       ruby2_keywords (~> 0.0.1)
     netrc (0.11.0)
-    public_suffix (5.0.3)
-    rack (2.2.8)
-    rack-protection (2.2.4)
+    public_suffix (4.0.1)
+    purdytest (2.0.0)
+      minitest (~> 5.5)
+    rack (2.2.3)
+    rack-protection (2.2.0)
       rack
-    rack-test (2.1.0)
-      rack (>= 1.3)
-    rainbows (5.2.1)
+    rack-test (1.1.0)
+      rack (>= 1.0, < 3)
+    rainbows (5.2.0)
       kgio (~> 2.5)
       rack (>= 1.1, < 3.0)
       unicorn (~> 5.1)
-    raindrops (0.20.1)
-    rake (13.0.6)
+    raindrops (0.19.0)
+    rake (13.0.1)
     redis (4.6.0)
     rest-client (2.1.0)
       http-accept (>= 1.7.0, < 2.0)
       http-cookie (>= 1.0.2, < 2.0)
       mime-types (>= 1.16, < 4.0)
       netrc (~> 0.8)
-    rexml (3.2.6)
     ruby2_keywords (0.0.5)
-    sentry-raven (3.1.2)
-      faraday (>= 1.0)
-    sinatra (2.2.4)
-      mustermann (~> 2.0)
+    safe_yaml (1.0.5)
+    sentry-raven (2.12.3)
+      faraday (>= 0.7.6, < 1.0)
+    sinatra (2.2.0)
+      mustermann (~> 1.0)
       rack (~> 2.2)
-      rack-protection (= 2.2.4)
+      rack-protection (= 2.2.0)
       tilt (~> 2.0)
-    sinatra-contrib (2.2.4)
+    sinatra-contrib (2.2.0)
       multi_json
-      mustermann (~> 2.0)
-      rack-protection (= 2.2.4)
-      sinatra (= 2.2.4)
+      mustermann (~> 1.0)
+      rack-protection (= 2.2.0)
+      sinatra (= 2.2.0)
       tilt (~> 2.0)
-    tilt (2.3.0)
-    tzinfo (2.0.6)
-      concurrent-ruby (~> 1.0)
+    thread_safe (0.3.6)
+    tilt (2.0.10)
+    tzinfo (1.2.9)
+      thread_safe (~> 0.1)
     unf (0.1.4)
       unf_ext
-    unf_ext (0.0.8.2)
-    unicorn (5.8.0)
+    unf_ext (0.0.7.6)
+    unicorn (5.5.1)
       kgio (~> 2.6)
       raindrops (~> 0.7)
-    webmock (3.19.1)
-      addressable (>= 2.8.0)
+    webmock (3.7.6)
+      addressable (>= 2.3.6)
       crack (>= 0.3.2)
       hashdiff (>= 0.4.0, < 2.0.0)
-    zeitwerk (2.6.12)
+    zeitwerk (2.5.4)
 
 PLATFORMS
   ruby
 
 DEPENDENCIES
-  activesupport (~> 6.1.0)
-  aws-sigv4 (~> 1.0.0)
+  activesupport (~> 6.0.5)
   m
   mime-types
-  minitest
   minitest-stub_any_instance
+  purdytest
   rack-test
   rainbows
   rake
@@ -110,4 +108,4 @@ DEPENDENCIES
   webmock
 
 BUNDLED WITH
-   2.3.7
+   1.16.0

README.md

@@ -12,9 +12,10 @@ protocol, but does not include the Webfinger and OAuth parts. It is meant to be
 added to existing systems and user accounts, so you will have to add your own
 OAuth dialog for remoteStorage authorizations and persist the tokens in Redis.
 
-If you have any questions about this program, please [post to the RS
+If you have any questions about this program, drop by #remotestorage on
+Freenode, or [post to the RS
 forums](https://community.remotestorage.io/c/server-development), and we'll
-gladly answer them.
+happily answer them.
 
 ## Contributing
 
@@ -22,5 +23,5 @@ We love pull requests. If you want to submit a patch:
 
 * Fork the project.
 * Make your feature addition or bug fix.
-* Write specs for it. This is important so nobody breaks it in a future version.
+* Write specs for it. This is important so nobody breaks it in a future version unintentionally.
 * Push to your fork and send a pull request.

config.yml.erb.example

@@ -1,18 +0,0 @@
-development: &defaults
-  maintenance: false
-  redis:
-    host: <%= ENV["REDIS_HOST"] || "localhost" %>
-    port: <%= ENV["REDIS_PORT"] || 6379 %>
-    db: <%= ENV["REDIS_DB"] || 1 %>
-  s3: &s3_defaults
-    endpoint: <%= ENV["S3_ENDPOINT"] || "http://127.0.0.1:9000" %>
-    region: <%= ENV["S3_REGION"] || "us-east-1" %>
-    access_key_id: <%= ENV["S3_ACCESS_KEY"] || "minioadmin" %>
-    secret_key_id: <%= ENV["S3_SECRET_KEY"] || "minioadmin" %>
-    bucket: <%= ENV["S3_BUCKET"] || "rs-development" %>
-test:
-  <<: *defaults
-staging:
-  <<: *defaults
-production:
-  <<: *defaults

config.yml.example.s3

@@ -0,0 +1,26 @@
+development: &defaults
+  maintenance: false
+  s3:
+    endpoint: "https://some-endpoint"
+    region: "region"
+    access_key_id: ""
+    secret_key_id: ""
+    bucket: "test-bucket"
+  redis:
+    host: localhost
+    port: 6379
+
+test:
+  <<: *defaults
+  s3:
+    endpoint: "https://some-endpoint"
+    region: "region"
+    access_key_id: ""
+    secret_key_id: ""
+    bucket: "test-bucket"
+
+staging:
+  <<: *defaults
+
+production:
+  <<: *defaults

lib/remote_storage/s3.rb

@@ -1,8 +1,8 @@
 require "remote_storage/rest_provider"
 require "digest"
 require "base64"
-require "cgi"
 require "openssl"
+require "webrick/httputils"
 
 module RemoteStorage
   class S3
@@ -10,26 +10,18 @@ module RemoteStorage
 
     private
 
-    def s3_signer
-      signer ||= Aws::Sigv4::Signer.new(
-        service: 's3',
-        region: settings.s3["region"],
-        access_key_id: settings.s3["access_key_id"].to_s,
-        secret_access_key: settings.s3["secret_key_id"].to_s
-      )
-    end
-
-    # S3 already wraps the ETag with quotes
+    # S3 already wraps the ETag around quotes
     def format_etag(etag)
       etag
     end
 
     def do_put_request(url, data, content_type)
       deal_with_unauthorized_requests do
-        headers = { "Content-Type" => content_type }
-        auth_headers = auth_headers_for("PUT", url, headers, data)
-
-        res = RestClient.put(url, data, headers.merge(auth_headers))
+        md5 = Digest::MD5.base64digest(data)
+        authorization_headers = authorization_headers_for(
+          "PUT", url, md5, content_type
+        ).merge({ "Content-Type" => content_type, "Content-Md5" => md5 })
+        res = RestClient.put(url, data, authorization_headers)
 
         return [
           res.headers[:etag].delete('"'),
@@ -40,24 +32,24 @@ module RemoteStorage
 
     def do_get_request(url, &block)
       deal_with_unauthorized_requests do
-        headers = {}
+        headers = { }
         headers["Range"] = server.env["HTTP_RANGE"] if server.env["HTTP_RANGE"]
-        auth_headers = auth_headers_for("GET", url, headers)
-        RestClient.get(url, headers.merge(auth_headers), &block)
+        authorization_headers = authorization_headers_for("GET", url)
+        RestClient.get(url, authorization_headers.merge(headers), &block)
       end
     end
 
     def do_head_request(url, &block)
       deal_with_unauthorized_requests do
-        auth_headers = auth_headers_for("HEAD", url)
-        RestClient.head(url, auth_headers, &block)
+        authorization_headers = authorization_headers_for("HEAD", url)
+        RestClient.head(url, authorization_headers, &block)
       end
     end
 
     def do_delete_request(url)
       deal_with_unauthorized_requests do
-        auth_headers = auth_headers_for("DELETE", url)
-        RestClient.delete(url, auth_headers)
+        authorization_headers = authorization_headers_for("DELETE", url)
+        RestClient.delete(url, authorization_headers)
       end
     end
 
@@ -75,15 +67,38 @@ module RemoteStorage
       return found
     end
 
-    def auth_headers_for(http_method, url, headers = {}, data = nil)
-      signature = s3_signer.sign_request(
-        http_method: http_method, url: url, headers: headers, body: data
-      )
-      signature.headers
+    # This is using the S3 authorizations, not the newer AW V4 Signatures
+    # (https://s3.amazonaws.com/doc/s3-developer-guide/RESTAuthentication.html)
+    def authorization_headers_for(http_verb, url, md5 = nil, content_type = nil)
+      url = File.join("/", url.gsub(base_url, ""))
+      date = Time.now.httpdate
+      signed_data = generate_s3_signature(http_verb, md5, content_type, date, url)
+      {
+        "Authorization" => "AWS #{credentials[:access_key_id]}:#{signed_data}",
+        "Date" => date
+      }
+    end
+
+    def credentials
+      @credentials ||= { access_key_id: settings.s3["access_key_id"], secret_key_id: settings.s3["secret_key_id"] }
+    end
+
+    def digest(secret, string_to_sign)
+      Base64.encode64(hmac(secret, string_to_sign)).strip
+    end
+
+    def hmac(key, value)
+      OpenSSL::HMAC.digest(OpenSSL::Digest.new('sha1'), key, value)
     end
 
     def uri_escape(s)
-      CGI.escape(s).gsub('%5B', '[').gsub('%5D', ']')
+      WEBrick::HTTPUtils.escape(s).gsub('%5B', '[').gsub('%5D', ']')
+    end
+
+    def generate_s3_signature(http_verb, md5, content_type, date, url)
+      string_to_sign = [http_verb, md5, content_type, date, url].join "\n"
+      signature = digest(credentials[:secret_key_id], string_to_sign)
+      uri_escape(signature)
     end
 
     def base_url
@@ -91,7 +106,8 @@ module RemoteStorage
     end
 
     def container_url_for(user)
-      "#{base_url}/#{settings.s3["bucket"]}/#{user}"
+      "#{base_url}#{settings.s3["bucket"]}/#{user}"
     end
   end
+
 end

liquor-cabinet.rb

@@ -18,7 +18,7 @@ class LiquorCabinet < Sinatra::Base
 
     register Sinatra::ConfigFile
     set :environments, %w{development test production staging}
-    config_file 'config.yml.erb'
+    config_file 'config.yml'
   end
 
   configure :development do
@@ -64,7 +64,7 @@ class LiquorCabinet < Sinatra::Base
       headers 'Access-Control-Allow-Origin' => '*',
               'Access-Control-Allow-Methods' => 'GET, PUT, DELETE',
               'Access-Control-Allow-Headers' => 'Authorization, Content-Type, Origin, If-Match, If-None-Match, Range',
-              'Access-Control-Expose-Headers' => 'ETag, Content-Length, Content-Range, Content-Type',
+              'Access-Control-Expose-Headers' => 'ETag, Content-Length, Content-Range',
               'Accept-Ranges' => 'bytes'
       headers['Access-Control-Allow-Origin'] = env["HTTP_ORIGIN"] if env["HTTP_ORIGIN"]
       headers['Cache-Control'] = 'no-cache'

run-dev.sh

@@ -1,11 +0,0 @@
-#!/bin/bash
-
-RACK_ENV=development \
-REDIS_HOST=localhost \
-REDIS_PORT=6379 \
-REDIS_DB=1 \
-S3_ENDPOINT='http://localhost:9000' \
-S3_ACCESS_KEY='dev-key' \
-S3_SECRET_KEY='123456789' \
-S3_BUCKET=remotestorage \
-bundle exec rackup -p 4567

spec/s3/app_spec.rb

@@ -2,7 +2,7 @@ require_relative "../spec_helper"
 
 describe "S3 provider" do
   def container_url_for(user)
-    "#{app.settings.s3["endpoint"]}/#{app.settings.s3["bucket"]}/#{user}"
+    "#{app.settings.s3["endpoint"]}#{app.settings.s3["bucket"]}/#{user}"
   end
 
   def storage_class

spec/spec_helper.rb

@@ -8,6 +8,7 @@ require_relative '../liquor-cabinet'
 require 'minitest/autorun'
 require "minitest/stub_any_instance"
 require 'rack/test'
+require 'purdytest'
 require "redis"
 require "rest_client"
 require "ostruct"
@@ -35,22 +36,22 @@ if app.settings.respond_to? :redis
   end
 end
 
-Minitest::Spec.class_eval do
+MiniTest::Spec.class_eval do
   def self.shared_examples
     @shared_examples ||= {}
   end
 end
 
-module Minitest::Spec::SharedExamples
+module MiniTest::Spec::SharedExamples
   def shared_examples_for(desc, &block)
-    Minitest::Spec.shared_examples[desc] = block
+    MiniTest::Spec.shared_examples[desc] = block
   end
 
   def it_behaves_like(desc)
-    self.instance_eval(&Minitest::Spec.shared_examples[desc])
+    self.instance_eval(&MiniTest::Spec.shared_examples[desc])
   end
 end
 
-Object.class_eval { include(Minitest::Spec::SharedExamples) }
+Object.class_eval { include(MiniTest::Spec::SharedExamples) }
 
 require_relative 'shared_examples'