From f7f23b4a19a84371f44ec5297125e96ba81681a1 Mon Sep 17 00:00:00 2001
From: Eugen Rochko <eugen@zeonfederated.com>
Date: Wed, 19 Jun 2019 23:42:38 +0200
Subject: [PATCH] Add audio uploads (#11123)

* Add audio uploads

Fix #4827

Accept uploads of OGG, WAV, FLAC, OPUS and MP3 files, and converts
them to OGG. Media attachments get a new `audio` type. In the UI,
audio uploads are displayed identically to video uploads.

* Improve code style
---
 app/controllers/media_controller.rb           | 12 +++-
 app/javascript/mastodon/components/status.js  | 12 ++--
 .../compose/components/upload_button.js       |  8 ++-
 .../containers/upload_button_container.js     |  2 +-
 .../status/components/detailed_status.js      | 12 ++--
 app/javascript/mastodon/locales/en.json       |  2 +-
 app/models/media_attachment.rb                | 64 +++++++++++++------
 app/serializers/initial_state_serializer.rb   |  2 +-
 app/services/post_status_service.rb           |  2 +-
 .../stream_entries/_detailed_status.html.haml |  2 +-
 .../stream_entries/_simple_status.html.haml   |  2 +-
 11 files changed, 76 insertions(+), 44 deletions(-)

diff --git a/app/controllers/media_controller.rb b/app/controllers/media_controller.rb
index a245db2d1..d44b52d26 100644
--- a/app/controllers/media_controller.rb
+++ b/app/controllers/media_controller.rb
@@ -7,6 +7,8 @@ class MediaController < ApplicationController
 
   before_action :set_media_attachment
   before_action :verify_permitted_status!
+  before_action :check_playable, only: :player
+  before_action :allow_iframing, only: :player
 
   content_security_policy only: :player do |p|
     p.frame_ancestors(false)
@@ -18,8 +20,6 @@ class MediaController < ApplicationController
 
   def player
     @body_classes = 'player'
-    response.headers['X-Frame-Options'] = 'ALLOWALL'
-    raise ActiveRecord::RecordNotFound unless @media_attachment.video? || @media_attachment.gifv?
   end
 
   private
@@ -34,4 +34,12 @@ class MediaController < ApplicationController
     # Reraise in order to get a 404 instead of a 403 error code
     raise ActiveRecord::RecordNotFound
   end
+
+  def check_playable
+    not_found unless @media_attachment.larger_media_format?
+  end
+
+  def allow_iframing
+    response.headers['X-Frame-Options'] = 'ALLOWALL'
+  end
 end
diff --git a/app/javascript/mastodon/components/status.js b/app/javascript/mastodon/components/status.js
index aa5e870dc..9b1035649 100644
--- a/app/javascript/mastodon/components/status.js
+++ b/app/javascript/mastodon/components/status.js
@@ -333,17 +333,17 @@ class Status extends ImmutablePureComponent {
             media={status.get('media_attachments')}
           />
         );
-      } else if (status.getIn(['media_attachments', 0, 'type']) === 'video') {
-        const video = status.getIn(['media_attachments', 0]);
+      } else if (['video', 'audio'].includes(status.getIn(['media_attachments', 0, 'type']))) {
+        const attachment = status.getIn(['media_attachments', 0]);
 
         media = (
           <Bundle fetchComponent={Video} loading={this.renderLoadingVideoPlayer} >
             {Component => (
               <Component
-                preview={video.get('preview_url')}
-                blurhash={video.get('blurhash')}
-                src={video.get('url')}
-                alt={video.get('description')}
+                preview={attachment.get('preview_url')}
+                blurhash={attachment.get('blurhash')}
+                src={attachment.get('url')}
+                alt={attachment.get('description')}
                 width={this.props.cachedMediaWidth}
                 height={110}
                 inline
diff --git a/app/javascript/mastodon/features/compose/components/upload_button.js b/app/javascript/mastodon/features/compose/components/upload_button.js
index 90e2769f3..e141857be 100644
--- a/app/javascript/mastodon/features/compose/components/upload_button.js
+++ b/app/javascript/mastodon/features/compose/components/upload_button.js
@@ -7,9 +7,11 @@ import ImmutablePureComponent from 'react-immutable-pure-component';
 import ImmutablePropTypes from 'react-immutable-proptypes';
 
 const messages = defineMessages({
-  upload: { id: 'upload_button.label', defaultMessage: 'Add media (JPEG, PNG, GIF, WebM, MP4, MOV)' },
+  upload: { id: 'upload_button.label', defaultMessage: 'Add media ({formats})' },
 });
 
+const SUPPORTED_FORMATS = 'JPEG, PNG, GIF, WebM, MP4, MOV, OGG, WAV, MP3, FLAC';
+
 const makeMapStateToProps = () => {
   const mapStateToProps = state => ({
     acceptContentTypes: state.getIn(['media_attachments', 'accept_content_types']),
@@ -60,9 +62,9 @@ class UploadButton extends ImmutablePureComponent {
 
     return (
       <div className='compose-form__upload-button'>
-        <IconButton icon='camera' title={intl.formatMessage(messages.upload)} disabled={disabled} onClick={this.handleClick} className='compose-form__upload-button-icon' size={18} inverted style={iconStyle} />
+        <IconButton icon='camera' title={intl.formatMessage(messages.upload, { formats: SUPPORTED_FORMATS })} disabled={disabled} onClick={this.handleClick} className='compose-form__upload-button-icon' size={18} inverted style={iconStyle} />
         <label>
-          <span style={{ display: 'none' }}>{intl.formatMessage(messages.upload)}</span>
+          <span style={{ display: 'none' }}>{intl.formatMessage(messages.upload, { formats: SUPPORTED_FORMATS })}</span>
           <input
             key={resetFileKey}
             ref={this.setRef}
diff --git a/app/javascript/mastodon/features/compose/containers/upload_button_container.js b/app/javascript/mastodon/features/compose/containers/upload_button_container.js
index d8b8c4b6e..1471e628b 100644
--- a/app/javascript/mastodon/features/compose/containers/upload_button_container.js
+++ b/app/javascript/mastodon/features/compose/containers/upload_button_container.js
@@ -3,7 +3,7 @@ import UploadButton from '../components/upload_button';
 import { uploadCompose } from '../../../actions/compose';
 
 const mapStateToProps = state => ({
-  disabled: state.getIn(['compose', 'is_uploading']) || (state.getIn(['compose', 'media_attachments']).size > 3 || state.getIn(['compose', 'media_attachments']).some(m => m.get('type') === 'video')),
+  disabled: state.getIn(['compose', 'is_uploading']) || (state.getIn(['compose', 'media_attachments']).size > 3 || state.getIn(['compose', 'media_attachments']).some(m => ['video', 'audio'].includes(m.get('type')))),
   unavailable: state.getIn(['compose', 'poll']) !== null,
   resetFileKey: state.getIn(['compose', 'resetFileKey']),
 });
diff --git a/app/javascript/mastodon/features/status/components/detailed_status.js b/app/javascript/mastodon/features/status/components/detailed_status.js
index c7aa4d033..4af157af1 100644
--- a/app/javascript/mastodon/features/status/components/detailed_status.js
+++ b/app/javascript/mastodon/features/status/components/detailed_status.js
@@ -107,15 +107,15 @@ export default class DetailedStatus extends ImmutablePureComponent {
     }
 
     if (status.get('media_attachments').size > 0) {
-      if (status.getIn(['media_attachments', 0, 'type']) === 'video') {
-        const video = status.getIn(['media_attachments', 0]);
+      if (['video', 'audio'].includes(status.getIn(['media_attachments', 0, 'type']))) {
+        const attachment = status.getIn(['media_attachments', 0]);
 
         media = (
           <Video
-            preview={video.get('preview_url')}
-            blurhash={video.get('blurhash')}
-            src={video.get('url')}
-            alt={video.get('description')}
+            preview={attachment.get('preview_url')}
+            blurhash={attachment.get('blurhash')}
+            src={attachment.get('url')}
+            alt={attachment.get('description')}
             width={300}
             height={150}
             inline
diff --git a/app/javascript/mastodon/locales/en.json b/app/javascript/mastodon/locales/en.json
index 34ec1ca23..67e1e01f8 100644
--- a/app/javascript/mastodon/locales/en.json
+++ b/app/javascript/mastodon/locales/en.json
@@ -369,7 +369,7 @@
   "trends.count_by_accounts": "{count} {rawCount, plural, one {person} other {people}} talking",
   "ui.beforeunload": "Your draft will be lost if you leave Mastodon.",
   "upload_area.title": "Drag & drop to upload",
-  "upload_button.label": "Add media (JPEG, PNG, GIF, WebM, MP4, MOV)",
+  "upload_button.label": "Add media ({formats})",
   "upload_error.limit": "File upload limit exceeded.",
   "upload_error.poll": "File upload not allowed with polls.",
   "upload_form.description": "Describe for the visually impaired",
diff --git a/app/models/media_attachment.rb b/app/models/media_attachment.rb
index ab794faa0..5ca218411 100644
--- a/app/models/media_attachment.rb
+++ b/app/models/media_attachment.rb
@@ -24,14 +24,16 @@
 class MediaAttachment < ApplicationRecord
   self.inheritance_column = nil
 
-  enum type: [:image, :gifv, :video, :unknown]
+  enum type: [:image, :gifv, :video, :unknown, :audio]
 
   IMAGE_FILE_EXTENSIONS = ['.jpg', '.jpeg', '.png', '.gif', '.webp'].freeze
   VIDEO_FILE_EXTENSIONS = ['.webm', '.mp4', '.m4v', '.mov'].freeze
+  AUDIO_FILE_EXTENSIONS = ['.ogg', '.oga', '.mp3', '.wav', '.flac', '.opus'].freeze
 
   IMAGE_MIME_TYPES             = ['image/jpeg', 'image/png', 'image/gif', 'image/webp'].freeze
   VIDEO_MIME_TYPES             = ['video/webm', 'video/mp4', 'video/quicktime'].freeze
   VIDEO_CONVERTIBLE_MIME_TYPES = ['video/webm', 'video/quicktime'].freeze
+  AUDIO_MIME_TYPES             = ['audio/wave', 'audio/wav', 'audio/x-wav', 'audio/x-pn-wave', 'audio/ogg', 'audio/mpeg', 'audio/webm', 'audio/flac'].freeze
 
   BLURHASH_OPTIONS = {
     x_comp: 4,
@@ -65,6 +67,13 @@ class MediaAttachment < ApplicationRecord
     },
   }.freeze
 
+  AUDIO_STYLES = {
+    original: {
+      format: 'ogg',
+      convert_options: {},
+    },
+  }.freeze
+
   VIDEO_FORMAT = {
     format: 'mp4',
     convert_options: {
@@ -83,6 +92,11 @@ class MediaAttachment < ApplicationRecord
     },
   }.freeze
 
+  VIDEO_CONVERTED_STYLES = {
+    small: VIDEO_STYLES[:small],
+    original: VIDEO_FORMAT,
+  }.freeze
+
   IMAGE_LIMIT = 8.megabytes
   VIDEO_LIMIT = 40.megabytes
 
@@ -95,9 +109,9 @@ class MediaAttachment < ApplicationRecord
                     processors: ->(f) { file_processors f },
                     convert_options: { all: '-quality 90 -strip' }
 
-  validates_attachment_content_type :file, content_type: IMAGE_MIME_TYPES + VIDEO_MIME_TYPES
-  validates_attachment_size :file, less_than: IMAGE_LIMIT, unless: :video_or_gifv?
-  validates_attachment_size :file, less_than: VIDEO_LIMIT, if: :video_or_gifv?
+  validates_attachment_content_type :file, content_type: IMAGE_MIME_TYPES + VIDEO_MIME_TYPES + AUDIO_MIME_TYPES
+  validates_attachment_size :file, less_than: IMAGE_LIMIT, unless: :larger_media_format?
+  validates_attachment_size :file, less_than: VIDEO_LIMIT, if: :larger_media_format?
   remotable_attachment :file, VIDEO_LIMIT
 
   include Attachmentable
@@ -120,8 +134,12 @@ class MediaAttachment < ApplicationRecord
     file.blank? && remote_url.present?
   end
 
-  def video_or_gifv?
-    video? || gifv?
+  def larger_media_format?
+    video? || gifv? || audio?
+  end
+
+  def audio_or_video?
+    audio? || video?
   end
 
   def to_param
@@ -156,28 +174,24 @@ class MediaAttachment < ApplicationRecord
     private
 
     def file_styles(f)
-      if f.instance.file_content_type == 'image/gif'
-        {
-          small: IMAGE_STYLES[:small],
-          original: VIDEO_FORMAT,
-        }
-      elsif IMAGE_MIME_TYPES.include? f.instance.file_content_type
+      if f.instance.file_content_type == 'image/gif' || VIDEO_CONVERTIBLE_MIME_TYPES.include?(f.instance.file_content_type)
+        VIDEO_CONVERTED_STYLES
+      elsif IMAGE_MIME_TYPES.include?(f.instance.file_content_type)
         IMAGE_STYLES
-      elsif VIDEO_CONVERTIBLE_MIME_TYPES.include?(f.instance.file_content_type)
-        {
-          small: VIDEO_STYLES[:small],
-          original: VIDEO_FORMAT,
-        }
-      else
+      elsif VIDEO_MIME_TYPES.include?(f.instance.file_content_type)
         VIDEO_STYLES
+      else
+        AUDIO_STYLES
       end
     end
 
     def file_processors(f)
       if f.file_content_type == 'image/gif'
         [:gif_transcoder, :blurhash_transcoder]
-      elsif VIDEO_MIME_TYPES.include? f.file_content_type
+      elsif VIDEO_MIME_TYPES.include?(f.file_content_type)
         [:video_transcoder, :blurhash_transcoder]
+      elsif AUDIO_MIME_TYPES.include?(f.file_content_type)
+        [:transcoder]
       else
         [:lazy_thumbnail, :blurhash_transcoder]
       end
@@ -202,7 +216,15 @@ class MediaAttachment < ApplicationRecord
   end
 
   def set_type_and_extension
-    self.type = VIDEO_MIME_TYPES.include?(file_content_type) ? :video : :image
+    self.type = begin
+      if VIDEO_MIME_TYPES.include?(file_content_type)
+        :video
+      elsif AUDIO_MIME_TYPES.include?(file_content_type)
+        :audio
+      else
+        :image
+      end
+    end
   end
 
   def set_meta
@@ -245,7 +267,7 @@ class MediaAttachment < ApplicationRecord
       frame_rate: movie.frame_rate,
       duration: movie.duration,
       bitrate: movie.bitrate,
-    }
+    }.compact
   end
 
   def reset_parent_cache
diff --git a/app/serializers/initial_state_serializer.rb b/app/serializers/initial_state_serializer.rb
index 184ed867c..3d825410e 100644
--- a/app/serializers/initial_state_serializer.rb
+++ b/app/serializers/initial_state_serializer.rb
@@ -60,7 +60,7 @@ class InitialStateSerializer < ActiveModel::Serializer
   end
 
   def media_attachments
-    { accept_content_types: MediaAttachment::IMAGE_FILE_EXTENSIONS + MediaAttachment::VIDEO_FILE_EXTENSIONS + MediaAttachment::IMAGE_MIME_TYPES + MediaAttachment::VIDEO_MIME_TYPES }
+    { accept_content_types: MediaAttachment::IMAGE_FILE_EXTENSIONS + MediaAttachment::VIDEO_FILE_EXTENSIONS + MediaAttachment::AUDIO_FILE_EXTENSIONS + MediaAttachment::IMAGE_MIME_TYPES + MediaAttachment::VIDEO_MIME_TYPES + MediaAttachment::AUDIO_MIME_TYPES }
   end
 
   private
diff --git a/app/services/post_status_service.rb b/app/services/post_status_service.rb
index b470a40e9..7830aee11 100644
--- a/app/services/post_status_service.rb
+++ b/app/services/post_status_service.rb
@@ -100,7 +100,7 @@ class PostStatusService < BaseService
 
     @media = @account.media_attachments.where(status_id: nil).where(id: @options[:media_ids].take(4).map(&:to_i))
 
-    raise Mastodon::ValidationError, I18n.t('media_attachments.validations.images_and_video') if @media.size > 1 && @media.find(&:video?)
+    raise Mastodon::ValidationError, I18n.t('media_attachments.validations.images_and_video') if @media.size > 1 && @media.find(&:audio_or_video?)
   end
 
   def language_from_option(str)
diff --git a/app/views/stream_entries/_detailed_status.html.haml b/app/views/stream_entries/_detailed_status.html.haml
index 95b96feef..069d0053f 100644
--- a/app/views/stream_entries/_detailed_status.html.haml
+++ b/app/views/stream_entries/_detailed_status.html.haml
@@ -27,7 +27,7 @@
           = render partial: 'stream_entries/poll', locals: { status: status, poll: status.preloadable_poll, autoplay: autoplay }
 
   - if !status.media_attachments.empty?
-    - if status.media_attachments.first.video?
+    - if status.media_attachments.first.audio_or_video?
       - video = status.media_attachments.first
       = react_component :video, src: video.file.url(:original), preview: video.file.url(:small), blurhash: video.blurhash, sensitive: !current_account&.user&.show_all_media? && status.sensitive? || current_account&.user&.hide_all_media?, width: 670, height: 380, detailed: true, inline: true, alt: video.description do
         = render partial: 'stream_entries/attachment_list', locals: { attachments: status.media_attachments }
diff --git a/app/views/stream_entries/_simple_status.html.haml b/app/views/stream_entries/_simple_status.html.haml
index 0f9c65af0..0b924f72f 100644
--- a/app/views/stream_entries/_simple_status.html.haml
+++ b/app/views/stream_entries/_simple_status.html.haml
@@ -31,7 +31,7 @@
           = render partial: 'stream_entries/poll', locals: { status: status, poll: status.preloadable_poll, autoplay: autoplay }
 
   - if !status.media_attachments.empty?
-    - if status.media_attachments.first.video?
+    - if status.media_attachments.first.audio_or_video?
       - video = status.media_attachments.first
       = react_component :video, src: video.file.url(:original), preview: video.file.url(:small), blurhash: video.blurhash, sensitive: !current_account&.user&.show_all_media? && status.sensitive? || current_account&.user&.hide_all_media?, width: 610, height: 343, inline: true, alt: video.description do
         = render partial: 'stream_entries/attachment_list', locals: { attachments: status.media_attachments }