Merge branch 'main' into glitch-soc/merge-upstream

- `app/views/statuses/_simple_status.html.haml`:
  Small markup change in glitch-soc, on a line that has been modified by
  upstream. Ported upstream changes.

lib/exceptions.rb

@@ -12,7 +12,11 @@ module Mastodon
   class RateLimitExceededError < Error; end
 
   class UnexpectedResponseError < Error
+    attr_reader :response
+
     def initialize(response = nil)
+      @response = response
+
       if response.respond_to? :uri
         super("#{response.uri} returned code #{response.code}")
       else

lib/mastodon/migration_helpers.rb

@@ -95,7 +95,7 @@ module Mastodon
             allow_null: options[:null]
           )
         else
-          add_column(table_name, column_name, :datetime_with_timezone, options)
+          add_column(table_name, column_name, :datetime_with_timezone, **options)
         end
       end
     end
@@ -120,7 +120,7 @@ module Mastodon
       options = options.merge({ algorithm: :concurrently })
       disable_statement_timeout
 
-      add_index(table_name, column_name, options)
+      add_index(table_name, column_name, **options)
     end
 
     # Removes an existed index, concurrently when supported
@@ -144,7 +144,7 @@ module Mastodon
         disable_statement_timeout
       end
 
-      remove_index(table_name, options.merge({ column: column_name }))
+      remove_index(table_name, **options.merge({ column: column_name }))
     end
 
     # Removes an existing index, concurrently when supported
@@ -168,7 +168,7 @@ module Mastodon
         disable_statement_timeout
       end
 
-      remove_index(table_name, options.merge({ name: index_name }))
+      remove_index(table_name, **options.merge({ name: index_name }))
     end
 
     # Only available on Postgresql >= 9.2
@@ -472,7 +472,7 @@ module Mastodon
         col_opts[:limit] = old_col.limit
       end
 
-      add_column(table, new, new_type, col_opts)
+      add_column(table, new, new_type, **col_opts)
 
       # We set the default value _after_ adding the column so we don't end up
       # updating any existing data with the default value. This isn't
@@ -510,10 +510,10 @@ module Mastodon
         new_pk_index_name = "index_#{table}_on_#{column}_cm"
 
         unless indexes_for(table, column).find{|i| i.name == old_pk_index_name}
-          add_concurrent_index(table, [temp_column], {
+          add_concurrent_index(table, [temp_column],
             unique: true,
             name: new_pk_index_name
-          })
+          )
         end
       end
     end
@@ -763,7 +763,7 @@ module Mastodon
         options[:using] = index.using if index.using
         options[:where] = index.where if index.where
 
-        add_concurrent_index(table, new_columns, options)
+        add_concurrent_index(table, new_columns, **options)
       end
     end
 

lib/mastodon/redis_config.rb

@@ -28,7 +28,7 @@ cache_namespace = namespace ? namespace + '_cache' : 'cache'
 
 REDIS_CACHE_PARAMS = {
   driver: :hiredis,
-  url: ENV['REDIS_URL'],
+  url: ENV['CACHE_REDIS_URL'],
   expires_in: 10.minutes,
   namespace: cache_namespace,
 }.freeze

lib/mastodon/search_cli.rb

@@ -53,7 +53,9 @@ module Mastodon
         index.specification.lock!
       end
 
-      ActiveRecord::Base.configurations[Rails.env]['pool'] = options[:concurrency] + 1
+      db_config = ActiveRecord::Base.configurations[Rails.env].dup
+      db_config['pool'] = options[:concurrency] + 1
+      ActiveRecord::Base.establish_connection(db_config)
 
       pool    = Concurrent::FixedThreadPool.new(options[:concurrency])
       added   = Concurrent::AtomicFixnum.new(0)

lib/paperclip/attachment_extensions.rb

@@ -2,6 +2,10 @@
 
 module Paperclip
   module AttachmentExtensions
+    def meta
+      instance_read(:meta)
+    end
+
     # We overwrite this method to support delayed processing in
     # Sidekiq. Since we process the original file to reduce disk
     # usage, and we still want to generate thumbnails straight

lib/paperclip/gif_transcoder.rb

@@ -100,7 +100,8 @@ end
 
 module Paperclip
   # This transcoder is only to be used for the MediaAttachment model
-  # to convert animated gifs to webm
+  # to convert animated GIFs to videos
+
   class GifTranscoder < Paperclip::Processor
     def make
       return File.open(@file.path) unless needs_convert?

lib/paperclip/image_extractor.rb

@@ -31,21 +31,17 @@ module Paperclip
     private
 
     def extract_image_from_file!
-      ::Av.logger = Paperclip.logger
-
-      cli = ::Av.cli
       dst = Tempfile.new([File.basename(@file.path, '.*'), '.png'])
       dst.binmode
 
-      cli.add_source(@file.path)
-      cli.add_destination(dst.path)
-      cli.add_output_param loglevel: 'fatal'
-
       begin
-        cli.run
-      rescue Cocaine::ExitStatusError, ::Av::CommandError
+        command = Terrapin::CommandLine.new('ffmpeg', '-i :source -loglevel :loglevel -y :destination', logger: Paperclip.logger)
+        command.run(source: @file.path, destination: dst.path, loglevel: 'fatal')
+      rescue Terrapin::ExitStatusError
         dst.close(true)
         return nil
+      rescue Terrapin::CommandNotFoundError
+        raise Paperclip::Errors::CommandNotFoundError, 'Could not run the `ffmpeg` command. Please install ffmpeg.'
       end
 
       dst

lib/paperclip/schema_extensions.rb

@@ -0,0 +1,37 @@
+# frozen_string_literal: true
+
+# Monkey-patch various Paperclip methods for Ruby 3.0 compatibility
+
+module Paperclip
+  module Schema
+    module StatementsExtensions
+      def add_attachment(table_name, *attachment_names)
+        raise ArgumentError, 'Please specify attachment name in your add_attachment call in your migration.' if attachment_names.empty?
+
+        options = attachment_names.extract_options!
+
+        attachment_names.each do |attachment_name|
+          COLUMNS.each_pair do |column_name, column_type|
+            column_options = options.merge(options[column_name.to_sym] || {})
+            add_column(table_name, "#{attachment_name}_#{column_name}", column_type, **column_options)
+          end
+        end
+      end
+    end
+
+    module TableDefinitionExtensions
+      def attachment(*attachment_names)
+        options = attachment_names.extract_options!
+        attachment_names.each do |attachment_name|
+          COLUMNS.each_pair do |column_name, column_type|
+            column_options = options.merge(options[column_name.to_sym] || {})
+            column("#{attachment_name}_#{column_name}", column_type, **column_options)
+          end
+        end
+      end
+    end
+  end
+end
+
+Paperclip::Schema::Statements.prepend(Paperclip::Schema::StatementsExtensions)
+Paperclip::Schema::TableDefinition.prepend(Paperclip::Schema::TableDefinitionExtensions)

lib/paperclip/transcoder.rb

@@ -0,0 +1,102 @@
+# frozen_string_literal: true
+
+module Paperclip
+  # This transcoder is only to be used for the MediaAttachment model
+  # to check when uploaded videos are actually gifv's
+  class Transcoder < Paperclip::Processor
+    def initialize(file, options = {}, attachment = nil)
+      super
+
+      @current_format      = File.extname(@file.path)
+      @basename            = File.basename(@file.path, @current_format)
+      @format              = options[:format]
+      @time                = options[:time] || 3
+      @passthrough_options = options[:passthrough_options]
+      @convert_options     = options[:convert_options].dup
+    end
+
+    def make
+      metadata = VideoMetadataExtractor.new(@file.path)
+
+      unless metadata.valid?
+        log("Unsupported file #{@file.path}")
+        return File.open(@file.path)
+      end
+
+      update_attachment_type(metadata)
+      update_options_from_metadata(metadata)
+
+      destination = Tempfile.new([@basename, @format ? ".#{@format}" : ''])
+      destination.binmode
+
+      @output_options = @convert_options[:output]&.dup || {}
+      @input_options  = @convert_options[:input]&.dup  || {}
+
+      case @format.to_s
+      when /jpg$/, /jpeg$/, /png$/, /gif$/
+        @input_options['ss'] = @time
+
+        @output_options['f']       = 'image2'
+        @output_options['vframes'] = 1
+      when 'mp4'
+        @output_options['acodec'] = 'aac'
+        @output_options['strict'] = 'experimental'
+      end
+
+      command_arguments, interpolations = prepare_command(destination)
+
+      begin
+        command = Terrapin::CommandLine.new('ffmpeg', command_arguments.join(' '), logger: Paperclip.logger)
+        command.run(interpolations)
+      rescue Terrapin::ExitStatusError => e
+        raise Paperclip::Error, "Error while transcoding #{@basename}: #{e}"
+      rescue Terrapin::CommandNotFoundError
+        raise Paperclip::Errors::CommandNotFoundError, 'Could not run the `ffmpeg` command. Please install ffmpeg.'
+      end
+
+      destination
+    end
+
+    private
+
+    def prepare_command(destination)
+      command_arguments  = ['-nostdin']
+      interpolations     = {}
+      interpolation_keys = 0
+
+      @input_options.each_pair do |key, value|
+        interpolation_key = interpolation_keys
+        command_arguments << "-#{key} :#{interpolation_key}"
+        interpolations[interpolation_key] = value
+        interpolation_keys += 1
+      end
+
+      command_arguments << '-i :source'
+      interpolations[:source] = @file.path
+
+      @output_options.each_pair do |key, value|
+        interpolation_key = interpolation_keys
+        command_arguments << "-#{key} :#{interpolation_key}"
+        interpolations[interpolation_key] = value
+        interpolation_keys += 1
+      end
+
+      command_arguments << '-y :destination'
+      interpolations[:destination] = destination.path
+
+      [command_arguments, interpolations]
+    end
+
+    def update_options_from_metadata(metadata)
+      return unless @passthrough_options && @passthrough_options[:video_codecs].include?(metadata.video_codec) && @passthrough_options[:audio_codecs].include?(metadata.audio_codec) && @passthrough_options[:colorspaces].include?(metadata.colorspace)
+
+      @format          = @passthrough_options[:options][:format] || @format
+      @time            = @passthrough_options[:options][:time]   || @time
+      @convert_options = @passthrough_options[:options][:convert_options].dup
+    end
+
+    def update_attachment_type(metadata)
+      @attachment.instance.type = MediaAttachment.types[:gifv] unless metadata.audio_codec
+    end
+  end
+end

lib/paperclip/transcoder_extensions.rb

@@ -1,14 +0,0 @@
-# frozen_string_literal: true
-
-module Paperclip
-  module TranscoderExtensions
-    # Prevent the transcoder from modifying our meta hash
-    def initialize(file, options = {}, attachment = nil)
-      meta_value = attachment&.instance_read(:meta)
-      super
-      attachment&.instance_write(:meta, meta_value)
-    end
-  end
-end
-
-Paperclip::Transcoder.prepend(Paperclip::TranscoderExtensions)

lib/paperclip/validation_extensions.rb

@@ -0,0 +1,58 @@
+# frozen_string_literal: true
+
+# Monkey-patch various Paperclip validators for Ruby 3.0 compatibility
+
+module Paperclip
+  module Validators
+    module AttachmentSizeValidatorExtensions
+      def validate_each(record, attr_name, _value)
+        base_attr_name = attr_name
+        attr_name = "#{attr_name}_file_size".to_sym
+        value = record.send(:read_attribute_for_validation, attr_name)
+
+        if value.present?
+          options.slice(*Paperclip::Validators::AttachmentSizeValidator::AVAILABLE_CHECKS).each do |option, option_value|
+            option_value = option_value.call(record) if option_value.is_a?(Proc)
+            option_value = extract_option_value(option, option_value)
+
+            next if value.send(Paperclip::Validators::AttachmentSizeValidator::CHECKS[option], option_value)
+
+            error_message_key = options[:in] ? :in_between : option
+            [attr_name, base_attr_name].each do |error_attr_name|
+              record.errors.add(error_attr_name, error_message_key, **filtered_options(value).merge(
+                min: min_value_in_human_size(record),
+                max: max_value_in_human_size(record),
+                count: human_size(option_value)
+              ))
+            end
+          end
+        end
+      end
+    end
+
+    module AttachmentContentTypeValidatorExtensions
+      def mark_invalid(record, attribute, types)
+        record.errors.add attribute, :invalid, **options.merge({ types: types.join(', ') })
+      end
+    end
+
+    module AttachmentPresenceValidatorExtensions
+      def validate_each(record, attribute, _value)
+        if record.send("#{attribute}_file_name").blank?
+          record.errors.add(attribute, :blank, **options)
+        end
+      end
+    end
+
+    module AttachmentFileNameValidatorExtensions
+      def mark_invalid(record, attribute, patterns)
+        record.errors.add attribute, :invalid, options.merge({ names: patterns.join(', ') })
+      end
+    end
+  end
+end
+
+Paperclip::Validators::AttachmentSizeValidator.prepend(Paperclip::Validators::AttachmentSizeValidatorExtensions)
+Paperclip::Validators::AttachmentContentTypeValidator.prepend(Paperclip::Validators::AttachmentContentTypeValidatorExtensions)
+Paperclip::Validators::AttachmentPresenceValidator.prepend(Paperclip::Validators::AttachmentPresenceValidatorExtensions)
+Paperclip::Validators::AttachmentFileNameValidator.prepend(Paperclip::Validators::AttachmentFileNameValidatorExtensions)

lib/paperclip/video_transcoder.rb

@@ -1,26 +0,0 @@
-# frozen_string_literal: true
-
-module Paperclip
-  # This transcoder is only to be used for the MediaAttachment model
-  # to check when uploaded videos are actually gifv's
-  class VideoTranscoder < Paperclip::Processor
-    def make
-      movie = FFMPEG::Movie.new(@file.path)
-
-      attachment.instance.type = MediaAttachment.types[:gifv] unless movie.audio_codec
-
-      Paperclip::Transcoder.make(file, actual_options(movie), attachment)
-    end
-
-    private
-
-    def actual_options(movie)
-      opts = options[:passthrough_options]
-      if opts && opts[:video_codecs].include?(movie.video_codec) && opts[:audio_codecs].include?(movie.audio_codec) && opts[:colorspaces].include?(movie.colorspace)
-        opts[:options]
-      else
-        options
-      end
-    end
-  end
-end

lib/tasks/db.rake

@@ -22,7 +22,7 @@ namespace :db do
       unless %w(C POSIX).include?(ActiveRecord::Base.connection.select_one('SELECT datcollate FROM pg_database WHERE datname = current_database();')['datcollate'])
         warn <<~WARNING
           Your database collation is susceptible to index corruption.
-            (This warning does not indicate that index corruption has occured and can be ignored)
+            (This warning does not indicate that index corruption has occurred and can be ignored)
             (To learn more, visit: https://docs.joinmastodon.org/admin/troubleshooting/index-corruption/)
         WARNING
       end

lib/terrapin/multi_pipe_extensions.rb

@@ -0,0 +1,66 @@
+# frozen_string_literal: false
+
+require 'fcntl'
+
+module Terrapin
+  module MultiPipeExtensions
+    def initialize
+      @stdout_in, @stdout_out = IO.pipe
+      @stderr_in, @stderr_out = IO.pipe
+
+      clear_nonblocking_flags!
+    end
+
+    def pipe_options
+      # Add some flags to explicitly close the other end of the pipes
+      { out: @stdout_out, err: @stderr_out, @stdout_in => :close, @stderr_in => :close }
+    end
+
+    def read
+      # While we are patching Terrapin, fix child process potentially getting stuck on writing
+      # to stderr.
+
+      @stdout_output = +''
+      @stderr_output = +''
+
+      fds_to_read = [@stdout_in, @stderr_in]
+      until fds_to_read.empty?
+        rs, = IO.select(fds_to_read)
+
+        read_nonblocking!(@stdout_in, @stdout_output, fds_to_read) if rs.include?(@stdout_in)
+        read_nonblocking!(@stderr_in, @stderr_output, fds_to_read) if rs.include?(@stderr_in)
+      end
+    end
+
+    private
+
+    # @param [IO] io IO Stream to read until there is nothing to read
+    # @param [String] result Mutable string to which read values will be appended to
+    # @param [Array<IO>] fds_to_read Mutable array from which `io` should be removed on EOF
+    def read_nonblocking!(io, result, fds_to_read)
+      while (partial_result = io.read_nonblock(8192))
+        result << partial_result
+      end
+    rescue IO::WaitReadable
+      # Do nothing
+    rescue EOFError
+      fds_to_read.delete(io)
+    end
+
+    def clear_nonblocking_flags!
+      # Ruby 3.0 sets pipes to non-blocking mode, and resets the flags as
+      # needed when calling fork/exec-related syscalls, but posix-spawn does
+      # not currently do that, so we need to do it manually for the time being
+      # so that the child process do not error out when the buffers are full.
+      stdout_flags = @stdout_out.fcntl(Fcntl::F_GETFL)
+      @stdout_out.fcntl(Fcntl::F_SETFL, stdout_flags & ~Fcntl::O_NONBLOCK) if stdout_flags & Fcntl::O_NONBLOCK
+
+      stderr_flags = @stderr_out.fcntl(Fcntl::F_GETFL)
+      @stderr_out.fcntl(Fcntl::F_SETFL, stderr_flags & ~Fcntl::O_NONBLOCK) if stderr_flags & Fcntl::O_NONBLOCK
+    rescue NameError, NotImplementedError, Errno::EINVAL
+      # Probably on windows, where pipes are blocking by default
+    end
+  end
+end
+
+Terrapin::CommandLine::MultiPipe.prepend(Terrapin::MultiPipeExtensions)