Merge branch 'main' into glitch-soc/merge-upstream

Conflicts:
- `app/models/status.rb`:
  Upstream updated media and edit-related code textually close to glitch-soc
  additions (local-only and content-type).
  Ported upstream changes.
- `app/models/status_edit.rb`:
  Upstream changes textually close to glitch-soc additions (content-type).
  Ported upstream changes.
- `app/serializers/activitypub/note_serializer.rb`:
  Upstream changed how media attachments are handled. Not really a conflict,
  but textually close to glitch-soc additions (directMessage attribute).
  Ported upstream changes.
- `app/services/remove_status_service.rb`:
  Upstream changed how media attachments are handled. Not really a conflict,
  but textually close to glitch-soc additions (DM timeline).
  Ported upstream changes.
- `app/services/update_status_service.rb`:
  Upstream fixed an issue with language selection. Not really a conflict,
  but textually close to glitch-soc additions (content-type).
  Ported upstream changes.
- `db/schema.rb`:
  Upstream added columns to the `status_edits` table, the conflict is because
  of an additional column (`content-type`) in glitch-soc.
  Ported upstream changes.
- `package.json`:
  Upstream dependency (express) textually adjacent to a glitch-soc-specific one
  (favico.js) got updated.
  Updated it as well.

app/models/concerns/omniauthable.rb

@@ -13,7 +13,7 @@ module Omniauthable
       Devise.omniauth_configs.keys
     end
 
-    def email_verified?
+    def email_present?
       email && email !~ TEMP_EMAIL_REGEX
     end
   end
@@ -40,16 +40,14 @@ module Omniauthable
     end
 
     def create_for_oauth(auth)
-      # Check if the user exists with provided email if the provider gives us a
-      # verified email.  If no verified email was provided or the user already
-      # exists, we assign a temporary email and ask the user to verify it on
+      # Check if the user exists with provided email. If no email was provided,
+      # we assign a temporary email and ask the user to verify it on
       # the next step via Auth::SetupController.show
 
       strategy          = Devise.omniauth_configs[auth.provider.to_sym].strategy
       assume_verified   = strategy&.security&.assume_email_is_verified
-      email_is_verified = auth.info.verified || auth.info.verified_email || assume_verified
+      email_is_verified = auth.info.verified || auth.info.verified_email || auth.info.email_verified || assume_verified
       email             = auth.info.verified_email || auth.info.email
-      email             = nil unless email_is_verified
 
       user = User.find_by(email: email) if email_is_verified
 
@@ -58,7 +56,7 @@ module Omniauthable
       user = User.new(user_params_from_auth(email, auth))
 
       user.account.avatar_remote_url = auth.info.image if /\A#{URI::DEFAULT_PARSER.make_regexp(%w(http https))}\z/.match?(auth.info.image)
-      user.skip_confirmation!
+      user.skip_confirmation! if email_is_verified
       user.save!
       user
     end
@@ -71,8 +69,8 @@ module Omniauthable
         agreement: true,
         external: true,
         account_attributes: {
-          username: ensure_unique_username(auth.uid),
-          display_name: auth.info.full_name || [auth.info.first_name, auth.info.last_name].join(' '),
+          username: ensure_unique_username(ensure_valid_username(auth.uid)),
+          display_name: auth.info.full_name || auth.info.name || [auth.info.first_name, auth.info.last_name].join(' '),
         },
       }
     end
@@ -88,5 +86,12 @@ module Omniauthable
 
       username
     end
+
+    def ensure_valid_username(starting_username)
+      starting_username = starting_username.split('@')[0]
+      temp_username = starting_username.gsub(/[^a-z0-9_]+/i, '')
+      validated_username = temp_username.truncate(30, omission: '')
+      validated_username
+    end
   end
 end

app/models/domain_block.rb

@@ -30,6 +30,14 @@ class DomainBlock < ApplicationRecord
   scope :with_user_facing_limitations, -> { where(severity: [:silence, :suspend]).or(where(reject_media: true)) }
   scope :by_severity, -> { order(Arel.sql('(CASE severity WHEN 0 THEN 1 WHEN 1 THEN 2 WHEN 2 THEN 0 END), reject_media, domain')) }
 
+  def policies
+    if suspend?
+      [:suspend]
+    else
+      [severity.to_sym, reject_media? ? :reject_media : nil, reject_reports? ? :reject_reports : nil].reject { |policy| policy == :noop || policy.nil? }
+    end
+  end
+
   class << self
     def suspend?(domain)
       !!rule_for(domain)&.suspend?

app/models/featured_tag.rb

@@ -4,8 +4,8 @@
 # Table name: featured_tags
 #
 #  id             :bigint(8)        not null, primary key
-#  account_id     :bigint(8)
-#  tag_id         :bigint(8)
+#  account_id     :bigint(8)        not null
+#  tag_id         :bigint(8)        not null
 #  statuses_count :bigint(8)        default(0), not null
 #  last_status_at :datetime
 #  created_at     :datetime         not null

app/models/instance.rb

@@ -32,35 +32,27 @@ class Instance < ApplicationRecord
     @delivery_failure_tracker ||= DeliveryFailureTracker.new(domain)
   end
 
-  def following_count
-    @following_count ||= Follow.where(account: accounts).count
+  def unavailable?
+    unavailable_domain.present?
   end
 
-  def followers_count
-    @followers_count ||= Follow.where(target_account: accounts).count
-  end
-
-  def reports_count
-    @reports_count ||= Report.where(target_account: accounts).count
-  end
-
-  def blocks_count
-    @blocks_count ||= Block.where(target_account: accounts).count
-  end
-
-  def public_comment
-    domain_block&.public_comment
-  end
-
-  def private_comment
-    domain_block&.private_comment
-  end
-
-  def media_storage
-    @media_storage ||= MediaAttachment.where(account: accounts).sum(:file_file_size)
+  def failing?
+    failure_days.present? || unavailable?
   end
 
   def to_param
     domain
   end
+
+  delegate :exhausted_deliveries_days, to: :delivery_failure_tracker
+
+  def availability_over_days(num_days, end_date = Time.now.utc.to_date)
+    failures_map    = exhausted_deliveries_days.index_with { true }
+    period_end_at   = exhausted_deliveries_days.last || end_date
+    period_start_at = period_end_at - num_days.days
+
+    (period_start_at..period_end_at).map do |date|
+      [date, failures_map[date]]
+    end
+  end
 end

app/models/instance_filter.rb

@@ -4,8 +4,7 @@ class InstanceFilter
   KEYS = %i(
     limited
     by_domain
-    warning
-    unavailable
+    availability
   ).freeze
 
   attr_reader :params
@@ -34,12 +33,21 @@ class InstanceFilter
       Instance.joins(:domain_allow).reorder(Arel.sql('domain_allows.id desc'))
     when 'by_domain'
       Instance.matches_domain(value)
-    when 'warning'
-      Instance.where(domain: DeliveryFailureTracker.warning_domains)
-    when 'unavailable'
-      Instance.joins(:unavailable_domain)
+    when 'availability'
+      availability_scope(value)
     else
       raise "Unknown filter: #{key}"
     end
   end
+
+  def availability_scope(value)
+    case value
+    when 'failing'
+      Instance.where(domain: DeliveryFailureTracker.warning_domains)
+    when 'unavailable'
+      Instance.joins(:unavailable_domain)
+    else
+      raise "Unknown availability: #{value}"
+    end
+  end
 end

app/models/report.rb

@@ -63,8 +63,20 @@ class Report < ApplicationRecord
     Status.with_discarded.where(id: status_ids)
   end
 
-  def media_attachments
-    MediaAttachment.where(status_id: status_ids)
+  def media_attachments_count
+    statuses_to_query = []
+    count = 0
+
+    statuses.pluck(:id, :ordered_media_attachment_ids).each do |id, ordered_ids|
+      if ordered_ids.nil?
+        statuses_to_query << id
+      else
+        count += ordered_ids.size
+      end
+    end
+
+    count += MediaAttachment.where(status_id: statuses_to_query).count unless statuses_to_query.empty?
+    count
   end
 
   def rules

app/models/status.rb

@@ -3,31 +3,31 @@
 #
 # Table name: statuses
 #
-#  id                     :bigint(8)        not null, primary key
-#  uri                    :string
-#  text                   :text             default(""), not null
-#  created_at             :datetime         not null
-#  updated_at             :datetime         not null
-#  in_reply_to_id         :bigint(8)
-#  reblog_of_id           :bigint(8)
-#  url                    :string
-#  sensitive              :boolean          default(FALSE), not null
-#  visibility             :integer          default("public"), not null
-#  spoiler_text           :text             default(""), not null
-#  reply                  :boolean          default(FALSE), not null
-#  language               :string
-#  conversation_id        :bigint(8)
-#  local                  :boolean
-#  account_id             :bigint(8)        not null
-#  application_id         :bigint(8)
-#  in_reply_to_account_id :bigint(8)
-#  local_only             :boolean
-#  full_status_text       :text             default(""), not null
-#  poll_id                :bigint(8)
-#  content_type           :string
-#  deleted_at             :datetime
-#  edited_at              :datetime
-#  trendable              :boolean
+#  id                           :bigint(8)        not null, primary key
+#  uri                          :string
+#  text                         :text             default(""), not null
+#  created_at                   :datetime         not null
+#  updated_at                   :datetime         not null
+#  in_reply_to_id               :bigint(8)
+#  reblog_of_id                 :bigint(8)
+#  url                          :string
+#  sensitive                    :boolean          default(FALSE), not null
+#  visibility                   :integer          default("public"), not null
+#  spoiler_text                 :text             default(""), not null
+#  reply                        :boolean          default(FALSE), not null
+#  language                     :string
+#  conversation_id              :bigint(8)
+#  local                        :boolean
+#  account_id                   :bigint(8)        not null
+#  application_id               :bigint(8)
+#  in_reply_to_account_id       :bigint(8)
+#  local_only                   :boolean
+#  poll_id                      :bigint(8)
+#  content_type                 :string
+#  deleted_at                   :datetime
+#  edited_at                    :datetime
+#  trendable                    :boolean
+#  ordered_media_attachment_ids :bigint(8)        is an Array
 #
 
 class Status < ApplicationRecord
@@ -217,14 +217,18 @@ class Status < ApplicationRecord
     public_visibility? || unlisted_visibility?
   end
 
-  def snapshot!(media_attachments_changed: false, account_id: nil, at_time: nil)
+  def snapshot!(account_id: nil, at_time: nil, rate_limit: true)
     edits.create!(
       text: text,
       spoiler_text: spoiler_text,
-      media_attachments_changed: media_attachments_changed,
+      sensitive: sensitive,
+      ordered_media_attachment_ids: ordered_media_attachment_ids || media_attachments.pluck(:id),
+      media_descriptions: ordered_media_attachments.map(&:description),
+      poll_options: preloadable_poll&.options,
       account_id: account_id || self.account_id,
       content_type: content_type,
-      created_at: at_time || edited_at
+      created_at: at_time || edited_at,
+      rate_limit: rate_limit
     )
   end
 
@@ -235,7 +239,7 @@ class Status < ApplicationRecord
   alias sign? distributable?
 
   def with_media?
-    media_attachments.any?
+    ordered_media_attachments.any?
   end
 
   def with_preview_card?
@@ -259,6 +263,15 @@ class Status < ApplicationRecord
     @emojis = CustomEmoji.from_text(fields.join(' '), account.domain)
   end
 
+  def ordered_media_attachments
+    if ordered_media_attachment_ids.nil?
+      media_attachments
+    else
+      map = media_attachments.index_by(&:id)
+      ordered_media_attachment_ids.map { |media_attachment_id| map[media_attachment_id] }
+    end
+  end
+
   def replies_count
     status_stat&.replies_count || 0
   end
@@ -428,6 +441,63 @@ class Status < ApplicationRecord
     super || build_status_stat
   end
 
+  # Hack to use a "INSERT INTO ... SELECT ..." query instead of "INSERT INTO ... VALUES ..." query
+  def self._insert_record(values)
+    if values.is_a?(Hash) && values['reblog_of_id'].present?
+      primary_key = self.primary_key
+      primary_key_value = nil
+
+      if primary_key
+        primary_key_value = values[primary_key]
+
+        if !primary_key_value && prefetch_primary_key?
+          primary_key_value = next_sequence_value
+          values[primary_key] = primary_key_value
+        end
+      end
+
+      # The following line is where we differ from stock ActiveRecord implementation
+      im = _compile_reblog_insert(values)
+
+      # Since we are using SELECT instead of VALUES, a non-error `nil` return is possible.
+      # For our purposes, it's equivalent to a foreign key constraint violation
+      result = connection.insert(im, "#{self} Create", primary_key || false, primary_key_value)
+      raise ActiveRecord::InvalidForeignKey, "(reblog_of_id)=(#{values['reblog_of_id']}) is not present in table \"statuses\"" if result.nil?
+
+      result
+    else
+      super
+    end
+  end
+
+  def self._compile_reblog_insert(values)
+    # This is somewhat equivalent to the following code of ActiveRecord::Persistence:
+    # `arel_table.compile_insert(_substitute_values(values))`
+    # The main difference is that we use a `SELECT` instead of a `VALUES` clause,
+    # which means we have to build the `SELECT` clause ourselves and do a bit more
+    # manual work.
+
+    # Instead of using Arel::InsertManager#values, we are going to use Arel::InsertManager#select
+    im = Arel::InsertManager.new
+    im.into(arel_table)
+
+    binds = []
+    reblog_bind = nil
+    values.each do |name, value|
+      attr = arel_table[name]
+      bind = predicate_builder.build_bind_attribute(attr.name, value)
+
+      im.columns << attr
+      binds << bind
+
+      reblog_bind = bind if name == 'reblog_of_id'
+    end
+
+    im.select(arel_table.where(arel_table[:id].eq(reblog_bind)).where(arel_table[:deleted_at].eq(nil)).project(*binds))
+
+    im
+  end
+
   private
 
   def update_status_stat!(attrs)

app/models/status_edit.rb

@@ -3,18 +3,39 @@
 #
 # Table name: status_edits
 #
-#  id                        :bigint(8)        not null, primary key
-#  status_id                 :bigint(8)        not null
-#  account_id                :bigint(8)
-#  text                      :text             default(""), not null
-#  spoiler_text              :text             default(""), not null
-#  media_attachments_changed :boolean          default(FALSE), not null
-#  created_at                :datetime         not null
-#  updated_at                :datetime         not null
-#  content_type              :string
+#  id                           :bigint(8)        not null, primary key
+#  status_id                    :bigint(8)        not null
+#  account_id                   :bigint(8)
+#  text                         :text             default(""), not null
+#  spoiler_text                 :text             default(""), not null
+#  created_at                   :datetime         not null
+#  updated_at                   :datetime         not null
+#  content_type                 :string
+#  ordered_media_attachment_ids :bigint(8)        is an Array
+#  media_descriptions           :text             is an Array
+#  poll_options                 :string           is an Array
+#  sensitive                    :boolean
 #
 
 class StatusEdit < ApplicationRecord
+  include RateLimitable
+
+  self.ignored_columns = %w(
+    media_attachments_changed
+  )
+
+  class PreservedMediaAttachment < ActiveModelSerializers::Model
+    attributes :media_attachment, :description
+
+    delegate :id, :type, :url, :preview_url, :remote_url,
+             :preview_remote_url, :text_url, :meta, :blurhash,
+             :not_processed?, :needs_redownload?, :local?,
+             :file, :thumbnail, :thumbnail_remote_url,
+             :shortcode, to: :media_attachment
+  end
+
+  rate_limit by: :account, family: :statuses
+
   belongs_to :status
   belongs_to :account, optional: true
 
@@ -26,4 +47,17 @@ class StatusEdit < ApplicationRecord
     return @emojis if defined?(@emojis)
     @emojis = CustomEmoji.from_text([spoiler_text, text].join(' '), status.account.domain)
   end
+
+  def ordered_media_attachments
+    return @ordered_media_attachments if defined?(@ordered_media_attachments)
+
+    @ordered_media_attachments = begin
+      if ordered_media_attachment_ids.nil?
+        []
+      else
+        map = status.media_attachments.index_by(&:id)
+        ordered_media_attachment_ids.map.with_index { |media_attachment_id, index| PreservedMediaAttachment.new(media_attachment: map[media_attachment_id], description: media_descriptions[index]) }
+      end
+    end
+  end
 end