Merge branch 'main' into glitch-soc/merge-upstream

Conflicts:
- `app/models/custom_emoji.rb`:
  Not a real conflict, just upstream changing a line too close to
  a glitch-soc-specific validation.
  Applied upstream changes.
- `app/models/public_feed.rb`:
  Not a real conflict, just upstream changing a line too close to
  a glitch-soc-specific parameter documentation.
  Applied upstream changes.

app/models/account.rb

@@ -64,6 +64,7 @@ class Account < ApplicationRecord
   USERNAME_RE   = /[a-z0-9_]+([a-z0-9_\.-]+[a-z0-9_]+)?/i
   MENTION_RE    = /(?<=^|[^\/[:word:]])@((#{USERNAME_RE})(?:@[[:word:]\.\-]+[[:word:]]+)?)/i
   URL_PREFIX_RE = /\Ahttp(s?):\/\/[^\/]+/
+  USERNAME_ONLY_RE = /\A#{USERNAME_RE}\z/i
 
   include Attachmentable
   include AccountAssociations
@@ -88,7 +89,7 @@ class Account < ApplicationRecord
   validates_with UniqueUsernameValidator, if: -> { will_save_change_to_username? }
 
   # Remote user validations
-  validates :username, format: { with: /\A#{USERNAME_RE}\z/i }, if: -> { !local? && will_save_change_to_username? }
+  validates :username, format: { with: USERNAME_ONLY_RE }, if: -> { !local? && will_save_change_to_username? }
 
   # Local user validations
   validates :username, format: { with: /\A[a-z0-9_]+\z/i }, length: { maximum: 30 }, if: -> { local? && will_save_change_to_username? && actor_type != 'Application' }
@@ -295,7 +296,7 @@ class Account < ApplicationRecord
 
   def fields
     (self[:fields] || []).map do |f|
-      Field.new(self, f)
+      Account::Field.new(self, f)
     rescue
       nil
     end.compact
@@ -399,48 +400,6 @@ class Account < ApplicationRecord
     requires_review? && !requested_review?
   end
 
-  class Field < ActiveModelSerializers::Model
-    attributes :name, :value, :verified_at, :account
-
-    def initialize(account, attributes)
-      @original_field = attributes
-      string_limit = account.local? ? 255 : 2047
-      super(
-        account:     account,
-        name:        attributes['name'].strip[0, string_limit],
-        value:       attributes['value'].strip[0, string_limit],
-        verified_at: attributes['verified_at']&.to_datetime,
-      )
-    end
-
-    def verified?
-      verified_at.present?
-    end
-
-    def value_for_verification
-      @value_for_verification ||= begin
-        if account.local?
-          value
-        else
-          ActionController::Base.helpers.strip_tags(value)
-        end
-      end
-    end
-
-    def verifiable?
-      value_for_verification.present? && value_for_verification.start_with?('http://', 'https://')
-    end
-
-    def mark_verified!
-      self.verified_at = Time.now.utc
-      @original_field['verified_at'] = verified_at
-    end
-
-    def to_h
-      { name: name, value: value, verified_at: verified_at }
-    end
-  end
-
   class << self
     DISALLOWED_TSQUERY_CHARACTERS = /['?\\:‘’]/.freeze
     TEXTSEARCH = "(setweight(to_tsvector('simple', accounts.display_name), 'A') || setweight(to_tsvector('simple', accounts.username), 'B') || setweight(to_tsvector('simple', coalesce(accounts.domain, '')), 'C'))"

app/models/account/field.rb

@@ -0,0 +1,87 @@
+# frozen_string_literal: true
+
+class Account::Field < ActiveModelSerializers::Model
+  MAX_CHARACTERS_LOCAL  = 255
+  MAX_CHARACTERS_COMPAT = 2_047
+  ACCEPTED_SCHEMES      = %w(http https).freeze
+
+  attributes :name, :value, :verified_at, :account
+
+  def initialize(account, attributes)
+    # Keeping this as reference allows us to update the field on the account
+    # from methods in this class, so that changes can be saved.
+    @original_field = attributes
+    @account        = account
+
+    super(
+      name:        sanitize(attributes['name']),
+      value:       sanitize(attributes['value']),
+      verified_at: attributes['verified_at']&.to_datetime,
+    )
+  end
+
+  def verified?
+    verified_at.present?
+  end
+
+  def value_for_verification
+    @value_for_verification ||= begin
+      if account.local?
+        value
+      else
+        extract_url_from_html
+      end
+    end
+  end
+
+  def verifiable?
+    return false if value_for_verification.blank?
+
+    # This is slower than checking through a regular expression, but we
+    # need to confirm that it's not an IDN domain.
+
+    parsed_url = Addressable::URI.parse(value_for_verification)
+
+    ACCEPTED_SCHEMES.include?(parsed_url.scheme) &&
+      parsed_url.user.nil? &&
+      parsed_url.password.nil? &&
+      parsed_url.host.present? &&
+      parsed_url.normalized_host == parsed_url.host
+  rescue Addressable::URI::InvalidURIError, IDN::Idna::IdnaError
+    false
+  end
+
+  def requires_verification?
+    !verified? && verifiable?
+  end
+
+  def mark_verified!
+    @original_field['verified_at'] = self.verified_at = Time.now.utc
+  end
+
+  def to_h
+    { name: name, value: value, verified_at: verified_at }
+  end
+
+  private
+
+  def sanitize(str)
+    str.strip[0, character_limit]
+  end
+
+  def character_limit
+    account.local? ? MAX_CHARACTERS_LOCAL : MAX_CHARACTERS_COMPAT
+  end
+
+  def extract_url_from_html
+    doc = Nokogiri::HTML(value).at_xpath('//body')
+
+    return if doc.children.size > 1
+
+    element = doc.children.first
+
+    return if element.name != 'a' || element['href'] != element.text
+
+    element['href']
+  end
+end

app/models/custom_emoji.rb

@@ -31,6 +31,7 @@ class CustomEmoji < ApplicationRecord
   SCAN_RE = /(?<=[^[:alnum:]:]|\n|^)
     :(#{SHORTCODE_RE_FRAGMENT}):
     (?=[^[:alnum:]:]|$)/x
+  SHORTCODE_ONLY_RE = /\A#{SHORTCODE_RE_FRAGMENT}\z/
 
   IMAGE_MIME_TYPES = %w(image/png image/gif image/webp).freeze
 
@@ -44,7 +45,7 @@ class CustomEmoji < ApplicationRecord
   validates_attachment :image, content_type: { content_type: IMAGE_MIME_TYPES }, presence: true
   validates_attachment_size :image, less_than: LIMIT, unless: :local?
   validates_attachment_size :image, less_than: LOCAL_LIMIT, if: :local?
-  validates :shortcode, uniqueness: { scope: :domain }, format: { with: /\A#{SHORTCODE_RE_FRAGMENT}\z/ }, length: { minimum: 2 }
+  validates :shortcode, uniqueness: { scope: :domain }, format: { with: SHORTCODE_ONLY_RE }, length: { minimum: 2 }
 
   scope :local, -> { where(domain: nil) }
   scope :remote, -> { where.not(domain: nil) }

app/models/featured_tag.rb

@@ -17,7 +17,7 @@ class FeaturedTag < ApplicationRecord
   belongs_to :account, inverse_of: :featured_tags
   belongs_to :tag, inverse_of: :featured_tags, optional: true # Set after validation
 
-  validates :name, presence: true, format: { with: /\A(#{Tag::HASHTAG_NAME_RE})\z/i }, on: :create
+  validates :name, presence: true, format: { with: Tag::HASHTAG_NAME_RE }, on: :create
 
   validate :validate_tag_uniqueness, on: :create
   validate :validate_featured_tags_limit, on: :create

app/models/public_feed.rb

@@ -9,7 +9,6 @@ class PublicFeed
   # @option [Boolean] :remote
   # @option [Boolean] :only_media
   # @option [Boolean] :allow_local_only
-  # @option [String]  :locale
   def initialize(account, options = {})
     @account = account
     @options = options
@@ -30,7 +29,7 @@ class PublicFeed
     scope.merge!(remote_only_scope) if remote_only?
     scope.merge!(account_filters_scope) if account?
     scope.merge!(media_only_scope) if media_only?
-    scope.merge!(language_scope)
+    scope.merge!(language_scope) if account&.chosen_languages.present?
 
     scope.cache_ids.to_a_paginated_by_id(limit, max_id: max_id, since_id: since_id, min_id: min_id)
   end
@@ -100,13 +99,7 @@ class PublicFeed
   end
 
   def language_scope
-    if account&.chosen_languages.present?
-      Status.where(language: account.chosen_languages)
-    elsif @options[:locale].present?
-      Status.where(language: @options[:locale])
-    else
-      Status.all
-    end
+    Status.where(language: account.chosen_languages)
   end
 
   def account_filters_scope

app/models/tag.rb

@@ -27,11 +27,14 @@ class Tag < ApplicationRecord
   has_many :followers, through: :passive_relationships, source: :account
 
   HASHTAG_SEPARATORS = "_\u00B7\u200c"
-  HASHTAG_NAME_RE    = "([[:word:]_][[:word:]#{HASHTAG_SEPARATORS}]*[[:alpha:]#{HASHTAG_SEPARATORS}][[:word:]#{HASHTAG_SEPARATORS}]*[[:word:]_])|([[:word:]_]*[[:alpha:]][[:word:]_]*)"
-  HASHTAG_RE         = /(?:^|[^\/\)\w])#(#{HASHTAG_NAME_RE})/i
+  HASHTAG_NAME_PAT = "([[:word:]_][[:word:]#{HASHTAG_SEPARATORS}]*[[:alpha:]#{HASHTAG_SEPARATORS}][[:word:]#{HASHTAG_SEPARATORS}]*[[:word:]_])|([[:word:]_]*[[:alpha:]][[:word:]_]*)"
 
-  validates :name, presence: true, format: { with: /\A(#{HASHTAG_NAME_RE})\z/i }
-  validates :display_name, format: { with: /\A(#{HASHTAG_NAME_RE})\z/i }
+  HASHTAG_RE = /(?:^|[^\/\)\w])#(#{HASHTAG_NAME_PAT})/i
+  HASHTAG_NAME_RE = /\A(#{HASHTAG_NAME_PAT})\z/i
+  HASHTAG_INVALID_CHARS_RE = /[^[:alnum:]#{HASHTAG_SEPARATORS}]/
+
+  validates :name, presence: true, format: { with: HASHTAG_NAME_RE }
+  validates :display_name, format: { with: HASHTAG_NAME_RE }
   validate :validate_name_change, if: -> { !new_record? && name_changed? }
   validate :validate_display_name_change, if: -> { !new_record? && display_name_changed? }
 
@@ -102,7 +105,7 @@ class Tag < ApplicationRecord
       names = Array(name_or_names).map { |str| [normalize(str), str] }.uniq(&:first)
 
       names.map do |(normalized_name, display_name)|
-        tag = matching_name(normalized_name).first || create(name: normalized_name, display_name: display_name.gsub(/[^[:alnum:]#{HASHTAG_SEPARATORS}]/, ''))
+        tag = matching_name(normalized_name).first || create(name: normalized_name, display_name: display_name.gsub(HASHTAG_INVALID_CHARS_RE, ''))
 
         yield tag if block_given?
 

app/models/tag_feed.rb

@@ -12,7 +12,6 @@ class TagFeed < PublicFeed
   # @option [Boolean] :local
   # @option [Boolean] :remote
   # @option [Boolean] :only_media
-  # @option [String]  :locale
   def initialize(tag, account, options = {})
     @tag = tag
     super(account, options)