Merge commit '4aea3f88a6d30f102a79c2da7fcfac96465ba1a8' into merging-upstream

app/lib/activitypub/activity/announce.rb

@@ -11,7 +11,12 @@ class ActivityPub::Activity::Announce < ActivityPub::Activity
 
     return status unless status.nil?
 
-    status = Status.create!(account: @account, reblog: original_status, uri: @json['id'])
+    status = Status.create!(
+      account: @account,
+      reblog: original_status,
+      uri: @json['id'],
+      created_at: @json['published'] || Time.now.utc
+    )
     distribute(status)
     status
   end

app/lib/activitypub/activity/create.rb

@@ -4,26 +4,31 @@ class ActivityPub::Activity::Create < ActivityPub::Activity
   def perform
     return if delete_arrived_first?(object_uri) || unsupported_object_type?
 
-    status = find_existing_status
-
-    return status unless status.nil?
-
-    ApplicationRecord.transaction do
-      status = Status.create!(status_params)
-
-      process_tags(status)
-      process_attachments(status)
+    RedisLock.acquire(lock_options) do |lock|
+      if lock.acquired?
+        @status = find_existing_status
+        process_status if @status.nil?
+      end
     end
 
-    resolve_thread(status)
-    distribute(status)
-    forward_for_reply if status.public_visibility? || status.unlisted_visibility?
-
-    status
+    @status
   end
 
   private
 
+  def process_status
+    ApplicationRecord.transaction do
+      @status = Status.create!(status_params)
+
+      process_tags(@status)
+      process_attachments(@status)
+    end
+
+    resolve_thread(@status)
+    distribute(@status)
+    forward_for_reply if @status.public_visibility? || @status.unlisted_visibility?
+  end
+
   def find_existing_status
     status   = status_from_uri(object_uri)
     status ||= Status.find_by(uri: @object['atomUri']) if @object['atomUri'].present?
@@ -56,6 +61,8 @@ class ActivityPub::Activity::Create < ActivityPub::Activity
         process_hashtag tag, status
       when 'Mention'
         process_mention tag, status
+      when 'Emoji'
+        process_emoji tag, status
       end
     end
   end
@@ -74,6 +81,17 @@ class ActivityPub::Activity::Create < ActivityPub::Activity
     account.mentions.create(status: status)
   end
 
+  def process_emoji(tag, _status)
+    shortcode = tag['name'].delete(':')
+    emoji     = CustomEmoji.find_by(shortcode: shortcode, domain: @account.domain)
+
+    return if !emoji.nil? || skip_download?
+
+    emoji = CustomEmoji.new(domain: @account.domain, shortcode: shortcode)
+    emoji.image_remote_url = tag['href']
+    emoji.save
+  end
+
   def process_attachments(status)
     return unless @object['attachment'].is_a?(Array)
 
@@ -182,4 +200,8 @@ class ActivityPub::Activity::Create < ActivityPub::Activity
     return unless @json['signature'].present? && reply_to_local?
     ActivityPub::RawDistributionWorker.perform_async(Oj.dump(@json), replied_to_status.account_id)
   end
+
+  def lock_options
+    { redis: Redis.current, key: "create:#{@object['id']}" }
+  end
 end

app/lib/activitypub/adapter.rb

@@ -14,6 +14,8 @@ class ActivityPub::Adapter < ActiveModelSerializers::Adapter::Base
         'atomUri'                   => 'ostatus:atomUri',
         'inReplyToAtomUri'          => 'ostatus:inReplyToAtomUri',
         'conversation'              => 'ostatus:conversation',
+        'toot'                      => 'http://joinmastodon.org/ns#',
+        'Emoji'                     => 'toot:Emoji',
       },
     ],
   }.freeze
@@ -28,7 +30,7 @@ class ActivityPub::Adapter < ActiveModelSerializers::Adapter::Base
 
   def serializable_hash(options = nil)
     options = serialization_options(options)
-    serialized_hash = CONTEXT.merge(ActiveModelSerializers::Adapter::Attributes.new(serializer, instance_options).serializable_hash(options))
-    self.class.transform_key_casing!(serialized_hash, instance_options)
+    serialized_hash = ActiveModelSerializers::Adapter::Attributes.new(serializer, instance_options).serializable_hash(options)
+    CONTEXT.merge(self.class.transform_key_casing!(serialized_hash, instance_options))
   end
 end

app/lib/activitypub/tag_manager.rb

@@ -37,7 +37,7 @@ class ActivityPub::TagManager
   end
 
   def activity_uri_for(target)
-    return nil unless %i(note comment activity).include?(target.object_type) && target.local?
+    raise ArgumentError, 'target must be a local activity' unless %i(note comment activity).include?(target.object_type) && target.local?
 
     activity_account_status_url(target.account, target)
   end

app/lib/formatter.rb

@@ -9,7 +9,7 @@ class Formatter
 
   include ActionView::Helpers::TextHelper
 
-  def format(status)
+  def format(status, options = {})
     if status.reblog?
       prepend_reblog = status.reblog.account.acct
       status         = status.proper
@@ -19,7 +19,11 @@ class Formatter
 
     raw_content = status.text
 
-    return reformat(raw_content) unless status.local?
+    unless status.local?
+      html = reformat(raw_content)
+      html = encode_custom_emojis(html, status.emojis) if options[:custom_emojify]
+      return html
+    end
 
     linkable_accounts = status.mentions.map(&:account)
     linkable_accounts << status.account
@@ -27,6 +31,7 @@ class Formatter
     html = raw_content
     html = "RT @#{prepend_reblog} #{html}" if prepend_reblog
     html = encode_and_link_urls(html, linkable_accounts)
+    html = encode_custom_emojis(html, status.emojis) if options[:custom_emojify]
     html = simple_format(html, {}, sanitize: false)
     html = html.delete("\n")
 
@@ -39,7 +44,9 @@ class Formatter
 
   def plaintext(status)
     return status.text if status.local?
-    strip_tags(status.text)
+
+    text = status.text.gsub(/(<br \/>|<br>|<\/p>)+/) { |match| "#{match}\n" }
+    strip_tags(text)
   end
 
   def simplified_format(account)
@@ -76,6 +83,47 @@ class Formatter
     end
   end
 
+  def encode_custom_emojis(html, emojis)
+    return html if emojis.empty?
+
+    emoji_map = emojis.map { |e| [e.shortcode, full_asset_url(e.image.url)] }.to_h
+
+    i                     = -1
+    inside_tag            = false
+    inside_shortname      = false
+    shortname_start_index = -1
+
+    while i + 1 < html.size
+      i += 1
+
+      if inside_shortname && html[i] == ':'
+        shortcode = html[shortname_start_index + 1..i - 1]
+        emoji     = emoji_map[shortcode]
+
+        if emoji
+          replacement = "<img draggable=\"false\" class=\"emojione\" alt=\":#{shortcode}:\" title=\":#{shortcode}:\" src=\"#{emoji}\" />"
+          before_html = shortname_start_index.positive? ? html[0..shortname_start_index - 1] : ''
+          html        = before_html + replacement + html[i + 1..-1]
+          i          += replacement.size - (shortcode.size + 2) - 1
+        else
+          i -= 1
+        end
+
+        inside_shortname = false
+      elsif inside_tag && html[i] == '>'
+        inside_tag = false
+      elsif html[i] == '<'
+        inside_tag       = true
+        inside_shortname = false
+      elsif !inside_tag && html[i] == ':'
+        inside_shortname      = true
+        shortname_start_index = i
+      end
+    end
+
+    html
+  end
+
   def rewrite(text, entities)
     chars = text.to_s.to_char_a
 
@@ -131,13 +179,13 @@ class Formatter
   end
 
   def link_html(url)
-    url    = Addressable::URI.parse(url).display_uri.to_s
+    url    = Addressable::URI.parse(url).to_s
     prefix = url.match(/\Ahttps?:\/\/(www\.)?/).to_s
     text   = url[prefix.length, 30]
     suffix = url[prefix.length + 30..-1]
     cutoff = url[prefix.length..-1].length > 30
 
-    "<span class=\"invisible\">#{prefix}</span><span class=\"#{cutoff ? 'ellipsis' : ''}\">#{text}</span><span class=\"invisible\">#{suffix}</span>"
+    "<span class=\"invisible\">#{encode(prefix)}</span><span class=\"#{cutoff ? 'ellipsis' : ''}\">#{encode(text)}</span><span class=\"invisible\">#{encode(suffix)}</span>"
   end
 
   def hashtag_html(tag)

app/lib/language_detector.rb

@@ -1,26 +1,31 @@
 # frozen_string_literal: true
 
 class LanguageDetector
-  attr_reader :text, :account
+  include Singleton
 
-  def initialize(text, account = nil)
-    @text = text
-    @account = account
+  def initialize
     @identifier = CLD3::NNetLanguageIdentifier.new(1, 2048)
   end
 
-  def to_iso_s
-    detected_language_code || default_locale
+  def detect(text, account)
+    detect_language_code(text) || default_locale(account)
   end
 
-  def prepared_text
-    simplified_text.strip
+  def language_names
+    @language_names =
+      CLD3::TaskContextParams::LANGUAGE_NAMES.map { |name| iso6391(name.to_s).to_sym }
+                                             .uniq
   end
 
   private
 
-  def detected_language_code
-    iso6391(result.language).to_sym if detected_language_reliable?
+  def prepare_text(text)
+    simplify_text(text).strip
+  end
+
+  def detect_language_code(text)
+    result = @identifier.find_language(prepare_text(text))
+    iso6391(result.language.to_s).to_sym if result.reliable?
   end
 
   def iso6391(bcp47)
@@ -32,15 +37,7 @@ class LanguageDetector
     ISO_639.find(iso639).alpha2
   end
 
-  def result
-    @result ||= @identifier.find_language(prepared_text)
-  end
-
-  def detected_language_reliable?
-    result.reliable?
-  end
-
-  def simplified_text
+  def simplify_text(text)
     text.dup.tap do |new_text|
       new_text.gsub!(FetchLinkCardService::URL_PATTERN, '')
       new_text.gsub!(Account::MENTION_RE, '')
@@ -49,7 +46,7 @@ class LanguageDetector
     end
   end
 
-  def default_locale
-    account&.user_locale&.to_sym || nil
+  def default_locale(account)
+    account.user_locale&.to_sym
   end
 end

app/lib/ostatus/activity/creation.rb

@@ -42,6 +42,7 @@ class OStatus::Activity::Creation < OStatus::Activity::Base
       save_mentions(status)
       save_hashtags(status)
       save_media(status)
+      save_emojis(status)
     end
 
     if thread? && status.thread.nil?
@@ -150,6 +151,25 @@ class OStatus::Activity::Creation < OStatus::Activity::Base
     end
   end
 
+  def save_emojis(parent)
+    do_not_download = DomainBlock.find_by(domain: parent.account.domain)&.reject_media?
+
+    return if do_not_download
+
+    @xml.xpath('./xmlns:link[@rel="emoji"]', xmlns: TagManager::XMLNS).each do |link|
+      next unless link['href'] && link['name']
+
+      shortcode = link['name'].delete(':')
+      emoji     = CustomEmoji.find_by(shortcode: shortcode, domain: parent.account.domain)
+
+      next unless emoji.nil?
+
+      emoji = CustomEmoji.new(shortcode: shortcode, domain: parent.account.domain)
+      emoji.image_remote_url = link['href']
+      emoji.save
+    end
+  end
+
   def account_from_href(href)
     url = Addressable::URI.parse(href).normalize
 

app/lib/ostatus/atom_serializer.rb

@@ -368,5 +368,9 @@ class OStatus::AtomSerializer
     end
 
     append_element(entry, 'mastodon:scope', status.visibility)
+
+    status.emojis.each do |emoji|
+      append_element(entry, 'link', nil, rel: :emoji, href: full_asset_url(emoji.image.url), name: emoji.shortcode)
+    end
   end
 end

app/lib/request.rb

@@ -31,6 +31,8 @@ class Request
 
   def perform
     http_client.headers(headers).public_send(@verb, @url.to_s, @options)
+  rescue => e
+    raise e.class, "#{e.message} on #{@url}"
   end
 
   def headers

app/lib/tag_manager.rb

@@ -87,7 +87,7 @@ class TagManager
   def local_url?(url)
     uri    = Addressable::URI.parse(url).normalize
     domain = uri.host + (uri.port ? ":#{uri.port}" : '')
-    TagManager.instance.local_domain?(domain)
+    TagManager.instance.web_domain?(domain)
   end
 
   def uri_for(target)