Change how hashtags are normalized (#18795)

* Change how hashtags are normalized

* Fix tests

app/models/tag.rb

@@ -15,6 +15,7 @@
 #  last_status_at      :datetime
 #  max_score           :float
 #  max_score_at        :datetime
+#  display_name        :string
 #
 
 class Tag < ApplicationRecord
@@ -24,11 +25,12 @@ class Tag < ApplicationRecord
   has_many :featured_tags, dependent: :destroy, inverse_of: :tag
 
   HASHTAG_SEPARATORS = "_\u00B7\u200c"
-  HASHTAG_NAME_RE    = "([[:word:]_][[:word:]#{HASHTAG_SEPARATORS}]*[[:alpha:]#{HASHTAG_SEPARATORS}][[:word:]#{HASHTAG_SEPARATORS}]*[[:word:]_])|([[:word:]_]*[[:alpha:]][[:word:]_]*)"
+  HASHTAG_NAME_RE    = "([[:alnum:]_][[:alnum:]#{HASHTAG_SEPARATORS}]*[[:alpha:]#{HASHTAG_SEPARATORS}][[:alnum:]#{HASHTAG_SEPARATORS}]*[[:alnum:]_])|([[:alnum:]_]*[[:alpha:]][[:alnum:]_]*)"
   HASHTAG_RE         = /(?:^|[^\/\)\w])#(#{HASHTAG_NAME_RE})/i
 
   validates :name, presence: true, format: { with: /\A(#{HASHTAG_NAME_RE})\z/i }
   validate :validate_name_change, if: -> { !new_record? && name_changed? }
+  validate :validate_display_name_change, if: -> { !new_record? && display_name_changed? }
 
   scope :reviewed, -> { where.not(reviewed_at: nil) }
   scope :unreviewed, -> { where(reviewed_at: nil) }
@@ -46,6 +48,10 @@ class Tag < ApplicationRecord
     name
   end
 
+  def display_name
+    attributes['display_name'] || name
+  end
+
   def usable
     boolean_with_default('usable', true)
   end
@@ -90,8 +96,10 @@ class Tag < ApplicationRecord
 
   class << self
     def find_or_create_by_names(name_or_names)
-      Array(name_or_names).map(&method(:normalize)).uniq { |str| str.mb_chars.downcase.to_s }.map do |normalized_name|
-        tag = matching_name(normalized_name).first || create(name: normalized_name)
+      names = Array(name_or_names).map { |str| [normalize(str), str] }.uniq(&:first)
+
+      names.map do |(normalized_name, display_name)|
+        tag = matching_name(normalized_name).first || create(name: normalized_name, display_name: display_name)
 
         yield tag if block_given?
 
@@ -129,7 +137,7 @@ class Tag < ApplicationRecord
     end
 
     def normalize(str)
-      str.gsub(/\A#/, '')
+      HashtagNormalizer.new.normalize(str)
     end
   end
 
@@ -138,4 +146,8 @@ class Tag < ApplicationRecord
   def validate_name_change
     errors.add(:name, I18n.t('tags.does_not_match_previous_name')) unless name_was.mb_chars.casecmp(name.mb_chars).zero?
   end
+
+  def validate_display_name_change
+    errors.add(:display_name, I18n.t('tags.does_not_match_previous_name')) unless HashtagNormalizer.new.normalize(display_name).casecmp(name.mb_chars).zero?
+  end
 end