tarrien/Mastodon
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge branch 'main' into glitch-soc/merge-upstream

Browse Source 
Conflicts:
- `app/views/admin/tags/index.html.haml`:
  Removed upstream while it had changes in glitch-soc to accomodate for the
  theming system.

Additional changes to accomodate for the theming system:
- `app/views/admin/trends/links/preview_card_providers/index.html.haml`
- `app/views/admin/trends/links/index.html.haml`
- `app/views/admin/trends/tags/index.html.haml`
- `app/views/admin/tags/show.html.haml`
This commit is contained in:
Claire
2021-11-25 23:49:17 +01:00
parent 8c2fe2a846 b1fd6d4490
commit 443ec4f8ba
100 changed files with 2119 additions and 725 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
app/models/account_statuses_cleanup_policy.rb
View File

@ -4,8 +4,8 @@
#
# Table name: account_statuses_cleanup_policies
#
# id :bigint not null, primary key
# account_id :bigint not null
# id :bigint(8) not null, primary key
# account_id :bigint(8) not null
# enabled :boolean default(TRUE), not null
# min_status_age :integer default(1209600), not null
# keep_direct :boolean default(TRUE), not null

65
app/models/form/preview_card_batch.rb Normal file
View File

@ -0,0 +1,65 @@
# frozen_string_literal: true
class Form::PreviewCardBatch
include ActiveModel::Model
include Authorization
attr_accessor :preview_card_ids, :action, :current_account, :precision
def save
case action
when 'approve'
approve!
when 'approve_all'
approve_all!
when 'reject'
reject!
when 'reject_all'
reject_all!
end
end
private
def preview_cards
@preview_cards ||= PreviewCard.where(id: preview_card_ids)
end
def preview_card_providers
@preview_card_providers ||= preview_cards.map(&:domain).uniq.map { |domain| PreviewCardProvider.matching_domain(domain) || PreviewCardProvider.new(domain: domain) }
end
def approve!
preview_cards.each { |preview_card| authorize(preview_card, :update?) }
preview_cards.update_all(trendable: true)
end
def approve_all!
preview_card_providers.each do |provider|
authorize(provider, :update?)
provider.update(trendable: true, reviewed_at: action_time)
end
# Reset any individual overrides
preview_cards.update_all(trendable: nil)
end
def reject!
preview_cards.each { |preview_card| authorize(preview_card, :update?) }
preview_cards.update_all(trendable: false)
end
def reject_all!
preview_card_providers.each do |provider|
authorize(provider, :update?)
provider.update(trendable: false, reviewed_at: action_time)
end
# Reset any individual overrides
preview_cards.update_all(trendable: nil)
end
def action_time
@action_time ||= Time.now.utc
end
end

33
app/models/form/preview_card_provider_batch.rb Normal file
View File

@ -0,0 +1,33 @@
# frozen_string_literal: true
class Form::PreviewCardProviderBatch
include ActiveModel::Model
include Authorization
attr_accessor :preview_card_provider_ids, :action, :current_account
def save
case action
when 'approve'
approve!
when 'reject'
reject!
end
end
private
def preview_card_providers
PreviewCardProvider.where(id: preview_card_provider_ids)
end
def approve!
preview_card_providers.each { |provider| authorize(provider, :update?) }
preview_card_providers.update_all(trendable: true, reviewed_at: Time.now.utc)
end
def reject!
preview_card_providers.each { |provider| authorize(provider, :update?) }
preview_card_providers.update_all(trendable: false, reviewed_at: Time.now.utc)
end
end

8
app/models/form/tag_batch.rb
View File

@ -23,11 +23,15 @@ class Form::TagBatch
def approve!
tags.each { |tag| authorize(tag, :update?) }
tags.update_all(trendable: true, reviewed_at: Time.now.utc)
tags.update_all(trendable: true, reviewed_at: action_time)
end
def reject!
tags.each { |tag| authorize(tag, :update?) }
tags.update_all(trendable: false, reviewed_at: Time.now.utc)
tags.update_all(trendable: false, reviewed_at: action_time)
end
def action_time
@action_time ||= Time.now.utc
end
end

42
app/models/preview_card.rb
View File

@ -24,6 +24,11 @@
# embed_url :string default(""), not null
# image_storage_schema_version :integer
# blurhash :string
# language :string
# max_score :float
# max_score_at :datetime
# trendable :boolean
# link_type :integer
#
class PreviewCard < ApplicationRecord
@ -40,6 +45,7 @@ class PreviewCard < ApplicationRecord
self.inheritance_column = false
enum type: [:link, :photo, :video, :rich]
enum link_type: [:unknown, :article]
has_and_belongs_to_many :statuses
@ -54,6 +60,32 @@ class PreviewCard < ApplicationRecord
before_save :extract_dimensions, if: :link?
def appropriate_for_trends?
link? && article? && title.present? && description.present? && image.present? && provider_name.present?
end
def domain
@domain ||= Addressable::URI.parse(url).normalized_host
end
def provider
@provider ||= PreviewCardProvider.matching_domain(domain)
end
def trendable?
if attributes['trendable'].nil?
provider&.trendable?
else
attributes['trendable']
end
end
def requires_review_notification?
attributes['trendable'].nil? && (provider.nil? || provider.requires_review_notification?)
end
attr_writer :provider
def local?
false
end
@ -69,11 +101,14 @@ class PreviewCard < ApplicationRecord
save!
end
def history
@history ||= Trends::History.new('links', id)
end
class << self
private
# rubocop:disable Naming/MethodParameterName
def image_styles(f)
def image_styles(file)
styles = {
original: {
geometry: '400x400>',
@ -83,10 +118,9 @@ class PreviewCard < ApplicationRecord
},
}
styles[:original][:format] = 'jpg' if f.instance.image_content_type == 'image/gif'
styles[:original][:format] = 'jpg' if file.instance.image_content_type == 'image/gif'
styles
end
# rubocop:enable Naming/MethodParameterName
end
private

53
app/models/preview_card_filter.rb Normal file
View File

@ -0,0 +1,53 @@
# frozen_string_literal: true
class PreviewCardFilter
KEYS = %i(
trending
).freeze
attr_reader :params
def initialize(params)
@params = params
end
def results
scope = PreviewCard.unscoped
params.each do |key, value|
next if key.to_s == 'page'
scope.merge!(scope_for(key, value.to_s.strip)) if value.present?
end
scope
end
private
def scope_for(key, value)
case key.to_s
when 'trending'
trending_scope(value)
else
raise "Unknown filter: #{key}"
end
end
def trending_scope(value)
ids = begin
case value.to_s
when 'allowed'
Trends.links.currently_trending_ids(true, -1)
else
Trends.links.currently_trending_ids(false, -1)
end
end
if ids.empty?
PreviewCard.none
else
PreviewCard.joins("join unnest(array[#{ids.map(&:to_i).join(',')}]::integer[]) with ordinality as x (id, ordering) on preview_cards.id = x.id").order('x.ordering')
end
end
end

57
app/models/preview_card_provider.rb Normal file
View File

@ -0,0 +1,57 @@
# frozen_string_literal: true
# == Schema Information
#
# Table name: preview_card_providers
#
# id :bigint(8) not null, primary key
# domain :string default(""), not null
# icon_file_name :string
# icon_content_type :string
# icon_file_size :bigint(8)
# icon_updated_at :datetime
# trendable :boolean
# reviewed_at :datetime
# requested_review_at :datetime
# created_at :datetime not null
# updated_at :datetime not null
#
class PreviewCardProvider < ApplicationRecord
include DomainNormalizable
include Attachmentable
ICON_MIME_TYPES = %w(image/x-icon image/vnd.microsoft.icon image/png).freeze
LIMIT = 1.megabyte
validates :domain, presence: true, uniqueness: true, domain: true
has_attached_file :icon, styles: { static: { format: 'png', convert_options: '-coalesce -strip' } }, validate_media_type: false
validates_attachment :icon, content_type: { content_type: ICON_MIME_TYPES }, size: { less_than: LIMIT }
remotable_attachment :icon, LIMIT
scope :trendable, -> { where(trendable: true) }
scope :not_trendable, -> { where(trendable: false) }
scope :reviewed, -> { where.not(reviewed_at: nil) }
scope :pending_review, -> { where(reviewed_at: nil) }
def requires_review?
reviewed_at.nil?
end
def reviewed?
reviewed_at.present?
end
def requested_review?
requested_review_at.present?
end
def requires_review_notification?
requires_review? && !requested_review?
end
def self.matching_domain(domain)
segments = domain.split('.')
where(domain: segments.map.with_index { |_, i| segments[i..-1].join('.') }).order(Arel.sql('char_length(domain) desc')).first
end
end

49
app/models/preview_card_provider_filter.rb Normal file
View File

@ -0,0 +1,49 @@
# frozen_string_literal: true
class PreviewCardProviderFilter
KEYS = %i(
status
).freeze
attr_reader :params
def initialize(params)
@params = params
end
def results
scope = PreviewCardProvider.unscoped
params.each do |key, value|
next if key.to_s == 'page'
scope.merge!(scope_for(key, value.to_s.strip)) if value.present?
end
scope.order(domain: :asc)
end
private
def scope_for(key, value)
case key.to_s
when 'status'
status_scope(value)
else
raise "Unknown filter: #{key}"
end
end
def status_scope(value)
case value.to_s
when 'approved'
PreviewCardProvider.trendable
when 'rejected'
PreviewCardProvider.not_trendable
when 'pending_review'
PreviewCardProvider.pending_review
else
raise "Unknown status: #{value}"
end
end
end

23
app/models/tag.rb
View File

@ -36,6 +36,7 @@ class Tag < ApplicationRecord
scope :usable, -> { where(usable: [true, nil]) }
scope :listable, -> { where(listable: [true, nil]) }
scope :trendable, -> { Setting.trendable_by_default ? where(trendable: [true, nil]) : where(trendable: true) }
scope :not_trendable, -> { where(trendable: false) }
scope :recently_used, ->(account) { joins(:statuses).where(statuses: { id: account.statuses.select(:id).limit(1000) }).group(:id).order(Arel.sql('count(*) desc')) }
scope :matches_name, ->(term) { where(arel_table[:name].lower.matches(arel_table.lower("#{sanitize_sql_like(Tag.normalize(term))}%"), nil, true)) } # Search with case-sensitive to use B-tree index
@ -75,28 +76,12 @@ class Tag < ApplicationRecord
requested_review_at.present?
end
def use!(account, status: nil, at_time: Time.now.utc)
TrendingTags.record_use!(self, account, status: status, at_time: at_time)
end
def trending?
TrendingTags.trending?(self)
def requires_review_notification?
requires_review? && !requested_review?
end
def history
days = []
7.times do |i|
day = i.days.ago.beginning_of_day.to_i
days << {
day: day.to_s,
uses: Redis.current.get("activity:tags:#{id}:#{day}") || '0',
accounts: Redis.current.pfcount("activity:tags:#{id}:#{day}:accounts").to_s,
}
end
days
@history ||= Trends::History.new('tags', id)
end
class << self

56
app/models/tag_filter.rb
View File

@ -2,13 +2,8 @@
class TagFilter
KEYS = %i(
directory
reviewed
unreviewed
pending_review
popular
active
name
trending
status
).freeze
attr_reader :params
@ -18,7 +13,13 @@ class TagFilter
end
def results
scope = Tag.unscoped
scope = begin
if params[:status] == 'pending_review'
Tag.unscoped
else
trending_scope
end
end
params.each do |key, value|
next if key.to_s == 'page'
@ -26,27 +27,40 @@ class TagFilter
scope.merge!(scope_for(key, value.to_s.strip)) if value.present?
end
scope.order(id: :desc)
scope
end
private
def scope_for(key, value)
case key.to_s
when 'reviewed'
Tag.reviewed.order(reviewed_at: :desc)
when 'unreviewed'
Tag.unreviewed
when 'pending_review'
Tag.pending_review.order(requested_review_at: :desc)
when 'popular'
Tag.order('max_score DESC NULLS LAST')
when 'active'
Tag.order('last_status_at DESC NULLS LAST')
when 'name'
Tag.matches_name(value)
when 'status'
status_scope(value)
else
raise "Unknown filter: #{key}"
end
end
def trending_scope
ids = Trends.tags.currently_trending_ids(false, -1)
if ids.empty?
Tag.none
else
Tag.joins("join unnest(array[#{ids.map(&:to_i).join(',')}]::integer[]) with ordinality as x (id, ordering) on tags.id = x.id").order('x.ordering')
end
end
def status_scope(value)
case value.to_s
when 'approved'
Tag.trendable
when 'rejected'
Tag.not_trendable
when 'pending_review'
Tag.pending_review
else
raise "Unknown status: #{value}"
end
end
end

128
app/models/trending_tags.rb
View File

@ -1,128 +0,0 @@
# frozen_string_literal: true
class TrendingTags
KEY = 'trending_tags'
EXPIRE_HISTORY_AFTER = 7.days.seconds
EXPIRE_TRENDS_AFTER = 1.day.seconds
THRESHOLD = 5
LIMIT = 10
REVIEW_THRESHOLD = 3
MAX_SCORE_COOLDOWN = 2.days.freeze
MAX_SCORE_HALFLIFE = 2.hours.freeze
class << self
include Redisable
def record_use!(tag, account, status: nil, at_time: Time.now.utc)
return unless tag.usable? && !account.silenced?
# Even if a tag is not allowed to trend, we still need to
# record the stats since they can be displayed in other places
increment_historical_use!(tag.id, at_time)
increment_unique_use!(tag.id, account.id, at_time)
increment_use!(tag.id, at_time)
# Only update when the tag was last used once every 12 hours
# and only if a status is given (lets use ignore reblogs)
tag.update(last_status_at: at_time) if status.present? && (tag.last_status_at.nil? || (tag.last_status_at < at_time && tag.last_status_at < 12.hours.ago))
end
def update!(at_time = Time.now.utc)
tag_ids = redis.smembers("#{KEY}:used:#{at_time.beginning_of_day.to_i}") + redis.zrange(KEY, 0, -1)
tags = Tag.trendable.where(id: tag_ids.uniq)
# First pass to calculate scores and update the set
tags.each do |tag|
expected = redis.pfcount("activity:tags:#{tag.id}:#{(at_time - 1.day).beginning_of_day.to_i}:accounts").to_f
expected = 1.0 if expected.zero?
observed = redis.pfcount("activity:tags:#{tag.id}:#{at_time.beginning_of_day.to_i}:accounts").to_f
max_time = tag.max_score_at
max_score = tag.max_score
max_score = 0 if max_time.nil? || max_time < (at_time - MAX_SCORE_COOLDOWN)
score = begin
if expected > observed || observed < THRESHOLD
0
else
((observed - expected)**2) / expected
end
end
if score > max_score
max_score = score
max_time = at_time
# Not interested in triggering any callbacks for this
tag.update_columns(max_score: max_score, max_score_at: max_time)
end
decaying_score = max_score * (0.5**((at_time.to_f - max_time.to_f) / MAX_SCORE_HALFLIFE.to_f))
if decaying_score.zero?
redis.zrem(KEY, tag.id)
else
redis.zadd(KEY, decaying_score, tag.id)
end
end
users_for_review = User.staff.includes(:account).to_a.select(&:allows_trending_tag_emails?)
# Second pass to notify about previously unreviewed trends
tags.each do |tag|
current_rank = redis.zrevrank(KEY, tag.id)
needs_review_notification = tag.requires_review? && !tag.requested_review?
rank_passes_threshold = current_rank.present? && current_rank <= REVIEW_THRESHOLD
next unless !tag.trendable? && rank_passes_threshold && needs_review_notification
tag.touch(:requested_review_at)
users_for_review.each do |user|
AdminMailer.new_trending_tag(user.account, tag).deliver_later!
end
end
# Trim older items
redis.zremrangebyrank(KEY, 0, -(LIMIT + 1))
redis.zremrangebyscore(KEY, '(0.3', '-inf')
end
def get(limit, filtered: true)
tag_ids = redis.zrevrange(KEY, 0, LIMIT - 1).map(&:to_i)
tags = Tag.where(id: tag_ids)
tags = tags.trendable if filtered
tags = tags.index_by(&:id)
tag_ids.map { |tag_id| tags[tag_id] }.compact.take(limit)
end
def trending?(tag)
rank = redis.zrevrank(KEY, tag.id)
rank.present? && rank < LIMIT
end
private
def increment_historical_use!(tag_id, at_time)
key = "activity:tags:#{tag_id}:#{at_time.beginning_of_day.to_i}"
redis.incrby(key, 1)
redis.expire(key, EXPIRE_HISTORY_AFTER)
end
def increment_unique_use!(tag_id, account_id, at_time)
key = "activity:tags:#{tag_id}:#{at_time.beginning_of_day.to_i}:accounts"
redis.pfadd(key, account_id)
redis.expire(key, EXPIRE_HISTORY_AFTER)
end
def increment_use!(tag_id, at_time)
key = "#{KEY}:used:#{at_time.beginning_of_day.to_i}"
redis.sadd(key, tag_id)
redis.expire(key, EXPIRE_HISTORY_AFTER)
end
end
end

27
app/models/trends.rb Normal file
View File

@ -0,0 +1,27 @@
# frozen_string_literal: true
module Trends
def self.table_name_prefix
'trends_'
end
def self.links
@links ||= Trends::Links.new
end
def self.tags
@tags ||= Trends::Tags.new
end
def self.refresh!
[links, tags].each(&:refresh)
end
def self.request_review!
[links, tags].each(&:request_review) if enabled?
end
def self.enabled?
Setting.trends
end
end

80
app/models/trends/base.rb Normal file
View File

@ -0,0 +1,80 @@
# frozen_string_literal: true
class Trends::Base
include Redisable
class_attribute :default_options
attr_reader :options
# @param [Hash] options
# @option options [Integer] :threshold Minimum amount of uses by unique accounts to begin calculating the score
# @option options [Integer] :review_threshold Minimum rank (lower = better) before requesting a review
# @option options [ActiveSupport::Duration] :max_score_cooldown For this amount of time, the peak score (if bigger than current score) is decayed-from
# @option options [ActiveSupport::Duration] :max_score_halflife How quickly a peak score decays
def initialize(options = {})
@options = self.class.default_options.merge(options)
end
def register(_status)
raise NotImplementedError
end
def add(*)
raise NotImplementedError
end
def refresh(*)
raise NotImplementedError
end
def request_review
raise NotImplementedError
end
def get(*)
raise NotImplementedError
end
def score(id)
redis.zscore("#{key_prefix}:all", id) || 0
end
def rank(id)
redis.zrevrank("#{key_prefix}:allowed", id)
end
def currently_trending_ids(allowed, limit)
redis.zrevrange(allowed ? "#{key_prefix}:allowed" : "#{key_prefix}:all", 0, limit.positive? ? limit - 1 : limit).map(&:to_i)
end
protected
def key_prefix
raise NotImplementedError
end
def recently_used_ids(at_time = Time.now.utc)
redis.smembers(used_key(at_time)).map(&:to_i)
end
def record_used_id(id, at_time = Time.now.utc)
redis.sadd(used_key(at_time), id)
redis.expire(used_key(at_time), 1.day.seconds)
end
def trim_older_items
redis.zremrangebyscore("#{key_prefix}:all", '-inf', '(1')
redis.zremrangebyscore("#{key_prefix}:allowed", '-inf', '(1')
end
def score_at_rank(rank)
redis.zrevrange("#{key_prefix}:allowed", 0, rank, with_scores: true).last&.last || 0
end
private
def used_key(at_time)
"#{key_prefix}:used:#{at_time.beginning_of_day.to_i}"
end
end

98
app/models/trends/history.rb Normal file
View File

@ -0,0 +1,98 @@
# frozen_string_literal: true
class Trends::History
include Enumerable
class Aggregate
include Redisable
def initialize(prefix, id, date_range)
@days = date_range.map { |date| Day.new(prefix, id, date.to_time(:utc)) }
end
def uses
redis.mget(*@days.map { |day| day.key_for(:uses) }).map(&:to_i).sum
end
def accounts
redis.pfcount(*@days.map { |day| day.key_for(:accounts) })
end
end
class Day
include Redisable
EXPIRE_AFTER = 14.days.seconds
def initialize(prefix, id, day)
@prefix = prefix
@id = id
@day = day.beginning_of_day
end
attr_reader :day
def accounts
redis.pfcount(key_for(:accounts))
end
def uses
redis.get(key_for(:uses))&.to_i || 0
end
def add(account_id)
redis.pipelined do
redis.incrby(key_for(:uses), 1)
redis.pfadd(key_for(:accounts), account_id)
redis.expire(key_for(:uses), EXPIRE_AFTER)
redis.expire(key_for(:accounts), EXPIRE_AFTER)
end
end
def as_json
{ day: day.to_i.to_s, accounts: accounts.to_s, uses: uses.to_s }
end
def key_for(suffix)
case suffix
when :accounts
"#{key_prefix}:#{suffix}"
when :uses
key_prefix
end
end
def key_prefix
"activity:#{@prefix}:#{@id}:#{day.to_i}"
end
end
def initialize(prefix, id)
@prefix = prefix
@id = id
end
def get(date)
Day.new(@prefix, @id, date)
end
def add(account_id, at_time = Time.now.utc)
Day.new(@prefix, @id, at_time).add(account_id)
end
def aggregate(date_range)
Aggregate.new(@prefix, @id, date_range)
end
def each(&block)
if block_given?
(0...7).map { |i| block.call(get(i.days.ago)) }
else
to_enum(:each)
end
end
def as_json(*)
map(&:as_json)
end
end

117
app/models/trends/links.rb Normal file
View File

@ -0,0 +1,117 @@
# frozen_string_literal: true
class Trends::Links < Trends::Base
PREFIX = 'trending_links'
self.default_options = {
threshold: 15,
review_threshold: 10,
max_score_cooldown: 2.days.freeze,
max_score_halflife: 8.hours.freeze,
}
def register(status, at_time = Time.now.utc)
original_status = status.reblog? ? status.reblog : status
return unless original_status.public_visibility? && status.public_visibility? &&
!original_status.account.silenced? && !status.account.silenced? &&
!original_status.spoiler_text?
original_status.preview_cards.each do |preview_card|
add(preview_card, status.account_id, at_time) if preview_card.appropriate_for_trends?
end
end
def add(preview_card, account_id, at_time = Time.now.utc)
preview_card.history.add(account_id, at_time)
record_used_id(preview_card.id, at_time)
end
def get(allowed, limit)
preview_card_ids = currently_trending_ids(allowed, limit)
preview_cards = PreviewCard.where(id: preview_card_ids).index_by(&:id)
preview_card_ids.map { |id| preview_cards[id] }.compact
end
def refresh(at_time = Time.now.utc)
preview_cards = PreviewCard.where(id: (recently_used_ids(at_time) + currently_trending_ids(false, -1)).uniq)
calculate_scores(preview_cards, at_time)
trim_older_items
end
def request_review
preview_cards = PreviewCard.where(id: currently_trending_ids(false, -1))
preview_cards_requiring_review = preview_cards.filter_map do |preview_card|
next unless would_be_trending?(preview_card.id) && !preview_card.trendable? && preview_card.requires_review_notification?
if preview_card.provider.nil?
preview_card.provider = PreviewCardProvider.create(domain: preview_card.domain, requested_review_at: Time.now.utc)
else
preview_card.provider.touch(:requested_review_at)
end
preview_card
end
return if preview_cards_requiring_review.empty?
User.staff.includes(:account).find_each do |user|
AdminMailer.new_trending_links(user.account, preview_cards_requiring_review).deliver_later! if user.allows_trending_tag_emails?
end
end
protected
def key_prefix
PREFIX
end
private
def calculate_scores(preview_cards, at_time)
preview_cards.each do |preview_card|
expected = preview_card.history.get(at_time - 1.day).accounts.to_f
expected = 1.0 if expected.zero?
observed = preview_card.history.get(at_time).accounts.to_f
max_time = preview_card.max_score_at
max_score = preview_card.max_score
max_score = 0 if max_time.nil? || max_time < (at_time - options[:max_score_cooldown])
score = begin
if expected > observed || observed < options[:threshold]
0
else
((observed - expected)**2) / expected
end
end
if score > max_score
max_score = score
max_time = at_time
# Not interested in triggering any callbacks for this
preview_card.update_columns(max_score: max_score, max_score_at: max_time)
end
decaying_score = max_score * (0.5**((at_time.to_f - max_time.to_f) / options[:max_score_halflife].to_f))
if decaying_score.zero?
redis.zrem("#{PREFIX}:all", preview_card.id)
redis.zrem("#{PREFIX}:allowed", preview_card.id)
else
redis.zadd("#{PREFIX}:all", decaying_score, preview_card.id)
if preview_card.trendable?
redis.zadd("#{PREFIX}:allowed", decaying_score, preview_card.id)
else
redis.zrem("#{PREFIX}:allowed", preview_card.id)
end
end
end
end
def would_be_trending?(id)
score(id) > score_at_rank(options[:review_threshold] - 1)
end
end

111
app/models/trends/tags.rb Normal file
View File

@ -0,0 +1,111 @@
# frozen_string_literal: true
class Trends::Tags < Trends::Base
PREFIX = 'trending_tags'
self.default_options = {
threshold: 15,
review_threshold: 10,
max_score_cooldown: 2.days.freeze,
max_score_halflife: 4.hours.freeze,
}
def register(status, at_time = Time.now.utc)
original_status = status.reblog? ? status.reblog : status
return unless original_status.public_visibility? && status.public_visibility? &&
!original_status.account.silenced? && !status.account.silenced?
original_status.tags.each do |tag|
add(tag, status.account_id, at_time) if tag.usable?
end
end
def add(tag, account_id, at_time = Time.now.utc)
tag.history.add(account_id, at_time)
record_used_id(tag.id, at_time)
end
def refresh(at_time = Time.now.utc)
tags = Tag.where(id: (recently_used_ids(at_time) + currently_trending_ids(false, -1)).uniq)
calculate_scores(tags, at_time)
trim_older_items
end
def get(allowed, limit)
tag_ids = currently_trending_ids(allowed, limit)
tags = Tag.where(id: tag_ids).index_by(&:id)
tag_ids.map { |id| tags[id] }.compact
end
def request_review
tags = Tag.where(id: currently_trending_ids(false, -1))
tags_requiring_review = tags.filter_map do |tag|
next unless would_be_trending?(tag.id) && !tag.trendable? && tag.requires_review_notification?
tag.touch(:requested_review_at)
tag
end
return if tags_requiring_review.empty?
User.staff.includes(:account).find_each do |user|
AdminMailer.new_trending_tags(user.account, tags_requiring_review).deliver_later! if user.allows_trending_tag_emails?
end
end
protected
def key_prefix
PREFIX
end
private
def calculate_scores(tags, at_time)
tags.each do |tag|
expected = tag.history.get(at_time - 1.day).accounts.to_f
expected = 1.0 if expected.zero?
observed = tag.history.get(at_time).accounts.to_f
max_time = tag.max_score_at
max_score = tag.max_score
max_score = 0 if max_time.nil? || max_time < (at_time - options[:max_score_cooldown])
score = begin
if expected > observed || observed < options[:threshold]
0
else
((observed - expected)**2) / expected
end
end
if score > max_score
max_score = score
max_time = at_time
# Not interested in triggering any callbacks for this
tag.update_columns(max_score: max_score, max_score_at: max_time)
end
decaying_score = max_score * (0.5**((at_time.to_f - max_time.to_f) / options[:max_score_halflife].to_f))
if decaying_score.zero?
redis.zrem("#{PREFIX}:all", tag.id)
redis.zrem("#{PREFIX}:allowed", tag.id)
else
redis.zadd("#{PREFIX}:all", decaying_score, tag.id)
if tag.trendable?
redis.zadd("#{PREFIX}:allowed", decaying_score, tag.id)
else
redis.zrem("#{PREFIX}:allowed", tag.id)
end
end
end
end
def would_be_trending?(id)
score(id) > score_at_rank(options[:review_threshold] - 1)
end
end