tarrien/Mastodon
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Add support for language preferences for trending statuses and links (#18288)

Browse Source
This commit is contained in:
Eugen Rochko
2022-10-08 16:45:40 +02:00
committed by GitHub
parent 678fc4d292
commit 45ebdb72ca
29 changed files with 274 additions and 121 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
app/models/trends/base.rb
View File

@ -98,4 +98,8 @@ class Trends::Base
pipeline.rename(from_key, to_key)
end
end
def skip_review?
Setting.trendable_by_default
end
end

98
app/models/trends/links.rb
View File

@ -11,6 +11,40 @@ class Trends::Links < Trends::Base
decay_threshold: 1,
}
class Query < Trends::Query
def filtered_for!(account)
@account = account
self
end
def filtered_for(account)
clone.filtered_for!(account)
end
def to_arel
scope = PreviewCard.joins(:trend).reorder(score: :desc)
scope = scope.reorder(language_order_clause.desc, score: :desc) if preferred_languages.present?
scope = scope.merge(PreviewCardTrend.allowed) if @allowed
scope = scope.offset(@offset) if @offset.present?
scope = scope.limit(@limit) if @limit.present?
scope
end
private
def language_order_clause
Arel::Nodes::Case.new.when(PreviewCardTrend.arel_table[:language].in(preferred_languages)).then(1).else(0)
end
def preferred_languages
if @account&.chosen_languages.present?
@account.chosen_languages
else
@locale
end
end
end
def register(status, at_time = Time.now.utc)
original_status = status.proper
@ -28,24 +62,33 @@ class Trends::Links < Trends::Base
record_used_id(preview_card.id, at_time)
end
def query
Query.new(key_prefix, klass)
end
def refresh(at_time = Time.now.utc)
preview_cards = PreviewCard.where(id: (recently_used_ids(at_time) + currently_trending_ids(false, -1)).uniq)
preview_cards = PreviewCard.where(id: (recently_used_ids(at_time) + PreviewCardTrend.pluck(:preview_card_id)).uniq)
calculate_scores(preview_cards, at_time)
end
def request_review
preview_cards = PreviewCard.where(id: currently_trending_ids(false, -1))
PreviewCardTrend.pluck('distinct language').flat_map do |language|
score_at_threshold = PreviewCardTrend.where(language: language, allowed: true).order(rank: :desc).where('rank <= ?', options[:review_threshold]).first&.score || 0
preview_card_trends = PreviewCardTrend.where(language: language, allowed: false).joins(:preview_card)
preview_cards.filter_map do |preview_card|
next unless would_be_trending?(preview_card.id) && !preview_card.trendable? && preview_card.requires_review_notification?
preview_card_trends.filter_map do |trend|
preview_card = trend.preview_card
if preview_card.provider.nil?
preview_card.provider = PreviewCardProvider.create(domain: preview_card.domain, requested_review_at: Time.now.utc)
else
preview_card.provider.touch(:requested_review_at)
next unless trend.score > score_at_threshold && !preview_card.trendable? && preview_card.requires_review_notification?
if preview_card.provider.nil?
preview_card.provider = PreviewCardProvider.create(domain: preview_card.domain, requested_review_at: Time.now.utc)
else
preview_card.provider.touch(:requested_review_at)
end
preview_card
end
preview_card
end
end
@ -62,10 +105,7 @@ class Trends::Links < Trends::Base
private
def calculate_scores(preview_cards, at_time)
global_items = []
locale_items = Hash.new { |h, key| h[key] = [] }
preview_cards.each do |preview_card|
items = preview_cards.map do |preview_card|
expected = preview_card.history.get(at_time - 1.day).accounts.to_f
expected = 1.0 if expected.zero?
observed = preview_card.history.get(at_time).accounts.to_f
@ -89,26 +129,24 @@ class Trends::Links < Trends::Base
preview_card.update_columns(max_score: max_score, max_score_at: max_time)
end
decaying_score = max_score * (0.5**((at_time.to_f - max_time.to_f) / options[:max_score_halflife].to_f))
decaying_score = begin
if max_score.zero? || !valid_locale?(preview_card.language)
0
else
max_score * (0.5**((at_time.to_f - max_time.to_f) / options[:max_score_halflife].to_f))
end
end
next unless decaying_score >= options[:decay_threshold]
global_items << { score: decaying_score, item: preview_card }
locale_items[preview_card.language] << { score: decaying_score, item: preview_card } if valid_locale?(preview_card.language)
[decaying_score, preview_card]
end
replace_items('', global_items)
to_insert = items.filter { |(score, _)| score >= options[:decay_threshold] }
to_delete = items.filter { |(score, _)| score < options[:decay_threshold] }
Trends.available_locales.each do |locale|
replace_items(":#{locale}", locale_items[locale])
PreviewCardTrend.transaction do
PreviewCardTrend.upsert_all(to_insert.map { |(score, preview_card)| { preview_card_id: preview_card.id, score: score, language: preview_card.language, allowed: preview_card.trendable? || false } }, unique_by: :preview_card_id) if to_insert.any?
PreviewCardTrend.where(preview_card_id: to_delete.map { |(_, preview_card)| preview_card.id }).delete_all if to_delete.any?
PreviewCardTrend.connection.exec_update('UPDATE preview_card_trends SET rank = t0.calculated_rank FROM (SELECT id, row_number() OVER w AS calculated_rank FROM preview_card_trends WINDOW w AS (PARTITION BY language ORDER BY score DESC)) t0 WHERE preview_card_trends.id = t0.id')
end
end
def filter_for_allowed_items(items)
items.select { |item| item[:item].trendable? }
end
def would_be_trending?(id)
score(id) > score_at_rank(options[:review_threshold] - 1)
end
end

25
app/models/trends/preview_card_filter.rb
View File

@ -13,10 +13,10 @@ class Trends::PreviewCardFilter
end
def results
scope = PreviewCard.unscoped
scope = initial_scope
params.each do |key, value|
next if %w(page locale).include?(key.to_s)
next if %w(page).include?(key.to_s)
scope.merge!(scope_for(key, value.to_s.strip)) if value.present?
end
@ -26,21 +26,30 @@ class Trends::PreviewCardFilter
private
def initial_scope
PreviewCard.select(PreviewCard.arel_table[Arel.star])
.joins(:trend)
.eager_load(:trend)
.reorder(score: :desc)
end
def scope_for(key, value)
case key.to_s
when 'trending'
trending_scope(value)
when 'locale'
PreviewCardTrend.where(language: value)
else
raise "Unknown filter: #{key}"
end
end
def trending_scope(value)
scope = Trends.links.query
scope = scope.in_locale(@params[:locale].to_s) if @params[:locale].present?
scope = scope.allowed if value == 'allowed'
scope.to_arel
case value
when 'allowed'
PreviewCardTrend.allowed
else
PreviewCardTrend.all
end
end
end

25
app/models/trends/status_filter.rb
View File

@ -13,10 +13,10 @@ class Trends::StatusFilter
end
def results
scope = Status.unscoped.kept
scope = initial_scope
params.each do |key, value|
next if %w(page locale).include?(key.to_s)
next if %w(page).include?(key.to_s)
scope.merge!(scope_for(key, value.to_s.strip)) if value.present?
end
@ -26,21 +26,30 @@ class Trends::StatusFilter
private
def initial_scope
Status.select(Status.arel_table[Arel.star])
.joins(:trend)
.eager_load(:trend)
.reorder(score: :desc)
end
def scope_for(key, value)
case key.to_s
when 'trending'
trending_scope(value)
when 'locale'
StatusTrend.where(language: value)
else
raise "Unknown filter: #{key}"
end
end
def trending_scope(value)
scope = Trends.statuses.query
scope = scope.in_locale(@params[:locale].to_s) if @params[:locale].present?
scope = scope.allowed if value == 'allowed'
scope.to_arel
case value
when 'allowed'
StatusTrend.allowed
else
StatusTrend.all
end
end
end

82
app/models/trends/statuses.rb
View File

@ -20,13 +20,27 @@ class Trends::Statuses < Trends::Base
clone.filtered_for!(account)
end
def to_arel
scope = Status.joins(:trend).reorder(score: :desc)
scope = scope.reorder(language_order_clause.desc, score: :desc) if preferred_languages.present?
scope = scope.merge(StatusTrend.allowed) if @allowed
scope = scope.not_excluded_by_account(@account).not_domain_blocked_by_account(@account) if @account.present?
scope = scope.offset(@offset) if @offset.present?
scope = scope.limit(@limit) if @limit.present?
scope
end
private
def apply_scopes(scope)
if @account.nil?
scope
def language_order_clause
Arel::Nodes::Case.new.when(StatusTrend.arel_table[:language].in(preferred_languages)).then(1).else(0)
end
def preferred_languages
if @account&.chosen_languages.present?
@account.chosen_languages
else
scope.not_excluded_by_account(@account).not_domain_blocked_by_account(@account)
@locale
end
end
end
@ -36,9 +50,6 @@ class Trends::Statuses < Trends::Base
end
def add(status, _account_id, at_time = Time.now.utc)
# We rely on the total reblogs and favourites count, so we
# don't record which account did the what and when here
record_used_id(status.id, at_time)
end
@ -47,18 +58,23 @@ class Trends::Statuses < Trends::Base
end
def refresh(at_time = Time.now.utc)
statuses = Status.where(id: (recently_used_ids(at_time) + currently_trending_ids(false, -1)).uniq).includes(:account, :media_attachments)
statuses = Status.where(id: (recently_used_ids(at_time) + StatusTrend.pluck(:status_id)).uniq).includes(:status_stat, :account)
calculate_scores(statuses, at_time)
end
def request_review
statuses = Status.where(id: currently_trending_ids(false, -1)).includes(:account)
StatusTrend.pluck('distinct language').flat_map do |language|
score_at_threshold = StatusTrend.where(language: language, allowed: true).order(rank: :desc).where('rank <= ?', options[:review_threshold]).first&.score || 0
status_trends = StatusTrend.where(language: language, allowed: false).joins(:status).includes(status: :account)
statuses.filter_map do |status|
next unless would_be_trending?(status.id) && !status.trendable? && status.requires_review_notification?
status_trends.filter_map do |trend|
status = trend.status
status.account.touch(:requested_review_at)
status
if trend.score > score_at_threshold && !status.trendable? && status.requires_review_notification?
status.account.touch(:requested_review_at)
status
end
end
end
end
@ -75,14 +91,11 @@ class Trends::Statuses < Trends::Base
private
def eligible?(status)
status.public_visibility? && status.account.discoverable? && !status.account.silenced? && status.spoiler_text.blank? && !status.sensitive? && !status.reply?
status.public_visibility? && status.account.discoverable? && !status.account.silenced? && status.spoiler_text.blank? && !status.sensitive? && !status.reply? && valid_locale?(status.language)
end
def calculate_scores(statuses, at_time)
global_items = []
locale_items = Hash.new { |h, key| h[key] = [] }
statuses.each do |status|
items = statuses.map do |status|
expected = 1.0
observed = (status.reblogs_count + status.favourites_count).to_f
@ -94,29 +107,24 @@ class Trends::Statuses < Trends::Base
end
end
decaying_score = score * (0.5**((at_time.to_f - status.created_at.to_f) / options[:score_halflife].to_f))
decaying_score = begin
if score.zero? || !eligible?(status)
0
else
score * (0.5**((at_time.to_f - status.created_at.to_f) / options[:score_halflife].to_f))
end
end
next unless decaying_score >= options[:decay_threshold]
global_items << { score: decaying_score, item: status }
locale_items[status.language] << { account_id: status.account_id, score: decaying_score, item: status } if valid_locale?(status.language)
[decaying_score, status]
end
replace_items('', global_items)
to_insert = items.filter { |(score, _)| score >= options[:decay_threshold] }
to_delete = items.filter { |(score, _)| score < options[:decay_threshold] }
Trends.available_locales.each do |locale|
replace_items(":#{locale}", locale_items[locale])
StatusTrend.transaction do
StatusTrend.upsert_all(to_insert.map { |(score, status)| { status_id: status.id, account_id: status.account_id, score: score, language: status.language, allowed: status.trendable? || false } }, unique_by: :status_id) if to_insert.any?
StatusTrend.where(status_id: to_delete.map { |(_, status)| status.id }).delete_all if to_delete.any?
StatusTrend.connection.exec_update('UPDATE status_trends SET rank = t0.calculated_rank FROM (SELECT id, row_number() OVER w AS calculated_rank FROM status_trends WINDOW w AS (PARTITION BY language ORDER BY score DESC)) t0 WHERE status_trends.id = t0.id')
end
end
def filter_for_allowed_items(items)
# Show only one status per account, pick the one with the highest score
# that's also eligible to trend
items.group_by { |item| item[:account_id] }.values.filter_map { |account_items| account_items.select { |item| item[:item].trendable? && item[:item].account.discoverable? }.max_by { |item| item[:score] } }
end
def would_be_trending?(id)
score(id) > score_at_rank(options[:review_threshold] - 1)
end
end