tarrien/Mastodon
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge branch 'main' into glitch-soc/merge-upstream

Browse Source
This commit is contained in:
Claire
2022-10-28 11:36:25 +02:00
parent 371563b0e2 8dfe5179ee
commit cb19be67d1
567 changed files with 14361 additions and 20828 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
app/models/account.rb
View File

@ -138,6 +138,7 @@ class Account < ApplicationRecord
:role,
:locale,
:shows_application?,
:prefers_noindex?,
to: :user,
prefix: true,
allow_nil: true
@ -194,10 +195,6 @@ class Account < ApplicationRecord
"acct:#{local_username_and_domain}"
end
def searchable?
!(suspended? || moved?) && (!local? || (approved? && confirmed?))
end
def possibly_stale?
last_webfingered_at.nil? || last_webfingered_at <= 1.day.ago
end

4
app/models/admin/status_batch_action.rb
View File

@ -40,7 +40,7 @@ class Admin::StatusBatchAction
end
def handle_delete!
statuses.each { |status| authorize(status, :destroy?) }
statuses.each { |status| authorize([:admin, status], :destroy?) }
ApplicationRecord.transaction do
statuses.each do |status|
@ -75,7 +75,7 @@ class Admin::StatusBatchAction
statuses.includes(:media_attachments, :preview_cards).find_each do |status|
next unless status.with_media? || status.with_preview_card?
authorize(status, :update?)
authorize([:admin, status], :update?)
if target_account.local?
UpdateStatusService.new.call(status, representative_account.id, sensitive: true)

5
app/models/admin/status_filter.rb
View File

@ -3,7 +3,6 @@
class Admin::StatusFilter
KEYS = %i(
media
id
report_id
).freeze
@ -28,12 +27,10 @@ class Admin::StatusFilter
private
def scope_for(key, value)
def scope_for(key, _value)
case key.to_s
when 'media'
Status.joins(:media_attachments).merge(@account.media_attachments.reorder(nil)).group(:id).reorder('statuses.id desc')
when 'id'
Status.where(id: value)
else
raise "Unknown filter: #{key}"
end

4
app/models/domain_block.rb
View File

@ -28,8 +28,8 @@ class DomainBlock < ApplicationRecord
delegate :count, to: :accounts, prefix: true
scope :matches_domain, ->(value) { where(arel_table[:domain].matches("%#{value}%")) }
scope :with_user_facing_limitations, -> { where(severity: [:silence, :suspend]).or(where(reject_media: true)) }
scope :by_severity, -> { order(Arel.sql('(CASE severity WHEN 0 THEN 1 WHEN 1 THEN 2 WHEN 2 THEN 0 END), reject_media, domain')) }
scope :with_user_facing_limitations, -> { where(severity: [:silence, :suspend]) }
scope :by_severity, -> { order(Arel.sql('(CASE severity WHEN 0 THEN 1 WHEN 1 THEN 2 WHEN 2 THEN 0 END), domain')) }
def to_log_human_identifier
domain

15
app/models/extended_description.rb Normal file
View File

@ -0,0 +1,15 @@
# frozen_string_literal: true
class ExtendedDescription < ActiveModelSerializers::Model
attributes :updated_at, :text
def self.current
custom = Setting.find_by(var: 'site_extended_description')
if custom&.value.present?
new(text: custom.value, updated_at: custom.updated_at)
else
new
end
end
end

11
app/models/featured_tag.rb
View File

@ -22,10 +22,18 @@ class FeaturedTag < ApplicationRecord
before_create :set_tag
before_create :reset_data
scope :by_name, ->(name) { joins(:tag).where(tag: { name: HashtagNormalizer.new.normalize(name) }) }
delegate :display_name, to: :tag
attr_writer :name
LIMIT = 10
def sign?
true
end
def name
tag_id.present? ? tag.name : @name
end
@ -50,11 +58,12 @@ class FeaturedTag < ApplicationRecord
end
def validate_featured_tags_limit
errors.add(:base, I18n.t('featured_tags.errors.limit')) if account.featured_tags.count >= 10
errors.add(:base, I18n.t('featured_tags.errors.limit')) if account.featured_tags.count >= LIMIT
end
def validate_tag_name
errors.add(:name, :blank) if @name.blank?
errors.add(:name, :invalid) unless @name.match?(/\A(#{Tag::HASHTAG_NAME_RE})\z/i)
errors.add(:name, :taken) if FeaturedTag.by_name(@name).where(account_id: account_id).exists?
end
end

73
app/models/form/admin_settings.rb
View File

@ -8,26 +8,22 @@ class Form::AdminSettings
site_contact_email
site_title
site_short_description
site_description
site_extended_description
site_terms
registrations_mode
closed_registrations_message
open_deletion
timeline_preview
bootstrap_timeline_accounts
flavour
skin
activity_api_enabled
peers_api_enabled
show_known_fediverse_at_about_page
preview_sensitive_media
custom_css
profile_directory
hide_followers_count
flavour_and_skin
thumbnail
hero
mascot
show_reblogs_in_public_timelines
show_replies_in_public_timelines
@ -45,12 +41,16 @@ class Form::AdminSettings
backups_retention_period
).freeze
INTEGER_KEYS = %i(
media_cache_retention_period
content_cache_retention_period
backups_retention_period
).freeze
BOOLEAN_KEYS = %i(
open_deletion
timeline_preview
activity_api_enabled
peers_api_enabled
show_known_fediverse_at_about_page
preview_sensitive_media
profile_directory
hide_followers_count
@ -66,7 +66,6 @@ class Form::AdminSettings
UPLOAD_KEYS = %i(
thumbnail
hero
mascot
).freeze
@ -76,34 +75,49 @@ class Form::AdminSettings
attr_accessor(*KEYS)
validates :site_short_description, :site_description, html: { wrap_with: :p }
validates :site_extended_description, :site_terms, :closed_registrations_message, html: true
validates :registrations_mode, inclusion: { in: %w(open approved none) }
validates :site_contact_email, :site_contact_username, presence: true
validates :site_contact_username, existing_username: true
validates :bootstrap_timeline_accounts, existing_username: { multiple: true }
validates :show_domain_blocks, inclusion: { in: %w(disabled users all) }
validates :show_domain_blocks_rationale, inclusion: { in: %w(disabled users all) }
validates :media_cache_retention_period, :content_cache_retention_period, :backups_retention_period, numericality: { only_integer: true }, allow_blank: true
validates :registrations_mode, inclusion: { in: %w(open approved none) }, if: -> { defined?(@registrations_mode) }
validates :site_contact_email, :site_contact_username, presence: true, if: -> { defined?(@site_contact_username) || defined?(@site_contact_email) }
validates :site_contact_username, existing_username: true, if: -> { defined?(@site_contact_username) }
validates :bootstrap_timeline_accounts, existing_username: { multiple: true }, if: -> { defined?(@bootstrap_timeline_accounts) }
validates :show_domain_blocks, inclusion: { in: %w(disabled users all) }, if: -> { defined?(@show_domain_blocks) }
validates :show_domain_blocks_rationale, inclusion: { in: %w(disabled users all) }, if: -> { defined?(@show_domain_blocks_rationale) }
validates :media_cache_retention_period, :content_cache_retention_period, :backups_retention_period, numericality: { only_integer: true }, allow_blank: true, if: -> { defined?(@media_cache_retention_period) || defined?(@content_cache_retention_period) || defined?(@backups_retention_period) }
validates :site_short_description, length: { maximum: 200 }, if: -> { defined?(@site_short_description) }
def initialize(_attributes = {})
super
initialize_attributes
KEYS.each do |key|
define_method(key) do
return instance_variable_get("@#{key}") if instance_variable_defined?("@#{key}")
stored_value = begin
if UPLOAD_KEYS.include?(key)
SiteUpload.where(var: key).first_or_initialize(var: key)
else
Setting.public_send(key)
end
end
instance_variable_set("@#{key}", stored_value)
end
end
UPLOAD_KEYS.each do |key|
define_method("#{key}=") do |file|
value = public_send(key)
value.file = file
end
end
def save
return false unless valid?
KEYS.each do |key|
next if PSEUDO_KEYS.include?(key)
value = instance_variable_get("@#{key}")
next if PSEUDO_KEYS.include?(key) || !instance_variable_defined?("@#{key}")
if UPLOAD_KEYS.include?(key) && !value.nil?
upload = SiteUpload.where(var: key).first_or_initialize(var: key)
upload.update(file: value)
if UPLOAD_KEYS.include?(key)
public_send(key).save
else
setting = Setting.where(var: key).first_or_initialize(var: key)
setting.update(value: typecast_value(key, value))
setting.update(value: typecast_value(key, instance_variable_get("@#{key}")))
end
end
end
@ -118,16 +132,11 @@ class Form::AdminSettings
private
def initialize_attributes
KEYS.each do |key|
next if PSEUDO_KEYS.include?(key)
instance_variable_set("@#{key}", Setting.public_send(key)) if instance_variable_get("@#{key}").nil?
end
end
def typecast_value(key, value)
if BOOLEAN_KEYS.include?(key)
value == '1'
elsif INTEGER_KEYS.include?(key)
value.blank? ? value : Integer(value)
else
value
end

1
app/models/ip_block.rb
View File

@ -25,6 +25,7 @@ class IpBlock < ApplicationRecord
}
validates :ip, :severity, presence: true
validates :ip, uniqueness: true
after_commit :reset_cache

1
app/models/preview_card.rb
View File

@ -48,6 +48,7 @@ class PreviewCard < ApplicationRecord
enum link_type: [:unknown, :article]
has_and_belongs_to_many :statuses
has_one :trend, class_name: 'PreviewCardTrend', inverse_of: :preview_card, dependent: :destroy
has_attached_file :image, processors: [:thumbnail, :blurhash_transcoder], styles: ->(f) { image_styles(f) }, convert_options: { all: '-quality 80 -strip' }, validate_media_type: false

17
app/models/preview_card_trend.rb Normal file
View File

@ -0,0 +1,17 @@
# frozen_string_literal: true
# == Schema Information
#
# Table name: preview_card_trends
#
# id :bigint(8) not null, primary key
# preview_card_id :bigint(8) not null
# score :float default(0.0), not null
# rank :integer default(0), not null
# allowed :boolean default(FALSE), not null
# language :string
#
class PreviewCardTrend < ApplicationRecord
belongs_to :preview_card
scope :allowed, -> { where(allowed: true) }
end

77
app/models/privacy_policy.rb Normal file
View File

@ -0,0 +1,77 @@
# frozen_string_literal: true
class PrivacyPolicy < ActiveModelSerializers::Model
DEFAULT_PRIVACY_POLICY = <<~TXT
This privacy policy describes how %{domain} ("%{domain}", "we", "us") collects, protects and uses the personally identifiable information you may provide through the %{domain} website or its API. The policy also describes the choices available to you regarding our use of your personal information and how you can access and update this information. This policy does not apply to the practices of companies that %{domain} does not own or control, or to individuals that %{domain} does not employ or manage.
# What information do we collect?
- **Basic account information**: If you register on this server, you may be asked to enter a username, an e-mail address and a password. You may also enter additional profile information such as a display name and biography, and upload a profile picture and header image. The username, display name, biography, profile picture and header image are always listed publicly.
- **Posts, following and other public information**: The list of people you follow is listed publicly, the same is true for your followers. When you submit a message, the date and time is stored as well as the application you submitted the message from. Messages may contain media attachments, such as pictures and videos. Public and unlisted posts are available publicly. When you feature a post on your profile, that is also publicly available information. Your posts are delivered to your followers, in some cases it means they are delivered to different servers and copies are stored there. When you delete posts, this is likewise delivered to your followers. The action of reblogging or favouriting another post is always public.
- **Direct and followers-only posts**: All posts are stored and processed on the server. Followers-only posts are delivered to your followers and users who are mentioned in them, and direct posts are delivered only to users mentioned in them. In some cases it means they are delivered to different servers and copies are stored there. We make a good faith effort to limit the access to those posts only to authorized persons, but other servers may fail to do so. Therefore it's important to review servers your followers belong to. You may toggle an option to approve and reject new followers manually in the settings. **Please keep in mind that the operators of the server and any receiving server may view such messages**, and that recipients may screenshot, copy or otherwise re-share them. **Do not share any sensitive information over Mastodon.**
- **IPs and other metadata**: When you log in, we record the IP address you log in from, as well as the name of your browser application. All the logged in sessions are available for your review and revocation in the settings. The latest IP address used is stored for up to 12 months. We also may retain server logs which include the IP address of every request to our server.
# What do we use your information for?
Any of the information we collect from you may be used in the following ways:
- To provide the core functionality of Mastodon. You can only interact with other people's content and post your own content when you are logged in. For example, you may follow other people to view their combined posts in your own personalized home timeline.
- To aid moderation of the community, for example comparing your IP address with other known ones to determine ban evasion or other violations.
- The email address you provide may be used to send you information, notifications about other people interacting with your content or sending you messages, and to respond to inquiries, and/or other requests or questions.
# How do we protect your information?
We implement a variety of security measures to maintain the safety of your personal information when you enter, submit, or access your personal information. Among other things, your browser session, as well as the traffic between your applications and the API, are secured with SSL, and your password is hashed using a strong one-way algorithm. You may enable two-factor authentication to further secure access to your account.
# What is our data retention policy?
We will make a good faith effort to:
- Retain server logs containing the IP address of all requests to this server, in so far as such logs are kept, no more than 90 days.
- Retain the IP addresses associated with registered users no more than 12 months.
You can request and download an archive of your content, including your posts, media attachments, profile picture, and header image.
You may irreversibly delete your account at any time.
# Do we use cookies?
Yes. Cookies are small files that a site or its service provider transfers to your computer's hard drive through your Web browser (if you allow). These cookies enable the site to recognize your browser and, if you have a registered account, associate it with your registered account.
We use cookies to understand and save your preferences for future visits.
# Do we disclose any information to outside parties?
We do not sell, trade, or otherwise transfer to outside parties your personally identifiable information. This does not include trusted third parties who assist us in operating our site, conducting our business, or servicing you, so long as those parties agree to keep this information confidential. We may also release your information when we believe release is appropriate to comply with the law, enforce our site policies, or protect ours or others rights, property, or safety.
Your public content may be downloaded by other servers in the network. Your public and followers-only posts are delivered to the servers where your followers reside, and direct messages are delivered to the servers of the recipients, in so far as those followers or recipients reside on a different server than this.
When you authorize an application to use your account, depending on the scope of permissions you approve, it may access your public profile information, your following list, your followers, your lists, all your posts, and your favourites. Applications can never access your e-mail address or password.
# Site usage by children
If this server is in the EU or the EEA: Our site, products and services are all directed to people who are at least 16 years old. If you are under the age of 16, per the requirements of the GDPR (General Data Protection Regulation) do not use this site.
If this server is in the USA: Our site, products and services are all directed to people who are at least 13 years old. If you are under the age of 13, per the requirements of COPPA (Children's Online Privacy Protection Act) do not use this site.
Law requirements can be different if this server is in another jurisdiction.
___
This document is CC-BY-SA. Originally adapted from the [Discourse privacy policy](https://github.com/discourse/discourse).
TXT
DEFAULT_UPDATED_AT = DateTime.new(2022, 10, 7).freeze
attributes :updated_at, :text
def self.current
custom = Setting.find_by(var: 'site_terms')
if custom&.value.present?
new(text: custom.value, updated_at: custom.updated_at)
else
new(text: DEFAULT_PRIVACY_POLICY, updated_at: DEFAULT_UPDATED_AT)
end
end
end

13
app/models/public_feed.rb
View File

@ -9,6 +9,7 @@ class PublicFeed
# @option [Boolean] :remote
# @option [Boolean] :only_media
# @option [Boolean] :allow_local_only
# @option [String] :locale
def initialize(account, options = {})
@account = account
@options = options
@ -29,6 +30,7 @@ class PublicFeed
scope.merge!(remote_only_scope) if remote_only?
scope.merge!(account_filters_scope) if account?
scope.merge!(media_only_scope) if media_only?
scope.merge!(language_scope)
scope.cache_ids.to_a_paginated_by_id(limit, max_id: max_id, since_id: since_id, min_id: min_id)
end
@ -97,10 +99,19 @@ class PublicFeed
Status.not_local_only
end
def language_scope
if account&.chosen_languages.present?
Status.where(language: account.chosen_languages)
elsif @options[:locale].present?
Status.where(language: @options[:locale])
else
Status.all
end
end
def account_filters_scope
Status.not_excluded_by_account(account).tap do |scope|
scope.merge!(Status.not_domain_blocked_by_account(account)) unless local_only?
scope.merge!(Status.in_chosen_languages(account)) if account.chosen_languages.present?
end
end
end

1
app/models/report.rb
View File

@ -33,6 +33,7 @@ class Report < ApplicationRecord
belongs_to :assigned_account, class_name: 'Account', optional: true
has_many :notes, class_name: 'ReportNote', foreign_key: :report_id, inverse_of: :report, dependent: :destroy
has_many :notifications, as: :activity, dependent: :destroy
scope :unresolved, -> { where(action_taken_at: nil) }
scope :resolved, -> { where.not(action_taken_at: nil) }

27
app/models/site_upload.rb
View File

@ -12,10 +12,35 @@
# meta :json
# created_at :datetime not null
# updated_at :datetime not null
# blurhash :string
#
class SiteUpload < ApplicationRecord
has_attached_file :file
include Attachmentable
STYLES = {
thumbnail: {
'@1x': {
format: 'png',
geometry: '1200x630#',
file_geometry_parser: FastGeometryParser,
blurhash: {
x_comp: 4,
y_comp: 4,
}.freeze,
},
'@2x': {
format: 'png',
geometry: '2400x1260#',
file_geometry_parser: FastGeometryParser,
}.freeze,
}.freeze,
mascot: {}.freeze,
}.freeze
has_attached_file :file, styles: ->(file) { STYLES[file.instance.var.to_sym] }, convert_options: { all: '-coalesce -strip' }, processors: [:lazy_thumbnail, :blurhash_transcoder, :type_corrector]
validates_attachment_content_type :file, content_type: /\Aimage\/.*\z/
validates :file, presence: true

2
app/models/status.rb
View File

@ -77,6 +77,7 @@ class Status < ApplicationRecord
has_one :notification, as: :activity, dependent: :destroy
has_one :status_stat, inverse_of: :status
has_one :poll, inverse_of: :status, dependent: :destroy
has_one :trend, class_name: 'StatusTrend', inverse_of: :status
validates :uri, uniqueness: true, presence: true, unless: :local?
validates :text, presence: true, unless: -> { with_media? || reblog? }
@ -98,7 +99,6 @@ class Status < ApplicationRecord
scope :without_reblogs, -> { where('statuses.reblog_of_id IS NULL') }
scope :with_public_visibility, -> { where(visibility: :public) }
scope :tagged_with, ->(tag_ids) { joins(:statuses_tags).where(statuses_tags: { tag_id: tag_ids }) }
scope :in_chosen_languages, ->(account) { where(language: nil).or where(language: account.chosen_languages) }
scope :excluding_silenced_accounts, -> { left_outer_joins(:account).where(accounts: { silenced_at: nil }) }
scope :including_silenced_accounts, -> { left_outer_joins(:account).where.not(accounts: { silenced_at: nil }) }
scope :not_excluded_by_account, ->(account) { where.not(account_id: account.excluded_from_timeline_account_ids) }

13
app/models/status_edit.rb
View File

@ -31,7 +31,7 @@ class StatusEdit < ApplicationRecord
:preview_remote_url, :text_url, :meta, :blurhash,
:not_processed?, :needs_redownload?, :local?,
:file, :thumbnail, :thumbnail_remote_url,
:shortcode, to: :media_attachment
:shortcode, :video?, :audio?, to: :media_attachment
end
rate_limit by: :account, family: :statuses
@ -41,7 +41,8 @@ class StatusEdit < ApplicationRecord
default_scope { order(id: :asc) }
delegate :local?, to: :status
delegate :local?, :application, :edited?, :edited_at,
:discarded?, :visibility, to: :status
def emojis
return @emojis if defined?(@emojis)
@ -60,4 +61,12 @@ class StatusEdit < ApplicationRecord
end
end
end
def proper
self
end
def reblog?
false
end
end

21
app/models/status_trend.rb Normal file
View File

@ -0,0 +1,21 @@
# frozen_string_literal: true
# == Schema Information
#
# Table name: status_trends
#
# id :bigint(8) not null, primary key
# status_id :bigint(8) not null
# account_id :bigint(8) not null
# score :float default(0.0), not null
# rank :integer default(0), not null
# allowed :boolean default(FALSE), not null
# language :string
#
class StatusTrend < ApplicationRecord
belongs_to :status
belongs_to :account
scope :allowed, -> { joins('INNER JOIN (SELECT account_id, MAX(score) AS max_score FROM status_trends GROUP BY account_id) AS grouped_status_trends ON status_trends.account_id = grouped_status_trends.account_id AND status_trends.score = grouped_status_trends.max_score').where(allowed: true) }
end

2
app/models/tag_feed.rb
View File

@ -12,6 +12,7 @@ class TagFeed < PublicFeed
# @option [Boolean] :local
# @option [Boolean] :remote
# @option [Boolean] :only_media
# @option [String] :locale
def initialize(tag, account, options = {})
@tag = tag
super(account, options)
@ -33,6 +34,7 @@ class TagFeed < PublicFeed
scope.merge!(remote_only_scope) if remote_only?
scope.merge!(account_filters_scope) if account?
scope.merge!(media_only_scope) if media_only?
scope.merge!(language_scope)
scope.cache_ids.to_a_paginated_by_id(limit, max_id: max_id, since_id: since_id, min_id: min_id)
end

6
app/models/trends.rb
View File

@ -26,7 +26,7 @@ module Trends
end
def self.request_review!
return unless enabled?
return if skip_review? || !enabled?
links_requiring_review = links.request_review
tags_requiring_review = tags.request_review
@ -46,6 +46,10 @@ module Trends
Setting.trends
end
def self.skip_review?
Setting.trendable_by_default
end
def self.available_locales
@available_locales ||= I18n.available_locales.map { |locale| locale.to_s.split(/[_-]/).first }.uniq
end

4
app/models/trends/base.rb
View File

@ -98,4 +98,8 @@ class Trends::Base
pipeline.rename(from_key, to_key)
end
end
def skip_review?
Setting.trendable_by_default
end
end

98
app/models/trends/links.rb
View File

@ -11,6 +11,40 @@ class Trends::Links < Trends::Base
decay_threshold: 1,
}
class Query < Trends::Query
def filtered_for!(account)
@account = account
self
end
def filtered_for(account)
clone.filtered_for!(account)
end
def to_arel
scope = PreviewCard.joins(:trend).reorder(score: :desc)
scope = scope.reorder(language_order_clause.desc, score: :desc) if preferred_languages.present?
scope = scope.merge(PreviewCardTrend.allowed) if @allowed
scope = scope.offset(@offset) if @offset.present?
scope = scope.limit(@limit) if @limit.present?
scope
end
private
def language_order_clause
Arel::Nodes::Case.new.when(PreviewCardTrend.arel_table[:language].in(preferred_languages)).then(1).else(0)
end
def preferred_languages
if @account&.chosen_languages.present?
@account.chosen_languages
else
@locale
end
end
end
def register(status, at_time = Time.now.utc)
original_status = status.proper
@ -28,24 +62,33 @@ class Trends::Links < Trends::Base
record_used_id(preview_card.id, at_time)
end
def query
Query.new(key_prefix, klass)
end
def refresh(at_time = Time.now.utc)
preview_cards = PreviewCard.where(id: (recently_used_ids(at_time) + currently_trending_ids(false, -1)).uniq)
preview_cards = PreviewCard.where(id: (recently_used_ids(at_time) + PreviewCardTrend.pluck(:preview_card_id)).uniq)
calculate_scores(preview_cards, at_time)
end
def request_review
preview_cards = PreviewCard.where(id: currently_trending_ids(false, -1))
PreviewCardTrend.pluck('distinct language').flat_map do |language|
score_at_threshold = PreviewCardTrend.where(language: language, allowed: true).order(rank: :desc).where('rank <= ?', options[:review_threshold]).first&.score || 0
preview_card_trends = PreviewCardTrend.where(language: language, allowed: false).joins(:preview_card)
preview_cards.filter_map do |preview_card|
next unless would_be_trending?(preview_card.id) && !preview_card.trendable? && preview_card.requires_review_notification?
preview_card_trends.filter_map do |trend|
preview_card = trend.preview_card
if preview_card.provider.nil?
preview_card.provider = PreviewCardProvider.create(domain: preview_card.domain, requested_review_at: Time.now.utc)
else
preview_card.provider.touch(:requested_review_at)
next unless trend.score > score_at_threshold && !preview_card.trendable? && preview_card.requires_review_notification?
if preview_card.provider.nil?
preview_card.provider = PreviewCardProvider.create(domain: preview_card.domain, requested_review_at: Time.now.utc)
else
preview_card.provider.touch(:requested_review_at)
end
preview_card
end
preview_card
end
end
@ -62,10 +105,7 @@ class Trends::Links < Trends::Base
private
def calculate_scores(preview_cards, at_time)
global_items = []
locale_items = Hash.new { |h, key| h[key] = [] }
preview_cards.each do |preview_card|
items = preview_cards.map do |preview_card|
expected = preview_card.history.get(at_time - 1.day).accounts.to_f
expected = 1.0 if expected.zero?
observed = preview_card.history.get(at_time).accounts.to_f
@ -89,26 +129,24 @@ class Trends::Links < Trends::Base
preview_card.update_columns(max_score: max_score, max_score_at: max_time)
end
decaying_score = max_score * (0.5**((at_time.to_f - max_time.to_f) / options[:max_score_halflife].to_f))
decaying_score = begin
if max_score.zero? || !valid_locale?(preview_card.language)
0
else
max_score * (0.5**((at_time.to_f - max_time.to_f) / options[:max_score_halflife].to_f))
end
end
next unless decaying_score >= options[:decay_threshold]
global_items << { score: decaying_score, item: preview_card }
locale_items[preview_card.language] << { score: decaying_score, item: preview_card } if valid_locale?(preview_card.language)
[decaying_score, preview_card]
end
replace_items('', global_items)
to_insert = items.filter { |(score, _)| score >= options[:decay_threshold] }
to_delete = items.filter { |(score, _)| score < options[:decay_threshold] }
Trends.available_locales.each do |locale|
replace_items(":#{locale}", locale_items[locale])
PreviewCardTrend.transaction do
PreviewCardTrend.upsert_all(to_insert.map { |(score, preview_card)| { preview_card_id: preview_card.id, score: score, language: preview_card.language, allowed: preview_card.trendable? || false } }, unique_by: :preview_card_id) if to_insert.any?
PreviewCardTrend.where(preview_card_id: to_delete.map { |(_, preview_card)| preview_card.id }).delete_all if to_delete.any?
PreviewCardTrend.connection.exec_update('UPDATE preview_card_trends SET rank = t0.calculated_rank FROM (SELECT id, row_number() OVER w AS calculated_rank FROM preview_card_trends WINDOW w AS (PARTITION BY language ORDER BY score DESC)) t0 WHERE preview_card_trends.id = t0.id')
end
end
def filter_for_allowed_items(items)
items.select { |item| item[:item].trendable? }
end
def would_be_trending?(id)
score(id) > score_at_rank(options[:review_threshold] - 1)
end
end

25
app/models/trends/preview_card_filter.rb
View File

@ -13,10 +13,10 @@ class Trends::PreviewCardFilter
end
def results
scope = PreviewCard.unscoped
scope = initial_scope
params.each do |key, value|
next if %w(page locale).include?(key.to_s)
next if %w(page).include?(key.to_s)
scope.merge!(scope_for(key, value.to_s.strip)) if value.present?
end
@ -26,21 +26,30 @@ class Trends::PreviewCardFilter
private
def initial_scope
PreviewCard.select(PreviewCard.arel_table[Arel.star])
.joins(:trend)
.eager_load(:trend)
.reorder(score: :desc)
end
def scope_for(key, value)
case key.to_s
when 'trending'
trending_scope(value)
when 'locale'
PreviewCardTrend.where(language: value)
else
raise "Unknown filter: #{key}"
end
end
def trending_scope(value)
scope = Trends.links.query
scope = scope.in_locale(@params[:locale].to_s) if @params[:locale].present?
scope = scope.allowed if value == 'allowed'
scope.to_arel
case value
when 'allowed'
PreviewCardTrend.allowed
else
PreviewCardTrend.all
end
end
end

4
app/models/trends/status_batch.rb
View File

@ -30,7 +30,7 @@ class Trends::StatusBatch
end
def approve!
statuses.each { |status| authorize(status, :review?) }
statuses.each { |status| authorize([:admin, status], :review?) }
statuses.update_all(trendable: true)
end
@ -45,7 +45,7 @@ class Trends::StatusBatch
end
def reject!
statuses.each { |status| authorize(status, :review?) }
statuses.each { |status| authorize([:admin, status], :review?) }
statuses.update_all(trendable: false)
end

25
app/models/trends/status_filter.rb
View File

@ -13,10 +13,10 @@ class Trends::StatusFilter
end
def results
scope = Status.unscoped.kept
scope = initial_scope
params.each do |key, value|
next if %w(page locale).include?(key.to_s)
next if %w(page).include?(key.to_s)
scope.merge!(scope_for(key, value.to_s.strip)) if value.present?
end
@ -26,21 +26,30 @@ class Trends::StatusFilter
private
def initial_scope
Status.select(Status.arel_table[Arel.star])
.joins(:trend)
.eager_load(:trend)
.reorder(score: :desc)
end
def scope_for(key, value)
case key.to_s
when 'trending'
trending_scope(value)
when 'locale'
StatusTrend.where(language: value)
else
raise "Unknown filter: #{key}"
end
end
def trending_scope(value)
scope = Trends.statuses.query
scope = scope.in_locale(@params[:locale].to_s) if @params[:locale].present?
scope = scope.allowed if value == 'allowed'
scope.to_arel
case value
when 'allowed'
StatusTrend.allowed
else
StatusTrend.all
end
end
end

82
app/models/trends/statuses.rb
View File

@ -20,13 +20,27 @@ class Trends::Statuses < Trends::Base
clone.filtered_for!(account)
end
def to_arel
scope = Status.joins(:trend).reorder(score: :desc)
scope = scope.reorder(language_order_clause.desc, score: :desc) if preferred_languages.present?
scope = scope.merge(StatusTrend.allowed) if @allowed
scope = scope.not_excluded_by_account(@account).not_domain_blocked_by_account(@account) if @account.present?
scope = scope.offset(@offset) if @offset.present?
scope = scope.limit(@limit) if @limit.present?
scope
end
private
def apply_scopes(scope)
if @account.nil?
scope
def language_order_clause
Arel::Nodes::Case.new.when(StatusTrend.arel_table[:language].in(preferred_languages)).then(1).else(0)
end
def preferred_languages
if @account&.chosen_languages.present?
@account.chosen_languages
else
scope.not_excluded_by_account(@account).not_domain_blocked_by_account(@account)
@locale
end
end
end
@ -36,9 +50,6 @@ class Trends::Statuses < Trends::Base
end
def add(status, _account_id, at_time = Time.now.utc)
# We rely on the total reblogs and favourites count, so we
# don't record which account did the what and when here
record_used_id(status.id, at_time)
end
@ -47,18 +58,23 @@ class Trends::Statuses < Trends::Base
end
def refresh(at_time = Time.now.utc)
statuses = Status.where(id: (recently_used_ids(at_time) + currently_trending_ids(false, -1)).uniq).includes(:account, :media_attachments)
statuses = Status.where(id: (recently_used_ids(at_time) + StatusTrend.pluck(:status_id)).uniq).includes(:status_stat, :account)
calculate_scores(statuses, at_time)
end
def request_review
statuses = Status.where(id: currently_trending_ids(false, -1)).includes(:account)
StatusTrend.pluck('distinct language').flat_map do |language|
score_at_threshold = StatusTrend.where(language: language, allowed: true).order(rank: :desc).where('rank <= ?', options[:review_threshold]).first&.score || 0
status_trends = StatusTrend.where(language: language, allowed: false).joins(:status).includes(status: :account)
statuses.filter_map do |status|
next unless would_be_trending?(status.id) && !status.trendable? && status.requires_review_notification?
status_trends.filter_map do |trend|
status = trend.status
status.account.touch(:requested_review_at)
status
if trend.score > score_at_threshold && !status.trendable? && status.requires_review_notification?
status.account.touch(:requested_review_at)
status
end
end
end
end
@ -75,14 +91,11 @@ class Trends::Statuses < Trends::Base
private
def eligible?(status)
status.public_visibility? && status.account.discoverable? && !status.account.silenced? && (status.spoiler_text.blank? || Setting.trending_status_cw) && !status.sensitive? && !status.reply?
status.public_visibility? && status.account.discoverable? && !status.account.silenced? && (status.spoiler_text.blank? || Setting.trending_status_cw) && !status.sensitive? && !status.reply? && valid_locale?(status.language)
end
def calculate_scores(statuses, at_time)
global_items = []
locale_items = Hash.new { |h, key| h[key] = [] }
statuses.each do |status|
items = statuses.map do |status|
expected = 1.0
observed = (status.reblogs_count + status.favourites_count).to_f
@ -94,29 +107,24 @@ class Trends::Statuses < Trends::Base
end
end
decaying_score = score * (0.5**((at_time.to_f - status.created_at.to_f) / options[:score_halflife].to_f))
decaying_score = begin
if score.zero? || !eligible?(status)
0
else
score * (0.5**((at_time.to_f - status.created_at.to_f) / options[:score_halflife].to_f))
end
end
next unless decaying_score >= options[:decay_threshold]
global_items << { score: decaying_score, item: status }
locale_items[status.language] << { account_id: status.account_id, score: decaying_score, item: status } if valid_locale?(status.language)
[decaying_score, status]
end
replace_items('', global_items)
to_insert = items.filter { |(score, _)| score >= options[:decay_threshold] }
to_delete = items.filter { |(score, _)| score < options[:decay_threshold] }
Trends.available_locales.each do |locale|
replace_items(":#{locale}", locale_items[locale])
StatusTrend.transaction do
StatusTrend.upsert_all(to_insert.map { |(score, status)| { status_id: status.id, account_id: status.account_id, score: score, language: status.language, allowed: status.trendable? || false } }, unique_by: :status_id) if to_insert.any?
StatusTrend.where(status_id: to_delete.map { |(_, status)| status.id }).delete_all if to_delete.any?
StatusTrend.connection.exec_update('UPDATE status_trends SET rank = t0.calculated_rank FROM (SELECT id, row_number() OVER w AS calculated_rank FROM status_trends WINDOW w AS (PARTITION BY language ORDER BY score DESC)) t0 WHERE status_trends.id = t0.id')
end
end
def filter_for_allowed_items(items)
# Show only one status per account, pick the one with the highest score
# that's also eligible to trend
items.group_by { |item| item[:account_id] }.values.filter_map { |account_items| account_items.select { |item| item[:item].trendable? && item[:item].account.discoverable? }.max_by { |item| item[:score] } }
end
def would_be_trending?(id)
score(id) > score_at_rank(options[:review_threshold] - 1)
end
end

4
app/models/user.rb
View File

@ -281,6 +281,10 @@ class User < ApplicationRecord
save!
end
def prefers_noindex?
setting_noindex
end
def preferred_posting_language
valid_locale_cascade(settings.default_language, locale, I18n.locale)
end