tarrien/Mastodon
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge branch 'main' into glitch-soc/merge-upstream

Browse Source 
Conflicts:
- `.github/dependabot.yml`:
  Upstream made changes while we have dropped this file.
  Keep the file deleted.
- `.prettierignore`:
  Upstream made changes at the end of the file, where we
  had our extra lines.
  Just moved our extra lines back at the end.
- `app/serializers/initial_state_serializer.rb`:
  Upstream code style changes.
  Applied them.
- `app/services/backup_service.rb`:
  Upstream code style changes.
  Applied them.
This commit is contained in:
Claire
2023-02-19 10:42:55 +01:00
parent 4c68189d2b fef6c59b3a
commit 7452a95998
391 changed files with 6713 additions and 3145 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
lib/cli.rb
View File

@@ -121,7 +121,7 @@ module Mastodon
prompt.warn('Do NOT interrupt this process...')
delete_account = ->(account) do
delete_account = lambda do |account|
payload = ActiveModelSerializers::SerializableResource.new(
account,
serializer: ActivityPub::DeleteActorSerializer,

28
lib/mastodon/accounts_cli.rb
View File

@@ -490,14 +490,12 @@ module Mastodon
scope = Account.where(id: ::Follow.where(account: account).select(:target_account_id))
scope.find_each do |target_account|
begin
UnfollowService.new.call(account, target_account)
rescue => e
progress.log pastel.red("Error processing #{target_account.id}: #{e}")
ensure
progress.increment
processed += 1
end
UnfollowService.new.call(account, target_account)
rescue => e
progress.log pastel.red("Error processing #{target_account.id}: #{e}")
ensure
progress.increment
processed += 1
end
BootstrapTimelineWorker.perform_async(account.id)
@@ -507,14 +505,12 @@ module Mastodon
scope = Account.where(id: ::Follow.where(target_account: account).select(:account_id))
scope.find_each do |target_account|
begin
UnfollowService.new.call(target_account, account)
rescue => e
progress.log pastel.red("Error processing #{target_account.id}: #{e}")
ensure
progress.increment
processed += 1
end
UnfollowService.new.call(target_account, account)
rescue => e
progress.log pastel.red("Error processing #{target_account.id}: #{e}")
ensure
progress.increment
processed += 1
end
end

42
lib/mastodon/cli_helper.rb
View File

@@ -42,30 +42,28 @@ module Mastodon
items.each do |item|
futures << Concurrent::Future.execute(executor: pool) do
begin
if !progress.total.nil? && progress.progress + 1 > progress.total
# The number of items has changed between start and now,
# since there is no good way to predict the final count from
# here, just change the progress bar to an indeterminate one
if !progress.total.nil? && progress.progress + 1 > progress.total
# The number of items has changed between start and now,
# since there is no good way to predict the final count from
# here, just change the progress bar to an indeterminate one
progress.total = nil
end
progress.log("Processing #{item.id}") if options[:verbose]
result = ActiveRecord::Base.connection_pool.with_connection do
yield(item)
ensure
RedisConfiguration.pool.checkin if Thread.current[:redis]
Thread.current[:redis] = nil
end
aggregate.increment(result) if result.is_a?(Integer)
rescue => e
progress.log pastel.red("Error processing #{item.id}: #{e}")
ensure
progress.increment
progress.total = nil
end
progress.log("Processing #{item.id}") if options[:verbose]
result = ActiveRecord::Base.connection_pool.with_connection do
yield(item)
ensure
RedisConfiguration.pool.checkin if Thread.current[:redis]
Thread.current[:redis] = nil
end
aggregate.increment(result) if result.is_a?(Integer)
rescue => e
progress.log pastel.red("Error processing #{item.id}: #{e}")
ensure
progress.increment
end
end

2
lib/mastodon/domains_cli.rb
View File

@@ -139,7 +139,7 @@ module Mastodon
pool = Concurrent::ThreadPoolExecutor.new(min_threads: 0, max_threads: options[:concurrency], idletime: 10, auto_terminate: true, max_queue: 0)
work_unit = ->(domain) do
work_unit = lambda do |domain|
next if stats.key?(domain)
next if options[:exclude_suspended] && domain.match?(blocked_domains)

2
lib/mastodon/emoji_cli.rb
View File

@@ -49,7 +49,7 @@ module Mastodon
next if filename.start_with?('._')
shortcode = [options[:prefix], filename, options[:suffix]].compact.join
custom_emoji = CustomEmoji.local.find_by("LOWER(shortcode) = ?", shortcode.downcase)
custom_emoji = CustomEmoji.local.find_by('LOWER(shortcode) = ?', shortcode.downcase)
if custom_emoji && !options[:overwrite]
skipped += 1

12
lib/mastodon/ip_blocks_cli.rb
View File

@@ -79,13 +79,11 @@ module Mastodon
skipped = 0
addresses.each do |address|
ip_blocks = begin
if options[:force]
IpBlock.where('ip >>= ?', address)
else
IpBlock.where('ip <<= ?', address)
end
end
ip_blocks = if options[:force]
IpBlock.where('ip >>= ?', address)
else
IpBlock.where('ip <<= ?', address)
end
if ip_blocks.empty?
say("#{address} is not yet blocked", :yellow)

84
lib/mastodon/maintenance_cli.rb
View File

@@ -13,8 +13,8 @@ module Mastodon
true
end
MIN_SUPPORTED_VERSION = 2019_10_01_213028 # rubocop:disable Style/NumericLiterals
MAX_SUPPORTED_VERSION = 2022_11_04_133904 # rubocop:disable Style/NumericLiterals
MIN_SUPPORTED_VERSION = 2019_10_01_213028
MAX_SUPPORTED_VERSION = 2022_11_04_133904
# Stubs to enjoy ActiveRecord queries while not depending on a particular
# version of the code/database
@@ -98,11 +98,9 @@ module Mastodon
owned_classes.each do |klass|
klass.where(account_id: other_account.id).find_each do |record|
begin
record.update_attribute(:account_id, id)
rescue ActiveRecord::RecordNotUnique
next
end
record.update_attribute(:account_id, id)
rescue ActiveRecord::RecordNotUnique
next
end
end
@@ -111,11 +109,9 @@ module Mastodon
target_classes.each do |klass|
klass.where(target_account_id: other_account.id).find_each do |record|
begin
record.update_attribute(:target_account_id, id)
rescue ActiveRecord::RecordNotUnique
next
end
record.update_attribute(:target_account_id, id)
rescue ActiveRecord::RecordNotUnique
next
end
end
@@ -209,7 +205,7 @@ module Mastodon
end
@prompt.say 'Restoring index_accounts_on_username_and_domain_lower…'
if ActiveRecord::Migrator.current_version < 20200620164023 # rubocop:disable Style/NumericLiterals
if ActiveRecord::Migrator.current_version < 2020_06_20_164023
ActiveRecord::Base.connection.add_index :accounts, 'lower (username), lower(domain)', name: 'index_accounts_on_username_and_domain_lower', unique: true
else
ActiveRecord::Base.connection.add_index :accounts, "lower (username), COALESCE(lower(domain), '')", name: 'index_accounts_on_username_and_domain_lower', unique: true
@@ -252,7 +248,7 @@ module Mastodon
end
end
if ActiveRecord::Migrator.current_version < 20220118183010 # rubocop:disable Style/NumericLiterals
if ActiveRecord::Migrator.current_version < 2022_01_18_183010
ActiveRecord::Base.connection.select_all("SELECT string_agg(id::text, ',') AS ids FROM users WHERE remember_token IS NOT NULL GROUP BY remember_token HAVING count(*) > 1").each do |row|
users = User.where(id: row['ids'].split(',')).sort_by(&:updated_at).reverse.drop(1)
@prompt.warn "Unsetting remember token for those accounts: #{users.map(&:account).map(&:acct).join(', ')}"
@@ -275,9 +271,9 @@ module Mastodon
@prompt.say 'Restoring users indexes…'
ActiveRecord::Base.connection.add_index :users, ['confirmation_token'], name: 'index_users_on_confirmation_token', unique: true
ActiveRecord::Base.connection.add_index :users, ['email'], name: 'index_users_on_email', unique: true
ActiveRecord::Base.connection.add_index :users, ['remember_token'], name: 'index_users_on_remember_token', unique: true if ActiveRecord::Migrator.current_version < 20220118183010
ActiveRecord::Base.connection.add_index :users, ['remember_token'], name: 'index_users_on_remember_token', unique: true if ActiveRecord::Migrator.current_version < 2022_01_18_183010
if ActiveRecord::Migrator.current_version < 20220310060641 # rubocop:disable Style/NumericLiterals
if ActiveRecord::Migrator.current_version < 2022_03_10_060641
ActiveRecord::Base.connection.add_index :users, ['reset_password_token'], name: 'index_users_on_reset_password_token', unique: true
else
ActiveRecord::Base.connection.add_index :users, ['reset_password_token'], name: 'index_users_on_reset_password_token', unique: true, where: 'reset_password_token IS NOT NULL', opclass: :text_pattern_ops
@@ -340,7 +336,7 @@ module Mastodon
end
@prompt.say 'Restoring conversations indexes…'
if ActiveRecord::Migrator.current_version < 20220307083603 # rubocop:disable Style/NumericLiterals
if ActiveRecord::Migrator.current_version < 2022_03_07_083603
ActiveRecord::Base.connection.add_index :conversations, ['uri'], name: 'index_conversations_on_uri', unique: true
else
ActiveRecord::Base.connection.add_index :conversations, ['uri'], name: 'index_conversations_on_uri', unique: true, where: 'uri IS NOT NULL', opclass: :text_pattern_ops
@@ -457,7 +453,7 @@ module Mastodon
end
@prompt.say 'Restoring media_attachments indexes…'
if ActiveRecord::Migrator.current_version < 20220310060626 # rubocop:disable Style/NumericLiterals
if ActiveRecord::Migrator.current_version < 2022_03_10_060626
ActiveRecord::Base.connection.add_index :media_attachments, ['shortcode'], name: 'index_media_attachments_on_shortcode', unique: true
else
ActiveRecord::Base.connection.add_index :media_attachments, ['shortcode'], name: 'index_media_attachments_on_shortcode', unique: true, where: 'shortcode IS NOT NULL', opclass: :text_pattern_ops
@@ -490,7 +486,7 @@ module Mastodon
end
@prompt.say 'Restoring statuses indexes…'
if ActiveRecord::Migrator.current_version < 20220310060706 # rubocop:disable Style/NumericLiterals
if ActiveRecord::Migrator.current_version < 2022_03_10_060706
ActiveRecord::Base.connection.add_index :statuses, ['uri'], name: 'index_statuses_on_uri', unique: true
else
ActiveRecord::Base.connection.add_index :statuses, ['uri'], name: 'index_statuses_on_uri', unique: true, where: 'uri IS NOT NULL', opclass: :text_pattern_ops
@@ -512,7 +508,7 @@ module Mastodon
end
@prompt.say 'Restoring tags indexes…'
if ActiveRecord::Migrator.current_version < 20210421121431
if ActiveRecord::Migrator.current_version < 2021_04_21_121431
ActiveRecord::Base.connection.add_index :tags, 'lower((name)::text)', name: 'index_tags_on_name_lower', unique: true
else
ActiveRecord::Base.connection.execute 'CREATE UNIQUE INDEX CONCURRENTLY index_tags_on_name_lower_btree ON tags (lower(name) text_pattern_ops)'
@@ -601,11 +597,9 @@ module Mastodon
owned_classes = [ConversationMute, AccountConversation]
owned_classes.each do |klass|
klass.where(conversation_id: duplicate_conv.id).find_each do |record|
begin
record.update_attribute(:account_id, main_conv.id)
rescue ActiveRecord::RecordNotUnique
next
end
record.update_attribute(:account_id, main_conv.id)
rescue ActiveRecord::RecordNotUnique
next
end
end
end
@@ -629,47 +623,37 @@ module Mastodon
owned_classes << Bookmark if ActiveRecord::Base.connection.table_exists?(:bookmarks)
owned_classes.each do |klass|
klass.where(status_id: duplicate_status.id).find_each do |record|
begin
record.update_attribute(:status_id, main_status.id)
rescue ActiveRecord::RecordNotUnique
next
end
end
end
StatusPin.where(account_id: main_status.account_id, status_id: duplicate_status.id).find_each do |record|
begin
record.update_attribute(:status_id, main_status.id)
rescue ActiveRecord::RecordNotUnique
next
end
end
StatusPin.where(account_id: main_status.account_id, status_id: duplicate_status.id).find_each do |record|
record.update_attribute(:status_id, main_status.id)
rescue ActiveRecord::RecordNotUnique
next
end
Status.where(in_reply_to_id: duplicate_status.id).find_each do |record|
begin
record.update_attribute(:in_reply_to_id, main_status.id)
rescue ActiveRecord::RecordNotUnique
next
end
record.update_attribute(:in_reply_to_id, main_status.id)
rescue ActiveRecord::RecordNotUnique
next
end
Status.where(reblog_of_id: duplicate_status.id).find_each do |record|
begin
record.update_attribute(:reblog_of_id, main_status.id)
rescue ActiveRecord::RecordNotUnique
next
end
record.update_attribute(:reblog_of_id, main_status.id)
rescue ActiveRecord::RecordNotUnique
next
end
end
def merge_tags!(main_tag, duplicate_tag)
[FeaturedTag].each do |klass|
klass.where(tag_id: duplicate_tag.id).find_each do |record|
begin
record.update_attribute(:tag_id, main_tag.id)
rescue ActiveRecord::RecordNotUnique
next
end
record.update_attribute(:tag_id, main_tag.id)
rescue ActiveRecord::RecordNotUnique
next
end
end
end

16
lib/mastodon/media_cli.rb
View File

@@ -116,13 +116,11 @@ module Mastodon
loop do
objects = begin
begin
bucket.objects(start_after: last_key, prefix: prefix).limit(1000).map { |x| x }
rescue => e
progress.log(pastel.red("Error fetching list of files: #{e}"))
progress.log("If you want to continue from this point, add --start-after=#{last_key} to your command") if last_key
break
end
bucket.objects(start_after: last_key, prefix: prefix).limit(1000).map { |x| x }
rescue => e
progress.log(pastel.red("Error fetching list of files: #{e}"))
progress.log("If you want to continue from this point, add --start-after=#{last_key} to your command") if last_key
break
end
break if objects.empty?
@@ -277,9 +275,7 @@ module Mastodon
exit(1)
end
if options[:days].present?
scope = scope.where('media_attachments.id > ?', Mastodon::Snowflake.id_at(options[:days].days.ago, with_random: false))
end
scope = scope.where('media_attachments.id > ?', Mastodon::Snowflake.id_at(options[:days].days.ago, with_random: false)) if options[:days].present?
processed, aggregate = parallelize_with_progress(scope) do |media_attachment|
next if media_attachment.remote_url.blank? || (!options[:force] && media_attachment.file_file_name.present?)

4
lib/mastodon/migration_helpers.rb
View File

@@ -289,8 +289,6 @@ module Mastodon
# determines this method to be too complex while there's no way to make it
# less "complex" without introducing extra methods (which actually will
# make things _more_ complex).
#
# rubocop: disable Metrics/AbcSize
def update_column_in_batches(table_name, column, value)
if transaction_open?
raise 'update_column_in_batches can not be run inside a transaction, ' \
@@ -573,7 +571,7 @@ module Mastodon
o.conname as name,
o.confdeltype as on_delete
from pg_constraint o
left join pg_class f on f.oid = o.confrelid
left join pg_class f on f.oid = o.confrelid
left join pg_class c on c.oid = o.conrelid
left join pg_class m on m.oid = o.conrelid
where o.contype = 'f'

4
lib/mastodon/redis_config.rb
View File

@@ -46,6 +46,4 @@ REDIS_SIDEKIQ_PARAMS = {
namespace: sidekiq_namespace,
}.freeze
if Rails.env.test?
ENV['REDIS_NAMESPACE'] = "mastodon_test#{ENV['TEST_ENV_NUMBER']}"
end
ENV['REDIS_NAMESPACE'] = "mastodon_test#{ENV['TEST_ENV_NUMBER']}" if Rails.env.test?

12
lib/mastodon/search_cli.rb
View File

@@ -43,13 +43,11 @@ module Mastodon
exit(1)
end
indices = begin
if options[:only]
options[:only].map { |str| "#{str.camelize}Index".constantize }
else
INDICES
end
end
indices = if options[:only]
options[:only].map { |str| "#{str.camelize}Index".constantize }
else
INDICES
end
pool = Concurrent::FixedThreadPool.new(options[:concurrency], max_queue: options[:concurrency] * 10)
importers = indices.index_with { |index| "Importer::#{index.name}Importer".constantize.new(batch_size: options[:batch_size], executor: pool) }

18
lib/mastodon/upgrade_cli.rb
View File

@@ -50,16 +50,14 @@ module Mastodon
styles << :original unless styles.include?(:original)
styles.each do |style|
success = begin
case Paperclip::Attachment.default_options[:storage]
when :s3
upgrade_storage_s3(progress, attachment, style)
when :fog
upgrade_storage_fog(progress, attachment, style)
when :filesystem
upgrade_storage_filesystem(progress, attachment, style)
end
end
success = case Paperclip::Attachment.default_options[:storage]
when :s3
upgrade_storage_s3(progress, attachment, style)
when :fog
upgrade_storage_fog(progress, attachment, style)
when :filesystem
upgrade_storage_filesystem(progress, attachment, style)
end
upgraded = true if style == :original && success

10
lib/paperclip/attachment_extensions.rb
View File

@@ -8,7 +8,7 @@ module Paperclip
# monkey-patch to avoid unlinking too avoid unlinking source file too early
# see https://github.com/kreeti/kt-paperclip/issues/64
def post_process_style(name, style) #:nodoc:
def post_process_style(name, style) # :nodoc:
raise "Style #{name} has no processors defined." if style.processors.blank?
intermediate_files = []
@@ -16,16 +16,16 @@ module Paperclip
# if we're processing the original, close + unlink the source tempfile
intermediate_files << original if name == :original
@queued_for_write[name] = style.processors.
inject(original) do |file, processor|
@queued_for_write[name] = style.processors
.inject(original) do |file, processor|
file = Paperclip.processor(processor).make(file, style.processor_options, self)
intermediate_files << file unless file == original
file
end
unadapted_file = @queued_for_write[name]
@queued_for_write[name] = Paperclip.io_adapters.
for(@queued_for_write[name], @options[:adapter_options])
@queued_for_write[name] = Paperclip.io_adapters
.for(@queued_for_write[name], @options[:adapter_options])
unadapted_file.close if unadapted_file.respond_to?(:close)
@queued_for_write[name]
rescue Paperclip::Errors::NotIdentifiedByImageMagickError => e

34
lib/paperclip/color_extractor.rb
View File

@@ -79,8 +79,8 @@ module Paperclip
private
def w3c_contrast(color1, color2)
luminance1 = color1.to_xyz.y * 0.01 + 0.05
luminance2 = color2.to_xyz.y * 0.01 + 0.05
luminance1 = (color1.to_xyz.y * 0.01) + 0.05
luminance2 = (color2.to_xyz.y * 0.01) + 0.05
if luminance1 > luminance2
luminance1 / luminance2
@@ -109,11 +109,11 @@ module Paperclip
case max
when r
h = (g - b) / d + (g < b ? 6.0 : 0)
h = ((g - b) / d) + (g < b ? 6.0 : 0)
when g
h = (b - r) / d + 2.0
h = ((b - r) / d) + 2.0
when b
h = (r - g) / d + 4.0
h = ((r - g) / d) + 4.0
end
h /= 6.0
@@ -126,9 +126,9 @@ module Paperclip
t += 1 if t.negative?
t -= 1 if t > 1
return (p + (q - p) * 6 * t) if t < 1 / 6.0
return (p + ((q - p) * 6 * t)) if t < 1 / 6.0
return q if t < 1 / 2.0
return (p + (q - p) * (2 / 3.0 - t) * 6) if t < 2 / 3.0
return (p + ((q - p) * ((2 / 3.0) - t) * 6)) if t < 2 / 3.0
p
end
@@ -147,11 +147,11 @@ module Paperclip
g = l.to_f
b = l.to_f # achromatic
else
q = l < 0.5 ? l * (s + 1) : l + s - l * s
p = 2 * l - q
r = hue_to_rgb(p, q, h + 1 / 3.0)
q = l < 0.5 ? l * (s + 1) : l + s - (l * s)
p = (2 * l) - q
r = hue_to_rgb(p, q, h + (1 / 3.0))
g = hue_to_rgb(p, q, h)
b = hue_to_rgb(p, q, h - 1 / 3.0)
b = hue_to_rgb(p, q, h - (1 / 3.0))
end
[(r * 255).round, (g * 255).round, (b * 255).round]
@@ -161,13 +161,11 @@ module Paperclip
def lighten_or_darken(color, by)
hue, saturation, light = rgb_to_hsl(color.r, color.g, color.b)
light = begin
if light < 50
[100, light + by].min
else
[0, light - by].max
end
end
light = if light < 50
[100, light + by].min
else
[0, light - by].max
end
ColorDiff::Color::RGB.new(*hsl_to_rgb(hue, saturation, light))
end

2
lib/rails/engine_extensions.rb
View File

@@ -3,7 +3,7 @@ module Rails
# Rewrite task loading code to filter digitalocean.rake task
def run_tasks_blocks(app)
Railtie.instance_method(:run_tasks_blocks).bind_call(self, app)
paths["lib/tasks"].existent.reject { |ext| ext.end_with?('digitalocean.rake') }.sort.each { |ext| load(ext) }
paths['lib/tasks'].existent.reject { |ext| ext.end_with?('digitalocean.rake') }.sort.each { |ext| load(ext) }
end
end
end

12
lib/sanitize_ext/sanitize_config.rb
View File

@@ -60,13 +60,11 @@ class Sanitize
current_node = env[:node]
scheme = begin
if current_node['href'] =~ Sanitize::REGEX_PROTOCOL
Regexp.last_match(1).downcase
else
:relative
end
end
scheme = if current_node['href'] =~ Sanitize::REGEX_PROTOCOL
Regexp.last_match(1).downcase
else
:relative
end
current_node.replace(Nokogiri::XML::Text.new(current_node.text, current_node.document)) unless LINK_PROTOCOLS.include?(scheme)
end

14
lib/tasks/db.rake
View File

@@ -4,16 +4,14 @@ namespace :db do
namespace :migrate do
desc 'Setup the db or migrate depending on state of db'
task setup: :environment do
begin
if ActiveRecord::Migrator.current_version.zero?
Rake::Task['db:migrate'].invoke
Rake::Task['db:seed'].invoke
end
rescue ActiveRecord::NoDatabaseError
Rake::Task['db:setup'].invoke
else
if ActiveRecord::Migrator.current_version.zero?
Rake::Task['db:migrate'].invoke
Rake::Task['db:seed'].invoke
end
rescue ActiveRecord::NoDatabaseError
Rake::Task['db:setup'].invoke
else
Rake::Task['db:migrate'].invoke
end
end

6
lib/tasks/mastodon.rake
View File

@@ -264,7 +264,7 @@ namespace :mastodon do
env['S3_ENDPOINT'] = prompt.ask('Storj DCS endpoint URL:') do |q|
q.required true
q.default "https://gateway.storjshare.io"
q.default 'https://gateway.storjshare.io'
q.modify :strip
end
@@ -286,13 +286,13 @@ namespace :mastodon do
q.required true
q.modify :strip
end
linksharing_access_key = prompt.ask('Storj Linksharing access key (uplink share --register --public --readonly=true --disallow-lists --not-after=none sj://bucket):') do |q|
q.required true
q.modify :strip
end
env['S3_ALIAS_HOST'] = "link.storjshare.io/raw/#{linksharing_access_key}/#{env['S3_BUCKET']}"
when 'Google Cloud Storage'
env['S3_ENABLED'] = 'true'
env['S3_PROTOCOL'] = 'https'

2
lib/tasks/statistics.rake
View File

@@ -13,7 +13,7 @@ namespace :mastodon do
%w(Validators app/validators),
%w(Workers app/workers),
].each do |name, dir|
::STATS_DIRECTORIES << [name, Rails.root.join(dir)]
STATS_DIRECTORIES << [name, Rails.root.join(dir)]
end
end
end