tarrien/Mastodon
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge branch 'main' into glitch-soc/merge-upstream

Browse Source 
Conflicts:
- `.github/dependabot.yml`:
  Upstream made changes while we have dropped this file.
  Keep the file deleted.
- `.prettierignore`:
  Upstream made changes at the end of the file, where we
  had our extra lines.
  Just moved our extra lines back at the end.
- `app/serializers/initial_state_serializer.rb`:
  Upstream code style changes.
  Applied them.
- `app/services/backup_service.rb`:
  Upstream code style changes.
  Applied them.
This commit is contained in:
Claire
2023-02-19 10:42:55 +01:00
parent 4c68189d2b fef6c59b3a
commit 7452a95998
391 changed files with 6713 additions and 3145 deletions
Download Patch File Download Diff File Expand all files Collapse all files

28
lib/mastodon/accounts_cli.rb
View File

@ -490,14 +490,12 @@ module Mastodon
scope = Account.where(id: ::Follow.where(account: account).select(:target_account_id))
scope.find_each do |target_account|
begin
UnfollowService.new.call(account, target_account)
rescue => e
progress.log pastel.red("Error processing #{target_account.id}: #{e}")
ensure
progress.increment
processed += 1
end
UnfollowService.new.call(account, target_account)
rescue => e
progress.log pastel.red("Error processing #{target_account.id}: #{e}")
ensure
progress.increment
processed += 1
end
BootstrapTimelineWorker.perform_async(account.id)
@ -507,14 +505,12 @@ module Mastodon
scope = Account.where(id: ::Follow.where(target_account: account).select(:account_id))
scope.find_each do |target_account|
begin
UnfollowService.new.call(target_account, account)
rescue => e
progress.log pastel.red("Error processing #{target_account.id}: #{e}")
ensure
progress.increment
processed += 1
end
UnfollowService.new.call(target_account, account)
rescue => e
progress.log pastel.red("Error processing #{target_account.id}: #{e}")
ensure
progress.increment
processed += 1
end
end

42
lib/mastodon/cli_helper.rb
View File

@ -42,30 +42,28 @@ module Mastodon
items.each do |item|
futures << Concurrent::Future.execute(executor: pool) do
begin
if !progress.total.nil? && progress.progress + 1 > progress.total
# The number of items has changed between start and now,
# since there is no good way to predict the final count from
# here, just change the progress bar to an indeterminate one
if !progress.total.nil? && progress.progress + 1 > progress.total
# The number of items has changed between start and now,
# since there is no good way to predict the final count from
# here, just change the progress bar to an indeterminate one
progress.total = nil
end
progress.log("Processing #{item.id}") if options[:verbose]
result = ActiveRecord::Base.connection_pool.with_connection do
yield(item)
ensure
RedisConfiguration.pool.checkin if Thread.current[:redis]
Thread.current[:redis] = nil
end
aggregate.increment(result) if result.is_a?(Integer)
rescue => e
progress.log pastel.red("Error processing #{item.id}: #{e}")
ensure
progress.increment
progress.total = nil
end
progress.log("Processing #{item.id}") if options[:verbose]
result = ActiveRecord::Base.connection_pool.with_connection do
yield(item)
ensure
RedisConfiguration.pool.checkin if Thread.current[:redis]
Thread.current[:redis] = nil
end
aggregate.increment(result) if result.is_a?(Integer)
rescue => e
progress.log pastel.red("Error processing #{item.id}: #{e}")
ensure
progress.increment
end
end

2
lib/mastodon/domains_cli.rb
View File

@ -139,7 +139,7 @@ module Mastodon
pool = Concurrent::ThreadPoolExecutor.new(min_threads: 0, max_threads: options[:concurrency], idletime: 10, auto_terminate: true, max_queue: 0)
work_unit = ->(domain) do
work_unit = lambda do |domain|
next if stats.key?(domain)
next if options[:exclude_suspended] && domain.match?(blocked_domains)

2
lib/mastodon/emoji_cli.rb
View File

@ -49,7 +49,7 @@ module Mastodon
next if filename.start_with?('._')
shortcode = [options[:prefix], filename, options[:suffix]].compact.join
custom_emoji = CustomEmoji.local.find_by("LOWER(shortcode) = ?", shortcode.downcase)
custom_emoji = CustomEmoji.local.find_by('LOWER(shortcode) = ?', shortcode.downcase)
if custom_emoji && !options[:overwrite]
skipped += 1

12
lib/mastodon/ip_blocks_cli.rb
View File

@ -79,13 +79,11 @@ module Mastodon
skipped = 0
addresses.each do |address|
ip_blocks = begin
if options[:force]
IpBlock.where('ip >>= ?', address)
else
IpBlock.where('ip <<= ?', address)
end
end
ip_blocks = if options[:force]
IpBlock.where('ip >>= ?', address)
else
IpBlock.where('ip <<= ?', address)
end
if ip_blocks.empty?
say("#{address} is not yet blocked", :yellow)

84
lib/mastodon/maintenance_cli.rb
View File

@ -13,8 +13,8 @@ module Mastodon
true
end
MIN_SUPPORTED_VERSION = 2019_10_01_213028 # rubocop:disable Style/NumericLiterals
MAX_SUPPORTED_VERSION = 2022_11_04_133904 # rubocop:disable Style/NumericLiterals
MIN_SUPPORTED_VERSION = 2019_10_01_213028
MAX_SUPPORTED_VERSION = 2022_11_04_133904
# Stubs to enjoy ActiveRecord queries while not depending on a particular
# version of the code/database
@ -98,11 +98,9 @@ module Mastodon
owned_classes.each do |klass|
klass.where(account_id: other_account.id).find_each do |record|
begin
record.update_attribute(:account_id, id)
rescue ActiveRecord::RecordNotUnique
next
end
record.update_attribute(:account_id, id)
rescue ActiveRecord::RecordNotUnique
next
end
end
@ -111,11 +109,9 @@ module Mastodon
target_classes.each do |klass|
klass.where(target_account_id: other_account.id).find_each do |record|
begin
record.update_attribute(:target_account_id, id)
rescue ActiveRecord::RecordNotUnique
next
end
record.update_attribute(:target_account_id, id)
rescue ActiveRecord::RecordNotUnique
next
end
end
@ -209,7 +205,7 @@ module Mastodon
end
@prompt.say 'Restoring index_accounts_on_username_and_domain_lower…'
if ActiveRecord::Migrator.current_version < 20200620164023 # rubocop:disable Style/NumericLiterals
if ActiveRecord::Migrator.current_version < 2020_06_20_164023
ActiveRecord::Base.connection.add_index :accounts, 'lower (username), lower(domain)', name: 'index_accounts_on_username_and_domain_lower', unique: true
else
ActiveRecord::Base.connection.add_index :accounts, "lower (username), COALESCE(lower(domain), '')", name: 'index_accounts_on_username_and_domain_lower', unique: true
@ -252,7 +248,7 @@ module Mastodon
end
end
if ActiveRecord::Migrator.current_version < 20220118183010 # rubocop:disable Style/NumericLiterals
if ActiveRecord::Migrator.current_version < 2022_01_18_183010
ActiveRecord::Base.connection.select_all("SELECT string_agg(id::text, ',') AS ids FROM users WHERE remember_token IS NOT NULL GROUP BY remember_token HAVING count(*) > 1").each do |row|
users = User.where(id: row['ids'].split(',')).sort_by(&:updated_at).reverse.drop(1)
@prompt.warn "Unsetting remember token for those accounts: #{users.map(&:account).map(&:acct).join(', ')}"
@ -275,9 +271,9 @@ module Mastodon
@prompt.say 'Restoring users indexes…'
ActiveRecord::Base.connection.add_index :users, ['confirmation_token'], name: 'index_users_on_confirmation_token', unique: true
ActiveRecord::Base.connection.add_index :users, ['email'], name: 'index_users_on_email', unique: true
ActiveRecord::Base.connection.add_index :users, ['remember_token'], name: 'index_users_on_remember_token', unique: true if ActiveRecord::Migrator.current_version < 20220118183010
ActiveRecord::Base.connection.add_index :users, ['remember_token'], name: 'index_users_on_remember_token', unique: true if ActiveRecord::Migrator.current_version < 2022_01_18_183010
if ActiveRecord::Migrator.current_version < 20220310060641 # rubocop:disable Style/NumericLiterals
if ActiveRecord::Migrator.current_version < 2022_03_10_060641
ActiveRecord::Base.connection.add_index :users, ['reset_password_token'], name: 'index_users_on_reset_password_token', unique: true
else
ActiveRecord::Base.connection.add_index :users, ['reset_password_token'], name: 'index_users_on_reset_password_token', unique: true, where: 'reset_password_token IS NOT NULL', opclass: :text_pattern_ops
@ -340,7 +336,7 @@ module Mastodon
end
@prompt.say 'Restoring conversations indexes…'
if ActiveRecord::Migrator.current_version < 20220307083603 # rubocop:disable Style/NumericLiterals
if ActiveRecord::Migrator.current_version < 2022_03_07_083603
ActiveRecord::Base.connection.add_index :conversations, ['uri'], name: 'index_conversations_on_uri', unique: true
else
ActiveRecord::Base.connection.add_index :conversations, ['uri'], name: 'index_conversations_on_uri', unique: true, where: 'uri IS NOT NULL', opclass: :text_pattern_ops
@ -457,7 +453,7 @@ module Mastodon
end
@prompt.say 'Restoring media_attachments indexes…'
if ActiveRecord::Migrator.current_version < 20220310060626 # rubocop:disable Style/NumericLiterals
if ActiveRecord::Migrator.current_version < 2022_03_10_060626
ActiveRecord::Base.connection.add_index :media_attachments, ['shortcode'], name: 'index_media_attachments_on_shortcode', unique: true
else
ActiveRecord::Base.connection.add_index :media_attachments, ['shortcode'], name: 'index_media_attachments_on_shortcode', unique: true, where: 'shortcode IS NOT NULL', opclass: :text_pattern_ops
@ -490,7 +486,7 @@ module Mastodon
end
@prompt.say 'Restoring statuses indexes…'
if ActiveRecord::Migrator.current_version < 20220310060706 # rubocop:disable Style/NumericLiterals
if ActiveRecord::Migrator.current_version < 2022_03_10_060706
ActiveRecord::Base.connection.add_index :statuses, ['uri'], name: 'index_statuses_on_uri', unique: true
else
ActiveRecord::Base.connection.add_index :statuses, ['uri'], name: 'index_statuses_on_uri', unique: true, where: 'uri IS NOT NULL', opclass: :text_pattern_ops
@ -512,7 +508,7 @@ module Mastodon
end
@prompt.say 'Restoring tags indexes…'
if ActiveRecord::Migrator.current_version < 20210421121431
if ActiveRecord::Migrator.current_version < 2021_04_21_121431
ActiveRecord::Base.connection.add_index :tags, 'lower((name)::text)', name: 'index_tags_on_name_lower', unique: true
else
ActiveRecord::Base.connection.execute 'CREATE UNIQUE INDEX CONCURRENTLY index_tags_on_name_lower_btree ON tags (lower(name) text_pattern_ops)'
@ -601,11 +597,9 @@ module Mastodon
owned_classes = [ConversationMute, AccountConversation]
owned_classes.each do |klass|
klass.where(conversation_id: duplicate_conv.id).find_each do |record|
begin
record.update_attribute(:account_id, main_conv.id)
rescue ActiveRecord::RecordNotUnique
next
end
record.update_attribute(:account_id, main_conv.id)
rescue ActiveRecord::RecordNotUnique
next
end
end
end
@ -629,47 +623,37 @@ module Mastodon
owned_classes << Bookmark if ActiveRecord::Base.connection.table_exists?(:bookmarks)
owned_classes.each do |klass|
klass.where(status_id: duplicate_status.id).find_each do |record|
begin
record.update_attribute(:status_id, main_status.id)
rescue ActiveRecord::RecordNotUnique
next
end
end
end
StatusPin.where(account_id: main_status.account_id, status_id: duplicate_status.id).find_each do |record|
begin
record.update_attribute(:status_id, main_status.id)
rescue ActiveRecord::RecordNotUnique
next
end
end
StatusPin.where(account_id: main_status.account_id, status_id: duplicate_status.id).find_each do |record|
record.update_attribute(:status_id, main_status.id)
rescue ActiveRecord::RecordNotUnique
next
end
Status.where(in_reply_to_id: duplicate_status.id).find_each do |record|
begin
record.update_attribute(:in_reply_to_id, main_status.id)
rescue ActiveRecord::RecordNotUnique
next
end
record.update_attribute(:in_reply_to_id, main_status.id)
rescue ActiveRecord::RecordNotUnique
next
end
Status.where(reblog_of_id: duplicate_status.id).find_each do |record|
begin
record.update_attribute(:reblog_of_id, main_status.id)
rescue ActiveRecord::RecordNotUnique
next
end
record.update_attribute(:reblog_of_id, main_status.id)
rescue ActiveRecord::RecordNotUnique
next
end
end
def merge_tags!(main_tag, duplicate_tag)
[FeaturedTag].each do |klass|
klass.where(tag_id: duplicate_tag.id).find_each do |record|
begin
record.update_attribute(:tag_id, main_tag.id)
rescue ActiveRecord::RecordNotUnique
next
end
record.update_attribute(:tag_id, main_tag.id)
rescue ActiveRecord::RecordNotUnique
next
end
end
end

16
lib/mastodon/media_cli.rb
View File

@ -116,13 +116,11 @@ module Mastodon
loop do
objects = begin
begin
bucket.objects(start_after: last_key, prefix: prefix).limit(1000).map { |x| x }
rescue => e
progress.log(pastel.red("Error fetching list of files: #{e}"))
progress.log("If you want to continue from this point, add --start-after=#{last_key} to your command") if last_key
break
end
bucket.objects(start_after: last_key, prefix: prefix).limit(1000).map { |x| x }
rescue => e
progress.log(pastel.red("Error fetching list of files: #{e}"))
progress.log("If you want to continue from this point, add --start-after=#{last_key} to your command") if last_key
break
end
break if objects.empty?
@ -277,9 +275,7 @@ module Mastodon
exit(1)
end
if options[:days].present?
scope = scope.where('media_attachments.id > ?', Mastodon::Snowflake.id_at(options[:days].days.ago, with_random: false))
end
scope = scope.where('media_attachments.id > ?', Mastodon::Snowflake.id_at(options[:days].days.ago, with_random: false)) if options[:days].present?
processed, aggregate = parallelize_with_progress(scope) do |media_attachment|
next if media_attachment.remote_url.blank? || (!options[:force] && media_attachment.file_file_name.present?)

4
lib/mastodon/migration_helpers.rb
View File

@ -289,8 +289,6 @@ module Mastodon
# determines this method to be too complex while there's no way to make it
# less "complex" without introducing extra methods (which actually will
# make things _more_ complex).
#
# rubocop: disable Metrics/AbcSize
def update_column_in_batches(table_name, column, value)
if transaction_open?
raise 'update_column_in_batches can not be run inside a transaction, ' \
@ -573,7 +571,7 @@ module Mastodon
o.conname as name,
o.confdeltype as on_delete
from pg_constraint o
left join pg_class f on f.oid = o.confrelid
left join pg_class f on f.oid = o.confrelid
left join pg_class c on c.oid = o.conrelid
left join pg_class m on m.oid = o.conrelid
where o.contype = 'f'

4
lib/mastodon/redis_config.rb
View File

@ -46,6 +46,4 @@ REDIS_SIDEKIQ_PARAMS = {
namespace: sidekiq_namespace,
}.freeze
if Rails.env.test?
ENV['REDIS_NAMESPACE'] = "mastodon_test#{ENV['TEST_ENV_NUMBER']}"
end
ENV['REDIS_NAMESPACE'] = "mastodon_test#{ENV['TEST_ENV_NUMBER']}" if Rails.env.test?

12
lib/mastodon/search_cli.rb
View File

@ -43,13 +43,11 @@ module Mastodon
exit(1)
end
indices = begin
if options[:only]
options[:only].map { |str| "#{str.camelize}Index".constantize }
else
INDICES
end
end
indices = if options[:only]
options[:only].map { |str| "#{str.camelize}Index".constantize }
else
INDICES
end
pool = Concurrent::FixedThreadPool.new(options[:concurrency], max_queue: options[:concurrency] * 10)
importers = indices.index_with { |index| "Importer::#{index.name}Importer".constantize.new(batch_size: options[:batch_size], executor: pool) }

18
lib/mastodon/upgrade_cli.rb
View File

@ -50,16 +50,14 @@ module Mastodon
styles << :original unless styles.include?(:original)
styles.each do |style|
success = begin
case Paperclip::Attachment.default_options[:storage]
when :s3
upgrade_storage_s3(progress, attachment, style)
when :fog
upgrade_storage_fog(progress, attachment, style)
when :filesystem
upgrade_storage_filesystem(progress, attachment, style)
end
end
success = case Paperclip::Attachment.default_options[:storage]
when :s3
upgrade_storage_s3(progress, attachment, style)
when :fog
upgrade_storage_fog(progress, attachment, style)
when :filesystem
upgrade_storage_filesystem(progress, attachment, style)
end
upgraded = true if style == :original && success