tarrien/Mastodon
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge commit '00c222377db0e305ac3f4a15bf1c18eb89c1f45f' into glitch-soc/merge-upstream

Browse Source 
Conflicts:
- `.rubocop_todo.yml`:
  Took upstream's changes.
This commit is contained in:
Claire
2023-06-05 13:25:22 +02:00
parent ee771f5bad 00c222377d
commit c22fc2fa80
33 changed files with 1640 additions and 820 deletions
Download Patch File Download Diff File Expand all files Collapse all files

52
lib/mastodon/cli/accounts.rb
View File

@@ -113,12 +113,7 @@ module Mastodon::CLI
say('OK', :green)
say("New password: #{password}")
else
user.errors.each do |error|
say('Failure/Error: ', :red)
say(error.attribute)
say(" #{error.type}", :red)
end
report_errors(user.errors)
exit(1)
end
end
@@ -189,12 +184,7 @@ module Mastodon::CLI
say('OK', :green)
say("New password: #{password}") if options[:reset_password]
else
user.errors.each do |error|
say('Failure/Error: ', :red)
say(error.attribute)
say(" #{error.type}", :red)
end
report_errors(user.errors)
exit(1)
end
end
@@ -217,7 +207,6 @@ module Mastodon::CLI
exit(1)
end
dry_run = options[:dry_run] ? ' (DRY RUN)' : ''
account = nil
if username.present?
@@ -234,9 +223,9 @@ module Mastodon::CLI
end
end
say("Deleting user with #{account.statuses_count} statuses, this might take a while...#{dry_run}")
DeleteAccountService.new.call(account, reserve_email: false) unless options[:dry_run]
say("OK#{dry_run}", :green)
say("Deleting user with #{account.statuses_count} statuses, this might take a while...#{dry_run_mode_suffix}")
DeleteAccountService.new.call(account, reserve_email: false) unless dry_run?
say("OK#{dry_run_mode_suffix}", :green)
end
option :force, type: :boolean, aliases: [:f], description: 'Override public key check'
@@ -291,7 +280,7 @@ module Mastodon::CLI
Account.remote.select(:uri, 'count(*)').group(:uri).having('count(*) > 1').pluck(:uri).each do |uri|
say("Duplicates found for #{uri}")
begin
ActivityPub::FetchRemoteAccountService.new.call(uri) unless options[:dry_run]
ActivityPub::FetchRemoteAccountService.new.call(uri) unless dry_run?
rescue => e
say("Error processing #{uri}: #{e}", :red)
end
@@ -332,7 +321,6 @@ module Mastodon::CLI
LONG_DESC
def cull(*domains)
skip_threshold = 7.days.ago
dry_run = options[:dry_run] ? ' (DRY RUN)' : ''
skip_domains = Concurrent::Set.new
query = Account.remote.where(protocol: :activitypub)
@@ -350,7 +338,7 @@ module Mastodon::CLI
end
if [404, 410].include?(code)
DeleteAccountService.new.call(account, reserve_username: false) unless options[:dry_run]
DeleteAccountService.new.call(account, reserve_username: false) unless dry_run?
1
else
# Touch account even during dry run to avoid getting the account into the window again
@@ -358,7 +346,7 @@ module Mastodon::CLI
end
end
say("Visited #{processed} accounts, removed #{culled}#{dry_run}", :green)
say("Visited #{processed} accounts, removed #{culled}#{dry_run_mode_suffix}", :green)
unless skip_domains.empty?
say('The following domains were not available during the check:', :yellow)
@@ -381,21 +369,19 @@ module Mastodon::CLI
specified with space-separated USERNAMES.
LONG_DESC
def refresh(*usernames)
dry_run = options[:dry_run] ? ' (DRY RUN)' : ''
if options[:domain] || options[:all]
scope = Account.remote
scope = scope.where(domain: options[:domain]) if options[:domain]
processed, = parallelize_with_progress(scope) do |account|
next if options[:dry_run]
next if dry_run?
account.reset_avatar!
account.reset_header!
account.save
end
say("Refreshed #{processed} accounts#{dry_run}", :green, true)
say("Refreshed #{processed} accounts#{dry_run_mode_suffix}", :green, true)
elsif !usernames.empty?
usernames.each do |user|
user, domain = user.split('@')
@@ -406,7 +392,7 @@ module Mastodon::CLI
exit(1)
end
next if options[:dry_run]
next if dry_run?
begin
account.reset_avatar!
@@ -417,7 +403,7 @@ module Mastodon::CLI
end
end
say("OK#{dry_run}", :green)
say("OK#{dry_run_mode_suffix}", :green)
else
say('No account(s) given', :red)
exit(1)
@@ -568,8 +554,6 @@ module Mastodon::CLI
- not muted/blocked by us
LONG_DESC
def prune
dry_run = options[:dry_run] ? ' (dry run)' : ''
query = Account.remote.where.not(actor_type: %i(Application Service))
query = query.where('NOT EXISTS (SELECT 1 FROM mentions WHERE account_id = accounts.id)')
query = query.where('NOT EXISTS (SELECT 1 FROM favourites WHERE account_id = accounts.id)')
@@ -585,11 +569,11 @@ module Mastodon::CLI
next if account.suspended?
next if account.silenced?
account.destroy unless options[:dry_run]
account.destroy unless dry_run?
1
end
say("OK, pruned #{deleted} accounts#{dry_run}", :green)
say("OK, pruned #{deleted} accounts#{dry_run_mode_suffix}", :green)
end
option :force, type: :boolean
@@ -667,6 +651,14 @@ module Mastodon::CLI
private
def report_errors(errors)
errors.each do |error|
say('Failure/Error: ', :red)
say(error.attribute)
say(" #{error.type}", :red)
end
end
def rotate_keys_for_account(account, delay = 0)
if account.nil?
say('No such account', :red)

15
lib/mastodon/cli/domains.rb
View File

@@ -34,7 +34,6 @@ module Mastodon::CLI
When the --purge-domain-blocks option is given, also purge matching domain blocks.
LONG_DESC
def purge(*domains)
dry_run = options[:dry_run] ? ' (DRY RUN)' : ''
domains = domains.map { |domain| TagManager.instance.normalize_domain(domain) }
account_scope = Account.none
domain_block_scope = DomainBlock.none
@@ -79,23 +78,23 @@ module Mastodon::CLI
# Actually perform the deletions
processed, = parallelize_with_progress(account_scope) do |account|
DeleteAccountService.new.call(account, reserve_username: false, skip_side_effects: true) unless options[:dry_run]
DeleteAccountService.new.call(account, reserve_username: false, skip_side_effects: true) unless dry_run?
end
say("Removed #{processed} accounts#{dry_run}", :green)
say("Removed #{processed} accounts#{dry_run_mode_suffix}", :green)
if options[:purge_domain_blocks]
domain_block_count = domain_block_scope.count
domain_block_scope.in_batches.destroy_all unless options[:dry_run]
say("Removed #{domain_block_count} domain blocks#{dry_run}", :green)
domain_block_scope.in_batches.destroy_all unless dry_run?
say("Removed #{domain_block_count} domain blocks#{dry_run_mode_suffix}", :green)
end
custom_emojis_count = emoji_scope.count
emoji_scope.in_batches.destroy_all unless options[:dry_run]
emoji_scope.in_batches.destroy_all unless dry_run?
Instance.refresh unless options[:dry_run]
Instance.refresh unless dry_run?
say("Removed #{custom_emojis_count} custom emojis#{dry_run}", :green)
say("Removed #{custom_emojis_count} custom emojis#{dry_run_mode_suffix}", :green)
end
option :concurrency, type: :numeric, default: 50, aliases: [:c]

4
lib/mastodon/cli/email_domain_blocks.rb
View File

@@ -39,7 +39,7 @@ module Mastodon::CLI
processed = 0
domains.each do |domain|
if EmailDomainBlock.where(domain: domain).exists?
if EmailDomainBlock.exists?(domain: domain)
say("#{domain} is already blocked.", :yellow)
skipped += 1
next
@@ -60,7 +60,7 @@ module Mastodon::CLI
(email_domain_block.other_domains || []).uniq.each do |hostname|
another_email_domain_block = EmailDomainBlock.new(domain: hostname, parent: email_domain_block)
if EmailDomainBlock.where(domain: hostname).exists?
if EmailDomainBlock.exists?(domain: hostname)
say("#{hostname} is already blocked.", :yellow)
skipped += 1
next

10
lib/mastodon/cli/feeds.rb
View File

@@ -18,14 +18,12 @@ module Mastodon::CLI
Otherwise, a single user specified by USERNAME.
LONG_DESC
def build(username = nil)
dry_run = options[:dry_run] ? '(DRY RUN)' : ''
if options[:all] || username.nil?
processed, = parallelize_with_progress(Account.joins(:user).merge(User.active)) do |account|
PrecomputeFeedService.new.call(account) unless options[:dry_run]
PrecomputeFeedService.new.call(account) unless dry_run?
end
say("Regenerated feeds for #{processed} accounts #{dry_run}", :green, true)
say("Regenerated feeds for #{processed} accounts #{dry_run_mode_suffix}", :green, true)
elsif username.present?
account = Account.find_local(username)
@@ -34,9 +32,9 @@ module Mastodon::CLI
exit(1)
end
PrecomputeFeedService.new.call(account) unless options[:dry_run]
PrecomputeFeedService.new.call(account) unless dry_run?
say("OK #{dry_run}", :green, true)
say("OK #{dry_run_mode_suffix}", :green, true)
else
say('No account(s) given', :red)
exit(1)

4
lib/mastodon/cli/helper.rb
View File

@@ -15,6 +15,10 @@ module Mastodon::CLI
options[:dry_run]
end
def dry_run_mode_suffix
dry_run? ? ' (DRY RUN)' : ''
end
def create_progress_bar(total = nil)
ProgressBar.create(total: total, format: '%c/%u |%b%i| %e')
end

11
lib/mastodon/cli/main.rb
View File

@@ -94,7 +94,7 @@ module Mastodon::CLI
exit(1) unless prompt.ask('Type in the domain of the server to confirm:', required: true) == Rails.configuration.x.local_domain
unless options[:dry_run]
unless dry_run?
prompt.warn('This operation WILL NOT be reversible. It can also take a long time.')
prompt.warn('While the data won\'t be erased locally, the server will be in a BROKEN STATE afterwards.')
prompt.warn('A running Sidekiq process is required. Do not shut it down until queues clear.')
@@ -104,12 +104,11 @@ module Mastodon::CLI
inboxes = Account.inboxes
processed = 0
dry_run = options[:dry_run] ? ' (DRY RUN)' : ''
Setting.registrations_mode = 'none' unless options[:dry_run]
Setting.registrations_mode = 'none' unless dry_run?
if inboxes.empty?
Account.local.without_suspended.in_batches.update_all(suspended_at: Time.now.utc, suspension_origin: :local) unless options[:dry_run]
Account.local.without_suspended.in_batches.update_all(suspended_at: Time.now.utc, suspension_origin: :local) unless dry_run?
prompt.ok('It seems like your server has not federated with anything')
prompt.ok('You can shut it down and delete it any time')
return
@@ -126,7 +125,7 @@ module Mastodon::CLI
json = Oj.dump(ActivityPub::LinkedDataSignature.new(payload).sign!(account))
unless options[:dry_run]
unless dry_run?
ActivityPub::DeliveryWorker.push_bulk(inboxes, limit: 1_000) do |inbox_url|
[json, account.id, inbox_url]
end
@@ -140,7 +139,7 @@ module Mastodon::CLI
Account.local.without_suspended.find_each { |account| delete_account.call(account) }
Account.local.suspended.joins(:deletion_request).find_each { |account| delete_account.call(account) }
prompt.ok("Queued #{inboxes.size * processed} items into Sidekiq for #{processed} accounts#{dry_run}")
prompt.ok("Queued #{inboxes.size * processed} items into Sidekiq for #{processed} accounts#{dry_run_mode_suffix}")
prompt.ok('Wait until Sidekiq processes all items, then you can shut everything down and delete the data')
rescue TTY::Reader::InputInterrupt
exit(1)

131
lib/mastodon/cli/maintenance.rb
View File

@@ -1,6 +1,5 @@
# frozen_string_literal: true
require 'tty-prompt'
require_relative 'base'
module Mastodon::CLI
@@ -134,25 +133,23 @@ module Mastodon::CLI
Mastodon has to be stopped to run this task, which will take a long time and may be destructive.
LONG_DESC
def fix_duplicates
@prompt = TTY::Prompt.new
if ActiveRecord::Migrator.current_version < MIN_SUPPORTED_VERSION
@prompt.error 'Your version of the database schema is too old and is not supported by this script.'
@prompt.error 'Please update to at least Mastodon 3.0.0 before running this script.'
say 'Your version of the database schema is too old and is not supported by this script.', :red
say 'Please update to at least Mastodon 3.0.0 before running this script.', :red
exit(1)
elsif ActiveRecord::Migrator.current_version > MAX_SUPPORTED_VERSION
@prompt.warn 'Your version of the database schema is more recent than this script, this may cause unexpected errors.'
exit(1) unless @prompt.yes?('Continue anyway? (Yes/No)')
say 'Your version of the database schema is more recent than this script, this may cause unexpected errors.', :yellow
exit(1) unless yes?('Continue anyway? (Yes/No)')
end
if Sidekiq::ProcessSet.new.any?
@prompt.error 'It seems Sidekiq is running. All Mastodon processes need to be stopped when using this script.'
say 'It seems Sidekiq is running. All Mastodon processes need to be stopped when using this script.', :red
exit(1)
end
@prompt.warn 'This task will take a long time to run and is potentially destructive.'
@prompt.warn 'Please make sure to stop Mastodon and have a backup.'
exit(1) unless @prompt.yes?('Continue? (Yes/No)')
say 'This task will take a long time to run and is potentially destructive.', :yellow
say 'Please make sure to stop Mastodon and have a backup.', :yellow
exit(1) unless yes?('Continue? (Yes/No)')
deduplicate_users!
deduplicate_account_domain_blocks!
@@ -176,7 +173,7 @@ module Mastodon::CLI
Scenic.database.refresh_materialized_view('instances', concurrently: true, cascade: false) if ActiveRecord::Migrator.current_version >= 2020_12_06_004238
Rails.cache.clear
@prompt.say 'Finished!'
say 'Finished!'
end
private
@@ -184,7 +181,7 @@ module Mastodon::CLI
def deduplicate_accounts!
remove_index_if_exists!(:accounts, 'index_accounts_on_username_and_domain_lower')
@prompt.say 'Deduplicating accounts… for local accounts, you will be asked to chose which account to keep unchanged.'
say 'Deduplicating accounts… for local accounts, you will be asked to chose which account to keep unchanged.'
find_duplicate_accounts.each do |row|
accounts = Account.where(id: row['ids'].split(',')).to_a
@@ -196,14 +193,14 @@ module Mastodon::CLI
end
end
@prompt.say 'Restoring index_accounts_on_username_and_domain_lower…'
say 'Restoring index_accounts_on_username_and_domain_lower…'
if ActiveRecord::Migrator.current_version < 2020_06_20_164023
ActiveRecord::Base.connection.add_index :accounts, 'lower (username), lower(domain)', name: 'index_accounts_on_username_and_domain_lower', unique: true
else
ActiveRecord::Base.connection.add_index :accounts, "lower (username), COALESCE(lower(domain), '')", name: 'index_accounts_on_username_and_domain_lower', unique: true
end
@prompt.say 'Reindexing textual indexes on accounts…'
say 'Reindexing textual indexes on accounts…'
ActiveRecord::Base.connection.execute('REINDEX INDEX search_index;')
ActiveRecord::Base.connection.execute('REINDEX INDEX index_accounts_on_uri;')
ActiveRecord::Base.connection.execute('REINDEX INDEX index_accounts_on_url;')
@@ -215,19 +212,18 @@ module Mastodon::CLI
remove_index_if_exists!(:users, 'index_users_on_remember_token')
remove_index_if_exists!(:users, 'index_users_on_reset_password_token')
@prompt.say 'Deduplicating user records…'
say 'Deduplicating user records…'
# Deduplicating email
ActiveRecord::Base.connection.select_all("SELECT string_agg(id::text, ',') AS ids FROM users GROUP BY email HAVING count(*) > 1").each do |row|
users = User.where(id: row['ids'].split(',')).sort_by(&:updated_at).reverse
ref_user = users.shift
@prompt.warn "Multiple users registered with e-mail address #{ref_user.email}."
@prompt.warn "e-mail will be disabled for the following accounts: #{user.map(&:account).map(&:acct).join(', ')}"
@prompt.warn 'Please reach out to them and set another address with `tootctl account modify` or delete them.'
say "Multiple users registered with e-mail address #{ref_user.email}.", :yellow
say "e-mail will be disabled for the following accounts: #{user.map(&:account).map(&:acct).join(', ')}", :yellow
say 'Please reach out to them and set another address with `tootctl account modify` or delete them.', :yellow
i = 0
users.each do |user|
user.update!(email: "#{i} " + user.email)
users.each_with_index do |user, index|
user.update!(email: "#{index} " + user.email)
end
end
@@ -235,7 +231,7 @@ module Mastodon::CLI
deduplicate_users_process_remember_token
deduplicate_users_process_password_token
@prompt.say 'Restoring users indexes…'
say 'Restoring users indexes…'
ActiveRecord::Base.connection.add_index :users, ['confirmation_token'], name: 'index_users_on_confirmation_token', unique: true
ActiveRecord::Base.connection.add_index :users, ['email'], name: 'index_users_on_email', unique: true
ActiveRecord::Base.connection.add_index :users, ['remember_token'], name: 'index_users_on_remember_token', unique: true if ActiveRecord::Migrator.current_version < 2022_01_18_183010
@@ -250,7 +246,7 @@ module Mastodon::CLI
def deduplicate_users_process_confirmation_token
ActiveRecord::Base.connection.select_all("SELECT string_agg(id::text, ',') AS ids FROM users WHERE confirmation_token IS NOT NULL GROUP BY confirmation_token HAVING count(*) > 1").each do |row|
users = User.where(id: row['ids'].split(',')).sort_by(&:created_at).reverse.drop(1)
@prompt.warn "Unsetting confirmation token for those accounts: #{users.map(&:account).map(&:acct).join(', ')}"
say "Unsetting confirmation token for those accounts: #{users.map(&:account).map(&:acct).join(', ')}", :yellow
users.each do |user|
user.update!(confirmation_token: nil)
@@ -262,7 +258,7 @@ module Mastodon::CLI
if ActiveRecord::Migrator.current_version < 2022_01_18_183010
ActiveRecord::Base.connection.select_all("SELECT string_agg(id::text, ',') AS ids FROM users WHERE remember_token IS NOT NULL GROUP BY remember_token HAVING count(*) > 1").each do |row|
users = User.where(id: row['ids'].split(',')).sort_by(&:updated_at).reverse.drop(1)
@prompt.warn "Unsetting remember token for those accounts: #{users.map(&:account).map(&:acct).join(', ')}"
say "Unsetting remember token for those accounts: #{users.map(&:account).map(&:acct).join(', ')}", :yellow
users.each do |user|
user.update!(remember_token: nil)
@@ -274,7 +270,7 @@ module Mastodon::CLI
def deduplicate_users_process_password_token
ActiveRecord::Base.connection.select_all("SELECT string_agg(id::text, ',') AS ids FROM users WHERE reset_password_token IS NOT NULL GROUP BY reset_password_token HAVING count(*) > 1").each do |row|
users = User.where(id: row['ids'].split(',')).sort_by(&:updated_at).reverse.drop(1)
@prompt.warn "Unsetting password reset token for those accounts: #{users.map(&:account).map(&:acct).join(', ')}"
say "Unsetting password reset token for those accounts: #{users.map(&:account).map(&:acct).join(', ')}", :yellow
users.each do |user|
user.update!(reset_password_token: nil)
@@ -285,12 +281,12 @@ module Mastodon::CLI
def deduplicate_account_domain_blocks!
remove_index_if_exists!(:account_domain_blocks, 'index_account_domain_blocks_on_account_id_and_domain')
@prompt.say 'Removing duplicate account domain blocks…'
say 'Removing duplicate account domain blocks…'
ActiveRecord::Base.connection.select_all("SELECT string_agg(id::text, ',') AS ids FROM account_domain_blocks GROUP BY account_id, domain HAVING count(*) > 1").each do |row|
AccountDomainBlock.where(id: row['ids'].split(',').drop(1)).delete_all
end
@prompt.say 'Restoring account domain blocks indexes…'
say 'Restoring account domain blocks indexes…'
ActiveRecord::Base.connection.add_index :account_domain_blocks, %w(account_id domain), name: 'index_account_domain_blocks_on_account_id_and_domain', unique: true
end
@@ -299,12 +295,12 @@ module Mastodon::CLI
remove_index_if_exists!(:account_identity_proofs, 'index_account_proofs_on_account_and_provider_and_username')
@prompt.say 'Removing duplicate account identity proofs…'
say 'Removing duplicate account identity proofs…'
ActiveRecord::Base.connection.select_all("SELECT string_agg(id::text, ',') AS ids FROM account_identity_proofs GROUP BY account_id, provider, provider_username HAVING count(*) > 1").each do |row|
AccountIdentityProof.where(id: row['ids'].split(',')).sort_by(&:id).reverse.drop(1).each(&:destroy)
end
@prompt.say 'Restoring account identity proofs indexes…'
say 'Restoring account identity proofs indexes…'
ActiveRecord::Base.connection.add_index :account_identity_proofs, %w(account_id provider provider_username), name: 'index_account_proofs_on_account_and_provider_and_username', unique: true
end
@@ -313,19 +309,19 @@ module Mastodon::CLI
remove_index_if_exists!(:announcement_reactions, 'index_announcement_reactions_on_account_id_and_announcement_id')
@prompt.say 'Removing duplicate account identity proofs…'
say 'Removing duplicate account identity proofs…'
ActiveRecord::Base.connection.select_all("SELECT string_agg(id::text, ',') AS ids FROM announcement_reactions GROUP BY account_id, announcement_id, name HAVING count(*) > 1").each do |row|
AnnouncementReaction.where(id: row['ids'].split(',')).sort_by(&:id).reverse.drop(1).each(&:destroy)
end
@prompt.say 'Restoring announcement_reactions indexes…'
say 'Restoring announcement_reactions indexes…'
ActiveRecord::Base.connection.add_index :announcement_reactions, %w(account_id announcement_id name), name: 'index_announcement_reactions_on_account_id_and_announcement_id', unique: true
end
def deduplicate_conversations!
remove_index_if_exists!(:conversations, 'index_conversations_on_uri')
@prompt.say 'Deduplicating conversations…'
say 'Deduplicating conversations…'
ActiveRecord::Base.connection.select_all("SELECT string_agg(id::text, ',') AS ids FROM conversations WHERE uri IS NOT NULL GROUP BY uri HAVING count(*) > 1").each do |row|
conversations = Conversation.where(id: row['ids'].split(',')).sort_by(&:id).reverse
@@ -337,7 +333,7 @@ module Mastodon::CLI
end
end
@prompt.say 'Restoring conversations indexes…'
say 'Restoring conversations indexes…'
if ActiveRecord::Migrator.current_version < 2022_03_07_083603
ActiveRecord::Base.connection.add_index :conversations, ['uri'], name: 'index_conversations_on_uri', unique: true
else
@@ -348,7 +344,7 @@ module Mastodon::CLI
def deduplicate_custom_emojis!
remove_index_if_exists!(:custom_emojis, 'index_custom_emojis_on_shortcode_and_domain')
@prompt.say 'Deduplicating custom_emojis…'
say 'Deduplicating custom_emojis…'
ActiveRecord::Base.connection.select_all("SELECT string_agg(id::text, ',') AS ids FROM custom_emojis GROUP BY shortcode, domain HAVING count(*) > 1").each do |row|
emojis = CustomEmoji.where(id: row['ids'].split(',')).sort_by(&:id).reverse
@@ -360,14 +356,14 @@ module Mastodon::CLI
end
end
@prompt.say 'Restoring custom_emojis indexes…'
say 'Restoring custom_emojis indexes…'
ActiveRecord::Base.connection.add_index :custom_emojis, %w(shortcode domain), name: 'index_custom_emojis_on_shortcode_and_domain', unique: true
end
def deduplicate_custom_emoji_categories!
remove_index_if_exists!(:custom_emoji_categories, 'index_custom_emoji_categories_on_name')
@prompt.say 'Deduplicating custom_emoji_categories…'
say 'Deduplicating custom_emoji_categories…'
ActiveRecord::Base.connection.select_all("SELECT string_agg(id::text, ',') AS ids FROM custom_emoji_categories GROUP BY name HAVING count(*) > 1").each do |row|
categories = CustomEmojiCategory.where(id: row['ids'].split(',')).sort_by(&:id).reverse
@@ -379,26 +375,26 @@ module Mastodon::CLI
end
end
@prompt.say 'Restoring custom_emoji_categories indexes…'
say 'Restoring custom_emoji_categories indexes…'
ActiveRecord::Base.connection.add_index :custom_emoji_categories, ['name'], name: 'index_custom_emoji_categories_on_name', unique: true
end
def deduplicate_domain_allows!
remove_index_if_exists!(:domain_allows, 'index_domain_allows_on_domain')
@prompt.say 'Deduplicating domain_allows…'
say 'Deduplicating domain_allows…'
ActiveRecord::Base.connection.select_all("SELECT string_agg(id::text, ',') AS ids FROM domain_allows GROUP BY domain HAVING count(*) > 1").each do |row|
DomainAllow.where(id: row['ids'].split(',')).sort_by(&:id).reverse.drop(1).each(&:destroy)
end
@prompt.say 'Restoring domain_allows indexes…'
say 'Restoring domain_allows indexes…'
ActiveRecord::Base.connection.add_index :domain_allows, ['domain'], name: 'index_domain_allows_on_domain', unique: true
end
def deduplicate_domain_blocks!
remove_index_if_exists!(:domain_blocks, 'index_domain_blocks_on_domain')
@prompt.say 'Deduplicating domain_allows…'
say 'Deduplicating domain_allows…'
ActiveRecord::Base.connection.select_all("SELECT string_agg(id::text, ',') AS ids FROM domain_blocks GROUP BY domain HAVING count(*) > 1").each do |row|
domain_blocks = DomainBlock.where(id: row['ids'].split(',')).by_severity.reverse.to_a
@@ -415,7 +411,7 @@ module Mastodon::CLI
domain_blocks.each(&:destroy)
end
@prompt.say 'Restoring domain_blocks indexes…'
say 'Restoring domain_blocks indexes…'
ActiveRecord::Base.connection.add_index :domain_blocks, ['domain'], name: 'index_domain_blocks_on_domain', unique: true
end
@@ -424,37 +420,37 @@ module Mastodon::CLI
remove_index_if_exists!(:unavailable_domains, 'index_unavailable_domains_on_domain')
@prompt.say 'Deduplicating unavailable_domains…'
say 'Deduplicating unavailable_domains…'
ActiveRecord::Base.connection.select_all("SELECT string_agg(id::text, ',') AS ids FROM unavailable_domains GROUP BY domain HAVING count(*) > 1").each do |row|
UnavailableDomain.where(id: row['ids'].split(',')).sort_by(&:id).reverse.drop(1).each(&:destroy)
end
@prompt.say 'Restoring domain_allows indexes…'
say 'Restoring domain_allows indexes…'
ActiveRecord::Base.connection.add_index :unavailable_domains, ['domain'], name: 'index_unavailable_domains_on_domain', unique: true
end
def deduplicate_email_domain_blocks!
remove_index_if_exists!(:email_domain_blocks, 'index_email_domain_blocks_on_domain')
@prompt.say 'Deduplicating email_domain_blocks…'
say 'Deduplicating email_domain_blocks…'
ActiveRecord::Base.connection.select_all("SELECT string_agg(id::text, ',') AS ids FROM email_domain_blocks GROUP BY domain HAVING count(*) > 1").each do |row|
domain_blocks = EmailDomainBlock.where(id: row['ids'].split(',')).sort_by { |b| b.parent.nil? ? 1 : 0 }.to_a
domain_blocks.drop(1).each(&:destroy)
end
@prompt.say 'Restoring email_domain_blocks indexes…'
say 'Restoring email_domain_blocks indexes…'
ActiveRecord::Base.connection.add_index :email_domain_blocks, ['domain'], name: 'index_email_domain_blocks_on_domain', unique: true
end
def deduplicate_media_attachments!
remove_index_if_exists!(:media_attachments, 'index_media_attachments_on_shortcode')
@prompt.say 'Deduplicating media_attachments…'
say 'Deduplicating media_attachments…'
ActiveRecord::Base.connection.select_all("SELECT string_agg(id::text, ',') AS ids FROM media_attachments WHERE shortcode IS NOT NULL GROUP BY shortcode HAVING count(*) > 1").each do |row|
MediaAttachment.where(id: row['ids'].split(',').drop(1)).update_all(shortcode: nil)
end
@prompt.say 'Restoring media_attachments indexes…'
say 'Restoring media_attachments indexes…'
if ActiveRecord::Migrator.current_version < 2022_03_10_060626
ActiveRecord::Base.connection.add_index :media_attachments, ['shortcode'], name: 'index_media_attachments_on_shortcode', unique: true
else
@@ -465,19 +461,19 @@ module Mastodon::CLI
def deduplicate_preview_cards!
remove_index_if_exists!(:preview_cards, 'index_preview_cards_on_url')
@prompt.say 'Deduplicating preview_cards…'
say 'Deduplicating preview_cards…'
ActiveRecord::Base.connection.select_all("SELECT string_agg(id::text, ',') AS ids FROM preview_cards GROUP BY url HAVING count(*) > 1").each do |row|
PreviewCard.where(id: row['ids'].split(',')).sort_by(&:id).reverse.drop(1).each(&:destroy)
end
@prompt.say 'Restoring preview_cards indexes…'
say 'Restoring preview_cards indexes…'
ActiveRecord::Base.connection.add_index :preview_cards, ['url'], name: 'index_preview_cards_on_url', unique: true
end
def deduplicate_statuses!
remove_index_if_exists!(:statuses, 'index_statuses_on_uri')
@prompt.say 'Deduplicating statuses…'
say 'Deduplicating statuses…'
ActiveRecord::Base.connection.select_all("SELECT string_agg(id::text, ',') AS ids FROM statuses WHERE uri IS NOT NULL GROUP BY uri HAVING count(*) > 1").each do |row|
statuses = Status.where(id: row['ids'].split(',')).sort_by(&:id)
ref_status = statuses.shift
@@ -487,7 +483,7 @@ module Mastodon::CLI
end
end
@prompt.say 'Restoring statuses indexes…'
say 'Restoring statuses indexes…'
if ActiveRecord::Migrator.current_version < 2022_03_10_060706
ActiveRecord::Base.connection.add_index :statuses, ['uri'], name: 'index_statuses_on_uri', unique: true
else
@@ -499,7 +495,7 @@ module Mastodon::CLI
remove_index_if_exists!(:tags, 'index_tags_on_name_lower')
remove_index_if_exists!(:tags, 'index_tags_on_name_lower_btree')
@prompt.say 'Deduplicating tags…'
say 'Deduplicating tags…'
ActiveRecord::Base.connection.select_all("SELECT string_agg(id::text, ',') AS ids FROM tags GROUP BY lower((name)::text) HAVING count(*) > 1").each do |row|
tags = Tag.where(id: row['ids'].split(',')).sort_by { |t| [t.usable?, t.trendable?, t.listable?].count(false) }
ref_tag = tags.shift
@@ -509,7 +505,7 @@ module Mastodon::CLI
end
end
@prompt.say 'Restoring tags indexes…'
say 'Restoring tags indexes…'
if ActiveRecord::Migrator.current_version < 2021_04_21_121431
ActiveRecord::Base.connection.add_index :tags, 'lower((name)::text)', name: 'index_tags_on_name_lower', unique: true
else
@@ -522,12 +518,12 @@ module Mastodon::CLI
remove_index_if_exists!(:webauthn_credentials, 'index_webauthn_credentials_on_external_id')
@prompt.say 'Deduplicating webauthn_credentials…'
say 'Deduplicating webauthn_credentials…'
ActiveRecord::Base.connection.select_all("SELECT string_agg(id::text, ',') AS ids FROM webauthn_credentials GROUP BY external_id HAVING count(*) > 1").each do |row|
WebauthnCredential.where(id: row['ids'].split(',')).sort_by(&:id).reverse.drop(1).each(&:destroy)
end
@prompt.say 'Restoring webauthn_credentials indexes…'
say 'Restoring webauthn_credentials indexes…'
ActiveRecord::Base.connection.add_index :webauthn_credentials, ['external_id'], name: 'index_webauthn_credentials_on_external_id', unique: true
end
@@ -536,28 +532,37 @@ module Mastodon::CLI
remove_index_if_exists!(:webhooks, 'index_webhooks_on_url')
@prompt.say 'Deduplicating webhooks…'
say 'Deduplicating webhooks…'
ActiveRecord::Base.connection.select_all("SELECT string_agg(id::text, ',') AS ids FROM webhooks GROUP BY url HAVING count(*) > 1").each do |row|
Webhooks.where(id: row['ids'].split(',')).sort_by(&:id).reverse.drop(1).each(&:destroy)
end
@prompt.say 'Restoring webhooks indexes…'
say 'Restoring webhooks indexes…'
ActiveRecord::Base.connection.add_index :webhooks, ['url'], name: 'index_webhooks_on_url', unique: true
end
def deduplicate_local_accounts!(accounts)
accounts = accounts.sort_by(&:id).reverse
@prompt.warn "Multiple local accounts were found for username '#{accounts.first.username}'."
@prompt.warn 'All those accounts are distinct accounts but only the most recently-created one is fully-functional.'
say "Multiple local accounts were found for username '#{accounts.first.username}'.", :yellow
say 'All those accounts are distinct accounts but only the most recently-created one is fully-functional.', :yellow
accounts.each_with_index do |account, idx|
@prompt.say format('%2d. %s: created at: %s; updated at: %s; last logged in at: %s; statuses: %5d; last status at: %s', idx, account.username, account.created_at, account.updated_at, account.user&.last_sign_in_at&.to_s || 'N/A', account.account_stat&.statuses_count || 0, account.account_stat&.last_status_at || 'N/A')
say format(
'%<index>2d. %<username>s: created at: %<created_at>s; updated at: %<updated_at>s; last logged in at: %<last_log_in_at>s; statuses: %<status_count>5d; last status at: %<last_status_at>s',
index: idx,
username: account.username,
created_at: account.created_at,
updated_at: account.updated_at,
last_log_in_at: account.user&.last_sign_in_at&.to_s || 'N/A',
status_count: account.account_stat&.statuses_count || 0,
last_status_at: account.account_stat&.last_status_at || 'N/A'
)
end
@prompt.say 'Please chose the one to keep unchanged, other ones will be automatically renamed.'
say 'Please chose the one to keep unchanged, other ones will be automatically renamed.'
ref_id = @prompt.ask('Account to keep unchanged:') do |q|
ref_id = ask('Account to keep unchanged:') do |q|
q.required true
q.default 0
q.convert :int

27
lib/mastodon/cli/media.rb
View File

@@ -35,12 +35,12 @@ module Mastodon::CLI
say('--prune-profiles and --remove-headers should not be specified simultaneously', :red, true)
exit(1)
end
if options[:include_follows] && !(options[:prune_profiles] || options[:remove_headers])
say('--include-follows can only be used with --prune-profiles or --remove-headers', :red, true)
exit(1)
end
time_ago = options[:days].days.ago
dry_run = options[:dry_run] ? ' (DRY RUN)' : ''
time_ago = options[:days].days.ago
if options[:prune_profiles] || options[:remove_headers]
processed, aggregate = parallelize_with_progress(Account.remote.where({ last_webfingered_at: ..time_ago, updated_at: ..time_ago })) do |account|
@@ -51,7 +51,7 @@ module Mastodon::CLI
size = (account.header_file_size || 0)
size += (account.avatar_file_size || 0) if options[:prune_profiles]
unless options[:dry_run]
unless dry_run?
account.header.destroy
account.avatar.destroy if options[:prune_profiles]
account.save!
@@ -60,7 +60,7 @@ module Mastodon::CLI
size
end
say("Visited #{processed} accounts and removed profile media totaling #{number_to_human_size(aggregate)}#{dry_run}", :green, true)
say("Visited #{processed} accounts and removed profile media totaling #{number_to_human_size(aggregate)}#{dry_run_mode_suffix}", :green, true)
end
unless options[:prune_profiles] || options[:remove_headers]
@@ -69,7 +69,7 @@ module Mastodon::CLI
size = (media_attachment.file_file_size || 0) + (media_attachment.thumbnail_file_size || 0)
unless options[:dry_run]
unless dry_run?
media_attachment.file.destroy
media_attachment.thumbnail.destroy
media_attachment.save
@@ -78,7 +78,7 @@ module Mastodon::CLI
size
end
say("Removed #{processed} media attachments (approx. #{number_to_human_size(aggregate)})#{dry_run}", :green, true)
say("Removed #{processed} media attachments (approx. #{number_to_human_size(aggregate)})#{dry_run_mode_suffix}", :green, true)
end
end
@@ -97,7 +97,6 @@ module Mastodon::CLI
progress = create_progress_bar(nil)
reclaimed_bytes = 0
removed = 0
dry_run = options[:dry_run] ? ' (DRY RUN)' : ''
prefix = options[:prefix]
case Paperclip::Attachment.default_options[:storage]
@@ -123,7 +122,7 @@ module Mastodon::CLI
record_map = preload_records_from_mixed_objects(objects)
objects.each do |object|
object.acl.put(acl: s3_permissions) if options[:fix_permissions] && !options[:dry_run]
object.acl.put(acl: s3_permissions) if options[:fix_permissions] && !dry_run?
path_segments = object.key.split('/')
path_segments.delete('cache')
@@ -145,7 +144,7 @@ module Mastodon::CLI
next unless attachment.blank? || !attachment.variant?(file_name)
begin
object.delete unless options[:dry_run]
object.delete unless dry_run?
reclaimed_bytes += object.size
removed += 1
@@ -194,7 +193,7 @@ module Mastodon::CLI
begin
size = File.size(path)
unless options[:dry_run]
unless dry_run?
File.delete(path)
begin
FileUtils.rmdir(File.dirname(path), parents: true)
@@ -216,7 +215,7 @@ module Mastodon::CLI
progress.total = progress.progress
progress.finish
say("Removed #{removed} orphans (approx. #{number_to_human_size(reclaimed_bytes)})#{dry_run}", :green, true)
say("Removed #{removed} orphans (approx. #{number_to_human_size(reclaimed_bytes)})#{dry_run_mode_suffix}", :green, true)
end
option :account, type: :string
@@ -246,8 +245,6 @@ module Mastodon::CLI
not be re-downloaded. To force re-download of every URL, use --force.
DESC
def refresh
dry_run = options[:dry_run] ? ' (DRY RUN)' : ''
if options[:status]
scope = MediaAttachment.where(status_id: options[:status])
elsif options[:account]
@@ -274,7 +271,7 @@ module Mastodon::CLI
next if media_attachment.remote_url.blank? || (!options[:force] && media_attachment.file_file_name.present?)
next if DomainBlock.reject_media?(media_attachment.account.domain)
unless options[:dry_run]
unless dry_run?
media_attachment.reset_file!
media_attachment.reset_thumbnail!
media_attachment.save
@@ -283,7 +280,7 @@ module Mastodon::CLI
media_attachment.file_file_size + (media_attachment.thumbnail_file_size || 0)
end
say("Downloaded #{processed} media attachments (approx. #{number_to_human_size(aggregate)})#{dry_run}", :green, true)
say("Downloaded #{processed} media attachments (approx. #{number_to_human_size(aggregate)})#{dry_run_mode_suffix}", :green, true)
end
desc 'usage', 'Calculate disk space consumed by Mastodon'

5
lib/mastodon/cli/preview_cards.rb
View File

@@ -27,7 +27,6 @@ module Mastodon::CLI
DESC
def remove
time_ago = options[:days].days.ago
dry_run = options[:dry_run] ? ' (DRY RUN)' : ''
link = options[:link] ? 'link-type ' : ''
scope = PreviewCard.cached
scope = scope.where(type: :link) if options[:link]
@@ -38,7 +37,7 @@ module Mastodon::CLI
size = preview_card.image_file_size
unless options[:dry_run]
unless dry_run?
preview_card.image.destroy
preview_card.save
end
@@ -46,7 +45,7 @@ module Mastodon::CLI
size
end
say("Removed #{processed} #{link}preview cards (approx. #{number_to_human_size(aggregate)})#{dry_run}", :green, true)
say("Removed #{processed} #{link}preview cards (approx. #{number_to_human_size(aggregate)})#{dry_run_mode_suffix}", :green, true)
end
end
end

3
lib/mastodon/cli/upgrade.rb
View File

@@ -17,7 +17,6 @@ module Mastodon::CLI
LONG_DESC
def storage_schema
progress = create_progress_bar(nil)
dry_run = dry_run? ? ' (DRY RUN)' : ''
records = 0
klasses = [
@@ -69,7 +68,7 @@ module Mastodon::CLI
progress.total = progress.progress
progress.finish
say("Upgraded storage schema of #{records} records#{dry_run}", :green, true)
say("Upgraded storage schema of #{records} records#{dry_run_mode_suffix}", :green, true)
end
private