tarrien/Mastodon
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Move the mastodon/*_cli files to mastodon/cli/* (#24139)

Browse Source
This commit is contained in:
Matt Jankowski
2023-05-23 10:08:26 -04:00
committed by GitHub
parent c9f980b268
commit b6b4ea4ca5
23 changed files with 149 additions and 151 deletions
Download Patch File Download Diff File Expand all files Collapse all files

690
lib/mastodon/cli/accounts.rb Normal file
View File

@ -0,0 +1,690 @@
# frozen_string_literal: true
require 'set'
require_relative '../../../config/boot'
require_relative '../../../config/environment'
require_relative 'helper'
module Mastodon::CLI
class Accounts < Thor
include Helper
def self.exit_on_failure?
true
end
option :all, type: :boolean
desc 'rotate [USERNAME]', 'Generate and broadcast new keys'
long_desc <<-LONG_DESC
Generate and broadcast new RSA keys as part of security
maintenance.
With the --all option, all local accounts will be subject
to the rotation. Otherwise, and by default, only a single
account specified by the USERNAME argument will be
processed.
LONG_DESC
def rotate(username = nil)
if options[:all]
processed = 0
delay = 0
scope = Account.local.without_suspended
progress = create_progress_bar(scope.count)
scope.find_in_batches do |accounts|
accounts.each do |account|
rotate_keys_for_account(account, delay)
progress.increment
processed += 1
end
delay += 5.minutes
end
progress.finish
say("OK, rotated keys for #{processed} accounts", :green)
elsif username.present?
rotate_keys_for_account(Account.find_local(username))
say('OK', :green)
else
say('No account(s) given', :red)
exit(1)
end
end
option :email, required: true
option :confirmed, type: :boolean
option :role
option :reattach, type: :boolean
option :force, type: :boolean
option :approve, type: :boolean
desc 'create USERNAME', 'Create a new user account'
long_desc <<-LONG_DESC
Create a new user account with a given USERNAME and an
e-mail address provided with --email.
With the --confirmed option, the confirmation e-mail will
be skipped and the account will be active straight away.
With the --role option, the role can be supplied.
With the --reattach option, the new user will be reattached
to a given existing username of an old account. If the old
account is still in use by someone else, you can supply
the --force option to delete the old record and reattach the
username to the new account anyway.
With the --approve option, the account will be approved.
LONG_DESC
def create(username)
role_id = nil
if options[:role]
role = UserRole.find_by(name: options[:role])
if role.nil?
say('Cannot find user role with that name', :red)
exit(1)
end
role_id = role.id
end
account = Account.new(username: username)
password = SecureRandom.hex
user = User.new(email: options[:email], password: password, agreement: true, role_id: role_id, confirmed_at: options[:confirmed] ? Time.now.utc : nil, bypass_invite_request_check: true)
if options[:reattach]
account = Account.find_local(username) || Account.new(username: username)
if account.user.present? && !options[:force]
say('The chosen username is currently in use', :red)
say('Use --force to reattach it anyway and delete the other user')
return
elsif account.user.present?
DeleteAccountService.new.call(account, reserve_email: false, reserve_username: false)
account = Account.new(username: username)
end
end
account.suspended_at = nil
user.account = account
if user.save
if options[:confirmed]
user.confirmed_at = nil
user.confirm!
end
user.approve! if options[:approve]
say('OK', :green)
say("New password: #{password}")
else
user.errors.each do |error|
say('Failure/Error: ', :red)
say(error.attribute)
say(" #{error.type}", :red)
end
exit(1)
end
end
option :role
option :remove_role, type: :boolean
option :email
option :confirm, type: :boolean
option :enable, type: :boolean
option :disable, type: :boolean
option :disable_2fa, type: :boolean
option :approve, type: :boolean
option :reset_password, type: :boolean
desc 'modify USERNAME', 'Modify a user account'
long_desc <<-LONG_DESC
Modify a user account.
With the --role option, update the user's role. To remove the user's
role, i.e. demote to normal user, use --remove-role.
With the --email option, update the user's e-mail address. With
the --confirm option, mark the user's e-mail as confirmed.
With the --disable option, lock the user out of their account. The
--enable option is the opposite.
With the --approve option, the account will be approved, if it was
previously not due to not having open registrations.
With the --disable-2fa option, the two-factor authentication
requirement for the user can be removed.
With the --reset-password option, the user's password is replaced by
a randomly-generated one, printed in the output.
LONG_DESC
def modify(username)
user = Account.find_local(username)&.user
if user.nil?
say('No user with such username', :red)
exit(1)
end
if options[:role]
role = UserRole.find_by(name: options[:role])
if role.nil?
say('Cannot find user role with that name', :red)
exit(1)
end
user.role_id = role.id
elsif options[:remove_role]
user.role_id = nil
end
password = SecureRandom.hex if options[:reset_password]
user.password = password if options[:reset_password]
user.email = options[:email] if options[:email]
user.disabled = false if options[:enable]
user.disabled = true if options[:disable]
user.approved = true if options[:approve]
user.otp_required_for_login = false if options[:disable_2fa]
if user.save
user.confirm if options[:confirm]
say('OK', :green)
say("New password: #{password}") if options[:reset_password]
else
user.errors.each do |error|
say('Failure/Error: ', :red)
say(error.attribute)
say(" #{error.type}", :red)
end
exit(1)
end
end
option :email
option :dry_run, type: :boolean
desc 'delete [USERNAME]', 'Delete a user'
long_desc <<-LONG_DESC
Remove a user account with a given USERNAME.
With the --email option, the user is selected based on email
rather than username.
LONG_DESC
def delete(username = nil)
if username.present? && options[:email].present?
say('Use username or --email, not both', :red)
exit(1)
elsif username.blank? && options[:email].blank?
say('No username provided', :red)
exit(1)
end
dry_run = options[:dry_run] ? ' (DRY RUN)' : ''
account = nil
if username.present?
account = Account.find_local(username)
if account.nil?
say('No user with such username', :red)
exit(1)
end
else
account = Account.left_joins(:user).find_by(user: { email: options[:email] })
if account.nil?
say('No user with such email', :red)
exit(1)
end
end
say("Deleting user with #{account.statuses_count} statuses, this might take a while...#{dry_run}")
DeleteAccountService.new.call(account, reserve_email: false) unless options[:dry_run]
say("OK#{dry_run}", :green)
end
option :force, type: :boolean, aliases: [:f], description: 'Override public key check'
desc 'merge FROM TO', 'Merge two remote accounts into one'
long_desc <<-LONG_DESC
Merge two remote accounts specified by their username@domain
into one, whereby the TO account is the one being merged into
and kept, while the FROM one is removed. It is primarily meant
to fix duplicates caused by other servers changing their domain.
The command by default only works if both accounts have the same
public key to prevent mistakes. To override this, use the --force.
LONG_DESC
def merge(from_acct, to_acct)
username, domain = from_acct.split('@')
from_account = Account.find_remote(username, domain)
if from_account.nil? || from_account.local?
say("No such account (#{from_acct})", :red)
exit(1)
end
username, domain = to_acct.split('@')
to_account = Account.find_remote(username, domain)
if to_account.nil? || to_account.local?
say("No such account (#{to_acct})", :red)
exit(1)
end
if from_account.public_key != to_account.public_key && !options[:force]
say("Accounts don't have the same public key, might not be duplicates!", :red)
say('Override with --force', :red)
exit(1)
end
to_account.merge_with!(from_account)
from_account.destroy
say('OK', :green)
end
desc 'fix-duplicates', 'Find duplicate remote accounts and merge them'
option :dry_run, type: :boolean
long_desc <<-LONG_DESC
Merge known remote accounts sharing an ActivityPub actor identifier.
Such duplicates can occur when a remote server admin misconfigures their
domain configuration.
LONG_DESC
def fix_duplicates
Account.remote.select(:uri, 'count(*)').group(:uri).having('count(*) > 1').pluck(:uri).each do |uri|
say("Duplicates found for #{uri}")
begin
ActivityPub::FetchRemoteAccountService.new.call(uri) unless options[:dry_run]
rescue => e
say("Error processing #{uri}: #{e}", :red)
end
end
end
desc 'backup USERNAME', 'Request a backup for a user'
long_desc <<-LONG_DESC
Request a new backup for an account with a given USERNAME.
The backup will be created in Sidekiq asynchronously, and
the user will receive an e-mail with a link to it once
it's done.
LONG_DESC
def backup(username)
account = Account.find_local(username)
if account.nil?
say('No user with such username', :red)
exit(1)
end
backup = account.user.backups.create!
BackupWorker.perform_async(backup.id)
say('OK', :green)
end
option :concurrency, type: :numeric, default: 5, aliases: [:c]
option :dry_run, type: :boolean
desc 'cull [DOMAIN...]', 'Remove remote accounts that no longer exist'
long_desc <<-LONG_DESC
Query every single remote account in the database to determine
if it still exists on the origin server, and if it doesn't,
remove it from the database.
Accounts that have had confirmed activity within the last week
are excluded from the checks.
LONG_DESC
def cull(*domains)
skip_threshold = 7.days.ago
dry_run = options[:dry_run] ? ' (DRY RUN)' : ''
skip_domains = Concurrent::Set.new
query = Account.remote.where(protocol: :activitypub)
query = query.where(domain: domains) unless domains.empty?
processed, culled = parallelize_with_progress(query.partitioned) do |account|
next if account.updated_at >= skip_threshold || (account.last_webfingered_at.present? && account.last_webfingered_at >= skip_threshold) || skip_domains.include?(account.domain)
code = 0
begin
code = Request.new(:head, account.uri).perform(&:code)
rescue HTTP::TimeoutError, HTTP::ConnectionError, OpenSSL::SSL::SSLError, Mastodon::PrivateNetworkAddressError
skip_domains << account.domain
end
if [404, 410].include?(code)
DeleteAccountService.new.call(account, reserve_username: false) unless options[:dry_run]
1
else
# Touch account even during dry run to avoid getting the account into the window again
account.touch
end
end
say("Visited #{processed} accounts, removed #{culled}#{dry_run}", :green)
unless skip_domains.empty?
say('The following domains were not available during the check:', :yellow)
skip_domains.each { |domain| say(" #{domain}") }
end
end
option :all, type: :boolean
option :domain
option :concurrency, type: :numeric, default: 5, aliases: [:c]
option :verbose, type: :boolean, aliases: [:v]
option :dry_run, type: :boolean
desc 'refresh [USERNAMES]', 'Fetch remote user data and files'
long_desc <<-LONG_DESC
Fetch remote user data and files for one or multiple accounts.
With the --all option, all remote accounts will be processed.
Through the --domain option, this can be narrowed down to a
specific domain only. Otherwise, remote accounts must be
specified with space-separated USERNAMES.
LONG_DESC
def refresh(*usernames)
dry_run = options[:dry_run] ? ' (DRY RUN)' : ''
if options[:domain] || options[:all]
scope = Account.remote
scope = scope.where(domain: options[:domain]) if options[:domain]
processed, = parallelize_with_progress(scope) do |account|
next if options[:dry_run]
account.reset_avatar!
account.reset_header!
account.save
end
say("Refreshed #{processed} accounts#{dry_run}", :green, true)
elsif !usernames.empty?
usernames.each do |user|
user, domain = user.split('@')
account = Account.find_remote(user, domain)
if account.nil?
say('No such account', :red)
exit(1)
end
next if options[:dry_run]
begin
account.reset_avatar!
account.reset_header!
account.save
rescue Mastodon::UnexpectedResponseError
say("Account failed: #{user}@#{domain}", :red)
end
end
say("OK#{dry_run}", :green)
else
say('No account(s) given', :red)
exit(1)
end
end
option :concurrency, type: :numeric, default: 5, aliases: [:c]
option :verbose, type: :boolean, aliases: [:v]
desc 'follow USERNAME', 'Make all local accounts follow account specified by USERNAME'
def follow(username)
target_account = Account.find_local(username)
if target_account.nil?
say('No such account', :red)
exit(1)
end
processed, = parallelize_with_progress(Account.local.without_suspended) do |account|
FollowService.new.call(account, target_account, bypass_limit: true)
end
say("OK, followed target from #{processed} accounts", :green)
end
option :concurrency, type: :numeric, default: 5, aliases: [:c]
option :verbose, type: :boolean, aliases: [:v]
desc 'unfollow ACCT', 'Make all local accounts unfollow account specified by ACCT'
def unfollow(acct)
username, domain = acct.split('@')
target_account = Account.find_remote(username, domain)
if target_account.nil?
say('No such account', :red)
exit(1)
end
processed, = parallelize_with_progress(target_account.followers.local) do |account|
UnfollowService.new.call(account, target_account)
end
say("OK, unfollowed target from #{processed} accounts", :green)
end
option :follows, type: :boolean, default: false
option :followers, type: :boolean, default: false
desc 'reset-relationships USERNAME', 'Reset all follows and/or followers for a user'
long_desc <<-LONG_DESC
Reset all follows and/or followers for a user specified by USERNAME.
With the --follows option, the command unfollows everyone that the account follows,
and then re-follows the users that would be followed by a brand new account.
With the --followers option, the command removes all followers of the account.
LONG_DESC
def reset_relationships(username)
unless options[:follows] || options[:followers]
say('Please specify either --follows or --followers, or both', :red)
exit(1)
end
account = Account.find_local(username)
if account.nil?
say('No such account', :red)
exit(1)
end
total = 0
total += Account.where(id: ::Follow.where(account: account).select(:target_account_id)).count if options[:follows]
total += Account.where(id: ::Follow.where(target_account: account).select(:account_id)).count if options[:followers]
progress = create_progress_bar(total)
processed = 0
if options[:follows]
scope = Account.where(id: ::Follow.where(account: account).select(:target_account_id))
scope.find_each do |target_account|
UnfollowService.new.call(account, target_account)
rescue => e
progress.log pastel.red("Error processing #{target_account.id}: #{e}")
ensure
progress.increment
processed += 1
end
BootstrapTimelineWorker.perform_async(account.id)
end
if options[:followers]
scope = Account.where(id: ::Follow.where(target_account: account).select(:account_id))
scope.find_each do |target_account|
UnfollowService.new.call(target_account, account)
rescue => e
progress.log pastel.red("Error processing #{target_account.id}: #{e}")
ensure
progress.increment
processed += 1
end
end
progress.finish
say("Processed #{processed} relationships", :green, true)
end
option :number, type: :numeric, aliases: [:n]
option :all, type: :boolean
desc 'approve [USERNAME]', 'Approve pending accounts'
long_desc <<~LONG_DESC
When registrations require review from staff, approve pending accounts,
either all of them with the --all option, or a specific number of them
specified with the --number (-n) option, or only a single specific
account identified by its username.
LONG_DESC
def approve(username = nil)
if options[:all]
User.pending.find_each(&:approve!)
say('OK', :green)
elsif options[:number]&.positive?
User.pending.order(created_at: :asc).limit(options[:number]).each(&:approve!)
say('OK', :green)
elsif username.present?
account = Account.find_local(username)
if account.nil?
say('No such account', :red)
exit(1)
end
account.user&.approve!
say('OK', :green)
else
say('Number must be positive', :red) if options[:number]
exit(1)
end
end
option :concurrency, type: :numeric, default: 5, aliases: [:c]
option :dry_run, type: :boolean
desc 'prune', 'Prune remote accounts that never interacted with local users'
long_desc <<-LONG_DESC
Prune remote account that
- follows no local accounts
- is not followed by any local accounts
- has no statuses on local
- has not been mentioned
- has not been favourited local posts
- not muted/blocked by us
LONG_DESC
def prune
dry_run = options[:dry_run] ? ' (dry run)' : ''
query = Account.remote.where.not(actor_type: %i(Application Service))
query = query.where('NOT EXISTS (SELECT 1 FROM mentions WHERE account_id = accounts.id)')
query = query.where('NOT EXISTS (SELECT 1 FROM favourites WHERE account_id = accounts.id)')
query = query.where('NOT EXISTS (SELECT 1 FROM statuses WHERE account_id = accounts.id)')
query = query.where('NOT EXISTS (SELECT 1 FROM follows WHERE account_id = accounts.id OR target_account_id = accounts.id)')
query = query.where('NOT EXISTS (SELECT 1 FROM blocks WHERE account_id = accounts.id OR target_account_id = accounts.id)')
query = query.where('NOT EXISTS (SELECT 1 FROM mutes WHERE target_account_id = accounts.id)')
query = query.where('NOT EXISTS (SELECT 1 FROM reports WHERE target_account_id = accounts.id)')
query = query.where('NOT EXISTS (SELECT 1 FROM follow_requests WHERE account_id = accounts.id OR target_account_id = accounts.id)')
_, deleted = parallelize_with_progress(query) do |account|
next if account.bot? || account.group?
next if account.suspended?
next if account.silenced?
account.destroy unless options[:dry_run]
1
end
say("OK, pruned #{deleted} accounts#{dry_run}", :green)
end
option :force, type: :boolean
option :replay, type: :boolean
option :target
desc 'migrate USERNAME', 'Migrate a local user to another account'
long_desc <<~LONG_DESC
With --replay, replay the last migration of the specified account, in
case some remote server may not have properly processed the associated
`Move` activity.
With --target, specify another account to migrate to.
With --force, perform the migration even if the selected account
redirects to a different account that the one specified.
LONG_DESC
def migrate(username)
if options[:replay].present? && options[:target].present?
say('Use --replay or --target, not both', :red)
exit(1)
end
if options[:replay].blank? && options[:target].blank?
say('Use either --replay or --target', :red)
exit(1)
end
account = Account.find_local(username)
if account.nil?
say("No such account: #{username}", :red)
exit(1)
end
migration = nil
if options[:replay]
migration = account.migrations.last
if migration.nil?
say('The specified account has not performed any migration', :red)
exit(1)
end
unless options[:force] || migration.target_account_id == account.moved_to_account_id
say('The specified account is not redirecting to its last migration target. Use --force if you want to replay the migration anyway', :red)
exit(1)
end
end
if options[:target]
target_account = ResolveAccountService.new.call(options[:target])
if target_account.nil?
say("The specified target account could not be found: #{options[:target]}", :red)
exit(1)
end
unless options[:force] || account.moved_to_account_id.nil? || account.moved_to_account_id == target_account.id
say('The specified account is redirecting to a different target account. Use --force if you want to change the migration target', :red)
exit(1)
end
begin
migration = account.migrations.create!(acct: target_account.acct)
rescue ActiveRecord::RecordInvalid => e
say("Error: #{e.message}", :red)
exit(1)
end
end
MoveService.new.call(migration)
say("OK, migrated #{account.acct} to #{migration.target_account.acct}", :green)
end
private
def rotate_keys_for_account(account, delay = 0)
if account.nil?
say('No such account', :red)
exit(1)
end
old_key = account.private_key
new_key = OpenSSL::PKey::RSA.new(2048)
account.update(private_key: new_key.to_pem, public_key: new_key.public_key.to_pem)
ActivityPub::UpdateDistributionWorker.perform_in(delay, account.id, { 'sign_with' => old_key })
end
end
end

60
lib/mastodon/cli/cache.rb Normal file
View File

@ -0,0 +1,60 @@
# frozen_string_literal: true
require_relative '../../../config/boot'
require_relative '../../../config/environment'
require_relative 'helper'
module Mastodon::CLI
class Cache < Thor
include Helper
def self.exit_on_failure?
true
end
desc 'clear', 'Clear out the cache storage'
def clear
Rails.cache.clear
say('OK', :green)
end
option :concurrency, type: :numeric, default: 5, aliases: [:c]
option :verbose, type: :boolean, aliases: [:v]
desc 'recount TYPE', 'Update hard-cached counters'
long_desc <<~LONG_DESC
Update hard-cached counters of TYPE by counting referenced
records from scratch. TYPE can be "accounts" or "statuses".
It may take a very long time to finish, depending on the
size of the database.
LONG_DESC
def recount(type)
case type
when 'accounts'
processed, = parallelize_with_progress(Account.local.includes(:account_stat)) do |account|
account_stat = account.account_stat
account_stat.following_count = account.active_relationships.count
account_stat.followers_count = account.passive_relationships.count
account_stat.statuses_count = account.statuses.where.not(visibility: :direct).count
account_stat.save if account_stat.changed?
end
when 'statuses'
processed, = parallelize_with_progress(Status.includes(:status_stat)) do |status|
status_stat = status.status_stat
status_stat.replies_count = status.replies.where.not(visibility: :direct).count
status_stat.reblogs_count = status.reblogs.count
status_stat.favourites_count = status.favourites.count
status_stat.save if status_stat.changed?
end
else
say("Unknown type: #{type}", :red)
exit(1)
end
say
say("OK, recounted #{processed} records", :green)
end
end
end

51
lib/mastodon/cli/canonical_email_blocks.rb Normal file
View File

@ -0,0 +1,51 @@
# frozen_string_literal: true
require 'concurrent'
require_relative '../../../config/boot'
require_relative '../../../config/environment'
require_relative 'helper'
module Mastodon::CLI
class CanonicalEmailBlocks < Thor
include Helper
def self.exit_on_failure?
true
end
desc 'find EMAIL', 'Find a given e-mail address in the canonical e-mail blocks'
long_desc <<-LONG_DESC
When suspending a local user, a hash of a "canonical" version of their e-mail
address is stored to prevent them from signing up again.
This command can be used to find whether a known email address is blocked.
LONG_DESC
def find(email)
accts = CanonicalEmailBlock.matching_email(email)
if accts.empty?
say("#{email} is not blocked", :green)
else
say("#{email} is blocked", :red)
end
end
desc 'remove EMAIL', 'Remove a canonical e-mail block'
long_desc <<-LONG_DESC
When suspending a local user, a hash of a "canonical" version of their e-mail
address is stored to prevent them from signing up again.
This command allows removing a canonical email block.
LONG_DESC
def remove(email)
blocks = CanonicalEmailBlock.matching_email(email)
if blocks.empty?
say("#{email} is not blocked", :green)
else
blocks.destroy_all
say("Unblocked #{email}", :green)
end
end
end
end

225
lib/mastodon/cli/domains.rb Normal file
View File

@ -0,0 +1,225 @@
# frozen_string_literal: true
require 'concurrent'
require_relative '../../../config/boot'
require_relative '../../../config/environment'
require_relative 'helper'
module Mastodon::CLI
class Domains < Thor
include Helper
def self.exit_on_failure?
true
end
option :concurrency, type: :numeric, default: 5, aliases: [:c]
option :verbose, type: :boolean, aliases: [:v]
option :dry_run, type: :boolean
option :limited_federation_mode, type: :boolean
option :by_uri, type: :boolean
option :include_subdomains, type: :boolean
option :purge_domain_blocks, type: :boolean
desc 'purge [DOMAIN...]', 'Remove accounts from a DOMAIN without a trace'
long_desc <<-LONG_DESC
Remove all accounts from a given DOMAIN without leaving behind any
records. Unlike a suspension, if the DOMAIN still exists in the wild,
it means the accounts could return if they are resolved again.
When the --limited-federation-mode option is given, instead of purging accounts
from a single domain, all accounts from domains that have not been explicitly allowed
are removed from the database.
When the --by-uri option is given, DOMAIN is used to match the domain part of actor
URIs rather than the domain part of the webfinger handle. For instance, an account
that has the handle `foo@bar.com` but whose profile is at the URL
`https://mastodon-bar.com/users/foo`, would be purged by either
`tootctl domains purge bar.com` or `tootctl domains purge --by-uri mastodon-bar.com`.
When the --include-subdomains option is given, not only DOMAIN is deleted, but all
subdomains as well. Note that this may be considerably slower.
When the --purge-domain-blocks option is given, also purge matching domain blocks.
LONG_DESC
def purge(*domains)
dry_run = options[:dry_run] ? ' (DRY RUN)' : ''
domains = domains.map { |domain| TagManager.instance.normalize_domain(domain) }
account_scope = Account.none
domain_block_scope = DomainBlock.none
emoji_scope = CustomEmoji.none
# Sanity check on command arguments
if options[:limited_federation_mode] && !domains.empty?
say('DOMAIN parameter not supported with --limited-federation-mode', :red)
exit(1)
elsif domains.empty? && !options[:limited_federation_mode]
say('No domain(s) given', :red)
exit(1)
end
# Build scopes from command arguments
if options[:limited_federation_mode]
account_scope = Account.remote.where.not(domain: DomainAllow.select(:domain))
emoji_scope = CustomEmoji.remote.where.not(domain: DomainAllow.select(:domain))
else
# Handle wildcard subdomains
subdomain_patterns = domains.filter_map { |domain| "%.#{Account.sanitize_sql_like(domain[2..])}" if domain.start_with?('*.') }
domains = domains.filter { |domain| !domain.start_with?('*.') }
# Handle --include-subdomains
subdomain_patterns += domains.map { |domain| "%.#{Account.sanitize_sql_like(domain)}" } if options[:include_subdomains]
uri_patterns = (domains.map { |domain| Account.sanitize_sql_like(domain) } + subdomain_patterns).map { |pattern| "https://#{pattern}/%" }
if options[:purge_domain_blocks]
domain_block_scope = DomainBlock.where(domain: domains)
domain_block_scope = domain_block_scope.or(DomainBlock.where(DomainBlock.arel_table[:domain].matches_any(subdomain_patterns))) unless subdomain_patterns.empty?
end
if options[:by_uri]
account_scope = Account.remote.where(Account.arel_table[:uri].matches_any(uri_patterns, false, true))
emoji_scope = CustomEmoji.remote.where(CustomEmoji.arel_table[:uri].matches_any(uri_patterns, false, true))
else
account_scope = Account.remote.where(domain: domains)
account_scope = account_scope.or(Account.remote.where(Account.arel_table[:domain].matches_any(subdomain_patterns))) unless subdomain_patterns.empty?
emoji_scope = CustomEmoji.where(domain: domains)
emoji_scope = emoji_scope.or(CustomEmoji.remote.where(CustomEmoji.arel_table[:uri].matches_any(subdomain_patterns))) unless subdomain_patterns.empty?
end
end
# Actually perform the deletions
processed, = parallelize_with_progress(account_scope) do |account|
DeleteAccountService.new.call(account, reserve_username: false, skip_side_effects: true) unless options[:dry_run]
end
say("Removed #{processed} accounts#{dry_run}", :green)
if options[:purge_domain_blocks]
domain_block_count = domain_block_scope.count
domain_block_scope.in_batches.destroy_all unless options[:dry_run]
say("Removed #{domain_block_count} domain blocks#{dry_run}", :green)
end
custom_emojis_count = emoji_scope.count
emoji_scope.in_batches.destroy_all unless options[:dry_run]
Instance.refresh unless options[:dry_run]
say("Removed #{custom_emojis_count} custom emojis#{dry_run}", :green)
end
option :concurrency, type: :numeric, default: 50, aliases: [:c]
option :format, type: :string, default: 'summary', aliases: [:f]
option :exclude_suspended, type: :boolean, default: false, aliases: [:x]
desc 'crawl [START]', 'Crawl all known peers, optionally beginning at START'
long_desc <<-LONG_DESC
Crawl the fediverse by using the Mastodon REST API endpoints that expose
all known peers, and collect statistics from those peers, as long as those
peers support those API endpoints. When no START is given, the command uses
this server's own database of known peers to seed the crawl.
The --concurrency (-c) option controls the number of threads performing HTTP
requests at the same time. More threads means the crawl may complete faster.
The --format (-f) option controls how the data is displayed at the end. By
default (`summary`), a summary of the statistics is returned. The other options
are `domains`, which returns a newline-delimited list of all discovered peers,
and `json`, which dumps all the aggregated data raw.
The --exclude-suspended (-x) option means that domains that are suspended
instance-wide do not appear in the output and are not included in summaries.
This also excludes subdomains of any of those domains.
LONG_DESC
def crawl(start = nil)
stats = Concurrent::Hash.new
processed = Concurrent::AtomicFixnum.new(0)
failed = Concurrent::AtomicFixnum.new(0)
start_at = Time.now.to_f
seed = start ? [start] : Instance.pluck(:domain)
blocked_domains = /\.?(#{DomainBlock.where(severity: 1).pluck(:domain).map { |domain| Regexp.escape(domain) }.join('|')})$/
progress = create_progress_bar
pool = Concurrent::ThreadPoolExecutor.new(min_threads: 0, max_threads: options[:concurrency], idletime: 10, auto_terminate: true, max_queue: 0)
work_unit = lambda do |domain|
next if stats.key?(domain)
next if options[:exclude_suspended] && domain.match?(blocked_domains)
stats[domain] = nil
begin
Request.new(:get, "https://#{domain}/api/v1/instance").perform do |res|
next unless res.code == 200
stats[domain] = Oj.load(res.to_s)
end
Request.new(:get, "https://#{domain}/api/v1/instance/peers").perform do |res|
next unless res.code == 200
Oj.load(res.to_s).reject { |peer| stats.key?(peer) }.each do |peer|
pool.post(peer, &work_unit)
end
end
Request.new(:get, "https://#{domain}/api/v1/instance/activity").perform do |res|
next unless res.code == 200
stats[domain]['activity'] = Oj.load(res.to_s)
end
rescue
failed.increment
ensure
processed.increment
progress.increment unless progress.finished?
end
end
seed.each do |domain|
pool.post(domain, &work_unit)
end
sleep 20
sleep 20 until pool.queue_length.zero?
pool.shutdown
pool.wait_for_termination(20)
ensure
progress.finish
pool.shutdown
case options[:format]
when 'summary'
stats_to_summary(stats, processed, failed, start_at)
when 'domains'
stats_to_domains(stats)
when 'json'
stats_to_json(stats)
end
end
private
def stats_to_summary(stats, processed, failed, start_at)
stats.compact!
total_domains = stats.size
total_users = stats.reduce(0) { |sum, (_key, val)| val.is_a?(Hash) && val['stats'].is_a?(Hash) ? sum + val['stats']['user_count'].to_i : sum }
total_active = stats.reduce(0) { |sum, (_key, val)| val.is_a?(Hash) && val['activity'].is_a?(Array) && val['activity'].size > 2 && val['activity'][1].is_a?(Hash) ? sum + val['activity'][1]['logins'].to_i : sum }
total_joined = stats.reduce(0) { |sum, (_key, val)| val.is_a?(Hash) && val['activity'].is_a?(Array) && val['activity'].size > 2 && val['activity'][1].is_a?(Hash) ? sum + val['activity'][1]['registrations'].to_i : sum }
say("Visited #{processed.value} domains, #{failed.value} failed (#{(Time.now.to_f - start_at).round}s elapsed)", :green)
say("Total servers: #{total_domains}", :green)
say("Total registered: #{total_users}", :green)
say("Total active last week: #{total_active}", :green)
say("Total joined last week: #{total_joined}", :green)
end
def stats_to_domains(stats)
say(stats.keys.join("\n"))
end
def stats_to_json(stats)
stats.compact!
say(Oj.dump(stats))
end
end
end

131
lib/mastodon/cli/email_domain_blocks.rb Normal file
View File

@ -0,0 +1,131 @@
# frozen_string_literal: true
require 'concurrent'
require_relative '../../../config/boot'
require_relative '../../../config/environment'
require_relative 'helper'
module Mastodon::CLI
class EmailDomainBlocks < Thor
include Helper
def self.exit_on_failure?
true
end
desc 'list', 'List blocked e-mail domains'
def list
EmailDomainBlock.where(parent_id: nil).order(id: 'DESC').find_each do |entry|
say(entry.domain.to_s, :white)
EmailDomainBlock.where(parent_id: entry.id).order(id: 'DESC').find_each do |child|
say(" #{child.domain}", :cyan)
end
end
end
option :with_dns_records, type: :boolean
desc 'add DOMAIN...', 'Block e-mail domain(s)'
long_desc <<-LONG_DESC
Blocking an e-mail domain prevents users from signing up
with e-mail addresses from that domain. You can provide one or
multiple domains to the command.
When the --with-dns-records option is given, an attempt to resolve the
given domains' MX records will be made and the results will also be blocked.
This can be helpful if you are blocking an e-mail server that has many
different domains pointing to it as it allows you to essentially block
it at the root.
LONG_DESC
def add(*domains)
if domains.empty?
say('No domain(s) given', :red)
exit(1)
end
skipped = 0
processed = 0
domains.each do |domain|
if EmailDomainBlock.where(domain: domain).exists?
say("#{domain} is already blocked.", :yellow)
skipped += 1
next
end
other_domains = []
if options[:with_dns_records]
Resolv::DNS.open do |dns|
dns.timeouts = 5
other_domains = dns.getresources(@email_domain_block.domain, Resolv::DNS::Resource::IN::MX).to_a
end
end
email_domain_block = EmailDomainBlock.new(domain: domain, other_domains: other_domains)
email_domain_block.save!
processed += 1
(email_domain_block.other_domains || []).uniq.each do |hostname|
another_email_domain_block = EmailDomainBlock.new(domain: hostname, parent: email_domain_block)
if EmailDomainBlock.where(domain: hostname).exists?
say("#{hostname} is already blocked.", :yellow)
skipped += 1
next
end
another_email_domain_block.save!
processed += 1
end
end
say("Added #{processed}, skipped #{skipped}", color(processed, 0))
end
desc 'remove DOMAIN...', 'Remove e-mail domain blocks'
def remove(*domains)
if domains.empty?
say('No domain(s) given', :red)
exit(1)
end
skipped = 0
processed = 0
failed = 0
domains.each do |domain|
entry = EmailDomainBlock.find_by(domain: domain)
if entry.nil?
say("#{domain} is not yet blocked.", :yellow)
skipped += 1
next
end
children_count = EmailDomainBlock.where(parent_id: entry.id).count
result = entry.destroy
if result
processed += children_count + 1
else
say("#{domain} could not be unblocked.", :red)
failed += 1
end
end
say("Removed #{processed}, skipped #{skipped}, failed #{failed}", color(processed, failed))
end
private
def color(processed, failed)
if !processed.zero? && failed.zero?
:green
elsif failed.zero?
:yellow
else
:red
end
end
end
end

146
lib/mastodon/cli/emoji.rb Normal file
View File

@ -0,0 +1,146 @@
# frozen_string_literal: true
require 'rubygems/package'
require_relative '../../../config/boot'
require_relative '../../../config/environment'
require_relative 'helper'
module Mastodon::CLI
class Emoji < Thor
def self.exit_on_failure?
true
end
option :prefix
option :suffix
option :overwrite, type: :boolean
option :unlisted, type: :boolean
option :category
desc 'import PATH', 'Import emoji from a TAR GZIP archive at PATH'
long_desc <<-LONG_DESC
Imports custom emoji from a TAR GZIP archive specified by PATH.
Existing emoji will be skipped unless the --overwrite option
is provided, in which case they will be overwritten.
You can specify a --category under which the emojis will be
grouped together.
With the --prefix option, a prefix can be added to all
generated shortcodes. Likewise, the --suffix option controls
the suffix of all shortcodes.
With the --unlisted option, the processed emoji will not be
visible in the emoji picker (but still usable via other means)
LONG_DESC
def import(path)
imported = 0
skipped = 0
failed = 0
category = options[:category] ? CustomEmojiCategory.find_or_create_by(name: options[:category]) : nil
Gem::Package::TarReader.new(Zlib::GzipReader.open(path)) do |tar|
tar.each do |entry|
next unless entry.file? && entry.full_name.end_with?('.png', '.gif')
filename = File.basename(entry.full_name, '.*')
# Skip macOS shadow files
next if filename.start_with?('._')
shortcode = [options[:prefix], filename, options[:suffix]].compact.join
custom_emoji = CustomEmoji.local.find_by('LOWER(shortcode) = ?', shortcode.downcase)
if custom_emoji && !options[:overwrite]
skipped += 1
next
end
custom_emoji ||= CustomEmoji.new(shortcode: shortcode, domain: nil)
custom_emoji.image = StringIO.new(entry.read)
custom_emoji.image_file_name = File.basename(entry.full_name)
custom_emoji.visible_in_picker = !options[:unlisted]
custom_emoji.category = category
if custom_emoji.save
imported += 1
else
failed += 1
say('Failure/Error: ', :red)
say(entry.full_name)
say(" #{custom_emoji.errors[:image].join(', ')}", :red)
end
end
end
say("Imported #{imported}, skipped #{skipped}, failed to import #{failed}", color(imported, skipped, failed))
end
option :category
option :overwrite, type: :boolean
desc 'export PATH', 'Export emoji to a TAR GZIP archive at PATH'
long_desc <<-LONG_DESC
Exports custom emoji to 'export.tar.gz' at PATH.
The --category option dumps only the specified category.
If this option is not specified, all emoji will be exported.
The --overwrite option will overwrite an existing archive.
LONG_DESC
def export(path)
exported = 0
category = CustomEmojiCategory.find_by(name: options[:category])
export_file_name = File.join(path, 'export.tar.gz')
if File.file?(export_file_name) && !options[:overwrite]
say("Archive already exists! Use '--overwrite' to overwrite it!")
exit 1
end
if category.nil? && options[:category]
say("Unable to find category '#{options[:category]}'!")
exit 1
end
File.open(export_file_name, 'wb') do |file|
Zlib::GzipWriter.wrap(file) do |gzip|
Gem::Package::TarWriter.new(gzip) do |tar|
scope = !options[:category] || category.nil? ? CustomEmoji.local : category.emojis
scope.find_each do |emoji|
say("Adding '#{emoji.shortcode}'...")
tar.add_file_simple(emoji.shortcode + File.extname(emoji.image_file_name), 0o644, emoji.image_file_size) do |io|
io.write Paperclip.io_adapters.for(emoji.image).read
exported += 1
end
end
end
end
end
say("Exported #{exported}")
end
option :remote_only, type: :boolean
desc 'purge', 'Remove all custom emoji'
long_desc <<-LONG_DESC
Removes all custom emoji.
With the --remote-only option, only remote emoji will be deleted.
LONG_DESC
def purge
scope = options[:remote_only] ? CustomEmoji.remote : CustomEmoji
scope.in_batches.destroy_all
say('OK', :green)
end
private
def color(green, _yellow, red)
if !green.zero? && red.zero?
:green
elsif red.zero?
:yellow
else
:red
end
end
end
end

60
lib/mastodon/cli/feeds.rb Normal file
View File

@ -0,0 +1,60 @@
# frozen_string_literal: true
require_relative '../../../config/boot'
require_relative '../../../config/environment'
require_relative 'helper'
module Mastodon::CLI
class Feeds < Thor
include Helper
include Redisable
def self.exit_on_failure?
true
end
option :all, type: :boolean, default: false
option :concurrency, type: :numeric, default: 5, aliases: [:c]
option :verbose, type: :boolean, aliases: [:v]
option :dry_run, type: :boolean, default: false
desc 'build [USERNAME]', 'Build home and list feeds for one or all users'
long_desc <<-LONG_DESC
Build home and list feeds that are stored in Redis from the database.
With the --all option, all active users will be processed.
Otherwise, a single user specified by USERNAME.
LONG_DESC
def build(username = nil)
dry_run = options[:dry_run] ? '(DRY RUN)' : ''
if options[:all] || username.nil?
processed, = parallelize_with_progress(Account.joins(:user).merge(User.active)) do |account|
PrecomputeFeedService.new.call(account) unless options[:dry_run]
end
say("Regenerated feeds for #{processed} accounts #{dry_run}", :green, true)
elsif username.present?
account = Account.find_local(username)
if account.nil?
say('No such account', :red)
exit(1)
end
PrecomputeFeedService.new.call(account) unless options[:dry_run]
say("OK #{dry_run}", :green, true)
else
say('No account(s) given', :red)
exit(1)
end
end
desc 'clear', 'Remove all home and list feeds from Redis'
def clear
keys = redis.keys('feed:*')
redis.del(keys)
say('OK', :green)
end
end
end

85
lib/mastodon/cli/helper.rb Normal file
View File

@ -0,0 +1,85 @@
# frozen_string_literal: true
dev_null = Logger.new('/dev/null')
Rails.logger = dev_null
ActiveRecord::Base.logger = dev_null
ActiveJob::Base.logger = dev_null
HttpLog.configuration.logger = dev_null
Paperclip.options[:log] = false
Chewy.logger = dev_null
module Mastodon::CLI
module Helper
def dry_run?
options[:dry_run]
end
def create_progress_bar(total = nil)
ProgressBar.create(total: total, format: '%c/%u |%b%i| %e')
end
def reset_connection_pools!
ActiveRecord::Base.establish_connection(ActiveRecord::Base.configurations[Rails.env].dup.tap { |config| config['pool'] = options[:concurrency] + 1 })
RedisConfiguration.establish_pool(options[:concurrency])
end
def parallelize_with_progress(scope)
if options[:concurrency] < 1
say('Cannot run with this concurrency setting, must be at least 1', :red)
exit(1)
end
reset_connection_pools!
progress = create_progress_bar(scope.count)
pool = Concurrent::FixedThreadPool.new(options[:concurrency])
total = Concurrent::AtomicFixnum.new(0)
aggregate = Concurrent::AtomicFixnum.new(0)
scope.reorder(nil).find_in_batches do |items|
futures = []
items.each do |item|
futures << Concurrent::Future.execute(executor: pool) do
if !progress.total.nil? && progress.progress + 1 > progress.total
# The number of items has changed between start and now,
# since there is no good way to predict the final count from
# here, just change the progress bar to an indeterminate one
progress.total = nil
end
progress.log("Processing #{item.id}") if options[:verbose]
Chewy.strategy(:mastodon) do
result = ActiveRecord::Base.connection_pool.with_connection do
yield(item)
ensure
RedisConfiguration.pool.checkin if Thread.current[:redis]
Thread.current[:redis] = nil
end
aggregate.increment(result) if result.is_a?(Integer)
end
rescue => e
progress.log pastel.red("Error processing #{item.id}: #{e}")
ensure
progress.increment
end
end
total.increment(items.size)
futures.map(&:value)
end
progress.stop
[total.value, aggregate.value]
end
def pastel
@pastel ||= Pastel.new
end
end
end

149
lib/mastodon/cli/ip_blocks.rb Normal file
View File

@ -0,0 +1,149 @@
# frozen_string_literal: true
require 'rubygems/package'
require_relative '../../../config/boot'
require_relative '../../../config/environment'
require_relative 'helper'
module Mastodon::CLI
class IpBlocks < Thor
def self.exit_on_failure?
true
end
option :severity, required: true, enum: %w(no_access sign_up_requires_approval sign_up_block), desc: 'Severity of the block'
option :comment, aliases: [:c], desc: 'Optional comment'
option :duration, aliases: [:d], type: :numeric, desc: 'Duration of the block in seconds'
option :force, type: :boolean, aliases: [:f], desc: 'Overwrite existing blocks'
desc 'add IP...', 'Add one or more IP blocks'
long_desc <<-LONG_DESC
Add one or more IP blocks. You can use CIDR syntax to
block IP ranges. You must specify --severity of the block. All
options will be copied for each IP block you create in one command.
You can add a --comment. If an IP block already exists for one of
the provided IPs, it will be skipped unless you use the --force
option to overwrite it.
LONG_DESC
def add(*addresses)
if addresses.empty?
say('No IP(s) given', :red)
exit(1)
end
skipped = 0
processed = 0
failed = 0
addresses.each do |address|
unless valid_ip_address?(address)
say("#{address} is invalid", :red)
failed += 1
next
end
ip_block = IpBlock.find_by(ip: address)
if ip_block.present? && !options[:force]
say("#{address} is already blocked", :yellow)
skipped += 1
next
end
ip_block ||= IpBlock.new(ip: address)
ip_block.severity = options[:severity]
ip_block.comment = options[:comment] if options[:comment].present?
ip_block.expires_in = options[:duration]
if ip_block.save
processed += 1
else
say("#{address} could not be saved", :red)
failed += 1
end
end
say("Added #{processed}, skipped #{skipped}, failed #{failed}", color(processed, failed))
end
option :force, type: :boolean, aliases: [:f], desc: 'Remove blocks for ranges that cover given IP(s)'
desc 'remove IP...', 'Remove one or more IP blocks'
long_desc <<-LONG_DESC
Remove one or more IP blocks. Normally, only exact matches are removed. If
you want to ensure that all of the given IP addresses are unblocked, you
can use --force which will also remove any blocks for IP ranges that would
cover the given IP(s).
LONG_DESC
def remove(*addresses)
if addresses.empty?
say('No IP(s) given', :red)
exit(1)
end
processed = 0
skipped = 0
addresses.each do |address|
unless valid_ip_address?(address)
say("#{address} is invalid", :yellow)
skipped += 1
next
end
ip_blocks = if options[:force]
IpBlock.where('ip >>= ?', address)
else
IpBlock.where('ip <<= ?', address)
end
if ip_blocks.empty?
say("#{address} is not yet blocked", :yellow)
skipped += 1
next
end
ip_blocks.in_batches.destroy_all
processed += 1
end
say("Removed #{processed}, skipped #{skipped}", color(processed, 0))
end
option :format, aliases: [:f], enum: %w(plain nginx), desc: 'Format of the output'
desc 'export', 'Export blocked IPs'
long_desc <<-LONG_DESC
Export blocked IPs. Different formats are supported for usage with other
tools. Only blocks with no_access severity are returned.
LONG_DESC
def export
IpBlock.where(severity: :no_access).find_each do |ip_block|
case options[:format]
when 'nginx'
say "deny #{ip_block.ip}/#{ip_block.ip.prefix};"
else
say "#{ip_block.ip}/#{ip_block.ip.prefix}"
end
end
end
private
def color(processed, failed)
if !processed.zero? && failed.zero?
:green
elsif failed.zero?
:yellow
else
:red
end
end
def valid_ip_address?(ip_address)
IPAddr.new(ip_address)
true
rescue IPAddr::InvalidAddressError
false
end
end
end

160
lib/mastodon/cli/main.rb Normal file
View File

@ -0,0 +1,160 @@
# frozen_string_literal: true
require 'thor'
require_relative 'media'
require_relative 'emoji'
require_relative 'accounts'
require_relative 'feeds'
require_relative 'search'
require_relative 'settings'
require_relative 'statuses'
require_relative 'domains'
require_relative 'preview_cards'
require_relative 'cache'
require_relative 'upgrade'
require_relative 'email_domain_blocks'
require_relative 'canonical_email_blocks'
require_relative 'ip_blocks'
require_relative 'maintenance'
require_relative '../version'
module Mastodon::CLI
class Main < Thor
def self.exit_on_failure?
true
end
desc 'media SUBCOMMAND ...ARGS', 'Manage media files'
subcommand 'media', Media
desc 'emoji SUBCOMMAND ...ARGS', 'Manage custom emoji'
subcommand 'emoji', Emoji
desc 'accounts SUBCOMMAND ...ARGS', 'Manage accounts'
subcommand 'accounts', Accounts
desc 'feeds SUBCOMMAND ...ARGS', 'Manage feeds'
subcommand 'feeds', Feeds
desc 'search SUBCOMMAND ...ARGS', 'Manage the search engine'
subcommand 'search', Search
desc 'settings SUBCOMMAND ...ARGS', 'Manage dynamic settings'
subcommand 'settings', Settings
desc 'statuses SUBCOMMAND ...ARGS', 'Manage statuses'
subcommand 'statuses', Statuses
desc 'domains SUBCOMMAND ...ARGS', 'Manage account domains'
subcommand 'domains', Domains
desc 'preview_cards SUBCOMMAND ...ARGS', 'Manage preview cards'
subcommand 'preview_cards', PreviewCards
desc 'cache SUBCOMMAND ...ARGS', 'Manage cache'
subcommand 'cache', Cache
desc 'upgrade SUBCOMMAND ...ARGS', 'Various version upgrade utilities'
subcommand 'upgrade', Upgrade
desc 'email_domain_blocks SUBCOMMAND ...ARGS', 'Manage e-mail domain blocks'
subcommand 'email_domain_blocks', EmailDomainBlocks
desc 'ip_blocks SUBCOMMAND ...ARGS', 'Manage IP blocks'
subcommand 'ip_blocks', IpBlocks
desc 'canonical_email_blocks SUBCOMMAND ...ARGS', 'Manage canonical e-mail blocks'
subcommand 'canonical_email_blocks', CanonicalEmailBlocks
desc 'maintenance SUBCOMMAND ...ARGS', 'Various maintenance utilities'
subcommand 'maintenance', Maintenance
option :dry_run, type: :boolean
desc 'self-destruct', 'Erase the server from the federation'
long_desc <<~LONG_DESC
Erase the server from the federation by broadcasting account delete
activities to all known other servers. This allows a "clean exit" from
running a Mastodon server, as it leaves next to no cache behind on
other servers.
This command is always interactive and requires confirmation twice.
No local data is actually deleted, because emptying the
database or removing files is much faster through other, external
means, such as e.g. deleting the entire VPS. However, because other
servers will delete data about local users, but no local data will be
updated (such as e.g. followers), there will be a state mismatch
that will lead to glitches and issues if you then continue to run and use
the server.
So either you know exactly what you are doing, or you are starting
from a blank slate afterwards by manually clearing out all the local
data!
LONG_DESC
def self_destruct
require 'tty-prompt'
prompt = TTY::Prompt.new
exit(1) unless prompt.ask('Type in the domain of the server to confirm:', required: true) == Rails.configuration.x.local_domain
unless options[:dry_run]
prompt.warn('This operation WILL NOT be reversible. It can also take a long time.')
prompt.warn('While the data won\'t be erased locally, the server will be in a BROKEN STATE afterwards.')
prompt.warn('A running Sidekiq process is required. Do not shut it down until queues clear.')
exit(1) if prompt.no?('Are you sure you want to proceed?')
end
inboxes = Account.inboxes
processed = 0
dry_run = options[:dry_run] ? ' (DRY RUN)' : ''
Setting.registrations_mode = 'none' unless options[:dry_run]
if inboxes.empty?
Account.local.without_suspended.in_batches.update_all(suspended_at: Time.now.utc, suspension_origin: :local) unless options[:dry_run]
prompt.ok('It seems like your server has not federated with anything')
prompt.ok('You can shut it down and delete it any time')
return
end
prompt.warn('Do NOT interrupt this process...')
delete_account = lambda do |account|
payload = ActiveModelSerializers::SerializableResource.new(
account,
serializer: ActivityPub::DeleteActorSerializer,
adapter: ActivityPub::Adapter
).as_json
json = Oj.dump(ActivityPub::LinkedDataSignature.new(payload).sign!(account))
unless options[:dry_run]
ActivityPub::DeliveryWorker.push_bulk(inboxes, limit: 1_000) do |inbox_url|
[json, account.id, inbox_url]
end
account.suspend!(block_email: false)
end
processed += 1
end
Account.local.without_suspended.find_each { |account| delete_account.call(account) }
Account.local.suspended.joins(:deletion_request).find_each { |account| delete_account.call(account) }
prompt.ok("Queued #{inboxes.size * processed} items into Sidekiq for #{processed} accounts#{dry_run}")
prompt.ok('Wait until Sidekiq processes all items, then you can shut everything down and delete the data')
rescue TTY::Reader::InputInterrupt
exit(1)
end
map %w(--version -v) => :version
desc 'version', 'Show version'
def version
say(Mastodon::Version.to_s)
end
end
end

671
lib/mastodon/cli/maintenance.rb Normal file
View File

@ -0,0 +1,671 @@
# frozen_string_literal: true
require 'tty-prompt'
require_relative '../../../config/boot'
require_relative '../../../config/environment'
require_relative 'helper'
module Mastodon::CLI
class Maintenance < Thor
include Helper
def self.exit_on_failure?
true
end
MIN_SUPPORTED_VERSION = 2019_10_01_213028
MAX_SUPPORTED_VERSION = 2022_11_04_133904
# Stubs to enjoy ActiveRecord queries while not depending on a particular
# version of the code/database
class Status < ApplicationRecord; end
class StatusPin < ApplicationRecord; end
class Poll < ApplicationRecord; end
class Report < ApplicationRecord; end
class Tombstone < ApplicationRecord; end
class Favourite < ApplicationRecord; end
class Follow < ApplicationRecord; end
class FollowRequest < ApplicationRecord; end
class Block < ApplicationRecord; end
class Mute < ApplicationRecord; end
class AccountIdentityProof < ApplicationRecord; end
class AccountModerationNote < ApplicationRecord; end
class AccountPin < ApplicationRecord; end
class ListAccount < ApplicationRecord; end
class PollVote < ApplicationRecord; end
class Mention < ApplicationRecord; end
class AccountDomainBlock < ApplicationRecord; end
class AnnouncementReaction < ApplicationRecord; end
class FeaturedTag < ApplicationRecord; end
class CustomEmoji < ApplicationRecord; end
class CustomEmojiCategory < ApplicationRecord; end
class Bookmark < ApplicationRecord; end
class WebauthnCredential < ApplicationRecord; end
class FollowRecommendationSuppression < ApplicationRecord; end
class CanonicalEmailBlock < ApplicationRecord; end
class Appeal < ApplicationRecord; end
class Webhook < ApplicationRecord; end
class PreviewCard < ApplicationRecord
self.inheritance_column = false
end
class MediaAttachment < ApplicationRecord
self.inheritance_column = nil
end
class AccountStat < ApplicationRecord
belongs_to :account, inverse_of: :account_stat
end
# Dummy class, to make migration possible across version changes
class Account < ApplicationRecord
has_one :user, inverse_of: :account
has_one :account_stat, inverse_of: :account
scope :local, -> { where(domain: nil) }
def local?
domain.nil?
end
def acct
local? ? username : "#{username}@#{domain}"
end
# This is a duplicate of the AccountMerging concern because we need it to
# be independent from code version.
def merge_with!(other_account)
# Since it's the same remote resource, the remote resource likely
# already believes we are following/blocking, so it's safe to
# re-attribute the relationships too. However, during the presence
# of the index bug users could have *also* followed the reference
# account already, therefore mass update will not work and we need
# to check for (and skip past) uniqueness errors
owned_classes = [
Status, StatusPin, MediaAttachment, Poll, Report, Tombstone, Favourite,
Follow, FollowRequest, Block, Mute,
AccountModerationNote, AccountPin, AccountStat, ListAccount,
PollVote, Mention
]
owned_classes << AccountDeletionRequest if ActiveRecord::Base.connection.table_exists?(:account_deletion_requests)
owned_classes << AccountNote if ActiveRecord::Base.connection.table_exists?(:account_notes)
owned_classes << FollowRecommendationSuppression if ActiveRecord::Base.connection.table_exists?(:follow_recommendation_suppressions)
owned_classes << AccountIdentityProof if ActiveRecord::Base.connection.table_exists?(:account_identity_proofs)
owned_classes << Appeal if ActiveRecord::Base.connection.table_exists?(:appeals)
owned_classes.each do |klass|
klass.where(account_id: other_account.id).find_each do |record|
record.update_attribute(:account_id, id)
rescue ActiveRecord::RecordNotUnique
next
end
end
target_classes = [Follow, FollowRequest, Block, Mute, AccountModerationNote, AccountPin]
target_classes << AccountNote if ActiveRecord::Base.connection.table_exists?(:account_notes)
target_classes.each do |klass|
klass.where(target_account_id: other_account.id).find_each do |record|
record.update_attribute(:target_account_id, id)
rescue ActiveRecord::RecordNotUnique
next
end
end
if ActiveRecord::Base.connection.table_exists?(:canonical_email_blocks)
CanonicalEmailBlock.where(reference_account_id: other_account.id).find_each do |record|
record.update_attribute(:reference_account_id, id)
end
end
if ActiveRecord::Base.connection.table_exists?(:appeals)
Appeal.where(account_warning_id: other_account.id).find_each do |record|
record.update_attribute(:account_warning_id, id)
end
end
end
end
class User < ApplicationRecord
belongs_to :account, inverse_of: :user
end
desc 'fix-duplicates', 'Fix duplicates in database and rebuild indexes'
long_desc <<~LONG_DESC
Delete or merge duplicate accounts, statuses, emojis, etc. and rebuild indexes.
This is useful if your database indexes are corrupted because of issues such as https://wiki.postgresql.org/wiki/Locale_data_changes
Mastodon has to be stopped to run this task, which will take a long time and may be destructive.
LONG_DESC
def fix_duplicates
@prompt = TTY::Prompt.new
if ActiveRecord::Migrator.current_version < MIN_SUPPORTED_VERSION
@prompt.error 'Your version of the database schema is too old and is not supported by this script.'
@prompt.error 'Please update to at least Mastodon 3.0.0 before running this script.'
exit(1)
elsif ActiveRecord::Migrator.current_version > MAX_SUPPORTED_VERSION
@prompt.warn 'Your version of the database schema is more recent than this script, this may cause unexpected errors.'
exit(1) unless @prompt.yes?('Continue anyway? (Yes/No)')
end
if Sidekiq::ProcessSet.new.any?
@prompt.error 'It seems Sidekiq is running. All Mastodon processes need to be stopped when using this script.'
exit(1)
end
@prompt.warn 'This task will take a long time to run and is potentially destructive.'
@prompt.warn 'Please make sure to stop Mastodon and have a backup.'
exit(1) unless @prompt.yes?('Continue? (Yes/No)')
deduplicate_users!
deduplicate_account_domain_blocks!
deduplicate_account_identity_proofs!
deduplicate_announcement_reactions!
deduplicate_conversations!
deduplicate_custom_emojis!
deduplicate_custom_emoji_categories!
deduplicate_domain_allows!
deduplicate_domain_blocks!
deduplicate_unavailable_domains!
deduplicate_email_domain_blocks!
deduplicate_media_attachments!
deduplicate_preview_cards!
deduplicate_statuses!
deduplicate_accounts!
deduplicate_tags!
deduplicate_webauthn_credentials!
deduplicate_webhooks!
Scenic.database.refresh_materialized_view('instances', concurrently: true, cascade: false) if ActiveRecord::Migrator.current_version >= 2020_12_06_004238
Rails.cache.clear
@prompt.say 'Finished!'
end
private
def deduplicate_accounts!
remove_index_if_exists!(:accounts, 'index_accounts_on_username_and_domain_lower')
@prompt.say 'Deduplicating accounts… for local accounts, you will be asked to chose which account to keep unchanged.'
find_duplicate_accounts.each do |row|
accounts = Account.where(id: row['ids'].split(',')).to_a
if accounts.first.local?
deduplicate_local_accounts!(accounts)
else
deduplicate_remote_accounts!(accounts)
end
end
@prompt.say 'Restoring index_accounts_on_username_and_domain_lower…'
if ActiveRecord::Migrator.current_version < 2020_06_20_164023
ActiveRecord::Base.connection.add_index :accounts, 'lower (username), lower(domain)', name: 'index_accounts_on_username_and_domain_lower', unique: true
else
ActiveRecord::Base.connection.add_index :accounts, "lower (username), COALESCE(lower(domain), '')", name: 'index_accounts_on_username_and_domain_lower', unique: true
end
@prompt.say 'Reindexing textual indexes on accounts…'
ActiveRecord::Base.connection.execute('REINDEX INDEX search_index;')
ActiveRecord::Base.connection.execute('REINDEX INDEX index_accounts_on_uri;')
ActiveRecord::Base.connection.execute('REINDEX INDEX index_accounts_on_url;')
end
def deduplicate_users!
remove_index_if_exists!(:users, 'index_users_on_confirmation_token')
remove_index_if_exists!(:users, 'index_users_on_email')
remove_index_if_exists!(:users, 'index_users_on_remember_token')
remove_index_if_exists!(:users, 'index_users_on_reset_password_token')
@prompt.say 'Deduplicating user records…'
# Deduplicating email
ActiveRecord::Base.connection.select_all("SELECT string_agg(id::text, ',') AS ids FROM users GROUP BY email HAVING count(*) > 1").each do |row|
users = User.where(id: row['ids'].split(',')).sort_by(&:updated_at).reverse
ref_user = users.shift
@prompt.warn "Multiple users registered with e-mail address #{ref_user.email}."
@prompt.warn "e-mail will be disabled for the following accounts: #{user.map(&:account).map(&:acct).join(', ')}"
@prompt.warn 'Please reach out to them and set another address with `tootctl account modify` or delete them.'
i = 0
users.each do |user|
user.update!(email: "#{i} " + user.email)
end
end
ActiveRecord::Base.connection.select_all("SELECT string_agg(id::text, ',') AS ids FROM users WHERE confirmation_token IS NOT NULL GROUP BY confirmation_token HAVING count(*) > 1").each do |row|
users = User.where(id: row['ids'].split(',')).sort_by(&:created_at).reverse.drop(1)
@prompt.warn "Unsetting confirmation token for those accounts: #{users.map(&:account).map(&:acct).join(', ')}"
users.each do |user|
user.update!(confirmation_token: nil)
end
end
if ActiveRecord::Migrator.current_version < 2022_01_18_183010
ActiveRecord::Base.connection.select_all("SELECT string_agg(id::text, ',') AS ids FROM users WHERE remember_token IS NOT NULL GROUP BY remember_token HAVING count(*) > 1").each do |row|
users = User.where(id: row['ids'].split(',')).sort_by(&:updated_at).reverse.drop(1)
@prompt.warn "Unsetting remember token for those accounts: #{users.map(&:account).map(&:acct).join(', ')}"
users.each do |user|
user.update!(remember_token: nil)
end
end
end
ActiveRecord::Base.connection.select_all("SELECT string_agg(id::text, ',') AS ids FROM users WHERE reset_password_token IS NOT NULL GROUP BY reset_password_token HAVING count(*) > 1").each do |row|
users = User.where(id: row['ids'].split(',')).sort_by(&:updated_at).reverse.drop(1)
@prompt.warn "Unsetting password reset token for those accounts: #{users.map(&:account).map(&:acct).join(', ')}"
users.each do |user|
user.update!(reset_password_token: nil)
end
end
@prompt.say 'Restoring users indexes…'
ActiveRecord::Base.connection.add_index :users, ['confirmation_token'], name: 'index_users_on_confirmation_token', unique: true
ActiveRecord::Base.connection.add_index :users, ['email'], name: 'index_users_on_email', unique: true
ActiveRecord::Base.connection.add_index :users, ['remember_token'], name: 'index_users_on_remember_token', unique: true if ActiveRecord::Migrator.current_version < 2022_01_18_183010
if ActiveRecord::Migrator.current_version < 2022_03_10_060641
ActiveRecord::Base.connection.add_index :users, ['reset_password_token'], name: 'index_users_on_reset_password_token', unique: true
else
ActiveRecord::Base.connection.add_index :users, ['reset_password_token'], name: 'index_users_on_reset_password_token', unique: true, where: 'reset_password_token IS NOT NULL', opclass: :text_pattern_ops
end
end
def deduplicate_account_domain_blocks!
remove_index_if_exists!(:account_domain_blocks, 'index_account_domain_blocks_on_account_id_and_domain')
@prompt.say 'Removing duplicate account domain blocks…'
ActiveRecord::Base.connection.select_all("SELECT string_agg(id::text, ',') AS ids FROM account_domain_blocks GROUP BY account_id, domain HAVING count(*) > 1").each do |row|
AccountDomainBlock.where(id: row['ids'].split(',').drop(1)).delete_all
end
@prompt.say 'Restoring account domain blocks indexes…'
ActiveRecord::Base.connection.add_index :account_domain_blocks, %w(account_id domain), name: 'index_account_domain_blocks_on_account_id_and_domain', unique: true
end
def deduplicate_account_identity_proofs!
return unless ActiveRecord::Base.connection.table_exists?(:account_identity_proofs)
remove_index_if_exists!(:account_identity_proofs, 'index_account_proofs_on_account_and_provider_and_username')
@prompt.say 'Removing duplicate account identity proofs…'
ActiveRecord::Base.connection.select_all("SELECT string_agg(id::text, ',') AS ids FROM account_identity_proofs GROUP BY account_id, provider, provider_username HAVING count(*) > 1").each do |row|
AccountIdentityProof.where(id: row['ids'].split(',')).sort_by(&:id).reverse.drop(1).each(&:destroy)
end
@prompt.say 'Restoring account identity proofs indexes…'
ActiveRecord::Base.connection.add_index :account_identity_proofs, %w(account_id provider provider_username), name: 'index_account_proofs_on_account_and_provider_and_username', unique: true
end
def deduplicate_announcement_reactions!
return unless ActiveRecord::Base.connection.table_exists?(:announcement_reactions)
remove_index_if_exists!(:announcement_reactions, 'index_announcement_reactions_on_account_id_and_announcement_id')
@prompt.say 'Removing duplicate account identity proofs…'
ActiveRecord::Base.connection.select_all("SELECT string_agg(id::text, ',') AS ids FROM announcement_reactions GROUP BY account_id, announcement_id, name HAVING count(*) > 1").each do |row|
AnnouncementReaction.where(id: row['ids'].split(',')).sort_by(&:id).reverse.drop(1).each(&:destroy)
end
@prompt.say 'Restoring announcement_reactions indexes…'
ActiveRecord::Base.connection.add_index :announcement_reactions, %w(account_id announcement_id name), name: 'index_announcement_reactions_on_account_id_and_announcement_id', unique: true
end
def deduplicate_conversations!
remove_index_if_exists!(:conversations, 'index_conversations_on_uri')
@prompt.say 'Deduplicating conversations…'
ActiveRecord::Base.connection.select_all("SELECT string_agg(id::text, ',') AS ids FROM conversations WHERE uri IS NOT NULL GROUP BY uri HAVING count(*) > 1").each do |row|
conversations = Conversation.where(id: row['ids'].split(',')).sort_by(&:id).reverse
ref_conversation = conversations.shift
conversations.each do |other|
merge_conversations!(ref_conversation, other)
other.destroy
end
end
@prompt.say 'Restoring conversations indexes…'
if ActiveRecord::Migrator.current_version < 2022_03_07_083603
ActiveRecord::Base.connection.add_index :conversations, ['uri'], name: 'index_conversations_on_uri', unique: true
else
ActiveRecord::Base.connection.add_index :conversations, ['uri'], name: 'index_conversations_on_uri', unique: true, where: 'uri IS NOT NULL', opclass: :text_pattern_ops
end
end
def deduplicate_custom_emojis!
remove_index_if_exists!(:custom_emojis, 'index_custom_emojis_on_shortcode_and_domain')
@prompt.say 'Deduplicating custom_emojis…'
ActiveRecord::Base.connection.select_all("SELECT string_agg(id::text, ',') AS ids FROM custom_emojis GROUP BY shortcode, domain HAVING count(*) > 1").each do |row|
emojis = CustomEmoji.where(id: row['ids'].split(',')).sort_by(&:id).reverse
ref_emoji = emojis.shift
emojis.each do |other|
merge_custom_emojis!(ref_emoji, other)
other.destroy
end
end
@prompt.say 'Restoring custom_emojis indexes…'
ActiveRecord::Base.connection.add_index :custom_emojis, %w(shortcode domain), name: 'index_custom_emojis_on_shortcode_and_domain', unique: true
end
def deduplicate_custom_emoji_categories!
remove_index_if_exists!(:custom_emoji_categories, 'index_custom_emoji_categories_on_name')
@prompt.say 'Deduplicating custom_emoji_categories…'
ActiveRecord::Base.connection.select_all("SELECT string_agg(id::text, ',') AS ids FROM custom_emoji_categories GROUP BY name HAVING count(*) > 1").each do |row|
categories = CustomEmojiCategory.where(id: row['ids'].split(',')).sort_by(&:id).reverse
ref_category = categories.shift
categories.each do |other|
merge_custom_emoji_categories!(ref_category, other)
other.destroy
end
end
@prompt.say 'Restoring custom_emoji_categories indexes…'
ActiveRecord::Base.connection.add_index :custom_emoji_categories, ['name'], name: 'index_custom_emoji_categories_on_name', unique: true
end
def deduplicate_domain_allows!
remove_index_if_exists!(:domain_allows, 'index_domain_allows_on_domain')
@prompt.say 'Deduplicating domain_allows…'
ActiveRecord::Base.connection.select_all("SELECT string_agg(id::text, ',') AS ids FROM domain_allows GROUP BY domain HAVING count(*) > 1").each do |row|
DomainAllow.where(id: row['ids'].split(',')).sort_by(&:id).reverse.drop(1).each(&:destroy)
end
@prompt.say 'Restoring domain_allows indexes…'
ActiveRecord::Base.connection.add_index :domain_allows, ['domain'], name: 'index_domain_allows_on_domain', unique: true
end
def deduplicate_domain_blocks!
remove_index_if_exists!(:domain_blocks, 'index_domain_blocks_on_domain')
@prompt.say 'Deduplicating domain_allows…'
ActiveRecord::Base.connection.select_all("SELECT string_agg(id::text, ',') AS ids FROM domain_blocks GROUP BY domain HAVING count(*) > 1").each do |row|
domain_blocks = DomainBlock.where(id: row['ids'].split(',')).by_severity.reverse.to_a
reject_media = domain_blocks.any?(&:reject_media?)
reject_reports = domain_blocks.any?(&:reject_reports?)
reference_block = domain_blocks.shift
private_comment = domain_blocks.reduce(reference_block.private_comment.presence) { |a, b| a || b.private_comment.presence }
public_comment = domain_blocks.reduce(reference_block.public_comment.presence) { |a, b| a || b.public_comment.presence }
reference_block.update!(reject_media: reject_media, reject_reports: reject_reports, private_comment: private_comment, public_comment: public_comment)
domain_blocks.each(&:destroy)
end
@prompt.say 'Restoring domain_blocks indexes…'
ActiveRecord::Base.connection.add_index :domain_blocks, ['domain'], name: 'index_domain_blocks_on_domain', unique: true
end
def deduplicate_unavailable_domains!
return unless ActiveRecord::Base.connection.table_exists?(:unavailable_domains)
remove_index_if_exists!(:unavailable_domains, 'index_unavailable_domains_on_domain')
@prompt.say 'Deduplicating unavailable_domains…'
ActiveRecord::Base.connection.select_all("SELECT string_agg(id::text, ',') AS ids FROM unavailable_domains GROUP BY domain HAVING count(*) > 1").each do |row|
UnavailableDomain.where(id: row['ids'].split(',')).sort_by(&:id).reverse.drop(1).each(&:destroy)
end
@prompt.say 'Restoring domain_allows indexes…'
ActiveRecord::Base.connection.add_index :unavailable_domains, ['domain'], name: 'index_unavailable_domains_on_domain', unique: true
end
def deduplicate_email_domain_blocks!
remove_index_if_exists!(:email_domain_blocks, 'index_email_domain_blocks_on_domain')
@prompt.say 'Deduplicating email_domain_blocks…'
ActiveRecord::Base.connection.select_all("SELECT string_agg(id::text, ',') AS ids FROM email_domain_blocks GROUP BY domain HAVING count(*) > 1").each do |row|
domain_blocks = EmailDomainBlock.where(id: row['ids'].split(',')).sort_by { |b| b.parent.nil? ? 1 : 0 }.to_a
domain_blocks.drop(1).each(&:destroy)
end
@prompt.say 'Restoring email_domain_blocks indexes…'
ActiveRecord::Base.connection.add_index :email_domain_blocks, ['domain'], name: 'index_email_domain_blocks_on_domain', unique: true
end
def deduplicate_media_attachments!
remove_index_if_exists!(:media_attachments, 'index_media_attachments_on_shortcode')
@prompt.say 'Deduplicating media_attachments…'
ActiveRecord::Base.connection.select_all("SELECT string_agg(id::text, ',') AS ids FROM media_attachments WHERE shortcode IS NOT NULL GROUP BY shortcode HAVING count(*) > 1").each do |row|
MediaAttachment.where(id: row['ids'].split(',').drop(1)).update_all(shortcode: nil)
end
@prompt.say 'Restoring media_attachments indexes…'
if ActiveRecord::Migrator.current_version < 2022_03_10_060626
ActiveRecord::Base.connection.add_index :media_attachments, ['shortcode'], name: 'index_media_attachments_on_shortcode', unique: true
else
ActiveRecord::Base.connection.add_index :media_attachments, ['shortcode'], name: 'index_media_attachments_on_shortcode', unique: true, where: 'shortcode IS NOT NULL', opclass: :text_pattern_ops
end
end
def deduplicate_preview_cards!
remove_index_if_exists!(:preview_cards, 'index_preview_cards_on_url')
@prompt.say 'Deduplicating preview_cards…'
ActiveRecord::Base.connection.select_all("SELECT string_agg(id::text, ',') AS ids FROM preview_cards GROUP BY url HAVING count(*) > 1").each do |row|
PreviewCard.where(id: row['ids'].split(',')).sort_by(&:id).reverse.drop(1).each(&:destroy)
end
@prompt.say 'Restoring preview_cards indexes…'
ActiveRecord::Base.connection.add_index :preview_cards, ['url'], name: 'index_preview_cards_on_url', unique: true
end
def deduplicate_statuses!
remove_index_if_exists!(:statuses, 'index_statuses_on_uri')
@prompt.say 'Deduplicating statuses…'
ActiveRecord::Base.connection.select_all("SELECT string_agg(id::text, ',') AS ids FROM statuses WHERE uri IS NOT NULL GROUP BY uri HAVING count(*) > 1").each do |row|
statuses = Status.where(id: row['ids'].split(',')).sort_by(&:id)
ref_status = statuses.shift
statuses.each do |status|
merge_statuses!(ref_status, status) if status.account_id == ref_status.account_id
status.destroy
end
end
@prompt.say 'Restoring statuses indexes…'
if ActiveRecord::Migrator.current_version < 2022_03_10_060706
ActiveRecord::Base.connection.add_index :statuses, ['uri'], name: 'index_statuses_on_uri', unique: true
else
ActiveRecord::Base.connection.add_index :statuses, ['uri'], name: 'index_statuses_on_uri', unique: true, where: 'uri IS NOT NULL', opclass: :text_pattern_ops
end
end
def deduplicate_tags!
remove_index_if_exists!(:tags, 'index_tags_on_name_lower')
remove_index_if_exists!(:tags, 'index_tags_on_name_lower_btree')
@prompt.say 'Deduplicating tags…'
ActiveRecord::Base.connection.select_all("SELECT string_agg(id::text, ',') AS ids FROM tags GROUP BY lower((name)::text) HAVING count(*) > 1").each do |row|
tags = Tag.where(id: row['ids'].split(',')).sort_by { |t| [t.usable?, t.trendable?, t.listable?].count(false) }
ref_tag = tags.shift
tags.each do |tag|
merge_tags!(ref_tag, tag)
tag.destroy
end
end
@prompt.say 'Restoring tags indexes…'
if ActiveRecord::Migrator.current_version < 2021_04_21_121431
ActiveRecord::Base.connection.add_index :tags, 'lower((name)::text)', name: 'index_tags_on_name_lower', unique: true
else
ActiveRecord::Base.connection.execute 'CREATE UNIQUE INDEX CONCURRENTLY index_tags_on_name_lower_btree ON tags (lower(name) text_pattern_ops)'
end
end
def deduplicate_webauthn_credentials!
return unless ActiveRecord::Base.connection.table_exists?(:webauthn_credentials)
remove_index_if_exists!(:webauthn_credentials, 'index_webauthn_credentials_on_external_id')
@prompt.say 'Deduplicating webauthn_credentials…'
ActiveRecord::Base.connection.select_all("SELECT string_agg(id::text, ',') AS ids FROM webauthn_credentials GROUP BY external_id HAVING count(*) > 1").each do |row|
WebauthnCredential.where(id: row['ids'].split(',')).sort_by(&:id).reverse.drop(1).each(&:destroy)
end
@prompt.say 'Restoring webauthn_credentials indexes…'
ActiveRecord::Base.connection.add_index :webauthn_credentials, ['external_id'], name: 'index_webauthn_credentials_on_external_id', unique: true
end
def deduplicate_webhooks!
return unless ActiveRecord::Base.connection.table_exists?(:webhooks)
remove_index_if_exists!(:webhooks, 'index_webhooks_on_url')
@prompt.say 'Deduplicating webhooks…'
ActiveRecord::Base.connection.select_all("SELECT string_agg(id::text, ',') AS ids FROM webhooks GROUP BY url HAVING count(*) > 1").each do |row|
Webhooks.where(id: row['ids'].split(',')).sort_by(&:id).reverse.drop(1).each(&:destroy)
end
@prompt.say 'Restoring webhooks indexes…'
ActiveRecord::Base.connection.add_index :webhooks, ['url'], name: 'index_webhooks_on_url', unique: true
end
def deduplicate_local_accounts!(accounts)
accounts = accounts.sort_by(&:id).reverse
@prompt.warn "Multiple local accounts were found for username '#{accounts.first.username}'."
@prompt.warn 'All those accounts are distinct accounts but only the most recently-created one is fully-functional.'
accounts.each_with_index do |account, idx|
@prompt.say format('%2d. %s: created at: %s; updated at: %s; last logged in at: %s; statuses: %5d; last status at: %s', idx, account.username, account.created_at, account.updated_at, account.user&.last_sign_in_at&.to_s || 'N/A', account.account_stat&.statuses_count || 0, account.account_stat&.last_status_at || 'N/A')
end
@prompt.say 'Please chose the one to keep unchanged, other ones will be automatically renamed.'
ref_id = @prompt.ask('Account to keep unchanged:') do |q|
q.required true
q.default 0
q.convert :int
end
accounts.delete_at(ref_id)
i = 0
accounts.each do |account|
i += 1
username = account.username + "_#{i}"
while Account.local.exists?(username: username)
i += 1
username = account.username + "_#{i}"
end
account.update!(username: username)
end
end
def deduplicate_remote_accounts!(accounts)
accounts = accounts.sort_by(&:updated_at).reverse
reference_account = accounts.shift
accounts.each do |other_account|
if other_account.public_key == reference_account.public_key
# The accounts definitely point to the same resource, so
# it's safe to re-attribute content and relationships
reference_account.merge_with!(other_account)
end
other_account.destroy
end
end
def merge_conversations!(main_conv, duplicate_conv)
owned_classes = [ConversationMute, AccountConversation]
owned_classes.each do |klass|
klass.where(conversation_id: duplicate_conv.id).find_each do |record|
record.update_attribute(:account_id, main_conv.id)
rescue ActiveRecord::RecordNotUnique
next
end
end
end
def merge_custom_emojis!(main_emoji, duplicate_emoji)
owned_classes = [AnnouncementReaction]
owned_classes.each do |klass|
klass.where(custom_emoji_id: duplicate_emoji.id).update_all(custom_emoji_id: main_emoji.id)
end
end
def merge_custom_emoji_categories!(main_category, duplicate_category)
owned_classes = [CustomEmoji]
owned_classes.each do |klass|
klass.where(category_id: duplicate_category.id).update_all(category_id: main_category.id)
end
end
def merge_statuses!(main_status, duplicate_status)
owned_classes = [Favourite, Mention, Poll]
owned_classes << Bookmark if ActiveRecord::Base.connection.table_exists?(:bookmarks)
owned_classes.each do |klass|
klass.where(status_id: duplicate_status.id).find_each do |record|
record.update_attribute(:status_id, main_status.id)
rescue ActiveRecord::RecordNotUnique
next
end
end
StatusPin.where(account_id: main_status.account_id, status_id: duplicate_status.id).find_each do |record|
record.update_attribute(:status_id, main_status.id)
rescue ActiveRecord::RecordNotUnique
next
end
Status.where(in_reply_to_id: duplicate_status.id).find_each do |record|
record.update_attribute(:in_reply_to_id, main_status.id)
rescue ActiveRecord::RecordNotUnique
next
end
Status.where(reblog_of_id: duplicate_status.id).find_each do |record|
record.update_attribute(:reblog_of_id, main_status.id)
rescue ActiveRecord::RecordNotUnique
next
end
end
def merge_tags!(main_tag, duplicate_tag)
[FeaturedTag].each do |klass|
klass.where(tag_id: duplicate_tag.id).find_each do |record|
record.update_attribute(:tag_id, main_tag.id)
rescue ActiveRecord::RecordNotUnique
next
end
end
end
def find_duplicate_accounts
ActiveRecord::Base.connection.select_all("SELECT string_agg(id::text, ',') AS ids FROM accounts GROUP BY lower(username), COALESCE(lower(domain), '') HAVING count(*) > 1")
end
def remove_index_if_exists!(table, name)
ActiveRecord::Base.connection.remove_index(table, name: name)
rescue ArgumentError, ActiveRecord::StatementInvalid
nil
end
end
end

383
lib/mastodon/cli/media.rb Normal file
View File

@ -0,0 +1,383 @@
# frozen_string_literal: true
require_relative '../../../config/boot'
require_relative '../../../config/environment'
require_relative 'helper'
module Mastodon::CLI
class Media < Thor
include ActionView::Helpers::NumberHelper
include Helper
VALID_PATH_SEGMENTS_SIZE = [7, 10].freeze
def self.exit_on_failure?
true
end
option :days, type: :numeric, default: 7, aliases: [:d]
option :prune_profiles, type: :boolean, default: false
option :remove_headers, type: :boolean, default: false
option :include_follows, type: :boolean, default: false
option :concurrency, type: :numeric, default: 5, aliases: [:c]
option :dry_run, type: :boolean, default: false
desc 'remove', 'Remove remote media files, headers or avatars'
long_desc <<-DESC
Removes locally cached copies of media attachments (and optionally profile
headers and avatars) from other servers. By default, only media attachments
are removed.
The --days option specifies how old media attachments have to be before
they are removed. In case of avatars and headers, it specifies how old
the last webfinger request and update to the user has to be before they
are pruned. It defaults to 7 days.
If --prune-profiles is specified, only avatars and headers are removed.
If --remove-headers is specified, only headers are removed.
If --include-follows is specified along with --prune-profiles or
--remove-headers, all non-local profiles will be pruned irrespective of
follow status. By default, only accounts that are not followed by or
following anyone locally are pruned.
DESC
def remove
if options[:prune_profiles] && options[:remove_headers]
say('--prune-profiles and --remove-headers should not be specified simultaneously', :red, true)
exit(1)
end
if options[:include_follows] && !(options[:prune_profiles] || options[:remove_headers])
say('--include-follows can only be used with --prune-profiles or --remove-headers', :red, true)
exit(1)
end
time_ago = options[:days].days.ago
dry_run = options[:dry_run] ? ' (DRY RUN)' : ''
if options[:prune_profiles] || options[:remove_headers]
processed, aggregate = parallelize_with_progress(Account.remote.where({ last_webfingered_at: ..time_ago, updated_at: ..time_ago })) do |account|
next if !options[:include_follows] && Follow.where(account: account).or(Follow.where(target_account: account)).exists?
next if account.avatar.blank? && account.header.blank?
next if options[:remove_headers] && account.header.blank?
size = (account.header_file_size || 0)
size += (account.avatar_file_size || 0) if options[:prune_profiles]
unless options[:dry_run]
account.header.destroy
account.avatar.destroy if options[:prune_profiles]
account.save!
end
size
end
say("Visited #{processed} accounts and removed profile media totaling #{number_to_human_size(aggregate)}#{dry_run}", :green, true)
end
unless options[:prune_profiles] || options[:remove_headers]
processed, aggregate = parallelize_with_progress(MediaAttachment.cached.where.not(remote_url: '').where(created_at: ..time_ago)) do |media_attachment|
next if media_attachment.file.blank?
size = (media_attachment.file_file_size || 0) + (media_attachment.thumbnail_file_size || 0)
unless options[:dry_run]
media_attachment.file.destroy
media_attachment.thumbnail.destroy
media_attachment.save
end
size
end
say("Removed #{processed} media attachments (approx. #{number_to_human_size(aggregate)})#{dry_run}", :green, true)
end
end
option :start_after
option :prefix
option :fix_permissions, type: :boolean, default: false
option :dry_run, type: :boolean, default: false
desc 'remove-orphans', 'Scan storage and check for files that do not belong to existing media attachments'
long_desc <<~LONG_DESC
Scans file storage for files that do not belong to existing media attachments. Because this operation
requires iterating over every single file individually, it will be slow.
Please mind that some storage providers charge for the necessary API requests to list objects.
LONG_DESC
def remove_orphans
progress = create_progress_bar(nil)
reclaimed_bytes = 0
removed = 0
dry_run = options[:dry_run] ? ' (DRY RUN)' : ''
prefix = options[:prefix]
case Paperclip::Attachment.default_options[:storage]
when :s3
paperclip_instance = MediaAttachment.new.file
s3_interface = paperclip_instance.s3_interface
s3_permissions = Paperclip::Attachment.default_options[:s3_permissions]
bucket = s3_interface.bucket(Paperclip::Attachment.default_options[:s3_credentials][:bucket])
last_key = options[:start_after]
loop do
objects = begin
bucket.objects(start_after: last_key, prefix: prefix).limit(1000).map { |x| x }
rescue => e
progress.log(pastel.red("Error fetching list of files: #{e}"))
progress.log("If you want to continue from this point, add --start-after=#{last_key} to your command") if last_key
break
end
break if objects.empty?
last_key = objects.last.key
record_map = preload_records_from_mixed_objects(objects)
objects.each do |object|
object.acl.put(acl: s3_permissions) if options[:fix_permissions] && !options[:dry_run]
path_segments = object.key.split('/')
path_segments.delete('cache')
unless VALID_PATH_SEGMENTS_SIZE.include?(path_segments.size)
progress.log(pastel.yellow("Unrecognized file found: #{object.key}"))
next
end
model_name = path_segments.first.classify
attachment_name = path_segments[1].singularize
record_id = path_segments[2..-2].join.to_i
file_name = path_segments.last
record = record_map.dig(model_name, record_id)
attachment = record&.public_send(attachment_name)
progress.increment
next unless attachment.blank? || !attachment.variant?(file_name)
begin
object.delete unless options[:dry_run]
reclaimed_bytes += object.size
removed += 1
progress.log("Found and removed orphan: #{object.key}")
rescue => e
progress.log(pastel.red("Error processing #{object.key}: #{e}"))
end
end
end
when :fog
say('The fog storage driver is not supported for this operation at this time', :red)
exit(1)
when :filesystem
require 'find'
root_path = ENV.fetch('PAPERCLIP_ROOT_PATH', File.join(':rails_root', 'public', 'system')).gsub(':rails_root', Rails.root.to_s)
Find.find(File.join(*[root_path, prefix].compact)) do |path|
next if File.directory?(path)
key = path.gsub("#{root_path}#{File::SEPARATOR}", '')
path_segments = key.split(File::SEPARATOR)
path_segments.delete('cache')
unless VALID_PATH_SEGMENTS_SIZE.include?(path_segments.size)
progress.log(pastel.yellow("Unrecognized file found: #{key}"))
next
end
model_name = path_segments.first.classify
record_id = path_segments[2..-2].join.to_i
attachment_name = path_segments[1].singularize
file_name = path_segments.last
next unless PRELOAD_MODEL_WHITELIST.include?(model_name)
record = model_name.constantize.find_by(id: record_id)
attachment = record&.public_send(attachment_name)
progress.increment
next unless attachment.blank? || !attachment.variant?(file_name)
begin
size = File.size(path)
unless options[:dry_run]
File.delete(path)
begin
FileUtils.rmdir(File.dirname(path), parents: true)
rescue Errno::ENOTEMPTY
# OK
end
end
reclaimed_bytes += size
removed += 1
progress.log("Found and removed orphan: #{key}")
rescue => e
progress.log(pastel.red("Error processing #{key}: #{e}"))
end
end
end
progress.total = progress.progress
progress.finish
say("Removed #{removed} orphans (approx. #{number_to_human_size(reclaimed_bytes)})#{dry_run}", :green, true)
end
option :account, type: :string
option :domain, type: :string
option :status, type: :numeric
option :days, type: :numeric
option :concurrency, type: :numeric, default: 5, aliases: [:c]
option :verbose, type: :boolean, default: false, aliases: [:v]
option :dry_run, type: :boolean, default: false
option :force, type: :boolean, default: false
desc 'refresh', 'Fetch remote media files'
long_desc <<-DESC
Re-downloads media attachments from other servers. You must specify the
source of media attachments with one of the following options:
Use the --status option to download attachments from a specific status,
using the status local numeric ID.
Use the --account option to download attachments from a specific account,
using username@domain handle of the account.
Use the --domain option to download attachments from a specific domain.
Use the --days option to limit attachments created within days.
By default, attachments that are believed to be already downloaded will
not be re-downloaded. To force re-download of every URL, use --force.
DESC
def refresh
dry_run = options[:dry_run] ? ' (DRY RUN)' : ''
if options[:status]
scope = MediaAttachment.where(status_id: options[:status])
elsif options[:account]
username, domain = options[:account].split('@')
account = Account.find_remote(username, domain)
if account.nil?
say('No such account', :red)
exit(1)
end
scope = MediaAttachment.where(account_id: account.id)
elsif options[:domain]
scope = MediaAttachment.joins(:account).merge(Account.by_domain_and_subdomains(options[:domain]))
elsif options[:days].present?
scope = MediaAttachment.remote
else
exit(1)
end
scope = scope.where('media_attachments.id > ?', Mastodon::Snowflake.id_at(options[:days].days.ago, with_random: false)) if options[:days].present?
processed, aggregate = parallelize_with_progress(scope) do |media_attachment|
next if media_attachment.remote_url.blank? || (!options[:force] && media_attachment.file_file_name.present?)
next if DomainBlock.reject_media?(media_attachment.account.domain)
unless options[:dry_run]
media_attachment.reset_file!
media_attachment.reset_thumbnail!
media_attachment.save
end
media_attachment.file_file_size + (media_attachment.thumbnail_file_size || 0)
end
say("Downloaded #{processed} media attachments (approx. #{number_to_human_size(aggregate)})#{dry_run}", :green, true)
end
desc 'usage', 'Calculate disk space consumed by Mastodon'
def usage
say("Attachments:\t#{number_to_human_size(MediaAttachment.sum(Arel.sql('COALESCE(file_file_size, 0) + COALESCE(thumbnail_file_size, 0)')))} (#{number_to_human_size(MediaAttachment.where(account: Account.local).sum(Arel.sql('COALESCE(file_file_size, 0) + COALESCE(thumbnail_file_size, 0)')))} local)")
say("Custom emoji:\t#{number_to_human_size(CustomEmoji.sum(:image_file_size))} (#{number_to_human_size(CustomEmoji.local.sum(:image_file_size))} local)")
say("Preview cards:\t#{number_to_human_size(PreviewCard.sum(:image_file_size))}")
say("Avatars:\t#{number_to_human_size(Account.sum(:avatar_file_size))} (#{number_to_human_size(Account.local.sum(:avatar_file_size))} local)")
say("Headers:\t#{number_to_human_size(Account.sum(:header_file_size))} (#{number_to_human_size(Account.local.sum(:header_file_size))} local)")
say("Backups:\t#{number_to_human_size(Backup.sum(:dump_file_size))}")
say("Imports:\t#{number_to_human_size(Import.sum(:data_file_size))}")
say("Settings:\t#{number_to_human_size(SiteUpload.sum(:file_file_size))}")
end
desc 'lookup URL', 'Lookup where media is displayed by passing a media URL'
def lookup(url)
path = Addressable::URI.parse(url).path
path_segments = path.split('/')[2..]
path_segments.delete('cache')
unless VALID_PATH_SEGMENTS_SIZE.include?(path_segments.size)
say('Not a media URL', :red)
exit(1)
end
model_name = path_segments.first.classify
record_id = path_segments[2..-2].join.to_i
unless PRELOAD_MODEL_WHITELIST.include?(model_name)
say("Cannot find corresponding model: #{model_name}", :red)
exit(1)
end
record = model_name.constantize.find_by(id: record_id)
record = record.status if record.respond_to?(:status)
unless record
say('Cannot find corresponding record', :red)
exit(1)
end
display_url = ActivityPub::TagManager.instance.url_for(record)
if display_url.blank?
say('No public URL for this type of record', :red)
exit(1)
end
say(display_url, :blue)
rescue Addressable::URI::InvalidURIError
say('Invalid URL', :red)
exit(1)
end
private
PRELOAD_MODEL_WHITELIST = %w(
Account
Backup
CustomEmoji
Import
MediaAttachment
PreviewCard
SiteUpload
).freeze
def preload_records_from_mixed_objects(objects)
preload_map = Hash.new { |hash, key| hash[key] = [] }
objects.map do |object|
segments = object.key.split('/')
segments.delete('cache')
next unless VALID_PATH_SEGMENTS_SIZE.include?(segments.size)
model_name = segments.first.classify
record_id = segments[2..-2].join.to_i
next unless PRELOAD_MODEL_WHITELIST.include?(model_name)
preload_map[model_name] << record_id
end
preload_map.each_with_object({}) do |(model_name, record_ids), model_map|
model_map[model_name] = model_name.constantize.where(id: record_ids).index_by(&:id)
end
end
end
end

59
lib/mastodon/cli/preview_cards.rb Normal file
View File

@ -0,0 +1,59 @@
# frozen_string_literal: true
require 'tty-prompt'
require_relative '../../../config/boot'
require_relative '../../../config/environment'
require_relative 'helper'
module Mastodon::CLI
class PreviewCards < Thor
include ActionView::Helpers::NumberHelper
include Helper
def self.exit_on_failure?
true
end
option :days, type: :numeric, default: 180
option :concurrency, type: :numeric, default: 5, aliases: [:c]
option :verbose, type: :boolean, aliases: [:v]
option :dry_run, type: :boolean, default: false
option :link, type: :boolean, default: false
desc 'remove', 'Remove preview cards'
long_desc <<-DESC
Removes local thumbnails for preview cards.
The --days option specifies how old preview cards have to be before
they are removed. It defaults to 180 days. Since preview cards will
not be re-fetched unless the link is re-posted after 2 weeks from
last time, it is not recommended to delete preview cards within the
last 14 days.
With the --link option, only link-type preview cards will be deleted,
leaving video and photo cards untouched.
DESC
def remove
time_ago = options[:days].days.ago
dry_run = options[:dry_run] ? ' (DRY RUN)' : ''
link = options[:link] ? 'link-type ' : ''
scope = PreviewCard.cached
scope = scope.where(type: :link) if options[:link]
scope = scope.where('updated_at < ?', time_ago)
processed, aggregate = parallelize_with_progress(scope) do |preview_card|
next if preview_card.image.blank?
size = preview_card.image_file_size
unless options[:dry_run]
preview_card.image.destroy
preview_card.save
end
size
end
say("Removed #{processed} #{link}preview cards (approx. #{number_to_human_size(aggregate)})#{dry_run}", :green, true)
end
end
end

106
lib/mastodon/cli/search.rb Normal file
View File

@ -0,0 +1,106 @@
# frozen_string_literal: true
require_relative '../../../config/boot'
require_relative '../../../config/environment'
require_relative 'helper'
module Mastodon::CLI
class Search < Thor
include Helper
# Indices are sorted by amount of data to be expected in each, so that
# smaller indices can go online sooner
INDICES = [
AccountsIndex,
TagsIndex,
StatusesIndex,
].freeze
option :concurrency, type: :numeric, default: 5, aliases: [:c], desc: 'Workload will be split between this number of threads'
option :batch_size, type: :numeric, default: 100, aliases: [:b], desc: 'Number of records in each batch'
option :only, type: :array, enum: %w(accounts tags statuses), desc: 'Only process these indices'
option :import, type: :boolean, default: true, desc: 'Import data from the database to the index'
option :clean, type: :boolean, default: true, desc: 'Remove outdated documents from the index'
desc 'deploy', 'Create or upgrade Elasticsearch indices and populate them'
long_desc <<~LONG_DESC
If Elasticsearch is empty, this command will create the necessary indices
and then import data from the database into those indices.
This command will also upgrade indices if the underlying schema has been
changed since the last run. Index upgrades erase index data.
Even if creating or upgrading indices is not necessary, data from the
database will be imported into the indices, unless overridden with --no-import.
LONG_DESC
def deploy
if options[:concurrency] < 1
say('Cannot run with this concurrency setting, must be at least 1', :red)
exit(1)
end
if options[:batch_size] < 1
say('Cannot run with this batch_size setting, must be at least 1', :red)
exit(1)
end
indices = if options[:only]
options[:only].map { |str| "#{str.camelize}Index".constantize }
else
INDICES
end
pool = Concurrent::FixedThreadPool.new(options[:concurrency], max_queue: options[:concurrency] * 10)
importers = indices.index_with { |index| "Importer::#{index.name}Importer".constantize.new(batch_size: options[:batch_size], executor: pool) }
progress = ProgressBar.create(total: nil, format: '%t%c/%u |%b%i| %e (%r docs/s)', autofinish: false)
# First, ensure all indices are created and have the correct
# structure, so that live data can already be written
indices.select { |index| index.specification.changed? }.each do |index|
progress.title = "Upgrading #{index} "
index.purge
index.specification.lock!
end
progress.title = 'Estimating workload '
progress.total = indices.sum { |index| importers[index].estimate! }
reset_connection_pools!
added = 0
removed = 0
indices.each do |index|
importer = importers[index]
importer.optimize_for_import!
importer.on_progress do |(indexed, deleted)|
progress.total = nil if progress.progress + indexed + deleted > progress.total
progress.progress += indexed + deleted
added += indexed
removed += deleted
end
importer.on_failure do |reason|
progress.log(pastel.red("Error while importing #{index}: #{reason}"))
end
if options[:import]
progress.title = "Importing #{index} "
importer.import!
end
if options[:clean]
progress.title = "Cleaning #{index} "
importer.clean_up!
end
ensure
importer.optimize_for_search!
end
progress.title = 'Done! '
progress.finish
say("Indexed #{added} records, de-indexed #{removed}", :green, true)
end
end
end

44
lib/mastodon/cli/settings.rb Normal file
View File

@ -0,0 +1,44 @@
# frozen_string_literal: true
require_relative '../../../config/boot'
require_relative '../../../config/environment'
require_relative 'helper'
module Mastodon::CLI
class Registrations < Thor
def self.exit_on_failure?
true
end
desc 'open', 'Open registrations'
def open
Setting.registrations_mode = 'open'
say('OK', :green)
end
desc 'approved', 'Open approval-based registrations'
option :require_reason, type: :boolean, aliases: [:require_invite_text]
long_desc <<~LONG_DESC
Set registrations to require review from staff.
With --require-reason, require users to enter a reason when registering,
otherwise this field is optional.
LONG_DESC
def approved
Setting.registrations_mode = 'approved'
Setting.require_invite_text = options[:require_reason] unless options[:require_reason].nil?
say('OK', :green)
end
desc 'close', 'Close registrations'
def close
Setting.registrations_mode = 'none'
say('OK', :green)
end
end
class Settings < Thor
desc 'registrations SUBCOMMAND ...ARGS', 'Manage state of registrations'
subcommand 'registrations', Registrations
end
end

226
lib/mastodon/cli/statuses.rb Normal file
View File

@ -0,0 +1,226 @@
# frozen_string_literal: true
require_relative '../../../config/boot'
require_relative '../../../config/environment'
require_relative 'helper'
module Mastodon::CLI
class Statuses < Thor
include Helper
include ActionView::Helpers::NumberHelper
def self.exit_on_failure?
true
end
option :days, type: :numeric, default: 90
option :batch_size, type: :numeric, default: 1_000, aliases: [:b], desc: 'Number of records in each batch'
option :continue, type: :boolean, default: false, desc: 'If remove is not completed, execute from the previous continuation'
option :clean_followed, type: :boolean, default: false, desc: 'Include the status of remote accounts that are followed by local accounts as candidates for remove'
option :skip_status_remove, type: :boolean, default: false, desc: 'Skip status remove (run only cleanup tasks)'
option :skip_media_remove, type: :boolean, default: false, desc: 'Skip remove orphaned media attachments'
option :compress_database, type: :boolean, default: false, desc: 'Compress database and update the statistics. This option locks the table for a long time, so run it offline'
desc 'remove', 'Remove unreferenced statuses'
long_desc <<~LONG_DESC
Remove statuses that are not referenced by local user activity, such as
ones that came from relays, or belonging to users that were once followed
by someone locally but no longer are.
It also removes orphaned records and performs additional cleanup tasks
such as updating statistics and recovering disk space.
This is a computationally heavy procedure that creates extra database
indices before commencing, and removes them afterward.
LONG_DESC
def remove
if options[:batch_size] < 1
say('Cannot run with this batch_size setting, must be at least 1', :red)
exit(1)
end
remove_statuses
vacuum_and_analyze_statuses
remove_orphans_media_attachments
remove_orphans_conversations
vacuum_and_analyze_conversations
end
private
def remove_statuses
return if options[:skip_status_remove]
say('Creating temporary database indices...')
ActiveRecord::Base.connection.add_index(:media_attachments, :remote_url, name: :index_media_attachments_remote_url, where: 'remote_url is not null', algorithm: :concurrently, if_not_exists: true)
max_id = Mastodon::Snowflake.id_at(options[:days].days.ago, with_random: false)
start_at = Time.now.to_f
unless options[:continue] && ActiveRecord::Base.connection.table_exists?('statuses_to_be_deleted')
ActiveRecord::Base.connection.add_index(:accounts, :id, name: :index_accounts_local, where: 'domain is null', algorithm: :concurrently, if_not_exists: true)
ActiveRecord::Base.connection.add_index(:status_pins, :status_id, name: :index_status_pins_status_id, algorithm: :concurrently, if_not_exists: true)
say('Extract the deletion target from statuses... This might take a while...')
ActiveRecord::Base.connection.create_table('statuses_to_be_deleted', force: true)
# Skip accounts followed by local accounts
clean_followed_sql = 'AND NOT EXISTS (SELECT 1 FROM follows WHERE statuses.account_id = follows.target_account_id)' unless options[:clean_followed]
ActiveRecord::Base.connection.exec_insert(<<-SQL.squish, 'SQL', [[nil, max_id]])
INSERT INTO statuses_to_be_deleted (id)
SELECT statuses.id FROM statuses WHERE deleted_at IS NULL AND NOT local AND uri IS NOT NULL AND (id < $1)
AND NOT EXISTS (SELECT 1 FROM statuses AS statuses1 WHERE statuses.id = statuses1.in_reply_to_id)
AND NOT EXISTS (SELECT 1 FROM statuses AS statuses1 WHERE statuses1.id = statuses.reblog_of_id AND (statuses1.uri IS NULL OR statuses1.local))
AND NOT EXISTS (SELECT 1 FROM statuses AS statuses1 WHERE statuses.id = statuses1.reblog_of_id AND (statuses1.uri IS NULL OR statuses1.local OR statuses1.id >= $1))
AND NOT EXISTS (SELECT 1 FROM status_pins WHERE statuses.id = status_id)
AND NOT EXISTS (SELECT 1 FROM mentions WHERE statuses.id = mentions.status_id AND mentions.account_id IN (SELECT accounts.id FROM accounts WHERE domain IS NULL))
AND NOT EXISTS (SELECT 1 FROM favourites WHERE statuses.id = favourites.status_id AND favourites.account_id IN (SELECT accounts.id FROM accounts WHERE domain IS NULL))
AND NOT EXISTS (SELECT 1 FROM bookmarks WHERE statuses.id = bookmarks.status_id AND bookmarks.account_id IN (SELECT accounts.id FROM accounts WHERE domain IS NULL))
#{clean_followed_sql}
SQL
say('Removing temporary database indices to restore write performance...')
ActiveRecord::Base.connection.remove_index(:accounts, name: :index_accounts_local, if_exists: true)
ActiveRecord::Base.connection.remove_index(:status_pins, name: :index_status_pins_status_id, if_exists: true)
end
say('Beginning statuses removal... This might take a while...')
klass = Class.new(ApplicationRecord) do |c|
c.table_name = 'statuses_to_be_deleted'
end
Object.const_set(:StatusToBeDeleted, klass)
scope = StatusToBeDeleted
processed = 0
removed = 0
progress = create_progress_bar(scope.count.fdiv(options[:batch_size]).ceil)
scope.reorder(nil).in_batches(of: options[:batch_size]) do |relation|
ids = relation.pluck(:id)
processed += ids.count
removed += Status.unscoped.where(id: ids).delete_all
progress.increment
end
progress.stop
ActiveRecord::Base.connection.drop_table('statuses_to_be_deleted')
say("Done after #{Time.now.to_f - start_at}s, removed #{removed} out of #{processed} statuses.", :green)
ensure
say('Removing temporary database indices to restore write performance...')
ActiveRecord::Base.connection.remove_index(:accounts, name: :index_accounts_local, if_exists: true)
ActiveRecord::Base.connection.remove_index(:status_pins, name: :index_status_pins_status_id, if_exists: true)
ActiveRecord::Base.connection.remove_index(:media_attachments, name: :index_media_attachments_remote_url, if_exists: true)
end
def remove_orphans_media_attachments
return if options[:skip_media_remove]
start_at = Time.now.to_f
say('Beginning removal of now-orphaned media attachments to free up disk space...')
scope = MediaAttachment.reorder(nil).unattached.where('created_at < ?', options[:days].pred.days.ago)
processed = 0
removed = 0
progress = create_progress_bar(scope.count)
scope.find_each do |media_attachment|
media_attachment.destroy!
removed += 1
rescue => e
progress.log pastel.red("Error processing #{media_attachment.id}: #{e}")
ensure
progress.increment
processed += 1
end
progress.stop
say("Done after #{Time.now.to_f - start_at}s, removed #{removed} out of #{processed} media_attachments.", :green)
end
def remove_orphans_conversations
start_at = Time.now.to_f
unless options[:continue] && ActiveRecord::Base.connection.table_exists?('conversations_to_be_deleted')
say('Creating temporary database indices...')
ActiveRecord::Base.connection.add_index(:statuses, :conversation_id, name: :index_statuses_conversation_id, algorithm: :concurrently, if_not_exists: true)
say('Extract the deletion target from conversations... This might take a while...')
ActiveRecord::Base.connection.create_table('conversations_to_be_deleted', force: true)
ActiveRecord::Base.connection.exec_insert(<<-SQL.squish, 'SQL')
INSERT INTO conversations_to_be_deleted (id)
SELECT id FROM conversations WHERE NOT EXISTS (SELECT 1 FROM statuses WHERE statuses.conversation_id = conversations.id)
SQL
say('Removing temporary database indices to restore write performance...')
ActiveRecord::Base.connection.remove_index(:statuses, name: :index_statuses_conversation_id, if_exists: true)
end
say('Beginning orphans removal... This might take a while...')
klass = Class.new(ApplicationRecord) do |c|
c.table_name = 'conversations_to_be_deleted'
end
Object.const_set(:ConversationsToBeDeleted, klass)
scope = ConversationsToBeDeleted
processed = 0
removed = 0
progress = create_progress_bar(scope.count.fdiv(options[:batch_size]).ceil)
scope.in_batches(of: options[:batch_size]) do |relation|
ids = relation.pluck(:id)
processed += ids.count
removed += Conversation.unscoped.where(id: ids).delete_all
progress.increment
end
progress.stop
ActiveRecord::Base.connection.drop_table('conversations_to_be_deleted')
say("Done after #{Time.now.to_f - start_at}s, removed #{removed} out of #{processed} conversations.", :green)
ensure
say('Removing temporary database indices to restore write performance...')
ActiveRecord::Base.connection.remove_index(:statuses, name: :index_statuses_conversation_id, if_exists: true)
end
def vacuum_and_analyze_statuses
if options[:compress_database]
say('Run VACUUM FULL ANALYZE to statuses...')
ActiveRecord::Base.connection.execute('VACUUM FULL ANALYZE statuses')
say('Run REINDEX to statuses...')
ActiveRecord::Base.connection.execute('REINDEX TABLE statuses')
else
say('Run ANALYZE to statuses...')
ActiveRecord::Base.connection.execute('ANALYZE statuses')
end
end
def vacuum_and_analyze_conversations
if options[:compress_database]
say('Run VACUUM FULL ANALYZE to conversations...')
ActiveRecord::Base.connection.execute('VACUUM FULL ANALYZE conversations')
say('Run REINDEX to conversations...')
ActiveRecord::Base.connection.execute('REINDEX TABLE conversations')
else
say('Run ANALYZE to conversations...')
ActiveRecord::Base.connection.execute('ANALYZE conversations')
end
end
end
end

161
lib/mastodon/cli/upgrade.rb Normal file
View File

@ -0,0 +1,161 @@
# frozen_string_literal: true
require_relative '../../../config/boot'
require_relative '../../../config/environment'
require_relative 'helper'
module Mastodon::CLI
class Upgrade < Thor
include Helper
def self.exit_on_failure?
true
end
CURRENT_STORAGE_SCHEMA_VERSION = 1
option :dry_run, type: :boolean, default: false
option :verbose, type: :boolean, default: false, aliases: [:v]
desc 'storage-schema', 'Upgrade storage schema of various file attachments to the latest version'
long_desc <<~LONG_DESC
Iterates over every file attachment of every record and, if its storage schema is outdated, performs the
necessary upgrade to the latest one. In practice this means e.g. moving files to different directories.
Will most likely take a long time.
LONG_DESC
def storage_schema
progress = create_progress_bar(nil)
dry_run = dry_run? ? ' (DRY RUN)' : ''
records = 0
klasses = [
Account,
CustomEmoji,
MediaAttachment,
PreviewCard,
]
klasses.each do |klass|
attachment_names = klass.attachment_definitions.keys
klass.find_each do |record|
attachment_names.each do |attachment_name|
attachment = record.public_send(attachment_name)
upgraded = false
next if attachment.blank? || attachment.storage_schema_version >= CURRENT_STORAGE_SCHEMA_VERSION
styles = attachment.styles.keys
styles << :original unless styles.include?(:original)
styles.each do |style|
success = case Paperclip::Attachment.default_options[:storage]
when :s3
upgrade_storage_s3(progress, attachment, style)
when :fog
upgrade_storage_fog(progress, attachment, style)
when :filesystem
upgrade_storage_filesystem(progress, attachment, style)
end
upgraded = true if style == :original && success
progress.increment
end
attachment.instance_write(:storage_schema_version, CURRENT_STORAGE_SCHEMA_VERSION) if upgraded
end
if record.changed?
record.save unless dry_run?
records += 1
end
end
end
progress.total = progress.progress
progress.finish
say("Upgraded storage schema of #{records} records#{dry_run}", :green, true)
end
private
def upgrade_storage_s3(progress, attachment, style)
previous_storage_schema_version = attachment.storage_schema_version
object = attachment.s3_object(style)
success = true
attachment.instance_write(:storage_schema_version, CURRENT_STORAGE_SCHEMA_VERSION)
new_object = attachment.s3_object(style)
if new_object.key != object.key && object.exists?
progress.log("Moving #{object.key} to #{new_object.key}") if options[:verbose]
begin
object.move_to(new_object, acl: attachment.s3_permissions(style)) unless dry_run?
rescue => e
progress.log(pastel.red("Error processing #{object.key}: #{e}"))
success = false
end
end
# Because we move files style-by-style, it's important to restore
# previous version at the end. The upgrade will be recorded after
# all styles are updated
attachment.instance_write(:storage_schema_version, previous_storage_schema_version)
success
end
def upgrade_storage_fog(_progress, _attachment, _style)
say('The fog storage driver is not supported for this operation at this time', :red)
exit(1)
end
def upgrade_storage_filesystem(progress, attachment, style)
previous_storage_schema_version = attachment.storage_schema_version
previous_path = attachment.path(style)
success = true
attachment.instance_write(:storage_schema_version, CURRENT_STORAGE_SCHEMA_VERSION)
upgraded_path = attachment.path(style)
if upgraded_path != previous_path && File.exist?(previous_path)
progress.log("Moving #{previous_path} to #{upgraded_path}") if options[:verbose]
begin
unless dry_run?
FileUtils.mkdir_p(File.dirname(upgraded_path))
FileUtils.mv(previous_path, upgraded_path)
begin
FileUtils.rmdir(File.dirname(previous_path), parents: true)
rescue Errno::ENOTEMPTY
# OK
end
end
rescue => e
progress.log(pastel.red("Error processing #{previous_path}: #{e}"))
success = false
unless dry_run?
begin
FileUtils.rmdir(File.dirname(upgraded_path), parents: true)
rescue Errno::ENOTEMPTY
# OK
end
end
end
end
# Because we move files style-by-style, it's important to restore
# previous version at the end. The upgrade will be recorded after
# all styles are updated
attachment.instance_write(:storage_schema_version, previous_storage_schema_version)
success
end
end
end