tarrien/Mastodon
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Rewrite import feature (#21054)

Browse Source
This commit is contained in:
Claire
2023-05-02 12:08:48 +02:00
committed by GitHub
parent 0ad2413b35
commit 32a030dd74
40 changed files with 2059 additions and 113 deletions
Download Patch File Download Diff File Expand all files Collapse all files

316
spec/controllers/settings/imports_controller_spec.rb
View File

@@ -5,13 +5,22 @@ require 'rails_helper'
RSpec.describe Settings::ImportsController, type: :controller do
render_views
let(:user) { Fabricate(:user) }
before do
sign_in Fabricate(:user), scope: :user
sign_in user, scope: :user
end
describe 'GET #show' do
describe 'GET #index' do
let!(:import) { Fabricate(:bulk_import, account: user.account) }
let!(:other_import) { Fabricate(:bulk_import) }
before do
get :show
get :index
end
it 'assigns the expected imports' do
expect(assigns(:recent_imports)).to eq [import]
end
it 'returns http success' do
@@ -23,31 +32,288 @@ RSpec.describe Settings::ImportsController, type: :controller do
end
end
describe 'POST #create' do
it 'redirects to settings path with successful following import' do
service = double(call: nil)
allow(ResolveAccountService).to receive(:new).and_return(service)
post :create, params: {
import: {
type: 'following',
data: fixture_file_upload('imports.txt'),
},
}
expect(response).to redirect_to(settings_import_path)
describe 'GET #show' do
before do
get :show, params: { id: bulk_import.id }
end
it 'redirects to settings path with successful blocking import' do
service = double(call: nil)
allow(ResolveAccountService).to receive(:new).and_return(service)
post :create, params: {
import: {
type: 'blocking',
data: fixture_file_upload('imports.txt'),
},
}
context 'with someone else\'s import' do
let(:bulk_import) { Fabricate(:bulk_import, state: :unconfirmed) }
expect(response).to redirect_to(settings_import_path)
it 'returns http not found' do
expect(response).to have_http_status(404)
end
end
context 'with an already-confirmed import' do
let(:bulk_import) { Fabricate(:bulk_import, account: user.account, state: :in_progress) }
it 'returns http not found' do
expect(response).to have_http_status(404)
end
end
context 'with an unconfirmed import' do
let(:bulk_import) { Fabricate(:bulk_import, account: user.account, state: :unconfirmed) }
it 'returns http success' do
expect(response).to have_http_status(200)
end
end
end
describe 'POST #confirm' do
subject { post :confirm, params: { id: bulk_import.id } }
before do
allow(BulkImportWorker).to receive(:perform_async)
end
context 'with someone else\'s import' do
let(:bulk_import) { Fabricate(:bulk_import, state: :unconfirmed) }
it 'does not change the import\'s state' do
expect { subject }.to_not(change { bulk_import.reload.state })
end
it 'does not fire the import worker' do
subject
expect(BulkImportWorker).to_not have_received(:perform_async)
end
it 'returns http not found' do
subject
expect(response).to have_http_status(404)
end
end
context 'with an already-confirmed import' do
let(:bulk_import) { Fabricate(:bulk_import, account: user.account, state: :in_progress) }
it 'does not change the import\'s state' do
expect { subject }.to_not(change { bulk_import.reload.state })
end
it 'does not fire the import worker' do
subject
expect(BulkImportWorker).to_not have_received(:perform_async)
end
it 'returns http not found' do
subject
expect(response).to have_http_status(404)
end
end
context 'with an unconfirmed import' do
let(:bulk_import) { Fabricate(:bulk_import, account: user.account, state: :unconfirmed) }
it 'changes the import\'s state to scheduled' do
expect { subject }.to change { bulk_import.reload.state.to_sym }.from(:unconfirmed).to(:scheduled)
end
it 'fires the import worker on the expected import' do
subject
expect(BulkImportWorker).to have_received(:perform_async).with(bulk_import.id)
end
it 'redirects to imports path' do
subject
expect(response).to redirect_to(settings_imports_path)
end
end
end
describe 'DELETE #destroy' do
subject { delete :destroy, params: { id: bulk_import.id } }
context 'with someone else\'s import' do
let(:bulk_import) { Fabricate(:bulk_import, state: :unconfirmed) }
it 'does not delete the import' do
expect { subject }.to_not(change { BulkImport.exists?(bulk_import.id) })
end
it 'returns http not found' do
subject
expect(response).to have_http_status(404)
end
end
context 'with an already-confirmed import' do
let(:bulk_import) { Fabricate(:bulk_import, account: user.account, state: :in_progress) }
it 'does not delete the import' do
expect { subject }.to_not(change { BulkImport.exists?(bulk_import.id) })
end
it 'returns http not found' do
subject
expect(response).to have_http_status(404)
end
end
context 'with an unconfirmed import' do
let(:bulk_import) { Fabricate(:bulk_import, account: user.account, state: :unconfirmed) }
it 'deletes the import' do
expect { subject }.to change { BulkImport.exists?(bulk_import.id) }.from(true).to(false)
end
it 'redirects to imports path' do
subject
expect(response).to redirect_to(settings_imports_path)
end
end
end
describe 'GET #failures' do
subject { get :failures, params: { id: bulk_import.id }, format: :csv }
shared_examples 'export failed rows' do |expected_contents|
let(:bulk_import) { Fabricate(:bulk_import, account: user.account, type: import_type, state: :finished) }
before do
bulk_import.update(total_items: bulk_import.rows.count, processed_items: bulk_import.rows.count, imported_items: 0)
end
it 'returns http success' do
subject
expect(response).to have_http_status(200)
end
it 'returns expected contents' do
subject
expect(response.body).to eq expected_contents
end
end
context 'with follows' do
let(:import_type) { 'following' }
let!(:rows) do
[
{ 'acct' => 'foo@bar' },
{ 'acct' => 'user@bar', 'show_reblogs' => false, 'notify' => true, 'languages' => ['fr', 'de'] },
].map { |data| Fabricate(:bulk_import_row, bulk_import: bulk_import, data: data) }
end
include_examples 'export failed rows', "Account address,Show boosts,Notify on new posts,Languages\nfoo@bar,true,false,\nuser@bar,false,true,\"fr, de\"\n"
end
context 'with blocks' do
let(:import_type) { 'blocking' }
let!(:rows) do
[
{ 'acct' => 'foo@bar' },
{ 'acct' => 'user@bar' },
].map { |data| Fabricate(:bulk_import_row, bulk_import: bulk_import, data: data) }
end
include_examples 'export failed rows', "foo@bar\nuser@bar\n"
end
context 'with mutes' do
let(:import_type) { 'muting' }
let!(:rows) do
[
{ 'acct' => 'foo@bar' },
{ 'acct' => 'user@bar', 'hide_notifications' => false },
].map { |data| Fabricate(:bulk_import_row, bulk_import: bulk_import, data: data) }
end
include_examples 'export failed rows', "Account address,Hide notifications\nfoo@bar,true\nuser@bar,false\n"
end
context 'with domain blocks' do
let(:import_type) { 'domain_blocking' }
let!(:rows) do
[
{ 'domain' => 'bad.domain' },
{ 'domain' => 'evil.domain' },
].map { |data| Fabricate(:bulk_import_row, bulk_import: bulk_import, data: data) }
end
include_examples 'export failed rows', "bad.domain\nevil.domain\n"
end
context 'with bookmarks' do
let(:import_type) { 'bookmarks' }
let!(:rows) do
[
{ 'uri' => 'https://foo.com/1' },
{ 'uri' => 'https://foo.com/2' },
].map { |data| Fabricate(:bulk_import_row, bulk_import: bulk_import, data: data) }
end
include_examples 'export failed rows', "https://foo.com/1\nhttps://foo.com/2\n"
end
end
describe 'POST #create' do
subject do
post :create, params: {
form_import: {
type: import_type,
mode: import_mode,
data: fixture_file_upload(import_file),
},
}
end
shared_examples 'successful import' do |type, file, mode|
let(:import_type) { type }
let(:import_file) { file }
let(:import_mode) { mode }
it 'creates an unconfirmed bulk_import with expected type' do
expect { subject }.to change { user.account.bulk_imports.pluck(:state, :type) }.from([]).to([['unconfirmed', import_type]])
end
it 'redirects to confirmation page for the import' do
subject
expect(response).to redirect_to(settings_import_path(user.account.bulk_imports.first))
end
end
shared_examples 'unsuccessful import' do |type, file, mode|
let(:import_type) { type }
let(:import_file) { file }
let(:import_mode) { mode }
it 'does not creates an unconfirmed bulk_import' do
expect { subject }.to_not(change { user.account.bulk_imports.count })
end
it 'sets error to the import' do
subject
expect(assigns(:import).errors).to_not be_empty
end
end
it_behaves_like 'successful import', 'following', 'imports.txt', 'merge'
it_behaves_like 'successful import', 'following', 'imports.txt', 'overwrite'
it_behaves_like 'successful import', 'blocking', 'imports.txt', 'merge'
it_behaves_like 'successful import', 'blocking', 'imports.txt', 'overwrite'
it_behaves_like 'successful import', 'muting', 'imports.txt', 'merge'
it_behaves_like 'successful import', 'muting', 'imports.txt', 'overwrite'
it_behaves_like 'successful import', 'domain_blocking', 'domain_blocks.csv', 'merge'
it_behaves_like 'successful import', 'domain_blocking', 'domain_blocks.csv', 'overwrite'
it_behaves_like 'successful import', 'bookmarks', 'bookmark-imports.txt', 'merge'
it_behaves_like 'successful import', 'bookmarks', 'bookmark-imports.txt', 'overwrite'
it_behaves_like 'unsuccessful import', 'following', 'domain_blocks.csv', 'merge'
it_behaves_like 'unsuccessful import', 'following', 'domain_blocks.csv', 'overwrite'
it_behaves_like 'unsuccessful import', 'blocking', 'domain_blocks.csv', 'merge'
it_behaves_like 'unsuccessful import', 'blocking', 'domain_blocks.csv', 'overwrite'
it_behaves_like 'unsuccessful import', 'muting', 'domain_blocks.csv', 'merge'
it_behaves_like 'unsuccessful import', 'muting', 'domain_blocks.csv', 'overwrite'
it_behaves_like 'unsuccessful import', 'following', 'empty.csv', 'merge'
it_behaves_like 'unsuccessful import', 'following', 'empty.csv', 'overwrite'
end
end

12
spec/fabricators/bulk_import_fabricator.rb Normal file
View File

@@ -0,0 +1,12 @@
# frozen_string_literal: true
Fabricator(:bulk_import) do
type 1
state 1
total_items 1
processed_items 1
imported_items 1
finished_at '2022-11-18 14:55:07'
overwrite false
account
end

6
spec/fabricators/bulk_import_row_fabricator.rb Normal file
View File

@@ -0,0 +1,6 @@
# frozen_string_literal: true
Fabricator(:bulk_import_row) do
bulk_import
data ''
end

0
spec/fixtures/files/empty.csv vendored Normal file
View File

5
spec/fixtures/files/following_accounts.csv vendored Normal file
View File

@@ -0,0 +1,5 @@
Account address,Show boosts,Notify on new posts,Languages
user@example.com,true,false,
user@test.com,true,true,"en,fr"
1 Account address Show boosts Notify on new posts Languages
2 user@example.com true false
3 user@test.com true true en,fr

5
spec/fixtures/files/muted_accounts.csv vendored Normal file
View File

@@ -0,0 +1,5 @@
Account address,Hide notifications
user@example.com,true
user@test.com,false
1 Account address Hide notifications
2 user@example.com true
3 user@test.com false

19
spec/lib/vacuum/imports_vacuum_spec.rb Normal file
View File

@@ -0,0 +1,19 @@
# frozen_string_literal: true
require 'rails_helper'
RSpec.describe Vacuum::ImportsVacuum do
subject { described_class.new }
let!(:old_unconfirmed) { Fabricate(:bulk_import, state: :unconfirmed, created_at: 2.days.ago) }
let!(:new_unconfirmed) { Fabricate(:bulk_import, state: :unconfirmed, created_at: 10.seconds.ago) }
let!(:recent_ongoing) { Fabricate(:bulk_import, state: :in_progress, created_at: 20.minutes.ago) }
let!(:recent_finished) { Fabricate(:bulk_import, state: :finished, created_at: 1.day.ago) }
let!(:old_finished) { Fabricate(:bulk_import, state: :finished, created_at: 2.months.ago) }
describe '#perform' do
it 'cleans up the expected imports' do
expect { subject.perform }.to change { BulkImport.all.pluck(:id) }.from([old_unconfirmed, new_unconfirmed, recent_ongoing, recent_finished, old_finished].map(&:id)).to([new_unconfirmed, recent_ongoing, recent_finished].map(&:id))
end
end
end

281
spec/models/form/import_spec.rb Normal file
View File

@@ -0,0 +1,281 @@
# frozen_string_literal: true
require 'rails_helper'
RSpec.describe Form::Import do
subject { described_class.new(current_account: account, type: import_type, mode: import_mode, data: data) }
let(:account) { Fabricate(:account) }
let(:data) { fixture_file_upload(import_file) }
let(:import_mode) { 'merge' }
describe 'validations' do
shared_examples 'incompatible import type' do |type, file|
let(:import_file) { file }
let(:import_type) { type }
it 'has errors' do
subject.validate
expect(subject.errors[:data]).to include(I18n.t('imports.errors.incompatible_type'))
end
end
shared_examples 'too many CSV rows' do |type, file, allowed_rows|
let(:import_file) { file }
let(:import_type) { type }
before do
stub_const 'Form::Import::ROWS_PROCESSING_LIMIT', allowed_rows
end
it 'has errors' do
subject.validate
expect(subject.errors[:data]).to include(I18n.t('imports.errors.over_rows_processing_limit', count: Form::Import::ROWS_PROCESSING_LIMIT))
end
end
shared_examples 'valid import' do |type, file|
let(:import_file) { file }
let(:import_type) { type }
it 'passes validation' do
expect(subject).to be_valid
end
end
context 'when the file too large' do
let(:import_type) { 'following' }
let(:import_file) { 'imports.txt' }
before do
stub_const 'Form::Import::FILE_SIZE_LIMIT', 5
end
it 'has errors' do
subject.validate
expect(subject.errors[:data]).to include(I18n.t('imports.errors.too_large'))
end
end
context 'when the CSV file is malformed CSV' do
let(:import_type) { 'following' }
let(:import_file) { 'boop.ogg' }
it 'has errors' do
# NOTE: not testing more specific error because we don't know the string to match
expect(subject).to model_have_error_on_field(:data)
end
end
context 'when importing more follows than allowed' do
let(:import_type) { 'following' }
let(:import_file) { 'imports.txt' }
before do
allow(FollowLimitValidator).to receive(:limit_for_account).with(account).and_return(1)
end
it 'has errors' do
subject.validate
expect(subject.errors[:data]).to include(I18n.t('users.follow_limit_reached', limit: 1))
end
end
it_behaves_like 'too many CSV rows', 'following', 'imports.txt', 1
it_behaves_like 'too many CSV rows', 'blocking', 'imports.txt', 1
it_behaves_like 'too many CSV rows', 'muting', 'imports.txt', 1
it_behaves_like 'too many CSV rows', 'domain_blocking', 'domain_blocks.csv', 2
it_behaves_like 'too many CSV rows', 'bookmarks', 'bookmark-imports.txt', 3
# Importing list of addresses with no headers into various types
it_behaves_like 'valid import', 'following', 'imports.txt'
it_behaves_like 'valid import', 'blocking', 'imports.txt'
it_behaves_like 'valid import', 'muting', 'imports.txt'
# Importing domain blocks with headers into expected type
it_behaves_like 'valid import', 'domain_blocking', 'domain_blocks.csv'
# Importing bookmarks list with no headers into expected type
it_behaves_like 'valid import', 'bookmarks', 'bookmark-imports.txt'
# Importing followed accounts with headers into various compatible types
it_behaves_like 'valid import', 'following', 'following_accounts.csv'
it_behaves_like 'valid import', 'blocking', 'following_accounts.csv'
it_behaves_like 'valid import', 'muting', 'following_accounts.csv'
# Importing domain blocks with headers into incompatible types
it_behaves_like 'incompatible import type', 'following', 'domain_blocks.csv'
it_behaves_like 'incompatible import type', 'blocking', 'domain_blocks.csv'
it_behaves_like 'incompatible import type', 'muting', 'domain_blocks.csv'
it_behaves_like 'incompatible import type', 'bookmarks', 'domain_blocks.csv'
# Importing followed accounts with headers into incompatible types
it_behaves_like 'incompatible import type', 'domain_blocking', 'following_accounts.csv'
it_behaves_like 'incompatible import type', 'bookmarks', 'following_accounts.csv'
end
describe '#guessed_type' do
shared_examples 'with enough information' do |type, file, original_filename, expected_guess|
let(:import_file) { file }
let(:import_type) { type }
before do
allow(data).to receive(:original_filename).and_return(original_filename)
end
it 'guesses the expected type' do
expect(subject.guessed_type).to eq expected_guess
end
end
context 'when the headers are enough to disambiguate' do
it_behaves_like 'with enough information', 'following', 'following_accounts.csv', 'import.csv', :following
it_behaves_like 'with enough information', 'blocking', 'following_accounts.csv', 'import.csv', :following
it_behaves_like 'with enough information', 'muting', 'following_accounts.csv', 'import.csv', :following
it_behaves_like 'with enough information', 'following', 'muted_accounts.csv', 'imports.csv', :muting
it_behaves_like 'with enough information', 'blocking', 'muted_accounts.csv', 'imports.csv', :muting
it_behaves_like 'with enough information', 'muting', 'muted_accounts.csv', 'imports.csv', :muting
end
context 'when the file name is enough to disambiguate' do
it_behaves_like 'with enough information', 'following', 'imports.txt', 'following_accounts.csv', :following
it_behaves_like 'with enough information', 'blocking', 'imports.txt', 'following_accounts.csv', :following
it_behaves_like 'with enough information', 'muting', 'imports.txt', 'following_accounts.csv', :following
it_behaves_like 'with enough information', 'following', 'imports.txt', 'follows.csv', :following
it_behaves_like 'with enough information', 'blocking', 'imports.txt', 'follows.csv', :following
it_behaves_like 'with enough information', 'muting', 'imports.txt', 'follows.csv', :following
it_behaves_like 'with enough information', 'following', 'imports.txt', 'blocked_accounts.csv', :blocking
it_behaves_like 'with enough information', 'blocking', 'imports.txt', 'blocked_accounts.csv', :blocking
it_behaves_like 'with enough information', 'muting', 'imports.txt', 'blocked_accounts.csv', :blocking
it_behaves_like 'with enough information', 'following', 'imports.txt', 'blocks.csv', :blocking
it_behaves_like 'with enough information', 'blocking', 'imports.txt', 'blocks.csv', :blocking
it_behaves_like 'with enough information', 'muting', 'imports.txt', 'blocks.csv', :blocking
it_behaves_like 'with enough information', 'following', 'imports.txt', 'muted_accounts.csv', :muting
it_behaves_like 'with enough information', 'blocking', 'imports.txt', 'muted_accounts.csv', :muting
it_behaves_like 'with enough information', 'muting', 'imports.txt', 'muted_accounts.csv', :muting
it_behaves_like 'with enough information', 'following', 'imports.txt', 'mutes.csv', :muting
it_behaves_like 'with enough information', 'blocking', 'imports.txt', 'mutes.csv', :muting
it_behaves_like 'with enough information', 'muting', 'imports.txt', 'mutes.csv', :muting
end
end
describe '#likely_mismatched?' do
shared_examples 'with matching types' do |type, file, original_filename = nil|
let(:import_file) { file }
let(:import_type) { type }
before do
allow(data).to receive(:original_filename).and_return(original_filename) if original_filename.present?
end
it 'returns false' do
expect(subject.likely_mismatched?).to be false
end
end
shared_examples 'with mismatching types' do |type, file, original_filename = nil|
let(:import_file) { file }
let(:import_type) { type }
before do
allow(data).to receive(:original_filename).and_return(original_filename) if original_filename.present?
end
it 'returns true' do
expect(subject.likely_mismatched?).to be true
end
end
it_behaves_like 'with matching types', 'following', 'following_accounts.csv'
it_behaves_like 'with matching types', 'following', 'following_accounts.csv', 'imports.txt'
it_behaves_like 'with matching types', 'following', 'imports.txt'
it_behaves_like 'with matching types', 'blocking', 'imports.txt', 'blocks.csv'
it_behaves_like 'with matching types', 'blocking', 'imports.txt'
it_behaves_like 'with matching types', 'muting', 'muted_accounts.csv'
it_behaves_like 'with matching types', 'muting', 'muted_accounts.csv', 'imports.txt'
it_behaves_like 'with matching types', 'muting', 'imports.txt'
it_behaves_like 'with matching types', 'domain_blocking', 'domain_blocks.csv'
it_behaves_like 'with matching types', 'domain_blocking', 'domain_blocks.csv', 'imports.txt'
it_behaves_like 'with matching types', 'bookmarks', 'bookmark-imports.txt'
it_behaves_like 'with matching types', 'bookmarks', 'bookmark-imports.txt', 'imports.txt'
it_behaves_like 'with mismatching types', 'following', 'imports.txt', 'blocks.csv'
it_behaves_like 'with mismatching types', 'following', 'imports.txt', 'blocked_accounts.csv'
it_behaves_like 'with mismatching types', 'following', 'imports.txt', 'mutes.csv'
it_behaves_like 'with mismatching types', 'following', 'imports.txt', 'muted_accounts.csv'
it_behaves_like 'with mismatching types', 'following', 'muted_accounts.csv'
it_behaves_like 'with mismatching types', 'following', 'muted_accounts.csv', 'imports.txt'
it_behaves_like 'with mismatching types', 'blocking', 'following_accounts.csv'
it_behaves_like 'with mismatching types', 'blocking', 'following_accounts.csv', 'imports.txt'
it_behaves_like 'with mismatching types', 'blocking', 'muted_accounts.csv'
it_behaves_like 'with mismatching types', 'blocking', 'muted_accounts.csv', 'imports.txt'
it_behaves_like 'with mismatching types', 'blocking', 'imports.txt', 'follows.csv'
it_behaves_like 'with mismatching types', 'blocking', 'imports.txt', 'following_accounts.csv'
it_behaves_like 'with mismatching types', 'blocking', 'imports.txt', 'mutes.csv'
it_behaves_like 'with mismatching types', 'blocking', 'imports.txt', 'muted_accounts.csv'
it_behaves_like 'with mismatching types', 'muting', 'following_accounts.csv'
it_behaves_like 'with mismatching types', 'muting', 'following_accounts.csv', 'imports.txt'
it_behaves_like 'with mismatching types', 'muting', 'imports.txt', 'follows.csv'
it_behaves_like 'with mismatching types', 'muting', 'imports.txt', 'following_accounts.csv'
it_behaves_like 'with mismatching types', 'muting', 'imports.txt', 'blocks.csv'
it_behaves_like 'with mismatching types', 'muting', 'imports.txt', 'blocked_accounts.csv'
end
describe 'save' do
shared_examples 'on successful import' do |type, mode, file, expected_rows|
let(:import_type) { type }
let(:import_file) { file }
let(:import_mode) { mode }
before do
subject.save
end
it 'creates the expected rows' do
expect(account.bulk_imports.first.rows.pluck(:data)).to match_array(expected_rows)
end
it 'creates a BulkImport with expected attributes' do
bulk_import = account.bulk_imports.first
expect(bulk_import).to_not be_nil
expect(bulk_import.type.to_sym).to eq subject.type.to_sym
expect(bulk_import.original_filename).to eq subject.data.original_filename
expect(bulk_import.likely_mismatched?).to eq subject.likely_mismatched?
expect(bulk_import.overwrite?).to eq !!subject.overwrite # rubocop:disable Style/DoubleNegation
expect(bulk_import.processed_items).to eq 0
expect(bulk_import.imported_items).to eq 0
expect(bulk_import.total_items).to eq bulk_import.rows.count
expect(bulk_import.unconfirmed?).to be true
end
end
it_behaves_like 'on successful import', 'following', 'merge', 'imports.txt', (%w(user@example.com user@test.com).map { |acct| { 'acct' => acct } })
it_behaves_like 'on successful import', 'following', 'overwrite', 'imports.txt', (%w(user@example.com user@test.com).map { |acct| { 'acct' => acct } })
it_behaves_like 'on successful import', 'blocking', 'merge', 'imports.txt', (%w(user@example.com user@test.com).map { |acct| { 'acct' => acct } })
it_behaves_like 'on successful import', 'blocking', 'overwrite', 'imports.txt', (%w(user@example.com user@test.com).map { |acct| { 'acct' => acct } })
it_behaves_like 'on successful import', 'muting', 'merge', 'imports.txt', (%w(user@example.com user@test.com).map { |acct| { 'acct' => acct } })
it_behaves_like 'on successful import', 'domain_blocking', 'merge', 'domain_blocks.csv', (%w(bad.domain worse.domain reject.media).map { |domain| { 'domain' => domain } })
it_behaves_like 'on successful import', 'bookmarks', 'merge', 'bookmark-imports.txt', (%w(https://example.com/statuses/1312 https://local.com/users/foo/statuses/42 https://unknown-remote.com/users/bar/statuses/1 https://example.com/statuses/direct).map { |uri| { 'uri' => uri } })
it_behaves_like 'on successful import', 'following', 'merge', 'following_accounts.csv', [
{ 'acct' => 'user@example.com', 'show_reblogs' => true, 'notify' => false, 'languages' => nil },
{ 'acct' => 'user@test.com', 'show_reblogs' => true, 'notify' => true, 'languages' => ['en', 'fr'] },
]
it_behaves_like 'on successful import', 'muting', 'merge', 'muted_accounts.csv', [
{ 'acct' => 'user@example.com', 'hide_notifications' => true },
{ 'acct' => 'user@test.com', 'hide_notifications' => false },
]
# Based on the bug report 20571 where UTF-8 encoded domains were rejecting import of their users
#
# https://github.com/mastodon/mastodon/issues/20571
it_behaves_like 'on successful import', 'following', 'merge', 'utf8-followers.txt', [{ 'acct' => 'nare@թութ.հայ' }]
end
end

15
spec/models/import_spec.rb
View File

@@ -22,20 +22,5 @@ RSpec.describe Import, type: :model do
import = Import.create(account: account, type: type)
expect(import).to model_have_error_on_field(:data)
end
it 'is invalid with malformed data' do
import = Import.create(account: account, type: type, data: StringIO.new('\"test'))
expect(import).to model_have_error_on_field(:data)
end
it 'is invalid with too many rows in data' do
import = Import.create(account: account, type: type, data: StringIO.new("foo@bar.com\n" * (ImportService::ROWS_PROCESSING_LIMIT + 10)))
expect(import).to model_have_error_on_field(:data)
end
it 'is invalid when there are more rows when following limit' do
import = Import.create(account: account, type: type, data: StringIO.new("foo@bar.com\n" * (FollowLimitValidator.limit_for_account(account) + 10)))
expect(import).to model_have_error_on_field(:data)
end
end
end

95
spec/services/bulk_import_row_service_spec.rb Normal file
View File

@@ -0,0 +1,95 @@
# frozen_string_literal: true
require 'rails_helper'
RSpec.describe BulkImportRowService do
subject { described_class.new }
let(:account) { Fabricate(:account) }
let(:import) { Fabricate(:bulk_import, account: account, type: import_type) }
let(:import_row) { Fabricate(:bulk_import_row, bulk_import: import, data: data) }
describe '#call' do
context 'when importing a follow' do
let(:import_type) { 'following' }
let(:target_account) { Fabricate(:account) }
let(:service_double) { instance_double(FollowService, call: nil) }
let(:data) do
{ 'acct' => target_account.acct }
end
before do
allow(FollowService).to receive(:new).and_return(service_double)
end
it 'calls FollowService with the expected arguments and returns true' do
expect(subject.call(import_row)).to be true
expect(service_double).to have_received(:call).with(account, target_account, { reblogs: nil, notify: nil, languages: nil })
end
end
context 'when importing a block' do
let(:import_type) { 'blocking' }
let(:target_account) { Fabricate(:account) }
let(:service_double) { instance_double(BlockService, call: nil) }
let(:data) do
{ 'acct' => target_account.acct }
end
before do
allow(BlockService).to receive(:new).and_return(service_double)
end
it 'calls BlockService with the expected arguments and returns true' do
expect(subject.call(import_row)).to be true
expect(service_double).to have_received(:call).with(account, target_account)
end
end
context 'when importing a mute' do
let(:import_type) { 'muting' }
let(:target_account) { Fabricate(:account) }
let(:service_double) { instance_double(MuteService, call: nil) }
let(:data) do
{ 'acct' => target_account.acct }
end
before do
allow(MuteService).to receive(:new).and_return(service_double)
end
it 'calls MuteService with the expected arguments and returns true' do
expect(subject.call(import_row)).to be true
expect(service_double).to have_received(:call).with(account, target_account, { notifications: nil })
end
end
context 'when importing a bookmark' do
let(:import_type) { 'bookmarks' }
let(:data) do
{ 'uri' => ActivityPub::TagManager.instance.uri_for(target_status) }
end
context 'when the status is public' do
let(:target_status) { Fabricate(:status) }
it 'bookmarks the status and returns true' do
expect(subject.call(import_row)).to be true
expect(account.bookmarked?(target_status)).to be true
end
end
context 'when the status is not accessible to the user' do
let(:target_status) { Fabricate(:status, visibility: :direct) }
it 'does not bookmark the status and returns false' do
expect(subject.call(import_row)).to be false
expect(account.bookmarked?(target_status)).to be false
end
end
end
end
end

417
spec/services/bulk_import_service_spec.rb Normal file
View File

@@ -0,0 +1,417 @@
# frozen_string_literal: true
require 'rails_helper'
RSpec.describe BulkImportService do
subject { described_class.new }
let(:account) { Fabricate(:account) }
let(:import) { Fabricate(:bulk_import, account: account, type: import_type, overwrite: overwrite, state: :in_progress, imported_items: 0, processed_items: 0) }
before do
import.update(total_items: import.rows.count)
end
describe '#call' do
around do |example|
Sidekiq::Testing.fake! do
example.run
Sidekiq::Worker.clear_all
end
end
context 'when importing follows' do
let(:import_type) { 'following' }
let(:overwrite) { false }
let!(:rows) do
[
{ 'acct' => 'user@foo.bar' },
{ 'acct' => 'unknown@unknown.bar' },
].map { |data| import.rows.create!(data: data) }
end
before do
account.follow!(Fabricate(:account))
end
it 'does not immediately change who the account follows' do
expect { subject.call(import) }.to_not(change { account.reload.active_relationships.to_a })
end
it 'enqueues workers for the expected rows' do
subject.call(import)
expect(Import::RowWorker.jobs.pluck('args').flatten).to match_array(rows.map(&:id))
end
it 'requests to follow all the listed users once the workers have run' do
subject.call(import)
resolve_account_service_double = double
allow(ResolveAccountService).to receive(:new).and_return(resolve_account_service_double)
allow(resolve_account_service_double).to receive(:call).with('user@foo.bar', any_args) { Fabricate(:account, username: 'user', domain: 'foo.bar', protocol: :activitypub) }
allow(resolve_account_service_double).to receive(:call).with('unknown@unknown.bar', any_args) { Fabricate(:account, username: 'unknown', domain: 'unknown.bar', protocol: :activitypub) }
Import::RowWorker.drain
expect(FollowRequest.includes(:target_account).where(account: account).map(&:target_account).map(&:acct)).to contain_exactly('user@foo.bar', 'unknown@unknown.bar')
end
end
context 'when importing follows with overwrite' do
let(:import_type) { 'following' }
let(:overwrite) { true }
let!(:followed) { Fabricate(:account, username: 'followed', domain: 'foo.bar', protocol: :activitypub) }
let!(:to_be_unfollowed) { Fabricate(:account, username: 'to_be_unfollowed', domain: 'foo.bar', protocol: :activitypub) }
let!(:rows) do
[
{ 'acct' => 'followed@foo.bar', 'show_reblogs' => false, 'notify' => true, 'languages' => ['en'] },
{ 'acct' => 'user@foo.bar' },
{ 'acct' => 'unknown@unknown.bar' },
].map { |data| import.rows.create!(data: data) }
end
before do
account.follow!(followed, reblogs: true, notify: false)
account.follow!(to_be_unfollowed)
end
it 'unfollows user not present on list' do
subject.call(import)
expect(account.following?(to_be_unfollowed)).to be false
end
it 'updates the existing follow relationship as expected' do
expect { subject.call(import) }.to change { Follow.where(account: account, target_account: followed).pick(:show_reblogs, :notify, :languages) }.from([true, false, nil]).to([false, true, ['en']])
end
it 'enqueues workers for the expected rows' do
subject.call(import)
expect(Import::RowWorker.jobs.pluck('args').flatten).to match_array(rows[1..].map(&:id))
end
it 'requests to follow all the expected users once the workers have run' do
subject.call(import)
resolve_account_service_double = double
allow(ResolveAccountService).to receive(:new).and_return(resolve_account_service_double)
allow(resolve_account_service_double).to receive(:call).with('user@foo.bar', any_args) { Fabricate(:account, username: 'user', domain: 'foo.bar', protocol: :activitypub) }
allow(resolve_account_service_double).to receive(:call).with('unknown@unknown.bar', any_args) { Fabricate(:account, username: 'unknown', domain: 'unknown.bar', protocol: :activitypub) }
Import::RowWorker.drain
expect(FollowRequest.includes(:target_account).where(account: account).map(&:target_account).map(&:acct)).to contain_exactly('user@foo.bar', 'unknown@unknown.bar')
end
end
context 'when importing blocks' do
let(:import_type) { 'blocking' }
let(:overwrite) { false }
let!(:rows) do
[
{ 'acct' => 'user@foo.bar' },
{ 'acct' => 'unknown@unknown.bar' },
].map { |data| import.rows.create!(data: data) }
end
before do
account.block!(Fabricate(:account, username: 'already_blocked', domain: 'remote.org'))
end
it 'does not immediately change who the account blocks' do
expect { subject.call(import) }.to_not(change { account.reload.blocking.to_a })
end
it 'enqueues workers for the expected rows' do
subject.call(import)
expect(Import::RowWorker.jobs.pluck('args').flatten).to match_array(rows.map(&:id))
end
it 'blocks all the listed users once the workers have run' do
subject.call(import)
resolve_account_service_double = double
allow(ResolveAccountService).to receive(:new).and_return(resolve_account_service_double)
allow(resolve_account_service_double).to receive(:call).with('user@foo.bar', any_args) { Fabricate(:account, username: 'user', domain: 'foo.bar', protocol: :activitypub) }
allow(resolve_account_service_double).to receive(:call).with('unknown@unknown.bar', any_args) { Fabricate(:account, username: 'unknown', domain: 'unknown.bar', protocol: :activitypub) }
Import::RowWorker.drain
expect(account.blocking.map(&:acct)).to contain_exactly('already_blocked@remote.org', 'user@foo.bar', 'unknown@unknown.bar')
end
end
context 'when importing blocks with overwrite' do
let(:import_type) { 'blocking' }
let(:overwrite) { true }
let!(:blocked) { Fabricate(:account, username: 'blocked', domain: 'foo.bar', protocol: :activitypub) }
let!(:to_be_unblocked) { Fabricate(:account, username: 'to_be_unblocked', domain: 'foo.bar', protocol: :activitypub) }
let!(:rows) do
[
{ 'acct' => 'blocked@foo.bar' },
{ 'acct' => 'user@foo.bar' },
{ 'acct' => 'unknown@unknown.bar' },
].map { |data| import.rows.create!(data: data) }
end
before do
account.block!(blocked)
account.block!(to_be_unblocked)
end
it 'unblocks user not present on list' do
subject.call(import)
expect(account.blocking?(to_be_unblocked)).to be false
end
it 'enqueues workers for the expected rows' do
subject.call(import)
expect(Import::RowWorker.jobs.pluck('args').flatten).to match_array(rows[1..].map(&:id))
end
it 'requests to follow all the expected users once the workers have run' do
subject.call(import)
resolve_account_service_double = double
allow(ResolveAccountService).to receive(:new).and_return(resolve_account_service_double)
allow(resolve_account_service_double).to receive(:call).with('user@foo.bar', any_args) { Fabricate(:account, username: 'user', domain: 'foo.bar', protocol: :activitypub) }
allow(resolve_account_service_double).to receive(:call).with('unknown@unknown.bar', any_args) { Fabricate(:account, username: 'unknown', domain: 'unknown.bar', protocol: :activitypub) }
Import::RowWorker.drain
expect(account.blocking.map(&:acct)).to contain_exactly('blocked@foo.bar', 'user@foo.bar', 'unknown@unknown.bar')
end
end
context 'when importing mutes' do
let(:import_type) { 'muting' }
let(:overwrite) { false }
let!(:rows) do
[
{ 'acct' => 'user@foo.bar' },
{ 'acct' => 'unknown@unknown.bar' },
].map { |data| import.rows.create!(data: data) }
end
before do
account.mute!(Fabricate(:account, username: 'already_muted', domain: 'remote.org'))
end
it 'does not immediately change who the account blocks' do
expect { subject.call(import) }.to_not(change { account.reload.muting.to_a })
end
it 'enqueues workers for the expected rows' do
subject.call(import)
expect(Import::RowWorker.jobs.pluck('args').flatten).to match_array(rows.map(&:id))
end
it 'mutes all the listed users once the workers have run' do
subject.call(import)
resolve_account_service_double = double
allow(ResolveAccountService).to receive(:new).and_return(resolve_account_service_double)
allow(resolve_account_service_double).to receive(:call).with('user@foo.bar', any_args) { Fabricate(:account, username: 'user', domain: 'foo.bar', protocol: :activitypub) }
allow(resolve_account_service_double).to receive(:call).with('unknown@unknown.bar', any_args) { Fabricate(:account, username: 'unknown', domain: 'unknown.bar', protocol: :activitypub) }
Import::RowWorker.drain
expect(account.muting.map(&:acct)).to contain_exactly('already_muted@remote.org', 'user@foo.bar', 'unknown@unknown.bar')
end
end
context 'when importing mutes with overwrite' do
let(:import_type) { 'muting' }
let(:overwrite) { true }
let!(:muted) { Fabricate(:account, username: 'muted', domain: 'foo.bar', protocol: :activitypub) }
let!(:to_be_unmuted) { Fabricate(:account, username: 'to_be_unmuted', domain: 'foo.bar', protocol: :activitypub) }
let!(:rows) do
[
{ 'acct' => 'muted@foo.bar', 'hide_notifications' => true },
{ 'acct' => 'user@foo.bar' },
{ 'acct' => 'unknown@unknown.bar' },
].map { |data| import.rows.create!(data: data) }
end
before do
account.mute!(muted, notifications: false)
account.mute!(to_be_unmuted)
end
it 'updates the existing mute as expected' do
expect { subject.call(import) }.to change { Mute.where(account: account, target_account: muted).pick(:hide_notifications) }.from(false).to(true)
end
it 'unblocks user not present on list' do
subject.call(import)
expect(account.muting?(to_be_unmuted)).to be false
end
it 'enqueues workers for the expected rows' do
subject.call(import)
expect(Import::RowWorker.jobs.pluck('args').flatten).to match_array(rows[1..].map(&:id))
end
it 'requests to follow all the expected users once the workers have run' do
subject.call(import)
resolve_account_service_double = double
allow(ResolveAccountService).to receive(:new).and_return(resolve_account_service_double)
allow(resolve_account_service_double).to receive(:call).with('user@foo.bar', any_args) { Fabricate(:account, username: 'user', domain: 'foo.bar', protocol: :activitypub) }
allow(resolve_account_service_double).to receive(:call).with('unknown@unknown.bar', any_args) { Fabricate(:account, username: 'unknown', domain: 'unknown.bar', protocol: :activitypub) }
Import::RowWorker.drain
expect(account.muting.map(&:acct)).to contain_exactly('muted@foo.bar', 'user@foo.bar', 'unknown@unknown.bar')
end
end
context 'when importing domain blocks' do
let(:import_type) { 'domain_blocking' }
let(:overwrite) { false }
let!(:rows) do
[
{ 'domain' => 'blocked.com' },
{ 'domain' => 'to_block.com' },
].map { |data| import.rows.create!(data: data) }
end
before do
account.block_domain!('alreadyblocked.com')
account.block_domain!('blocked.com')
end
it 'blocks all the new domains' do
subject.call(import)
expect(account.domain_blocks.pluck(:domain)).to contain_exactly('alreadyblocked.com', 'blocked.com', 'to_block.com')
end
it 'marks the import as finished' do
subject.call(import)
expect(import.reload.finished?).to be true
end
end
context 'when importing domain blocks with overwrite' do
let(:import_type) { 'domain_blocking' }
let(:overwrite) { true }
let!(:rows) do
[
{ 'domain' => 'blocked.com' },
{ 'domain' => 'to_block.com' },
].map { |data| import.rows.create!(data: data) }
end
before do
account.block_domain!('alreadyblocked.com')
account.block_domain!('blocked.com')
end
it 'blocks all the new domains' do
subject.call(import)
expect(account.domain_blocks.pluck(:domain)).to contain_exactly('blocked.com', 'to_block.com')
end
it 'marks the import as finished' do
subject.call(import)
expect(import.reload.finished?).to be true
end
end
context 'when importing bookmarks' do
let(:import_type) { 'bookmarks' }
let(:overwrite) { false }
let!(:already_bookmarked) { Fabricate(:status, uri: 'https://already.bookmarked/1') }
let!(:status) { Fabricate(:status, uri: 'https://foo.bar/posts/1') }
let!(:inaccessible_status) { Fabricate(:status, uri: 'https://foo.bar/posts/inaccessible', visibility: :direct) }
let!(:bookmarked) { Fabricate(:status, uri: 'https://foo.bar/posts/already-bookmarked') }
let!(:rows) do
[
{ 'uri' => status.uri },
{ 'uri' => inaccessible_status.uri },
{ 'uri' => bookmarked.uri },
{ 'uri' => 'https://domain.unknown/foo' },
{ 'uri' => 'https://domain.unknown/private' },
].map { |data| import.rows.create!(data: data) }
end
before do
account.bookmarks.create!(status: already_bookmarked)
account.bookmarks.create!(status: bookmarked)
end
it 'enqueues workers for the expected rows' do
subject.call(import)
expect(Import::RowWorker.jobs.pluck('args').flatten).to match_array(rows.map(&:id))
end
it 'updates the bookmarks as expected once the workers have run' do
subject.call(import)
service_double = double
allow(ActivityPub::FetchRemoteStatusService).to receive(:new).and_return(service_double)
allow(service_double).to receive(:call).with('https://domain.unknown/foo') { Fabricate(:status, uri: 'https://domain.unknown/foo') }
allow(service_double).to receive(:call).with('https://domain.unknown/private') { Fabricate(:status, uri: 'https://domain.unknown/private', visibility: :direct) }
Import::RowWorker.drain
expect(account.bookmarks.map(&:status).map(&:uri)).to contain_exactly(already_bookmarked.uri, status.uri, bookmarked.uri, 'https://domain.unknown/foo')
end
end
context 'when importing bookmarks with overwrite' do
let(:import_type) { 'bookmarks' }
let(:overwrite) { true }
let!(:already_bookmarked) { Fabricate(:status, uri: 'https://already.bookmarked/1') }
let!(:status) { Fabricate(:status, uri: 'https://foo.bar/posts/1') }
let!(:inaccessible_status) { Fabricate(:status, uri: 'https://foo.bar/posts/inaccessible', visibility: :direct) }
let!(:bookmarked) { Fabricate(:status, uri: 'https://foo.bar/posts/already-bookmarked') }
let!(:rows) do
[
{ 'uri' => status.uri },
{ 'uri' => inaccessible_status.uri },
{ 'uri' => bookmarked.uri },
{ 'uri' => 'https://domain.unknown/foo' },
{ 'uri' => 'https://domain.unknown/private' },
].map { |data| import.rows.create!(data: data) }
end
before do
account.bookmarks.create!(status: already_bookmarked)
account.bookmarks.create!(status: bookmarked)
end
it 'enqueues workers for the expected rows' do
subject.call(import)
expect(Import::RowWorker.jobs.pluck('args').flatten).to match_array(rows.map(&:id))
end
it 'updates the bookmarks as expected once the workers have run' do
subject.call(import)
service_double = double
allow(ActivityPub::FetchRemoteStatusService).to receive(:new).and_return(service_double)
allow(service_double).to receive(:call).with('https://domain.unknown/foo') { Fabricate(:status, uri: 'https://domain.unknown/foo') }
allow(service_double).to receive(:call).with('https://domain.unknown/private') { Fabricate(:status, uri: 'https://domain.unknown/private', visibility: :direct) }
Import::RowWorker.drain
expect(account.bookmarks.map(&:status).map(&:uri)).to contain_exactly(status.uri, bookmarked.uri, 'https://domain.unknown/foo')
end
end
end
end

26
spec/workers/bulk_import_worker_spec.rb Normal file
View File

@@ -0,0 +1,26 @@
# frozen_string_literal: true
require 'rails_helper'
describe BulkImportWorker do
subject { described_class.new }
let(:import) { Fabricate(:bulk_import, state: :scheduled) }
describe '#perform' do
let(:service_double) { instance_double(BulkImportService, call: nil) }
before do
allow(BulkImportService).to receive(:new).and_return(service_double)
end
it 'changes the import\'s state as appropriate' do
expect { subject.perform(import.id) }.to change { import.reload.state.to_sym }.from(:scheduled).to(:in_progress)
end
it 'calls BulkImportService' do
subject.perform(import.id)
expect(service_double).to have_received(:call).with(import)
end
end
end

127
spec/workers/import/row_worker_spec.rb Normal file
View File

@@ -0,0 +1,127 @@
# frozen_string_literal: true
require 'rails_helper'
describe Import::RowWorker do
subject { described_class.new }
let(:row) { Fabricate(:bulk_import_row, bulk_import: import) }
describe '#perform' do
before do
allow(BulkImportRowService).to receive(:new).and_return(service_double)
end
shared_examples 'clean failure' do
let(:service_double) { instance_double(BulkImportRowService, call: false) }
it 'calls BulkImportRowService' do
subject.perform(row.id)
expect(service_double).to have_received(:call).with(row)
end
it 'increases the number of processed items' do
expect { subject.perform(row.id) }.to(change { import.reload.processed_items }.by(+1))
end
it 'does not increase the number of imported items' do
expect { subject.perform(row.id) }.to_not(change { import.reload.imported_items })
end
it 'does not delete the row' do
subject.perform(row.id)
expect(BulkImportRow.exists?(row.id)).to be true
end
end
shared_examples 'unclean failure' do
let(:service_double) { instance_double(BulkImportRowService) }
before do
allow(service_double).to receive(:call) do
raise 'dummy error'
end
end
it 'raises an error and does not change processed items count' do
expect { subject.perform(row.id) }.to raise_error(StandardError, 'dummy error').and(not_change { import.reload.processed_items })
end
it 'does not delete the row' do
expect { subject.perform(row.id) }.to raise_error(StandardError, 'dummy error').and(not_change { BulkImportRow.exists?(row.id) })
end
end
shared_examples 'clean success' do
let(:service_double) { instance_double(BulkImportRowService, call: true) }
it 'calls BulkImportRowService' do
subject.perform(row.id)
expect(service_double).to have_received(:call).with(row)
end
it 'increases the number of processed items' do
expect { subject.perform(row.id) }.to(change { import.reload.processed_items }.by(+1))
end
it 'increases the number of imported items' do
expect { subject.perform(row.id) }.to(change { import.reload.imported_items }.by(+1))
end
it 'deletes the row' do
expect { subject.perform(row.id) }.to change { BulkImportRow.exists?(row.id) }.from(true).to(false)
end
end
context 'when there are multiple rows to process' do
let(:import) { Fabricate(:bulk_import, total_items: 2, processed_items: 0, imported_items: 0, state: :in_progress) }
context 'with a clean failure' do
include_examples 'clean failure'
it 'does not mark the import as finished' do
expect { subject.perform(row.id) }.to_not(change { import.reload.state.to_sym })
end
end
context 'with an unclean failure' do
include_examples 'unclean failure'
it 'does not mark the import as finished' do
expect { subject.perform(row.id) }.to raise_error(StandardError).and(not_change { import.reload.state.to_sym })
end
end
context 'with a clean success' do
include_examples 'clean success'
it 'does not mark the import as finished' do
expect { subject.perform(row.id) }.to_not(change { import.reload.state.to_sym })
end
end
end
context 'when this is the last row to process' do
let(:import) { Fabricate(:bulk_import, total_items: 2, processed_items: 1, imported_items: 0, state: :in_progress) }
context 'with a clean failure' do
include_examples 'clean failure'
it 'marks the import as finished' do
expect { subject.perform(row.id) }.to change { import.reload.state.to_sym }.from(:in_progress).to(:finished)
end
end
# NOTE: sidekiq retry logic may be a bit too difficult to test, so leaving this blind spot for now
it_behaves_like 'unclean failure'
context 'with a clean success' do
include_examples 'clean success'
it 'marks the import as finished' do
expect { subject.perform(row.id) }.to change { import.reload.state.to_sym }.from(:in_progress).to(:finished)
end
end
end
end
end