Allow import/export of instance-level domain blocks/allows (#1754)

* Allow import/export of instance-level domain blocks/allows.
Fixes #15095

* Pacify circleci

* Address simple code review feedback

* Add headers to exported CSV

* Extract common import/export functionality to
AdminExportControllerConcern

* Add additional fields to instance-blocked domain export

* Address review feedback

* Split instance domain block/allow import/export into separate pages/controllers

* Address code review feedback

* Pacify DeepSource

* Work around Paperclip::HasAttachmentFile for Rails 6

* Fix deprecated API warning in export tests

* Remove after_commit workaround

app/controllers/admin/export_domain_allows_controller.rb

@@ -0,0 +1,60 @@
+# frozen_string_literal: true
+
+require 'csv'
+
+module Admin
+  class ExportDomainAllowsController < BaseController
+    include AdminExportControllerConcern
+
+    before_action :set_dummy_import!, only: [:new]
+
+    ROWS_PROCESSING_LIMIT = 20_000
+
+    def new
+      authorize :domain_allow, :create?
+    end
+
+    def export
+      authorize :instance, :index?
+      send_export_file
+    end
+
+    def import
+      authorize :domain_allow, :create?
+      begin
+        @import = Admin::Import.new(import_params)
+        parse_import_data!(export_headers)
+
+        @data.take(ROWS_PROCESSING_LIMIT).each do |row|
+          domain = row['#domain'].strip
+          next if DomainAllow.allowed?(domain)
+
+          domain_allow = DomainAllow.new(domain: domain)
+          log_action :create, domain_allow if domain_allow.save
+        end
+        flash[:notice] = I18n.t('admin.domain_allows.created_msg')
+      rescue ActionController::ParameterMissing
+        flash[:error] = I18n.t('admin.export_domain_allows.no_file')
+      end
+      redirect_to admin_instances_path
+    end
+
+    private
+
+    def export_filename
+      'domain_allows.csv'
+    end
+
+    def export_headers
+      %w(#domain)
+    end
+
+    def export_data
+      CSV.generate(headers: export_headers, write_headers: true) do |content|
+        DomainAllow.allowed_domains.each do |instance|
+          content << [instance.domain]
+        end
+      end
+    end
+  end
+end

app/controllers/admin/export_domain_blocks_controller.rb

@@ -0,0 +1,68 @@
+# frozen_string_literal: true
+
+require 'csv'
+
+module Admin
+  class ExportDomainBlocksController < BaseController
+    include AdminExportControllerConcern
+
+    before_action :set_dummy_import!, only: [:new]
+
+    ROWS_PROCESSING_LIMIT = 20_000
+
+    def new
+      authorize :domain_block, :create?
+    end
+
+    def export
+      authorize :instance, :index?
+      send_export_file
+    end
+
+    def import
+      authorize :domain_block, :create?
+      begin
+        @import = Admin::Import.new(import_params)
+        parse_import_data!(export_headers)
+
+        @data.take(ROWS_PROCESSING_LIMIT).each do |row|
+          domain = row['#domain'].strip
+          next if DomainBlock.rule_for(domain).present?
+
+          domain_block = DomainBlock.new(domain: domain,
+                                         severity: row['#severity'].strip,
+                                         reject_media: row['#reject_media'].strip,
+                                         reject_reports: row['#reject_reports'].strip,
+                                         public_comment: row['#public_comment'].strip,
+                                         obfuscate: row['#obfuscate'].strip)
+          if domain_block.save
+            DomainBlockWorker.perform_async(domain_block.id)
+            log_action :create, domain_block
+          end
+        end
+        flash[:notice] = I18n.t('admin.domain_blocks.created_msg')
+      rescue ActionController::ParameterMissing
+        flash[:error] = I18n.t('admin.export_domain_blocks.no_file')
+      end
+      redirect_to admin_instances_path(limited: '1')
+    end
+
+    private
+
+    def export_filename
+      'domain_blocks.csv'
+    end
+
+    def export_headers
+      %w(#domain #severity #reject_media #reject_reports #public_comment #obfuscate)
+    end
+
+    def export_data
+      CSV.generate(headers: export_headers, write_headers: true) do |content|
+        DomainBlock.with_user_facing_limitations.each do |instance|
+          content << [instance.domain, instance.severity, instance.reject_media, instance.reject_reports, instance.public_comment, instance.obfuscate]
+        end
+      end
+    end
+  end
+end

app/controllers/concerns/admin_export_controller_concern.rb

@@ -0,0 +1,39 @@
+# frozen_string_literal: true
+
+module AdminExportControllerConcern
+  extend ActiveSupport::Concern
+
+  private
+
+  def send_export_file
+    respond_to do |format|
+      format.csv { send_data export_data, filename: export_filename }
+    end
+  end
+
+  def export_data
+    raise 'Override in controller'
+  end
+
+  def export_filename
+    raise 'Override in controller'
+  end
+
+  def set_dummy_import!
+    @import = Admin::Import.new
+  end
+
+  def import_params
+    params.require(:admin_import).permit(:data)
+  end
+
+  def import_data
+    Paperclip.io_adapters.for(@import.data).read
+  end
+
+  def parse_import_data!(default_headers)
+    data = CSV.parse(import_data, headers: true)
+    data = CSV.parse(import_data, headers: default_headers) unless data.headers&.first&.strip&.include?(default_headers[0])
+    @data = data.reject(&:blank?)
+  end
+end

app/models/admin/import.rb

@@ -0,0 +1,29 @@
+# frozen_string_literal: true
+
+# A non-activerecord helper class for csv upload
+class Admin::Import
+  extend ActiveModel::Callbacks
+  include ActiveModel::Model
+  include Paperclip::Glue
+
+  FILE_TYPES = %w(text/plain text/csv application/csv).freeze
+
+  # Paperclip required callbacks
+  define_model_callbacks :save, only: [:after]
+  define_model_callbacks :destroy, only: [:before, :after]
+
+  attr_accessor :data_file_name, :data_content_type
+
+  has_attached_file :data
+  validates_attachment_content_type :data, content_type: FILE_TYPES
+  validates_attachment_presence :data
+  validates_with AdminImportValidator, on: :create
+
+  def save
+    run_callbacks :save
+  end
+
+  def destroy
+    run_callbacks :destroy
+  end
+end

app/models/domain_allow.rb

@@ -23,6 +23,10 @@ class DomainAllow < ApplicationRecord
       !rule_for(domain).nil?
     end
 
+    def allowed_domains
+      select(:domain)
+    end
+
     def rule_for(domain)
       return if domain.blank?
 

app/validators/admin_import_validator.rb

@@ -0,0 +1,19 @@
+# frozen_string_literal: true
+
+class AdminImportValidator < ActiveModel::Validator
+  FIRST_HEADER = '#domain'
+
+  def validate(import)
+    return if import.type.blank? || import.data.blank?
+
+    # We parse because newlines could be part of individual rows. This
+    # runs on create so we should be reading the local file here before
+    # it is uploaded to object storage or moved anywhere...
+    csv_data = CSV.parse(import.data.queued_for_write[:original].read)
+
+    row_count  = csv_data.size
+    row_count -= 1 if csv_data.first&.first == FIRST_HEADER
+
+    import.errors.add(:data, I18n.t('imports.errors.over_rows_processing_limit', count: Admin::DomainBlocksController::ROWS_PROCESSING_LIMIT)) if row_count > Admin::DomainBlocksController::ROWS_PROCESSING_LIMIT
+  end
+end

app/views/admin/export_domain_allows/new.html.haml

@@ -0,0 +1,10 @@
+- content_for :page_title do
+  = t('.title')
+
+= simple_form_for @import, url: import_admin_export_domain_allows_path, html: { multipart: true } do |f|
+  .fields-row
+    .fields-group.fields-row__column.fields-row__column-6
+      = f.input :data, wrapper: :with_block_label, hint: t('simple_form.hints.imports.data'), as: :file
+
+  .actions
+    = f.button :button, t('imports.upload'), type: :submit

app/views/admin/export_domain_blocks/new.html.haml

@@ -0,0 +1,10 @@
+- content_for :page_title do
+  = t('.title')
+
+= simple_form_for @import, url: import_admin_export_domain_blocks_path, html: { multipart: true } do |f|
+  .fields-row
+    .fields-group.fields-row__column.fields-row__column-6
+      = f.input :data, wrapper: :with_block_label, hint: t('simple_form.hints.imports.data'), as: :file
+
+  .actions
+    = f.button :button, t('imports.upload'), type: :submit

app/views/admin/instances/index.html.haml

@@ -4,8 +4,12 @@
 - content_for :heading_actions do
   - if whitelist_mode?
     = link_to t('admin.domain_allows.add_new'), new_admin_domain_allow_path, class: 'button', id: 'add-instance-button'
+    = link_to t('admin.domain_allows.export'), export_admin_export_domain_allows_path(format: :csv), class: 'button'
+    = link_to t('admin.domain_allows.import'), new_admin_export_domain_allow_path, class: 'button'
   - else
     = link_to t('admin.domain_blocks.add_new'), new_admin_domain_block_path, class: 'button', id: 'add-instance-button'
+    = link_to t('admin.domain_blocks.export'), export_admin_export_domain_blocks_path(format: :csv), class: 'button'
+    = link_to t('admin.domain_blocks.import'), new_admin_export_domain_block_path, class: 'button'
 
 .filters
   .filter-subset