4.2.0-Beta2 test update

db/migrate/20170105224407_add_shortcode_to_media_attachments.rb

@@ -1,12 +1,19 @@
 # frozen_string_literal: true
 
 class AddShortcodeToMediaAttachments < ActiveRecord::Migration[5.0]
+  class MigrationMediaAttachment < ApplicationRecord
+    self.table_name = :media_attachments
+    scope :local, -> { where(remote_url: '') }
+  end
+
   def up
     add_column :media_attachments, :shortcode, :string, null: true, default: nil
     add_index :media_attachments, :shortcode, unique: true
 
+    MigrationMediaAttachment.reset_column_information
+
     # Migrate old links
-    MediaAttachment.local.update_all('shortcode = id')
+    MigrationMediaAttachment.local.update_all('shortcode = id')
   end
 
   def down

db/migrate/20170304202101_add_type_to_media_attachments.rb

@@ -1,11 +1,24 @@
 # frozen_string_literal: true
 
 class AddTypeToMediaAttachments < ActiveRecord::Migration[5.0]
+  class MigrationMediaAttachment < ApplicationRecord
+    self.table_name = :media_attachments
+    enum type: [:image, :gifv, :video]
+    IMAGE_MIME_TYPES = ['image/jpeg', 'image/png', 'image/gif'].freeze
+    VIDEO_MIME_TYPES = ['video/webm', 'video/mp4'].freeze
+  end
+
   def up
     add_column :media_attachments, :type, :integer, default: 0, null: false
 
-    MediaAttachment.where(file_content_type: MediaAttachment::IMAGE_MIME_TYPES).update_all(type: MediaAttachment.types[:image])
-    MediaAttachment.where(file_content_type: MediaAttachment::VIDEO_MIME_TYPES).update_all(type: MediaAttachment.types[:video])
+    MigrationMediaAttachment.reset_column_information
+
+    MigrationMediaAttachment
+      .where(file_content_type: MigrationMediaAttachment::IMAGE_MIME_TYPES)
+      .update_all(type: MigrationMediaAttachment.types[:image])
+    MigrationMediaAttachment
+      .where(file_content_type: MigrationMediaAttachment::VIDEO_MIME_TYPES)
+      .update_all(type: MigrationMediaAttachment.types[:video])
   end
 
   def down

db/migrate/20181024224956_migrate_account_conversations.rb

@@ -7,9 +7,29 @@ class MigrateAccountConversations < ActiveRecord::Migration[5.2]
 
   disable_ddl_transaction!
 
-  class Mention < ApplicationRecord
-    belongs_to :account, inverse_of: :mentions
-    belongs_to :status, -> { unscope(where: :deleted_at) }
+  class MigrationAccount < ApplicationRecord
+    self.table_name = :accounts
+    has_many :mentions, inverse_of: :account, dependent: :destroy, class_name: 'MigrationMention', foreign_key: :account_id
+  end
+
+  class MigrationConversation < ApplicationRecord
+    self.table_name = :conversations
+  end
+
+  class MigrationStatus < ApplicationRecord
+    self.table_name = :statuses
+    belongs_to :account, class_name: 'MigrationAccount'
+    has_many :mentions, dependent: :destroy, inverse_of: :status, class_name: 'MigrationMention', foreign_key: :status_id
+    scope :local, -> { where(local: true).or(where(uri: nil)) }
+    enum visibility: { public: 0, unlisted: 1, private: 2, direct: 3, limited: 4 }, _suffix: :visibility
+    has_many :active_mentions, -> { active }, class_name: 'MigrationMention', inverse_of: :status, foreign_key: :status_id
+  end
+
+  class MigrationMention < ApplicationRecord
+    self.table_name = :mentions
+    belongs_to :account, inverse_of: :mentions, class_name: 'MigrationAccount'
+    belongs_to :status, -> { unscope(where: :deleted_at) }, class_name: 'MigrationStatus'
+    scope :active, -> { where(silent: false) }
 
     delegate(
       :username,
@@ -19,22 +39,24 @@ class MigrateAccountConversations < ActiveRecord::Migration[5.2]
     )
   end
 
-  class Notification < ApplicationRecord
-    belongs_to :account, optional: true
+  class MigrationNotification < ApplicationRecord
+    self.table_name = :notifications
+    belongs_to :account, optional: true, class_name: 'MigrationAccount'
     belongs_to :activity, polymorphic: true, optional: true
 
-    belongs_to :status,  foreign_key: 'activity_id', optional: true
-    belongs_to :mention, foreign_key: 'activity_id', optional: true
+    belongs_to :status,  foreign_key: 'activity_id', optional: true, class_name: 'MigrationStatus'
+    belongs_to :mention, foreign_key: 'activity_id', optional: true, class_name: 'MigrationMention'
 
     def target_status
       mention&.status
     end
   end
 
-  class AccountConversation < ApplicationRecord
-    belongs_to :account
-    belongs_to :conversation
-    belongs_to :last_status, -> { unscope(where: :deleted_at) }, class_name: 'Status'
+  class MigrationAccountConversation < ApplicationRecord
+    self.table_name = :account_conversations
+    belongs_to :account, class_name: 'MigrationAccount'
+    belongs_to :conversation, class_name: 'MigrationConversation'
+    belongs_to :last_status, -> { unscope(where: :deleted_at) }, class_name: 'MigrationStatus'
 
     before_validation :set_last_status
 
@@ -74,7 +96,7 @@ class MigrateAccountConversations < ActiveRecord::Migration[5.2]
     last_time = Time.zone.now
 
     local_direct_statuses.includes(:account, mentions: :account).find_each do |status|
-      AccountConversation.add_status(status.account, status)
+      MigrationAccountConversation.add_status(status.account, status)
       migrated += 1
 
       if Time.zone.now - last_time > 1
@@ -84,7 +106,7 @@ class MigrateAccountConversations < ActiveRecord::Migration[5.2]
     end
 
     notifications_about_direct_statuses.includes(:account, mention: { status: [:account, mentions: :account] }).find_each do |notification|
-      AccountConversation.add_status(notification.account, notification.target_status)
+      MigrationAccountConversation.add_status(notification.account, notification.target_status)
       migrated += 1
 
       if Time.zone.now - last_time > 1
@@ -103,10 +125,10 @@ class MigrateAccountConversations < ActiveRecord::Migration[5.2]
   end
 
   def local_direct_statuses
-    Status.unscoped.local.where(visibility: :direct)
+    MigrationStatus.unscoped.local.where(visibility: :direct)
   end
 
   def notifications_about_direct_statuses
-    Notification.joins('INNER JOIN mentions ON mentions.id = notifications.activity_id INNER JOIN statuses ON statuses.id = mentions.status_id').where(activity_type: 'Mention', statuses: { visibility: :direct })
+    MigrationNotification.joins('INNER JOIN mentions ON mentions.id = notifications.activity_id INNER JOIN statuses ON statuses.id = mentions.status_id').where(activity_type: 'Mention', statuses: { visibility: :direct })
   end
 end

db/migrate/20181116173541_copy_account_stats.rb

@@ -3,6 +3,10 @@
 class CopyAccountStats < ActiveRecord::Migration[5.2]
   disable_ddl_transaction!
 
+  class MigrationAccount < ApplicationRecord
+    self.table_name = :accounts
+  end
+
   def up
     safety_assured do
       if supports_upsert?
@@ -27,7 +31,7 @@ class CopyAccountStats < ActiveRecord::Migration[5.2]
   def up_fast
     say 'Upsert is available, importing counters using the fast method'
 
-    Account.unscoped.select('id').find_in_batches(batch_size: 5_000) do |accounts|
+    MigrationAccount.unscoped.select('id').find_in_batches(batch_size: 5_000) do |accounts|
       execute <<-SQL.squish
         INSERT INTO account_stats (account_id, statuses_count, following_count, followers_count, created_at, updated_at)
         SELECT id, statuses_count, following_count, followers_count, created_at, updated_at
@@ -44,7 +48,7 @@ class CopyAccountStats < ActiveRecord::Migration[5.2]
 
     # We cannot use bulk INSERT or overarching transactions here because of possible
     # uniqueness violations that we need to skip over
-    Account.unscoped.select('id, statuses_count, following_count, followers_count, created_at, updated_at').find_each do |account|
+    MigrationAccount.unscoped.select('id, statuses_count, following_count, followers_count, created_at, updated_at').find_each do |account|
       params = [account.id, account[:statuses_count], account[:following_count], account[:followers_count], account.created_at, account.updated_at]
       exec_insert('INSERT INTO account_stats (account_id, statuses_count, following_count, followers_count, created_at, updated_at) VALUES ($1, $2, $3, $4, $5, $6)', nil, params)
     rescue ActiveRecord::RecordNotUnique

db/migrate/20181116173541_copy_account_stats.rb.orig

@@ -0,0 +1,62 @@
+# frozen_string_literal: true
+
+class CopyAccountStats < ActiveRecord::Migration[5.2]
+  disable_ddl_transaction!
+
+  class MigrationAccount < ApplicationRecord
+    self.table_name = :accounts
+  end
+
+  def up
+    safety_assured do
+      if supports_upsert?
+        up_fast
+      else
+        up_slow
+      end
+    end
+  end
+
+  def down
+    # Nothing
+  end
+
+  private
+
+  def supports_upsert?
+    version = select_one("SELECT current_setting('server_version_num') AS v")['v'].to_i
+    version >= 90_500
+  end
+
+  def up_fast
+    say 'Upsert is available, importing counters using the fast method'
+
+    MigrationAccount.unscoped.select('id').find_in_batches(batch_size: 5_000) do |accounts|
+      execute <<-SQL.squish
+        INSERT INTO account_stats (account_id, statuses_count, following_count, followers_count, created_at, updated_at)
+        SELECT id, statuses_count, following_count, followers_count, created_at, updated_at
+        FROM accounts
+        WHERE id IN (#{accounts.map(&:id).join(', ')})
+        ON CONFLICT (account_id) DO UPDATE
+        SET statuses_count = EXCLUDED.statuses_count, following_count = EXCLUDED.following_count, followers_count = EXCLUDED.followers_count
+      SQL
+    end
+  end
+
+  def up_slow
+    say 'Upsert is not available in PostgreSQL below 9.5, falling back to slow import of counters'
+
+    # We cannot use bulk INSERT or overarching transactions here because of possible
+    # uniqueness violations that we need to skip over
+<<<<<<< HEAD
+    Account.unscoped.select('id, statuses_count, following_count, followers_count, created_at, updated_at').find_each do |account|
+=======
+    MigrationAccount.unscoped.select('id, statuses_count, following_count, followers_count, created_at, updated_at').find_each do |account|
+>>>>>>> upstream/main
+      params = [account.id, account[:statuses_count], account[:following_count], account[:followers_count], account.created_at, account.updated_at]
+      exec_insert('INSERT INTO account_stats (account_id, statuses_count, following_count, followers_count, created_at, updated_at) VALUES ($1, $2, $3, $4, $5, $6)', nil, params)
+    rescue ActiveRecord::RecordNotUnique
+      next
+    end
+  end
+end

db/migrate/20230725213448_add_image_description_to_preview_cards.rb

@@ -0,0 +1,17 @@
+# frozen_string_literal: true
+
+require Rails.root.join('lib', 'mastodon', 'migration_helpers')
+
+class AddImageDescriptionToPreviewCards < ActiveRecord::Migration[7.0]
+  include Mastodon::MigrationHelpers
+
+  disable_ddl_transaction!
+
+  def up
+    safety_assured { add_column_with_default :preview_cards, :image_description, :string, default: '', allow_null: false }
+  end
+
+  def down
+    remove_column :preview_cards, :image_description
+  end
+end

db/migrate/20230814223300_add_indexable_to_accounts.rb

@@ -0,0 +1,17 @@
+# frozen_string_literal: true
+
+require Rails.root.join('lib', 'mastodon', 'migration_helpers')
+
+class AddIndexableToAccounts < ActiveRecord::Migration[7.0]
+  include Mastodon::MigrationHelpers
+
+  disable_ddl_transaction!
+
+  def up
+    safety_assured { add_column_with_default :accounts, :indexable, :boolean, default: false, allow_null: false }
+  end
+
+  def down
+    remove_column :accounts, :indexable
+  end
+end

db/migrate/20230818141056_create_global_follow_recommendations.rb

@@ -0,0 +1,8 @@
+# frozen_string_literal: true
+
+class CreateGlobalFollowRecommendations < ActiveRecord::Migration[7.0]
+  def change
+    create_view :global_follow_recommendations, materialized: { no_data: true }
+    safety_assured { add_index :global_follow_recommendations, :account_id, unique: true }
+  end
+end

db/post_migrate/20201017234926_fill_account_suspension_origin.rb

@@ -3,8 +3,15 @@
 class FillAccountSuspensionOrigin < ActiveRecord::Migration[5.2]
   disable_ddl_transaction!
 
+  class MigrationAccount < ApplicationRecord
+    self.table_name = :accounts
+    scope :suspended, -> { where.not(suspended_at: nil) }
+    enum suspension_origin: { local: 0, remote: 1 }, _prefix: true
+  end
+
   def up
-    Account.suspended.where(suspension_origin: nil).in_batches.update_all(suspension_origin: :local)
+    MigrationAccount.reset_column_information
+    MigrationAccount.suspended.where(suspension_origin: nil).in_batches.update_all(suspension_origin: :local)
   end
 
   def down; end

db/post_migrate/20230803082451_add_unique_index_on_preview_cards_statuses.rb

@@ -0,0 +1,39 @@
+# frozen_string_literal: true
+
+class AddUniqueIndexOnPreviewCardsStatuses < ActiveRecord::Migration[6.1]
+  disable_ddl_transaction!
+
+  def up
+    add_index :preview_cards_statuses, [:status_id, :preview_card_id], name: :preview_cards_statuses_pkey, algorithm: :concurrently, unique: true
+  rescue ActiveRecord::RecordNotUnique
+    deduplicate_and_reindex!
+  end
+
+  def down
+    remove_index :preview_cards_statuses, name: :preview_cards_statuses_pkey
+  end
+
+  private
+
+  def deduplicate_and_reindex!
+    deduplicate_preview_cards!
+
+    safety_assured { execute 'REINDEX INDEX CONCURRENTLY preview_cards_statuses_pkey' }
+  rescue ActiveRecord::RecordNotUnique
+    retry
+  end
+
+  def deduplicate_preview_cards!
+    # Statuses should have only one preview card at most, even if that's not the database
+    # constraint we will end up with
+    duplicate_ids = select_all('SELECT status_id FROM preview_cards_statuses GROUP BY status_id HAVING count(*) > 1;').rows
+
+    duplicate_ids.each_slice(1000) do |ids|
+      # This one is tricky: since we don't have primary keys to keep only one record,
+      # use the physical `ctid`
+      safety_assured do
+        execute "DELETE FROM preview_cards_statuses p WHERE p.status_id IN (#{ids.join(', ')}) AND p.ctid NOT IN (SELECT q.ctid FROM preview_cards_statuses q WHERE q.status_id = p.status_id LIMIT 1)"
+      end
+    end
+  end
+end

db/post_migrate/20230803112520_add_primary_key_to_preview_cards_statuses_join_table.rb

@@ -0,0 +1,20 @@
+# frozen_string_literal: true
+
+class AddPrimaryKeyToPreviewCardsStatusesJoinTable < ActiveRecord::Migration[6.1]
+  disable_ddl_transaction!
+
+  def up
+    safety_assured do
+      execute 'ALTER TABLE preview_cards_statuses ADD PRIMARY KEY USING INDEX preview_cards_statuses_pkey'
+    end
+  end
+
+  def down
+    safety_assured do
+      # I have found no way to demote the primary key to an index, instead, re-create the index
+      execute 'CREATE UNIQUE INDEX CONCURRENTLY preview_cards_statuses_pkey_tmp ON preview_cards_statuses (status_id, preview_card_id)'
+      execute 'ALTER TABLE preview_cards_statuses DROP CONSTRAINT preview_cards_statuses_pkey'
+      execute 'ALTER INDEX preview_cards_statuses_pkey_tmp RENAME TO preview_cards_statuses_pkey'
+    end
+  end
+end

db/post_migrate/20230811103651_remove_index_preview_cards_statuses_on_status_id_and_preview_card_id.rb

@@ -0,0 +1,9 @@
+# frozen_string_literal: true
+
+class RemoveIndexPreviewCardsStatusesOnStatusIdAndPreviewCardId < ActiveRecord::Migration[7.0]
+  disable_ddl_transaction!
+
+  def change
+    remove_index :preview_cards_statuses, column: [:status_id, :preview_card_id], name: :index_preview_cards_statuses_on_status_id_and_preview_card_id
+  end
+end

db/post_migrate/20230818142253_drop_follow_recommendations.rb

@@ -0,0 +1,12 @@
+# frozen_string_literal: true
+
+class DropFollowRecommendations < ActiveRecord::Migration[7.0]
+  def up
+    drop_view :follow_recommendations, materialized: true
+  end
+
+  def down
+    create_view :follow_recommendations, version: 2, materialized: { no_data: true }
+    safety_assured { add_index :follow_recommendations, :account_id, unique: true }
+  end
+end

db/schema.rb

@@ -10,7 +10,7 @@
 #
 # It's strongly recommended that you check this file into your version control system.
 
-ActiveRecord::Schema[7.0].define(version: 2023_07_24_160715) do
+ActiveRecord::Schema[7.0].define(version: 2023_08_18_142253) do
   # These are extensions that must be enabled in order to support this database
   enable_extension "plpgsql"
 
@@ -185,6 +185,7 @@ ActiveRecord::Schema[7.0].define(version: 2023_07_24_160715) do
     t.boolean "trendable"
     t.datetime "reviewed_at", precision: nil
     t.datetime "requested_review_at", precision: nil
+    t.boolean "indexable", default: false, null: false
     t.index "(((setweight(to_tsvector('simple'::regconfig, (display_name)::text), 'A'::\"char\") || setweight(to_tsvector('simple'::regconfig, (username)::text), 'B'::\"char\")) || setweight(to_tsvector('simple'::regconfig, (COALESCE(domain, ''::character varying))::text), 'C'::\"char\")))", name: "search_index", using: :gin
     t.index "lower((username)::text), COALESCE(lower((domain)::text), ''::text)", name: "index_accounts_on_username_and_domain_lower", unique: true
     t.index ["domain", "id"], name: "index_accounts_on_domain_and_id"
@@ -802,13 +803,13 @@ ActiveRecord::Schema[7.0].define(version: 2023_07_24_160715) do
     t.boolean "trendable"
     t.integer "link_type"
     t.datetime "published_at"
+    t.string "image_description", default: "", null: false
     t.index ["url"], name: "index_preview_cards_on_url", unique: true
   end
 
-  create_table "preview_cards_statuses", id: false, force: :cascade do |t|
+  create_table "preview_cards_statuses", primary_key: ["status_id", "preview_card_id"], force: :cascade do |t|
     t.bigint "preview_card_id", null: false
     t.bigint "status_id", null: false
-    t.index ["status_id", "preview_card_id"], name: "index_preview_cards_statuses_on_status_id_and_preview_card_id"
   end
 
   create_table "relays", force: :cascade do |t|
@@ -1333,34 +1334,36 @@ ActiveRecord::Schema[7.0].define(version: 2023_07_24_160715) do
   SQL
   add_index "account_summaries", ["account_id"], name: "index_account_summaries_on_account_id", unique: true
 
-  create_view "follow_recommendations", materialized: true, sql_definition: <<-SQL
+  create_view "global_follow_recommendations", materialized: true, sql_definition: <<-SQL
       SELECT t0.account_id,
       sum(t0.rank) AS rank,
       array_agg(t0.reason) AS reason
      FROM ( SELECT account_summaries.account_id,
               ((count(follows.id))::numeric / (1.0 + (count(follows.id))::numeric)) AS rank,
               'most_followed'::text AS reason
-             FROM (((follows
+             FROM ((follows
                JOIN account_summaries ON ((account_summaries.account_id = follows.target_account_id)))
                JOIN users ON ((users.account_id = follows.account_id)))
-               LEFT JOIN follow_recommendation_suppressions ON ((follow_recommendation_suppressions.account_id = follows.target_account_id)))
-            WHERE ((users.current_sign_in_at >= (now() - 'P30D'::interval)) AND (account_summaries.sensitive = false) AND (follow_recommendation_suppressions.id IS NULL))
+            WHERE ((users.current_sign_in_at >= (now() - 'P30D'::interval)) AND (account_summaries.sensitive = false) AND (NOT (EXISTS ( SELECT 1
+                     FROM follow_recommendation_suppressions
+                    WHERE (follow_recommendation_suppressions.account_id = follows.target_account_id)))))
             GROUP BY account_summaries.account_id
            HAVING (count(follows.id) >= 5)
           UNION ALL
            SELECT account_summaries.account_id,
               (sum((status_stats.reblogs_count + status_stats.favourites_count)) / (1.0 + sum((status_stats.reblogs_count + status_stats.favourites_count)))) AS rank,
               'most_interactions'::text AS reason
-             FROM (((status_stats
+             FROM ((status_stats
                JOIN statuses ON ((statuses.id = status_stats.status_id)))
                JOIN account_summaries ON ((account_summaries.account_id = statuses.account_id)))
-               LEFT JOIN follow_recommendation_suppressions ON ((follow_recommendation_suppressions.account_id = statuses.account_id)))
-            WHERE ((statuses.id >= (((date_part('epoch'::text, (now() - 'P30D'::interval)) * (1000)::double precision))::bigint << 16)) AND (account_summaries.sensitive = false) AND (follow_recommendation_suppressions.id IS NULL))
+            WHERE ((statuses.id >= (((date_part('epoch'::text, (now() - 'P30D'::interval)) * (1000)::double precision))::bigint << 16)) AND (account_summaries.sensitive = false) AND (NOT (EXISTS ( SELECT 1
+                     FROM follow_recommendation_suppressions
+                    WHERE (follow_recommendation_suppressions.account_id = statuses.account_id)))))
             GROUP BY account_summaries.account_id
            HAVING (sum((status_stats.reblogs_count + status_stats.favourites_count)) >= (5)::numeric)) t0
     GROUP BY t0.account_id
     ORDER BY (sum(t0.rank)) DESC;
   SQL
-  add_index "follow_recommendations", ["account_id"], name: "index_follow_recommendations_on_account_id", unique: true
+  add_index "global_follow_recommendations", ["account_id"], name: "index_global_follow_recommendations_on_account_id", unique: true
 
 end

db/views/global_follow_recommendations_v01.sql

@@ -0,0 +1,32 @@
+SELECT
+  account_id,
+  sum(rank) AS rank,
+  array_agg(reason) AS reason
+FROM (
+  SELECT
+    account_summaries.account_id AS account_id,
+    count(follows.id) / (1.0 + count(follows.id)) AS rank,
+    'most_followed' AS reason
+  FROM follows
+  INNER JOIN account_summaries ON account_summaries.account_id = follows.target_account_id
+  INNER JOIN users ON users.account_id = follows.account_id
+  WHERE users.current_sign_in_at >= (now() - interval '30 days')
+    AND account_summaries.sensitive = 'f'
+    AND NOT EXISTS (SELECT 1 FROM follow_recommendation_suppressions WHERE follow_recommendation_suppressions.account_id = follows.target_account_id)
+  GROUP BY account_summaries.account_id
+  HAVING count(follows.id) >= 5
+  UNION ALL
+  SELECT account_summaries.account_id AS account_id,
+         sum(status_stats.reblogs_count + status_stats.favourites_count) / (1.0 + sum(status_stats.reblogs_count + status_stats.favourites_count)) AS rank,
+         'most_interactions' AS reason
+  FROM status_stats
+  INNER JOIN statuses ON statuses.id = status_stats.status_id
+  INNER JOIN account_summaries ON account_summaries.account_id = statuses.account_id
+  WHERE statuses.id >= ((date_part('epoch', now() - interval '30 days') * 1000)::bigint << 16)
+    AND account_summaries.sensitive = 'f'
+    AND NOT EXISTS (SELECT 1 FROM follow_recommendation_suppressions WHERE follow_recommendation_suppressions.account_id = statuses.account_id)
+  GROUP BY account_summaries.account_id
+  HAVING sum(status_stats.reblogs_count + status_stats.favourites_count) >= 5
+) t0
+GROUP BY account_id
+ORDER BY rank DESC