Merge upstream 2.0ish #165

db/migrate/20170918125918_ids_to_bigints.rb

@@ -1,127 +1,119 @@
+require Rails.root.join('lib', 'mastodon', 'migration_helpers')
+
 class IdsToBigints < ActiveRecord::Migration[5.1]
+  include Mastodon::MigrationHelpers
+
+  disable_ddl_transaction!
+
+  INCLUDED_COLUMNS = [
+    [:account_domain_blocks, :account_id],
+    [:account_domain_blocks, :id],
+    [:accounts, :id],
+    [:blocks, :account_id],
+    [:blocks, :id],
+    [:blocks, :target_account_id],
+    [:conversation_mutes, :account_id],
+    [:conversation_mutes, :id],
+    [:domain_blocks, :id],
+    [:favourites, :account_id],
+    [:favourites, :id],
+    [:favourites, :status_id],
+    [:follow_requests, :account_id],
+    [:follow_requests, :id],
+    [:follow_requests, :target_account_id],
+    [:follows, :account_id],
+    [:follows, :id],
+    [:follows, :target_account_id],
+    [:imports, :account_id],
+    [:imports, :id],
+    [:media_attachments, :account_id],
+    [:media_attachments, :id],
+    [:mentions, :account_id],
+    [:mentions, :id],
+    [:mutes, :account_id],
+    [:mutes, :id],
+    [:mutes, :target_account_id],
+    [:notifications, :account_id],
+    [:notifications, :from_account_id],
+    [:notifications, :id],
+    [:oauth_access_grants, :application_id],
+    [:oauth_access_grants, :id],
+    [:oauth_access_grants, :resource_owner_id],
+    [:oauth_access_tokens, :application_id],
+    [:oauth_access_tokens, :id],
+    [:oauth_access_tokens, :resource_owner_id],
+    [:oauth_applications, :id],
+    [:oauth_applications, :owner_id],
+    [:reports, :account_id],
+    [:reports, :action_taken_by_account_id],
+    [:reports, :id],
+    [:reports, :target_account_id],
+    [:session_activations, :access_token_id],
+    [:session_activations, :user_id],
+    [:session_activations, :web_push_subscription_id],
+    [:settings, :id],
+    [:settings, :thing_id],
+    [:statuses, :account_id],
+    [:statuses, :application_id],
+    [:statuses, :in_reply_to_account_id],
+    [:stream_entries, :account_id],
+    [:stream_entries, :id],
+    [:subscriptions, :account_id],
+    [:subscriptions, :id],
+    [:tags, :id],
+    [:users, :account_id],
+    [:users, :id],
+    [:web_settings, :id],
+    [:web_settings, :user_id],
+  ]
+  INCLUDED_COLUMNS << [:deprecated_preview_cards, :id] if table_exists?(:deprecated_preview_cards)
+
+  def migrate_columns(to_type)
+    # Print out a warning that this will probably take a while.
+    say ''
+    say 'WARNING: This migration may take a *long* time for large instances'
+    say 'It will *not* lock tables for any significant time, but it may run'
+    say 'for a very long time. We will pause for 10 seconds to allow you to'
+    say 'interrupt this migration if you are not ready.'
+    say ''
+    say 'This migration has some sections that can be safely interrupted'
+    say 'and restarted later, and will tell you when those are occurring.'
+    say ''
+    say 'For more information, see https://github.com/tootsuite/mastodon/pull/5088'
+
+    10.downto(1) do |i|
+      say "Continuing in #{i} second#{i == 1 ? '' : 's'}...", true
+      sleep 1
+    end
+
+    tables = INCLUDED_COLUMNS.map(&:first).uniq
+    table_sizes = {}
+
+    # Sort tables by their size
+    tables.each do |table|
+      table_sizes[table] = estimate_rows_in_table(table)
+    end
+
+    ordered_columns = INCLUDED_COLUMNS.sort_by do |col_parts|
+      [-table_sizes[col_parts.first], col_parts.last]
+    end
+
+    ordered_columns.each do |column_parts|
+      table, column = column_parts
+
+      # Skip this if we're resuming and already did this one.
+      next if column_for(table, column).sql_type == to_type.to_s
+
+      change_column_type_concurrently table, column, to_type
+      cleanup_concurrent_column_type_change table, column
+    end
+  end
+
   def up
-    change_column :account_domain_blocks, :account_id, :bigint
-    change_column :account_domain_blocks, :id, :bigint
-    change_column :accounts, :id, :bigint
-    change_column :blocks, :account_id, :bigint
-    change_column :blocks, :id, :bigint
-    change_column :blocks, :target_account_id, :bigint
-    change_column :conversation_mutes, :account_id, :bigint
-    change_column :conversation_mutes, :id, :bigint
-    change_column :deprecated_preview_cards, :id, :bigint if table_exists?(:deprecated_preview_cards)
-    change_column :domain_blocks, :id, :bigint
-    change_column :favourites, :account_id, :bigint
-    change_column :favourites, :id, :bigint
-    change_column :favourites, :status_id, :bigint
-    change_column :follow_requests, :account_id, :bigint
-    change_column :follow_requests, :id, :bigint
-    change_column :follow_requests, :target_account_id, :bigint
-    change_column :follows, :account_id, :bigint
-    change_column :follows, :id, :bigint
-    change_column :follows, :target_account_id, :bigint
-    change_column :imports, :account_id, :bigint
-    change_column :imports, :id, :bigint
-    change_column :media_attachments, :account_id, :bigint
-    change_column :media_attachments, :id, :bigint
-    change_column :mentions, :account_id, :bigint
-    change_column :mentions, :id, :bigint
-    change_column :mutes, :account_id, :bigint
-    change_column :mutes, :id, :bigint
-    change_column :mutes, :target_account_id, :bigint
-    change_column :notifications, :account_id, :bigint
-    change_column :notifications, :from_account_id, :bigint
-    change_column :notifications, :id, :bigint
-    change_column :oauth_access_grants, :application_id, :bigint
-    change_column :oauth_access_grants, :id, :bigint
-    change_column :oauth_access_grants, :resource_owner_id, :bigint
-    change_column :oauth_access_tokens, :application_id, :bigint
-    change_column :oauth_access_tokens, :id, :bigint
-    change_column :oauth_access_tokens, :resource_owner_id, :bigint
-    change_column :oauth_applications, :id, :bigint
-    change_column :oauth_applications, :owner_id, :bigint
-    change_column :reports, :account_id, :bigint
-    change_column :reports, :action_taken_by_account_id, :bigint
-    change_column :reports, :id, :bigint
-    change_column :reports, :target_account_id, :bigint
-    change_column :session_activations, :access_token_id, :bigint
-    change_column :session_activations, :user_id, :bigint
-    change_column :session_activations, :web_push_subscription_id, :bigint
-    change_column :settings, :id, :bigint
-    change_column :settings, :thing_id, :bigint
-    change_column :statuses, :account_id, :bigint
-    change_column :statuses, :application_id, :bigint
-    change_column :statuses, :in_reply_to_account_id, :bigint
-    change_column :stream_entries, :account_id, :bigint
-    change_column :stream_entries, :id, :bigint
-    change_column :subscriptions, :account_id, :bigint
-    change_column :subscriptions, :id, :bigint
-    change_column :tags, :id, :bigint
-    change_column :users, :account_id, :bigint
-    change_column :users, :id, :bigint
-    change_column :web_settings, :id, :bigint
-    change_column :web_settings, :user_id, :bigint
+    migrate_columns(:bigint)
   end
 
   def down
-    change_column :account_domain_blocks, :account_id, :integer
-    change_column :account_domain_blocks, :id, :integer
-    change_column :accounts, :id, :integer
-    change_column :blocks, :account_id, :integer
-    change_column :blocks, :id, :integer
-    change_column :blocks, :target_account_id, :integer
-    change_column :conversation_mutes, :account_id, :integer
-    change_column :conversation_mutes, :id, :integer
-    change_column :deprecated_preview_cards, :id, :integer if table_exists?(:deprecated_preview_cards)
-    change_column :domain_blocks, :id, :integer
-    change_column :favourites, :account_id, :integer
-    change_column :favourites, :id, :integer
-    change_column :favourites, :status_id, :integer
-    change_column :follow_requests, :account_id, :integer
-    change_column :follow_requests, :id, :integer
-    change_column :follow_requests, :target_account_id, :integer
-    change_column :follows, :account_id, :integer
-    change_column :follows, :id, :integer
-    change_column :follows, :target_account_id, :integer
-    change_column :imports, :account_id, :integer
-    change_column :imports, :id, :integer
-    change_column :media_attachments, :account_id, :integer
-    change_column :media_attachments, :id, :integer
-    change_column :mentions, :account_id, :integer
-    change_column :mentions, :id, :integer
-    change_column :mutes, :account_id, :integer
-    change_column :mutes, :id, :integer
-    change_column :mutes, :target_account_id, :integer
-    change_column :notifications, :account_id, :integer
-    change_column :notifications, :from_account_id, :integer
-    change_column :notifications, :id, :integer
-    change_column :oauth_access_grants, :application_id, :integer
-    change_column :oauth_access_grants, :id, :integer
-    change_column :oauth_access_grants, :resource_owner_id, :integer
-    change_column :oauth_access_tokens, :application_id, :integer
-    change_column :oauth_access_tokens, :id, :integer
-    change_column :oauth_access_tokens, :resource_owner_id, :integer
-    change_column :oauth_applications, :id, :integer
-    change_column :oauth_applications, :owner_id, :integer
-    change_column :reports, :account_id, :integer
-    change_column :reports, :action_taken_by_account_id, :integer
-    change_column :reports, :id, :integer
-    change_column :reports, :target_account_id, :integer
-    change_column :session_activations, :access_token_id, :integer
-    change_column :session_activations, :user_id, :integer
-    change_column :session_activations, :web_push_subscription_id, :integer
-    change_column :settings, :id, :integer
-    change_column :settings, :thing_id, :integer
-    change_column :statuses, :account_id, :integer
-    change_column :statuses, :application_id, :integer
-    change_column :statuses, :in_reply_to_account_id, :integer
-    change_column :stream_entries, :account_id, :integer
-    change_column :stream_entries, :id, :integer
-    change_column :subscriptions, :account_id, :integer
-    change_column :subscriptions, :id, :integer
-    change_column :tags, :id, :integer
-    change_column :users, :account_id, :integer
-    change_column :users, :id, :integer
-    change_column :web_settings, :id, :integer
-    change_column :web_settings, :user_id, :integer
+    migrate_columns(:integer)
   end
 end

db/migrate/20170920024819_status_ids_to_timestamp_ids.rb

@@ -0,0 +1,32 @@
+class StatusIdsToTimestampIds < ActiveRecord::Migration[5.1]
+  def up
+    # Prepare the function we will use to generate IDs.
+    Rake::Task['db:define_timestamp_id'].execute
+
+    # Set up the statuses.id column to use our timestamp-based IDs.
+    ActiveRecord::Base.connection.execute(<<~SQL)
+      ALTER TABLE statuses
+      ALTER COLUMN id
+      SET DEFAULT timestamp_id('statuses')
+    SQL
+
+    # Make sure we have a sequence to use.
+    Rake::Task['db:ensure_id_sequences_exist'].execute
+  end
+
+  def down
+    # Revert the column to the old method of just using the sequence
+    # value for new IDs. Set the current ID sequence to the maximum
+    # existing ID, such that the next sequence will be one higher.
+
+    # We lock the table during this so that the ID won't get clobbered,
+    # but ID is indexed, so this should be a fast operation.
+    ActiveRecord::Base.connection.execute(<<~SQL)
+      LOCK statuses;
+      SELECT setval('statuses_id_seq', (SELECT MAX(id) FROM statuses));
+      ALTER TABLE statuses
+        ALTER COLUMN id
+        SET DEFAULT nextval('statuses_id_seq');"
+    SQL
+  end
+end

db/migrate/20170920032311_fix_reblogs_in_feeds.rb

@@ -0,0 +1,63 @@
+class FixReblogsInFeeds < ActiveRecord::Migration[5.1]
+  def up
+    redis = Redis.current
+    fm = FeedManager.instance
+
+    # find_each is batched on the database side.
+    User.includes(:account).find_each do |user|
+      account = user.account
+
+      # Old scheme:
+      # Each user's feed zset had a series of score:value entries,
+      # where "regular" statuses had the same score and value (their
+      # ID). Reblogs had a score of the reblogging status' ID, and a
+      # value of the reblogged status' ID.
+
+      # New scheme:
+      # The feed contains only entries with the same score and value.
+      # Reblogs result in the reblogging status being added to the
+      # feed, with an entry in a reblog tracking zset (where the score
+      # is once again set to the reblogging status' ID, and the value
+      # is set to the reblogged status' ID). This is safe for Redis'
+      # float coersion because in this reblog tracking zset, we only
+      # need the rebloggging status' ID to be able to stop tracking
+      # entries after they have gotten too far down the feed, which
+      # does not require an exact value.
+
+      # So, first, we iterate over the user's feed to find any reblogs.
+      timeline_key = fm.key(:home, account.id)
+      reblog_key = fm.key(:home, account.id, 'reblogs')
+      redis.zrange(timeline_key, 0, -1, with_scores: true).each do |entry|
+        next if entry[0] == entry[1]
+
+        # The score and value don't match, so this is a reblog.
+        # (note that we're transitioning from IDs < 53 bits so we
+        # don't have to worry about the loss of precision)
+
+        reblogged_id, reblogging_id = entry
+
+        # Remove the old entry
+        redis.zrem(timeline_key, reblogged_id)
+
+        # Add a new one for the reblogging status
+        redis.zadd(timeline_key, reblogging_id, reblogging_id)
+
+        # Track the fact that this was a reblog
+        redis.zadd(reblog_key, reblogging_id, reblogged_id)
+      end
+    end
+  end
+
+  def down
+    # We *deliberately* do nothing here. This means that reverting
+    # this and the associated changes to the FeedManager code could
+    # allow one superfluous reblog of any given status, but in the case
+    # where things have gone wrong and a revert is necessary, this
+    # appears preferable to requiring a database hit for every status
+    # in every users' feed simply to revert.
+
+    # Note that this is operating under the assumption that entries
+    # with >53-bit IDs have already been entered. Otherwise, we could
+    # just use the data in Redis to reverse this transition.
+  end
+end

db/migrate/20170927215609_add_description_to_media_attachments.rb

@@ -0,0 +1,5 @@
+class AddDescriptionToMediaAttachments < ActiveRecord::Migration[5.1]
+  def change
+    add_column :media_attachments, :description, :text
+  end
+end

db/migrate/20170928082043_create_email_domain_blocks.rb

@@ -0,0 +1,9 @@
+class CreateEmailDomainBlocks < ActiveRecord::Migration[5.1]
+  def change
+    create_table :email_domain_blocks do |t|
+      t.string :domain, null: false
+
+      t.timestamps
+    end
+  end
+end

db/migrate/20171005102658_create_account_moderation_notes.rb

@@ -0,0 +1,12 @@
+class CreateAccountModerationNotes < ActiveRecord::Migration[5.1]
+  def change
+    create_table :account_moderation_notes do |t|
+      t.text :content, null: false
+      t.references :account
+      t.references :target_account
+
+      t.timestamps
+    end
+    add_foreign_key :account_moderation_notes, :accounts, column: :target_account_id
+  end
+end

db/migrate/20171005171936_add_disabled_to_custom_emojis.rb

@@ -0,0 +1,15 @@
+require Rails.root.join('lib', 'mastodon', 'migration_helpers')
+
+class AddDisabledToCustomEmojis < ActiveRecord::Migration[5.1]
+  include Mastodon::MigrationHelpers
+
+  disable_ddl_transaction!
+
+  def up
+    safety_assured { add_column_with_default :custom_emojis, :disabled, :bool, default: false }
+  end
+
+  def down
+    remove_column :custom_emojis, :disabled
+  end
+end

db/migrate/20171006142024_add_uri_to_custom_emojis.rb

@@ -0,0 +1,6 @@
+class AddUriToCustomEmojis < ActiveRecord::Migration[5.1]
+  def change
+    add_column :custom_emojis, :uri, :string
+    add_column :custom_emojis, :image_remote_url, :string
+  end
+end

db/migrate/20171010023049_add_foreign_key_to_account_moderation_notes.rb

@@ -0,0 +1,5 @@
+class AddForeignKeyToAccountModerationNotes < ActiveRecord::Migration[5.1]
+  def change
+    add_foreign_key :account_moderation_notes, :accounts
+  end
+end

db/migrate/20171010025614_change_accounts_nonnullable_in_account_moderation_notes.rb

@@ -0,0 +1,6 @@
+class ChangeAccountsNonnullableInAccountModerationNotes < ActiveRecord::Migration[5.1]
+  def change
+    change_column_null :account_moderation_notes, :account_id, false
+    change_column_null :account_moderation_notes, :target_account_id, false
+  end
+end