Extract helper methods for db connection and table existence check in CLI::Maintenance task (#28281)

This commit is contained in:
Matt Jankowski 2024-01-25 08:49:33 -05:00 committed by GitHub
parent 2866106ec1
commit 274a48a9f4
No known key found for this signature in database
GPG key ID: B5690EEEBB952194

View file

@ -72,6 +72,10 @@ module Mastodon::CLI
local? ? username : "#{username}@#{domain}" local? ? username : "#{username}@#{domain}"
end end
def db_table_exists?(table)
ActiveRecord::Base.connection.table_exists?(table)
end
# This is a duplicate of the Account::Merging concern because we need it # This is a duplicate of the Account::Merging concern because we need it
# to be independent from code version. # to be independent from code version.
def merge_with!(other_account) def merge_with!(other_account)
@ -88,12 +92,12 @@ module Mastodon::CLI
AccountModerationNote, AccountPin, AccountStat, ListAccount, AccountModerationNote, AccountPin, AccountStat, ListAccount,
PollVote, Mention PollVote, Mention
] ]
owned_classes << AccountDeletionRequest if ActiveRecord::Base.connection.table_exists?(:account_deletion_requests) owned_classes << AccountDeletionRequest if db_table_exists?(:account_deletion_requests)
owned_classes << AccountNote if ActiveRecord::Base.connection.table_exists?(:account_notes) owned_classes << AccountNote if db_table_exists?(:account_notes)
owned_classes << FollowRecommendationSuppression if ActiveRecord::Base.connection.table_exists?(:follow_recommendation_suppressions) owned_classes << FollowRecommendationSuppression if db_table_exists?(:follow_recommendation_suppressions)
owned_classes << AccountIdentityProof if ActiveRecord::Base.connection.table_exists?(:account_identity_proofs) owned_classes << AccountIdentityProof if db_table_exists?(:account_identity_proofs)
owned_classes << Appeal if ActiveRecord::Base.connection.table_exists?(:appeals) owned_classes << Appeal if db_table_exists?(:appeals)
owned_classes << BulkImport if ActiveRecord::Base.connection.table_exists?(:bulk_imports) owned_classes << BulkImport if db_table_exists?(:bulk_imports)
owned_classes.each do |klass| owned_classes.each do |klass|
klass.where(account_id: other_account.id).find_each do |record| klass.where(account_id: other_account.id).find_each do |record|
@ -104,7 +108,7 @@ module Mastodon::CLI
end end
target_classes = [Follow, FollowRequest, Block, Mute, AccountModerationNote, AccountPin] target_classes = [Follow, FollowRequest, Block, Mute, AccountModerationNote, AccountPin]
target_classes << AccountNote if ActiveRecord::Base.connection.table_exists?(:account_notes) target_classes << AccountNote if db_table_exists?(:account_notes)
target_classes.each do |klass| target_classes.each do |klass|
klass.where(target_account_id: other_account.id).find_each do |record| klass.where(target_account_id: other_account.id).find_each do |record|
@ -114,13 +118,13 @@ module Mastodon::CLI
end end
end end
if ActiveRecord::Base.connection.table_exists?(:canonical_email_blocks) if db_table_exists?(:canonical_email_blocks)
CanonicalEmailBlock.where(reference_account_id: other_account.id).find_each do |record| CanonicalEmailBlock.where(reference_account_id: other_account.id).find_each do |record|
record.update_attribute(:reference_account_id, id) record.update_attribute(:reference_account_id, id)
end end
end end
if ActiveRecord::Base.connection.table_exists?(:appeals) if db_table_exists?(:appeals)
Appeal.where(account_warning_id: other_account.id).find_each do |record| Appeal.where(account_warning_id: other_account.id).find_each do |record|
record.update_attribute(:account_warning_id, id) record.update_attribute(:account_warning_id, id)
end end
@ -234,16 +238,16 @@ module Mastodon::CLI
say 'Restoring index_accounts_on_username_and_domain_lower…' say 'Restoring index_accounts_on_username_and_domain_lower…'
if migrator_version < 2020_06_20_164023 if migrator_version < 2020_06_20_164023
ActiveRecord::Base.connection.add_index :accounts, 'lower (username), lower(domain)', name: 'index_accounts_on_username_and_domain_lower', unique: true database_connection.add_index :accounts, 'lower (username), lower(domain)', name: 'index_accounts_on_username_and_domain_lower', unique: true
else else
ActiveRecord::Base.connection.add_index :accounts, "lower (username), COALESCE(lower(domain), '')", name: 'index_accounts_on_username_and_domain_lower', unique: true database_connection.add_index :accounts, "lower (username), COALESCE(lower(domain), '')", name: 'index_accounts_on_username_and_domain_lower', unique: true
end end
say 'Reindexing textual indexes on accounts…' say 'Reindexing textual indexes on accounts…'
ActiveRecord::Base.connection.execute('REINDEX INDEX search_index;') database_connection.execute('REINDEX INDEX search_index;')
ActiveRecord::Base.connection.execute('REINDEX INDEX index_accounts_on_uri;') database_connection.execute('REINDEX INDEX index_accounts_on_uri;')
ActiveRecord::Base.connection.execute('REINDEX INDEX index_accounts_on_url;') database_connection.execute('REINDEX INDEX index_accounts_on_url;')
ActiveRecord::Base.connection.execute('REINDEX INDEX index_accounts_on_domain_and_id;') if migrator_version >= 2023_05_24_190515 database_connection.execute('REINDEX INDEX index_accounts_on_domain_and_id;') if migrator_version >= 2023_05_24_190515
end end
def deduplicate_users! def deduplicate_users!
@ -260,21 +264,21 @@ module Mastodon::CLI
deduplicate_users_process_password_token deduplicate_users_process_password_token
say 'Restoring users indexes…' say 'Restoring users indexes…'
ActiveRecord::Base.connection.add_index :users, ['confirmation_token'], name: 'index_users_on_confirmation_token', unique: true database_connection.add_index :users, ['confirmation_token'], name: 'index_users_on_confirmation_token', unique: true
ActiveRecord::Base.connection.add_index :users, ['email'], name: 'index_users_on_email', unique: true database_connection.add_index :users, ['email'], name: 'index_users_on_email', unique: true
ActiveRecord::Base.connection.add_index :users, ['remember_token'], name: 'index_users_on_remember_token', unique: true if migrator_version < 2022_01_18_183010 database_connection.add_index :users, ['remember_token'], name: 'index_users_on_remember_token', unique: true if migrator_version < 2022_01_18_183010
if migrator_version < 2022_03_10_060641 if migrator_version < 2022_03_10_060641
ActiveRecord::Base.connection.add_index :users, ['reset_password_token'], name: 'index_users_on_reset_password_token', unique: true database_connection.add_index :users, ['reset_password_token'], name: 'index_users_on_reset_password_token', unique: true
else else
ActiveRecord::Base.connection.add_index :users, ['reset_password_token'], name: 'index_users_on_reset_password_token', unique: true, where: 'reset_password_token IS NOT NULL', opclass: :text_pattern_ops database_connection.add_index :users, ['reset_password_token'], name: 'index_users_on_reset_password_token', unique: true, where: 'reset_password_token IS NOT NULL', opclass: :text_pattern_ops
end end
ActiveRecord::Base.connection.execute('REINDEX INDEX index_users_on_unconfirmed_email;') if migrator_version >= 2023_07_02_151753 database_connection.execute('REINDEX INDEX index_users_on_unconfirmed_email;') if migrator_version >= 2023_07_02_151753
end end
def deduplicate_users_process_email def deduplicate_users_process_email
ActiveRecord::Base.connection.select_all("SELECT string_agg(id::text, ',') AS ids FROM users GROUP BY email HAVING count(*) > 1").each do |row| database_connection.select_all("SELECT string_agg(id::text, ',') AS ids FROM users GROUP BY email HAVING count(*) > 1").each do |row|
users = User.where(id: row['ids'].split(',')).order(updated_at: :desc).includes(:account).to_a users = User.where(id: row['ids'].split(',')).order(updated_at: :desc).includes(:account).to_a
ref_user = users.shift ref_user = users.shift
say "Multiple users registered with e-mail address #{ref_user.email}.", :yellow say "Multiple users registered with e-mail address #{ref_user.email}.", :yellow
@ -288,7 +292,7 @@ module Mastodon::CLI
end end
def deduplicate_users_process_confirmation_token def deduplicate_users_process_confirmation_token
ActiveRecord::Base.connection.select_all("SELECT string_agg(id::text, ',') AS ids FROM users WHERE confirmation_token IS NOT NULL GROUP BY confirmation_token HAVING count(*) > 1").each do |row| database_connection.select_all("SELECT string_agg(id::text, ',') AS ids FROM users WHERE confirmation_token IS NOT NULL GROUP BY confirmation_token HAVING count(*) > 1").each do |row|
users = User.where(id: row['ids'].split(',')).order(created_at: :desc).includes(:account).to_a.drop(1) users = User.where(id: row['ids'].split(',')).order(created_at: :desc).includes(:account).to_a.drop(1)
say "Unsetting confirmation token for those accounts: #{users.map { |user| user.account.acct }.join(', ')}", :yellow say "Unsetting confirmation token for those accounts: #{users.map { |user| user.account.acct }.join(', ')}", :yellow
@ -300,7 +304,7 @@ module Mastodon::CLI
def deduplicate_users_process_remember_token def deduplicate_users_process_remember_token
if migrator_version < 2022_01_18_183010 if migrator_version < 2022_01_18_183010
ActiveRecord::Base.connection.select_all("SELECT string_agg(id::text, ',') AS ids FROM users WHERE remember_token IS NOT NULL GROUP BY remember_token HAVING count(*) > 1").each do |row| database_connection.select_all("SELECT string_agg(id::text, ',') AS ids FROM users WHERE remember_token IS NOT NULL GROUP BY remember_token HAVING count(*) > 1").each do |row|
users = User.where(id: row['ids'].split(',')).order(updated_at: :desc).to_a.drop(1) users = User.where(id: row['ids'].split(',')).order(updated_at: :desc).to_a.drop(1)
say "Unsetting remember token for those accounts: #{users.map { |user| user.account.acct }.join(', ')}", :yellow say "Unsetting remember token for those accounts: #{users.map { |user| user.account.acct }.join(', ')}", :yellow
@ -312,7 +316,7 @@ module Mastodon::CLI
end end
def deduplicate_users_process_password_token def deduplicate_users_process_password_token
ActiveRecord::Base.connection.select_all("SELECT string_agg(id::text, ',') AS ids FROM users WHERE reset_password_token IS NOT NULL GROUP BY reset_password_token HAVING count(*) > 1").each do |row| database_connection.select_all("SELECT string_agg(id::text, ',') AS ids FROM users WHERE reset_password_token IS NOT NULL GROUP BY reset_password_token HAVING count(*) > 1").each do |row|
users = User.where(id: row['ids'].split(',')).order(updated_at: :desc).includes(:account).to_a.drop(1) users = User.where(id: row['ids'].split(',')).order(updated_at: :desc).includes(:account).to_a.drop(1)
say "Unsetting password reset token for those accounts: #{users.map { |user| user.account.acct }.join(', ')}", :yellow say "Unsetting password reset token for those accounts: #{users.map { |user| user.account.acct }.join(', ')}", :yellow
@ -326,47 +330,47 @@ module Mastodon::CLI
remove_index_if_exists!(:account_domain_blocks, 'index_account_domain_blocks_on_account_id_and_domain') remove_index_if_exists!(:account_domain_blocks, 'index_account_domain_blocks_on_account_id_and_domain')
say 'Removing duplicate account domain blocks…' say 'Removing duplicate account domain blocks…'
ActiveRecord::Base.connection.select_all("SELECT string_agg(id::text, ',') AS ids FROM account_domain_blocks GROUP BY account_id, domain HAVING count(*) > 1").each do |row| database_connection.select_all("SELECT string_agg(id::text, ',') AS ids FROM account_domain_blocks GROUP BY account_id, domain HAVING count(*) > 1").each do |row|
AccountDomainBlock.where(id: row['ids'].split(',').drop(1)).delete_all AccountDomainBlock.where(id: row['ids'].split(',').drop(1)).delete_all
end end
say 'Restoring account domain blocks indexes…' say 'Restoring account domain blocks indexes…'
ActiveRecord::Base.connection.add_index :account_domain_blocks, %w(account_id domain), name: 'index_account_domain_blocks_on_account_id_and_domain', unique: true database_connection.add_index :account_domain_blocks, %w(account_id domain), name: 'index_account_domain_blocks_on_account_id_and_domain', unique: true
end end
def deduplicate_account_identity_proofs! def deduplicate_account_identity_proofs!
return unless ActiveRecord::Base.connection.table_exists?(:account_identity_proofs) return unless db_table_exists?(:account_identity_proofs)
remove_index_if_exists!(:account_identity_proofs, 'index_account_proofs_on_account_and_provider_and_username') remove_index_if_exists!(:account_identity_proofs, 'index_account_proofs_on_account_and_provider_and_username')
say 'Removing duplicate account identity proofs…' say 'Removing duplicate account identity proofs…'
ActiveRecord::Base.connection.select_all("SELECT string_agg(id::text, ',') AS ids FROM account_identity_proofs GROUP BY account_id, provider, provider_username HAVING count(*) > 1").each do |row| database_connection.select_all("SELECT string_agg(id::text, ',') AS ids FROM account_identity_proofs GROUP BY account_id, provider, provider_username HAVING count(*) > 1").each do |row|
AccountIdentityProof.where(id: row['ids'].split(',')).order(id: :desc).to_a.drop(1).each(&:destroy) AccountIdentityProof.where(id: row['ids'].split(',')).order(id: :desc).to_a.drop(1).each(&:destroy)
end end
say 'Restoring account identity proofs indexes…' say 'Restoring account identity proofs indexes…'
ActiveRecord::Base.connection.add_index :account_identity_proofs, %w(account_id provider provider_username), name: 'index_account_proofs_on_account_and_provider_and_username', unique: true database_connection.add_index :account_identity_proofs, %w(account_id provider provider_username), name: 'index_account_proofs_on_account_and_provider_and_username', unique: true
end end
def deduplicate_announcement_reactions! def deduplicate_announcement_reactions!
return unless ActiveRecord::Base.connection.table_exists?(:announcement_reactions) return unless db_table_exists?(:announcement_reactions)
remove_index_if_exists!(:announcement_reactions, 'index_announcement_reactions_on_account_id_and_announcement_id') remove_index_if_exists!(:announcement_reactions, 'index_announcement_reactions_on_account_id_and_announcement_id')
say 'Removing duplicate announcement reactions…' say 'Removing duplicate announcement reactions…'
ActiveRecord::Base.connection.select_all("SELECT string_agg(id::text, ',') AS ids FROM announcement_reactions GROUP BY account_id, announcement_id, name HAVING count(*) > 1").each do |row| database_connection.select_all("SELECT string_agg(id::text, ',') AS ids FROM announcement_reactions GROUP BY account_id, announcement_id, name HAVING count(*) > 1").each do |row|
AnnouncementReaction.where(id: row['ids'].split(',')).order(id: :desc).to_a.drop(1).each(&:destroy) AnnouncementReaction.where(id: row['ids'].split(',')).order(id: :desc).to_a.drop(1).each(&:destroy)
end end
say 'Restoring announcement_reactions indexes…' say 'Restoring announcement_reactions indexes…'
ActiveRecord::Base.connection.add_index :announcement_reactions, %w(account_id announcement_id name), name: 'index_announcement_reactions_on_account_id_and_announcement_id', unique: true database_connection.add_index :announcement_reactions, %w(account_id announcement_id name), name: 'index_announcement_reactions_on_account_id_and_announcement_id', unique: true
end end
def deduplicate_conversations! def deduplicate_conversations!
remove_index_if_exists!(:conversations, 'index_conversations_on_uri') remove_index_if_exists!(:conversations, 'index_conversations_on_uri')
say 'Deduplicating conversations…' say 'Deduplicating conversations…'
ActiveRecord::Base.connection.select_all("SELECT string_agg(id::text, ',') AS ids FROM conversations WHERE uri IS NOT NULL GROUP BY uri HAVING count(*) > 1").each do |row| database_connection.select_all("SELECT string_agg(id::text, ',') AS ids FROM conversations WHERE uri IS NOT NULL GROUP BY uri HAVING count(*) > 1").each do |row|
conversations = Conversation.where(id: row['ids'].split(',')).order(id: :desc).to_a conversations = Conversation.where(id: row['ids'].split(',')).order(id: :desc).to_a
ref_conversation = conversations.shift ref_conversation = conversations.shift
@ -379,9 +383,9 @@ module Mastodon::CLI
say 'Restoring conversations indexes…' say 'Restoring conversations indexes…'
if migrator_version < 2022_03_07_083603 if migrator_version < 2022_03_07_083603
ActiveRecord::Base.connection.add_index :conversations, ['uri'], name: 'index_conversations_on_uri', unique: true database_connection.add_index :conversations, ['uri'], name: 'index_conversations_on_uri', unique: true
else else
ActiveRecord::Base.connection.add_index :conversations, ['uri'], name: 'index_conversations_on_uri', unique: true, where: 'uri IS NOT NULL', opclass: :text_pattern_ops database_connection.add_index :conversations, ['uri'], name: 'index_conversations_on_uri', unique: true, where: 'uri IS NOT NULL', opclass: :text_pattern_ops
end end
end end
@ -389,7 +393,7 @@ module Mastodon::CLI
remove_index_if_exists!(:custom_emojis, 'index_custom_emojis_on_shortcode_and_domain') remove_index_if_exists!(:custom_emojis, 'index_custom_emojis_on_shortcode_and_domain')
say 'Deduplicating custom_emojis…' say 'Deduplicating custom_emojis…'
ActiveRecord::Base.connection.select_all("SELECT string_agg(id::text, ',') AS ids FROM custom_emojis GROUP BY shortcode, domain HAVING count(*) > 1").each do |row| database_connection.select_all("SELECT string_agg(id::text, ',') AS ids FROM custom_emojis GROUP BY shortcode, domain HAVING count(*) > 1").each do |row|
emojis = CustomEmoji.where(id: row['ids'].split(',')).order(id: :desc).to_a emojis = CustomEmoji.where(id: row['ids'].split(',')).order(id: :desc).to_a
ref_emoji = emojis.shift ref_emoji = emojis.shift
@ -401,14 +405,14 @@ module Mastodon::CLI
end end
say 'Restoring custom_emojis indexes…' say 'Restoring custom_emojis indexes…'
ActiveRecord::Base.connection.add_index :custom_emojis, %w(shortcode domain), name: 'index_custom_emojis_on_shortcode_and_domain', unique: true database_connection.add_index :custom_emojis, %w(shortcode domain), name: 'index_custom_emojis_on_shortcode_and_domain', unique: true
end end
def deduplicate_custom_emoji_categories! def deduplicate_custom_emoji_categories!
remove_index_if_exists!(:custom_emoji_categories, 'index_custom_emoji_categories_on_name') remove_index_if_exists!(:custom_emoji_categories, 'index_custom_emoji_categories_on_name')
say 'Deduplicating custom_emoji_categories…' say 'Deduplicating custom_emoji_categories…'
ActiveRecord::Base.connection.select_all("SELECT string_agg(id::text, ',') AS ids FROM custom_emoji_categories GROUP BY name HAVING count(*) > 1").each do |row| database_connection.select_all("SELECT string_agg(id::text, ',') AS ids FROM custom_emoji_categories GROUP BY name HAVING count(*) > 1").each do |row|
categories = CustomEmojiCategory.where(id: row['ids'].split(',')).order(id: :desc).to_a categories = CustomEmojiCategory.where(id: row['ids'].split(',')).order(id: :desc).to_a
ref_category = categories.shift ref_category = categories.shift
@ -420,26 +424,26 @@ module Mastodon::CLI
end end
say 'Restoring custom_emoji_categories indexes…' say 'Restoring custom_emoji_categories indexes…'
ActiveRecord::Base.connection.add_index :custom_emoji_categories, ['name'], name: 'index_custom_emoji_categories_on_name', unique: true database_connection.add_index :custom_emoji_categories, ['name'], name: 'index_custom_emoji_categories_on_name', unique: true
end end
def deduplicate_domain_allows! def deduplicate_domain_allows!
remove_index_if_exists!(:domain_allows, 'index_domain_allows_on_domain') remove_index_if_exists!(:domain_allows, 'index_domain_allows_on_domain')
say 'Deduplicating domain_allows…' say 'Deduplicating domain_allows…'
ActiveRecord::Base.connection.select_all("SELECT string_agg(id::text, ',') AS ids FROM domain_allows GROUP BY domain HAVING count(*) > 1").each do |row| database_connection.select_all("SELECT string_agg(id::text, ',') AS ids FROM domain_allows GROUP BY domain HAVING count(*) > 1").each do |row|
DomainAllow.where(id: row['ids'].split(',')).order(id: :desc).to_a.drop(1).each(&:destroy) DomainAllow.where(id: row['ids'].split(',')).order(id: :desc).to_a.drop(1).each(&:destroy)
end end
say 'Restoring domain_allows indexes…' say 'Restoring domain_allows indexes…'
ActiveRecord::Base.connection.add_index :domain_allows, ['domain'], name: 'index_domain_allows_on_domain', unique: true database_connection.add_index :domain_allows, ['domain'], name: 'index_domain_allows_on_domain', unique: true
end end
def deduplicate_domain_blocks! def deduplicate_domain_blocks!
remove_index_if_exists!(:domain_blocks, 'index_domain_blocks_on_domain') remove_index_if_exists!(:domain_blocks, 'index_domain_blocks_on_domain')
say 'Deduplicating domain_blocks…' say 'Deduplicating domain_blocks…'
ActiveRecord::Base.connection.select_all("SELECT string_agg(id::text, ',') AS ids FROM domain_blocks GROUP BY domain HAVING count(*) > 1").each do |row| database_connection.select_all("SELECT string_agg(id::text, ',') AS ids FROM domain_blocks GROUP BY domain HAVING count(*) > 1").each do |row|
domain_blocks = DomainBlock.where(id: row['ids'].split(',')).by_severity.reverse.to_a domain_blocks = DomainBlock.where(id: row['ids'].split(',')).by_severity.reverse.to_a
reject_media = domain_blocks.any?(&:reject_media?) reject_media = domain_blocks.any?(&:reject_media?)
@ -456,49 +460,49 @@ module Mastodon::CLI
end end
say 'Restoring domain_blocks indexes…' say 'Restoring domain_blocks indexes…'
ActiveRecord::Base.connection.add_index :domain_blocks, ['domain'], name: 'index_domain_blocks_on_domain', unique: true database_connection.add_index :domain_blocks, ['domain'], name: 'index_domain_blocks_on_domain', unique: true
end end
def deduplicate_unavailable_domains! def deduplicate_unavailable_domains!
return unless ActiveRecord::Base.connection.table_exists?(:unavailable_domains) return unless db_table_exists?(:unavailable_domains)
remove_index_if_exists!(:unavailable_domains, 'index_unavailable_domains_on_domain') remove_index_if_exists!(:unavailable_domains, 'index_unavailable_domains_on_domain')
say 'Deduplicating unavailable_domains…' say 'Deduplicating unavailable_domains…'
ActiveRecord::Base.connection.select_all("SELECT string_agg(id::text, ',') AS ids FROM unavailable_domains GROUP BY domain HAVING count(*) > 1").each do |row| database_connection.select_all("SELECT string_agg(id::text, ',') AS ids FROM unavailable_domains GROUP BY domain HAVING count(*) > 1").each do |row|
UnavailableDomain.where(id: row['ids'].split(',')).order(id: :desc).to_a.drop(1).each(&:destroy) UnavailableDomain.where(id: row['ids'].split(',')).order(id: :desc).to_a.drop(1).each(&:destroy)
end end
say 'Restoring unavailable_domains indexes…' say 'Restoring unavailable_domains indexes…'
ActiveRecord::Base.connection.add_index :unavailable_domains, ['domain'], name: 'index_unavailable_domains_on_domain', unique: true database_connection.add_index :unavailable_domains, ['domain'], name: 'index_unavailable_domains_on_domain', unique: true
end end
def deduplicate_email_domain_blocks! def deduplicate_email_domain_blocks!
remove_index_if_exists!(:email_domain_blocks, 'index_email_domain_blocks_on_domain') remove_index_if_exists!(:email_domain_blocks, 'index_email_domain_blocks_on_domain')
say 'Deduplicating email_domain_blocks…' say 'Deduplicating email_domain_blocks…'
ActiveRecord::Base.connection.select_all("SELECT string_agg(id::text, ',') AS ids FROM email_domain_blocks GROUP BY domain HAVING count(*) > 1").each do |row| database_connection.select_all("SELECT string_agg(id::text, ',') AS ids FROM email_domain_blocks GROUP BY domain HAVING count(*) > 1").each do |row|
domain_blocks = EmailDomainBlock.where(id: row['ids'].split(',')).order(EmailDomainBlock.arel_table[:parent_id].asc.nulls_first).to_a domain_blocks = EmailDomainBlock.where(id: row['ids'].split(',')).order(EmailDomainBlock.arel_table[:parent_id].asc.nulls_first).to_a
domain_blocks.drop(1).each(&:destroy) domain_blocks.drop(1).each(&:destroy)
end end
say 'Restoring email_domain_blocks indexes…' say 'Restoring email_domain_blocks indexes…'
ActiveRecord::Base.connection.add_index :email_domain_blocks, ['domain'], name: 'index_email_domain_blocks_on_domain', unique: true database_connection.add_index :email_domain_blocks, ['domain'], name: 'index_email_domain_blocks_on_domain', unique: true
end end
def deduplicate_media_attachments! def deduplicate_media_attachments!
remove_index_if_exists!(:media_attachments, 'index_media_attachments_on_shortcode') remove_index_if_exists!(:media_attachments, 'index_media_attachments_on_shortcode')
say 'Deduplicating media_attachments…' say 'Deduplicating media_attachments…'
ActiveRecord::Base.connection.select_all("SELECT string_agg(id::text, ',') AS ids FROM media_attachments WHERE shortcode IS NOT NULL GROUP BY shortcode HAVING count(*) > 1").each do |row| database_connection.select_all("SELECT string_agg(id::text, ',') AS ids FROM media_attachments WHERE shortcode IS NOT NULL GROUP BY shortcode HAVING count(*) > 1").each do |row|
MediaAttachment.where(id: row['ids'].split(',').drop(1)).update_all(shortcode: nil) MediaAttachment.where(id: row['ids'].split(',').drop(1)).update_all(shortcode: nil)
end end
say 'Restoring media_attachments indexes…' say 'Restoring media_attachments indexes…'
if migrator_version < 2022_03_10_060626 if migrator_version < 2022_03_10_060626
ActiveRecord::Base.connection.add_index :media_attachments, ['shortcode'], name: 'index_media_attachments_on_shortcode', unique: true database_connection.add_index :media_attachments, ['shortcode'], name: 'index_media_attachments_on_shortcode', unique: true
else else
ActiveRecord::Base.connection.add_index :media_attachments, ['shortcode'], name: 'index_media_attachments_on_shortcode', unique: true, where: 'shortcode IS NOT NULL', opclass: :text_pattern_ops database_connection.add_index :media_attachments, ['shortcode'], name: 'index_media_attachments_on_shortcode', unique: true, where: 'shortcode IS NOT NULL', opclass: :text_pattern_ops
end end
end end
@ -506,19 +510,19 @@ module Mastodon::CLI
remove_index_if_exists!(:preview_cards, 'index_preview_cards_on_url') remove_index_if_exists!(:preview_cards, 'index_preview_cards_on_url')
say 'Deduplicating preview_cards…' say 'Deduplicating preview_cards…'
ActiveRecord::Base.connection.select_all("SELECT string_agg(id::text, ',') AS ids FROM preview_cards GROUP BY url HAVING count(*) > 1").each do |row| database_connection.select_all("SELECT string_agg(id::text, ',') AS ids FROM preview_cards GROUP BY url HAVING count(*) > 1").each do |row|
PreviewCard.where(id: row['ids'].split(',')).order(id: :desc).to_a.drop(1).each(&:destroy) PreviewCard.where(id: row['ids'].split(',')).order(id: :desc).to_a.drop(1).each(&:destroy)
end end
say 'Restoring preview_cards indexes…' say 'Restoring preview_cards indexes…'
ActiveRecord::Base.connection.add_index :preview_cards, ['url'], name: 'index_preview_cards_on_url', unique: true database_connection.add_index :preview_cards, ['url'], name: 'index_preview_cards_on_url', unique: true
end end
def deduplicate_statuses! def deduplicate_statuses!
remove_index_if_exists!(:statuses, 'index_statuses_on_uri') remove_index_if_exists!(:statuses, 'index_statuses_on_uri')
say 'Deduplicating statuses…' say 'Deduplicating statuses…'
ActiveRecord::Base.connection.select_all("SELECT string_agg(id::text, ',') AS ids FROM statuses WHERE uri IS NOT NULL GROUP BY uri HAVING count(*) > 1").each do |row| database_connection.select_all("SELECT string_agg(id::text, ',') AS ids FROM statuses WHERE uri IS NOT NULL GROUP BY uri HAVING count(*) > 1").each do |row|
statuses = Status.where(id: row['ids'].split(',')).order(id: :asc).to_a statuses = Status.where(id: row['ids'].split(',')).order(id: :asc).to_a
ref_status = statuses.shift ref_status = statuses.shift
statuses.each do |status| statuses.each do |status|
@ -529,9 +533,9 @@ module Mastodon::CLI
say 'Restoring statuses indexes…' say 'Restoring statuses indexes…'
if migrator_version < 2022_03_10_060706 if migrator_version < 2022_03_10_060706
ActiveRecord::Base.connection.add_index :statuses, ['uri'], name: 'index_statuses_on_uri', unique: true database_connection.add_index :statuses, ['uri'], name: 'index_statuses_on_uri', unique: true
else else
ActiveRecord::Base.connection.add_index :statuses, ['uri'], name: 'index_statuses_on_uri', unique: true, where: 'uri IS NOT NULL', opclass: :text_pattern_ops database_connection.add_index :statuses, ['uri'], name: 'index_statuses_on_uri', unique: true, where: 'uri IS NOT NULL', opclass: :text_pattern_ops
end end
end end
@ -540,7 +544,7 @@ module Mastodon::CLI
remove_index_if_exists!(:tags, 'index_tags_on_name_lower_btree') remove_index_if_exists!(:tags, 'index_tags_on_name_lower_btree')
say 'Deduplicating tags…' say 'Deduplicating tags…'
ActiveRecord::Base.connection.select_all("SELECT string_agg(id::text, ',') AS ids FROM tags GROUP BY lower((name)::text) HAVING count(*) > 1").each do |row| database_connection.select_all("SELECT string_agg(id::text, ',') AS ids FROM tags GROUP BY lower((name)::text) HAVING count(*) > 1").each do |row|
tags = Tag.where(id: row['ids'].split(',')).order(Arel.sql('(usable::int + trendable::int + listable::int) desc')).to_a tags = Tag.where(id: row['ids'].split(',')).order(Arel.sql('(usable::int + trendable::int + listable::int) desc')).to_a
ref_tag = tags.shift ref_tag = tags.shift
tags.each do |tag| tags.each do |tag|
@ -551,38 +555,38 @@ module Mastodon::CLI
say 'Restoring tags indexes…' say 'Restoring tags indexes…'
if migrator_version < 2021_04_21_121431 if migrator_version < 2021_04_21_121431
ActiveRecord::Base.connection.add_index :tags, 'lower((name)::text)', name: 'index_tags_on_name_lower', unique: true database_connection.add_index :tags, 'lower((name)::text)', name: 'index_tags_on_name_lower', unique: true
else else
ActiveRecord::Base.connection.execute 'CREATE UNIQUE INDEX index_tags_on_name_lower_btree ON tags (lower(name) text_pattern_ops)' database_connection.execute 'CREATE UNIQUE INDEX index_tags_on_name_lower_btree ON tags (lower(name) text_pattern_ops)'
end end
end end
def deduplicate_webauthn_credentials! def deduplicate_webauthn_credentials!
return unless ActiveRecord::Base.connection.table_exists?(:webauthn_credentials) return unless db_table_exists?(:webauthn_credentials)
remove_index_if_exists!(:webauthn_credentials, 'index_webauthn_credentials_on_external_id') remove_index_if_exists!(:webauthn_credentials, 'index_webauthn_credentials_on_external_id')
say 'Deduplicating webauthn_credentials…' say 'Deduplicating webauthn_credentials…'
ActiveRecord::Base.connection.select_all("SELECT string_agg(id::text, ',') AS ids FROM webauthn_credentials GROUP BY external_id HAVING count(*) > 1").each do |row| database_connection.select_all("SELECT string_agg(id::text, ',') AS ids FROM webauthn_credentials GROUP BY external_id HAVING count(*) > 1").each do |row|
WebauthnCredential.where(id: row['ids'].split(',')).order(id: :desc).to_a.drop(1).each(&:destroy) WebauthnCredential.where(id: row['ids'].split(',')).order(id: :desc).to_a.drop(1).each(&:destroy)
end end
say 'Restoring webauthn_credentials indexes…' say 'Restoring webauthn_credentials indexes…'
ActiveRecord::Base.connection.add_index :webauthn_credentials, ['external_id'], name: 'index_webauthn_credentials_on_external_id', unique: true database_connection.add_index :webauthn_credentials, ['external_id'], name: 'index_webauthn_credentials_on_external_id', unique: true
end end
def deduplicate_webhooks! def deduplicate_webhooks!
return unless ActiveRecord::Base.connection.table_exists?(:webhooks) return unless db_table_exists?(:webhooks)
remove_index_if_exists!(:webhooks, 'index_webhooks_on_url') remove_index_if_exists!(:webhooks, 'index_webhooks_on_url')
say 'Deduplicating webhooks…' say 'Deduplicating webhooks…'
ActiveRecord::Base.connection.select_all("SELECT string_agg(id::text, ',') AS ids FROM webhooks GROUP BY url HAVING count(*) > 1").each do |row| database_connection.select_all("SELECT string_agg(id::text, ',') AS ids FROM webhooks GROUP BY url HAVING count(*) > 1").each do |row|
Webhook.where(id: row['ids'].split(',')).order(id: :desc).drop(1).each(&:destroy) Webhook.where(id: row['ids'].split(',')).order(id: :desc).drop(1).each(&:destroy)
end end
say 'Restoring webhooks indexes…' say 'Restoring webhooks indexes…'
ActiveRecord::Base.connection.add_index :webhooks, ['url'], name: 'index_webhooks_on_url', unique: true database_connection.add_index :webhooks, ['url'], name: 'index_webhooks_on_url', unique: true
end end
def deduplicate_software_updates! def deduplicate_software_updates!
@ -672,7 +676,7 @@ module Mastodon::CLI
def merge_statuses!(main_status, duplicate_status) def merge_statuses!(main_status, duplicate_status)
owned_classes = [Favourite, Mention, Poll] owned_classes = [Favourite, Mention, Poll]
owned_classes << Bookmark if ActiveRecord::Base.connection.table_exists?(:bookmarks) owned_classes << Bookmark if db_table_exists?(:bookmarks)
owned_classes.each do |klass| owned_classes.each do |klass|
klass.where(status_id: duplicate_status.id).find_each do |record| klass.where(status_id: duplicate_status.id).find_each do |record|
record.update_attribute(:status_id, main_status.id) record.update_attribute(:status_id, main_status.id)
@ -715,13 +719,21 @@ module Mastodon::CLI
end end
def find_duplicate_accounts def find_duplicate_accounts
ActiveRecord::Base.connection.select_all("SELECT string_agg(id::text, ',') AS ids FROM accounts GROUP BY lower(username), COALESCE(lower(domain), '') HAVING count(*) > 1") database_connection.select_all("SELECT string_agg(id::text, ',') AS ids FROM accounts GROUP BY lower(username), COALESCE(lower(domain), '') HAVING count(*) > 1")
end end
def remove_index_if_exists!(table, name) def remove_index_if_exists!(table, name)
ActiveRecord::Base.connection.remove_index(table, name: name) if ActiveRecord::Base.connection.index_name_exists?(table, name) database_connection.remove_index(table, name: name) if database_connection.index_name_exists?(table, name)
rescue ArgumentError, ActiveRecord::StatementInvalid rescue ArgumentError, ActiveRecord::StatementInvalid
nil nil
end end
def database_connection
ActiveRecord::Base.connection
end
def db_table_exists?(table)
database_connection.table_exists?(table)
end
end end
end end