Merge branch '333507-integrations-type-migration-2-backfill-type_new-for-all-records' into 'master'

Backfill `type_new` column on integrations

See merge request gitlab-org/gitlab!66928

db/post_migrate/20210727113447_backfill_integrations_type_new.rb

+# frozen_string_literal: true
+
+class BackfillIntegrationsTypeNew < ActiveRecord::Migration[6.1]
+  include Gitlab::Database::MigrationHelpers
+
+  MIGRATION = 'BackfillIntegrationsTypeNew'
+  INTERVAL = 2.minutes
+
+  def up
+    queue_batched_background_migration(
+      MIGRATION,
+      :integrations,
+      :id,
+      job_interval: INTERVAL
+    )
+  end
+
+  def down
+    Gitlab::Database::BackgroundMigration::BatchedMigration
+      .for_configuration(MIGRATION, :integrations, :id, [])
+      .delete_all
+  end
+end

db/schema_migrations/20210727113447

+19e23131949e6056ea9837231fac6a2307fb52a8287eb34cc6e89eed11d52849
\ No newline at end of file

lib/gitlab/background_migration/backfill_integrations_type_new.rb

+# frozen_string_literal: true
+
+module Gitlab
+  module BackgroundMigration
+    # Backfills the new `integrations.type_new` column, which contains
+    # the real class name, rather than the legacy class name in `type`
+    # which is mapped via `Gitlab::Integrations::StiType`.
+    class BackfillIntegrationsTypeNew
+      def perform(start_id, stop_id, *args)
+        ActiveRecord::Base.connection.execute(<<~SQL)
+          WITH mapping(old_type, new_type) AS (VALUES
+            ('AsanaService',                   'Integrations::Asana'),
+            ('AssemblaService',                'Integrations::Assembla'),
+            ('BambooService',                  'Integrations::Bamboo'),
+            ('BugzillaService',                'Integrations::Bugzilla'),
+            ('BuildkiteService',               'Integrations::Buildkite'),
+            ('CampfireService',                'Integrations::Campfire'),
+            ('ConfluenceService',              'Integrations::Confluence'),
+            ('CustomIssueTrackerService',      'Integrations::CustomIssueTracker'),
+            ('DatadogService',                 'Integrations::Datadog'),
+            ('DiscordService',                 'Integrations::Discord'),
+            ('DroneCiService',                 'Integrations::DroneCi'),
+            ('EmailsOnPushService',            'Integrations::EmailsOnPush'),
+            ('EwmService',                     'Integrations::Ewm'),
+            ('ExternalWikiService',            'Integrations::ExternalWiki'),
+            ('FlowdockService',                'Integrations::Flowdock'),
+            ('HangoutsChatService',            'Integrations::HangoutsChat'),
+            ('IrkerService',                   'Integrations::Irker'),
+            ('JenkinsService',                 'Integrations::Jenkins'),
+            ('JiraService',                    'Integrations::Jira'),
+            ('MattermostService',              'Integrations::Mattermost'),
+            ('MattermostSlashCommandsService', 'Integrations::MattermostSlashCommands'),
+            ('MicrosoftTeamsService',          'Integrations::MicrosoftTeams'),
+            ('MockCiService',                  'Integrations::MockCi'),
+            ('MockMonitoringService',          'Integrations::MockMonitoring'),
+            ('PackagistService',               'Integrations::Packagist'),
+            ('PipelinesEmailService',          'Integrations::PipelinesEmail'),
+            ('PivotaltrackerService',          'Integrations::Pivotaltracker'),
+            ('PrometheusService',              'Integrations::Prometheus'),
+            ('PushoverService',                'Integrations::Pushover'),
+            ('RedmineService',                 'Integrations::Redmine'),
+            ('SlackService',                   'Integrations::Slack'),
+            ('SlackSlashCommandsService',      'Integrations::SlackSlashCommands'),
+            ('TeamcityService',                'Integrations::Teamcity'),
+            ('UnifyCircuitService',            'Integrations::UnifyCircuit'),
+            ('WebexTeamsService',              'Integrations::WebexTeams'),
+            ('YoutrackService',                'Integrations::Youtrack'),
+
+            -- EE-only integrations
+            ('GithubService',                  'Integrations::Github'),
+            ('GitlabSlackApplicationService',  'Integrations::GitlabSlackApplication')
+          )
+
+          UPDATE integrations SET type_new = mapping.new_type
+          FROM mapping
+          WHERE integrations.id BETWEEN #{start_id} AND #{stop_id}
+            AND integrations.type = mapping.old_type
+        SQL
+      end
+    end
+  end
+end

lib/gitlab/integrations/sti_type.rb

@@ -7,7 +7,7 @@ module Gitlab
         Asana Assembla Bamboo Bugzilla Buildkite Campfire Confluence CustomIssueTracker Datadog
         Discord DroneCi EmailsOnPush Ewm ExternalWiki Flowdock HangoutsChat Irker Jenkins Jira Mattermost
         MattermostSlashCommands MicrosoftTeams MockCi MockMonitoring Packagist PipelinesEmail Pivotaltracker
-        Prometheus Pushover Redmine Slack SlackSlashCommands Teamcity UnifyCircuit Youtrack WebexTeams
+        Prometheus Pushover Redmine Slack SlackSlashCommands Teamcity UnifyCircuit WebexTeams Youtrack
       )).freeze
 
       def self.namespaced_integrations

spec/lib/gitlab/background_migration/backfill_integrations_type_new_spec.rb

+# frozen_string_literal: true
+
+require 'spec_helper'
+
+RSpec.describe Gitlab::BackgroundMigration::BackfillIntegrationsTypeNew do
+  let(:integrations) { table(:integrations) }
+  let(:namespaced_integrations) { Gitlab::Integrations::StiType.namespaced_integrations }
+
+  before do
+    integrations.connection.execute 'ALTER TABLE integrations DISABLE TRIGGER "trigger_type_new_on_insert"'
+
+    namespaced_integrations.each_with_index do |type, i|
+      integrations.create!(id: i + 1, type: "#{type}Service")
+    end
+  ensure
+    integrations.connection.execute 'ALTER TABLE integrations ENABLE TRIGGER "trigger_type_new_on_insert"'
+  end
+
+  it 'backfills `type_new` for the selected records' do
+    described_class.new.perform(2, 10)
+
+    expect(integrations.where(id: 2..10).pluck(:type, :type_new)).to contain_exactly(
+      ['AssemblaService',           'Integrations::Assembla'],
+      ['BambooService',             'Integrations::Bamboo'],
+      ['BugzillaService',           'Integrations::Bugzilla'],
+      ['BuildkiteService',          'Integrations::Buildkite'],
+      ['CampfireService',           'Integrations::Campfire'],
+      ['ConfluenceService',         'Integrations::Confluence'],
+      ['CustomIssueTrackerService', 'Integrations::CustomIssueTracker'],
+      ['DatadogService',            'Integrations::Datadog'],
+      ['DiscordService',            'Integrations::Discord']
+    )
+
+    expect(integrations.where.not(id: 2..10)).to all(have_attributes(type_new: nil))
+  end
+end

spec/migrations/backfill_integrations_type_new_spec.rb

+# frozen_string_literal: true
+
+require 'spec_helper'
+require_migration!
+
+RSpec.describe BackfillIntegrationsTypeNew do
+  let_it_be(:migration) { described_class::MIGRATION }
+  let_it_be(:integrations) { table(:integrations) }
+
+  before do
+    integrations.create!(id: 1)
+    integrations.create!(id: 2)
+    integrations.create!(id: 3)
+    integrations.create!(id: 4)
+    integrations.create!(id: 5)
+  end
+
+  describe '#up' do
+    it 'schedules background jobs for each batch of integrations' do
+      migrate!
+
+      expect(migration).to have_scheduled_batched_migration(
+        table_name: :integrations,
+        column_name: :id,
+        interval: described_class::INTERVAL
+      )
+    end
+  end
+
+  describe '#down' do
+    it 'deletes all batched migration records' do
+      migrate!
+      schema_migrate_down!
+
+      expect(migration).not_to have_scheduled_batched_migration
+    end
+  end
+end

spec/support/matchers/background_migrations_matchers.rb

@@ -64,3 +64,33 @@ RSpec::Matchers.define :be_scheduled_migration_with_multiple_args do |*expected|
     arg.sort == expected.sort
   end
 end
+
+RSpec::Matchers.define :have_scheduled_batched_migration do |table_name: nil, column_name: nil, job_arguments: [], **attributes|
+  define_method :matches? do |migration|
+    # Default arguments passed by BatchedMigrationWrapper (values don't matter here)
+    expect(migration).to be_background_migration_with_arguments([
+      _start_id = 1,
+      _stop_id = 2,
+      table_name,
+      column_name,
+      _sub_batch_size = 10,
+      _pause_ms = 100,
+      *job_arguments
+    ])
+
+    batched_migrations =
+      Gitlab::Database::BackgroundMigration::BatchedMigration
+        .for_configuration(migration, table_name, column_name, job_arguments)
+
+    expect(batched_migrations.count).to be(1)
+    expect(batched_migrations).to all(have_attributes(attributes)) if attributes.present?
+  end
+
+  define_method :does_not_match? do |migration|
+    batched_migrations =
+      Gitlab::Database::BackgroundMigration::BatchedMigration
+        .where(job_class_name: migration)
+
+    expect(batched_migrations.count).to be(0)
+  end
+end