Merge branch...

Merge branch '243607-background-migration-for-setting-jira-tracker-data-deployment-type' into 'master'

Background migration for setting Jira tracker data deployment type

See merge request gitlab-org/gitlab!46368

changelogs/unreleased/243607-background-migration-for-setting-jira-tracker-data-deployment-type.yml

+---
+title: Background migration for setting Jira tracker data deployment type
+merge_request: 46368
+author:
+type: changed

db/post_migrate/20201028182809_backfill_jira_tracker_deployment_type2.rb

+# frozen_string_literal: true
+
+class BackfillJiraTrackerDeploymentType2 < ActiveRecord::Migration[6.0]
+  include Gitlab::Database::MigrationHelpers
+
+  DOWNTIME = false
+
+  MIGRATION = 'BackfillJiraTrackerDeploymentType2'
+  DELAY_INTERVAL = 2.minutes
+  BATCH_SIZE = 1000
+
+  disable_ddl_transaction!
+
+  class JiraTrackerData < ActiveRecord::Base
+    include EachBatch
+
+    self.table_name = 'jira_tracker_data'
+  end
+
+  def up
+    queue_background_migration_jobs_by_range_at_intervals(
+      JiraTrackerData.where(deployment_type: 0),
+      MIGRATION,
+      DELAY_INTERVAL,
+      batch_size: BATCH_SIZE,
+      track_jobs: true)
+  end
+
+  def down
+    # NOOP
+  end
+end

db/schema_migrations/20201028182809

+c51bf825045ef80714f3903f25321785883da3d612725f6fa67ec3ddd12d5808
\ No newline at end of file

lib/gitlab/background_migration/backfill_jira_tracker_deployment_type2.rb

+# frozen_string_literal: true
+
+# Based on https://community.developer.atlassian.com/t/get-rest-api-3-filter-search/29459/2,
+# it's enough at the moment to simply notice if the url is from `atlassian.net`
+module Gitlab
+  module BackgroundMigration
+    # Backfill the deployment_type in jira_tracker_data table
+    class BackfillJiraTrackerDeploymentType2
+      # Migration only version of jira_tracker_data table
+      class JiraTrackerDataTemp < ApplicationRecord
+        self.table_name = 'jira_tracker_data'
+
+        def self.encryption_options
+          {
+            key: Settings.attr_encrypted_db_key_base_32,
+            encode: true,
+            mode: :per_attribute_iv,
+            algorithm: 'aes-256-gcm'
+          }
+        end
+
+        attr_encrypted :url, encryption_options
+        attr_encrypted :api_url, encryption_options
+
+        enum deployment_type: { unknown: 0, server: 1, cloud: 2 }, _prefix: :deployment
+      end
+
+      # Migration only version of services table
+      class JiraServiceTemp < ApplicationRecord
+        self.table_name = 'services'
+        self.inheritance_column = :_type_disabled
+      end
+
+      def perform(start_id, stop_id)
+        @server_ids = []
+        @cloud_ids  = []
+
+        JiraTrackerDataTemp
+          .where(id: start_id..stop_id, deployment_type: 0)
+          .each do |jira_tracker_data|
+            collect_deployment_type(jira_tracker_data)
+          end
+
+        unless cloud_ids.empty?
+          JiraTrackerDataTemp.where(id: cloud_ids)
+            .update_all(deployment_type: JiraTrackerDataTemp.deployment_types[:cloud])
+        end
+
+        unless server_ids.empty?
+          JiraTrackerDataTemp.where(id: server_ids)
+            .update_all(deployment_type: JiraTrackerDataTemp.deployment_types[:server])
+        end
+
+        mark_jobs_as_succeeded(start_id, stop_id)
+      end
+
+      private
+
+      attr_reader :server_ids, :cloud_ids
+
+      def client_url(jira_tracker_data)
+        jira_tracker_data.api_url.presence || jira_tracker_data.url.presence
+      end
+
+      def server_type(url)
+        url.downcase.include?('.atlassian.net') ? :cloud : :server
+      end
+
+      def collect_deployment_type(jira_tracker_data)
+        url = client_url(jira_tracker_data)
+        return unless url
+
+        case server_type(url)
+        when :cloud
+          cloud_ids << jira_tracker_data.id
+        else
+          server_ids << jira_tracker_data.id
+        end
+      end
+
+      def mark_jobs_as_succeeded(*arguments)
+        Gitlab::Database::BackgroundMigrationJob.mark_all_as_succeeded(self.class.name, arguments)
+      end
+    end
+  end
+end

spec/lib/gitlab/background_migration/backfill_jira_tracker_deployment_type2_spec.rb

+# frozen_string_literal: true
+
+require 'spec_helper'
+
+RSpec.describe Gitlab::BackgroundMigration::BackfillJiraTrackerDeploymentType2, :migration, schema: 20201028182809 do
+  let_it_be(:jira_service_temp) { described_class::JiraServiceTemp }
+  let_it_be(:jira_tracker_data_temp) { described_class::JiraTrackerDataTemp }
+  let_it_be(:atlassian_host) { 'https://api.atlassian.net' }
+  let_it_be(:mixedcase_host) { 'https://api.AtlassiaN.nEt' }
+  let_it_be(:server_host) { 'https://my.server.net' }
+
+  let(:jira_service) { jira_service_temp.create!(type: 'JiraService', active: true, category: 'issue_tracker') }
+
+  subject { described_class.new }
+
+  def create_tracker_data(options = {})
+    jira_tracker_data_temp.create!({ service_id: jira_service.id }.merge(options))
+  end
+
+  describe '#perform' do
+    context do
+      it 'ignores if deployment already set' do
+        tracker_data = create_tracker_data(url: atlassian_host, deployment_type: 'server')
+
+        expect(subject).not_to receive(:collect_deployment_type)
+
+        subject.perform(tracker_data.id, tracker_data.id)
+
+        expect(tracker_data.reload.deployment_type).to eq 'server'
+      end
+
+      it 'ignores if no url is set' do
+        tracker_data = create_tracker_data(deployment_type: 'unknown')
+
+        expect(subject).to receive(:collect_deployment_type)
+
+        subject.perform(tracker_data.id, tracker_data.id)
+
+        expect(tracker_data.reload.deployment_type).to eq 'unknown'
+      end
+    end
+
+    context 'when tracker is valid' do
+      let!(:tracker_1) { create_tracker_data(url: atlassian_host, deployment_type: 0) }
+      let!(:tracker_2) { create_tracker_data(url: mixedcase_host, deployment_type: 0) }
+      let!(:tracker_3) { create_tracker_data(url: server_host, deployment_type: 0) }
+      let!(:tracker_4) { create_tracker_data(api_url: server_host, deployment_type: 0) }
+      let!(:tracker_nextbatch) { create_tracker_data(api_url: atlassian_host, deployment_type: 0) }
+
+      it 'sets the proper deployment_type', :aggregate_failures do
+        subject.perform(tracker_1.id, tracker_4.id)
+
+        expect(tracker_1.reload.deployment_cloud?).to be_truthy
+        expect(tracker_2.reload.deployment_cloud?).to be_truthy
+        expect(tracker_3.reload.deployment_server?).to be_truthy
+        expect(tracker_4.reload.deployment_server?).to be_truthy
+        expect(tracker_nextbatch.reload.deployment_unknown?).to be_truthy
+      end
+    end
+
+    it_behaves_like 'marks background migration job records' do
+      let(:arguments) { [1, 4] }
+    end
+  end
+end

spec/migrations/20201028182809_backfill_jira_tracker_deployment_type2_spec.rb

+# frozen_string_literal: true
+
+require 'spec_helper'
+require Rails.root.join('db', 'post_migrate', '20201028182809_backfill_jira_tracker_deployment_type2.rb')
+
+RSpec.describe BackfillJiraTrackerDeploymentType2, :sidekiq, schema: 20201028182809 do
+  let(:services) { table(:services) }
+  let(:jira_tracker_data) { table(:jira_tracker_data) }
+  let(:migration) { described_class.new }
+  let(:batch_interval) { described_class::DELAY_INTERVAL }
+
+  describe '#up' do
+    before do
+      stub_const("#{described_class}::BATCH_SIZE", 2)
+
+      active_service   = services.create!(type: 'JiraService', active: true)
+      inactive_service = services.create!(type: 'JiraService', active: false)
+
+      jira_tracker_data.create!(id: 1, service_id: active_service.id, deployment_type: 0)
+      jira_tracker_data.create!(id: 2, service_id: active_service.id, deployment_type: 1)
+      jira_tracker_data.create!(id: 3, service_id: inactive_service.id, deployment_type: 2)
+      jira_tracker_data.create!(id: 4, service_id: inactive_service.id, deployment_type: 0)
+      jira_tracker_data.create!(id: 5, service_id: active_service.id, deployment_type: 0)
+    end
+
+    it 'schedules BackfillJiraTrackerDeploymentType2 background jobs' do
+      Sidekiq::Testing.fake! do
+        freeze_time do
+          migration.up
+
+          expect(BackgroundMigrationWorker.jobs.size).to eq(2)
+          expect(described_class::MIGRATION).to be_scheduled_delayed_migration(batch_interval, 1, 4)
+          expect(described_class::MIGRATION).to be_scheduled_delayed_migration(batch_interval * 2, 5, 5)
+        end
+      end
+    end
+  end
+end

spec/support/shared_examples/lib/gitlab/database/background_migration_job_shared_examples.rb

+# frozen_string_literal: true
+
+RSpec.shared_examples 'marks background migration job records' do
+  it 'marks each job record as succeeded after processing' do
+    create(:background_migration_job, class_name: "::#{described_class.name}",
+           arguments: arguments)
+
+    expect(::Gitlab::Database::BackgroundMigrationJob).to receive(:mark_all_as_succeeded).and_call_original
+
+    expect do
+      subject.perform(*arguments)
+    end.to change { ::Gitlab::Database::BackgroundMigrationJob.succeeded.count }.from(0).to(1)
+  end
+
+  it 'returns the number of job records marked as succeeded' do
+    create(:background_migration_job, class_name: "::#{described_class.name}",
+           arguments: arguments)
+
+    jobs_updated = subject.perform(*arguments)
+
+    expect(jobs_updated).to eq(1)
+  end
+end