Merge branch '199388-propagate-instance-level-integration' into 'master'

Propagate instance-level integration to projects

See merge request gitlab-org/gitlab!32331

app/controllers/concerns/integrations_actions.rb

@@ -16,10 +16,12 @@ module IntegrationsActions
 
   def update
     saved = integration.update(service_params[:service])
+    overwrite = ActiveRecord::Type::Boolean.new.cast(params[:overwrite])
 
     respond_to do |format|
       format.html do
         if saved
+          PropagateIntegrationWorker.perform_async(integration.id, overwrite)
           redirect_to scoped_edit_integration_path(integration), notice: success_message
         else
           render 'shared/integrations/edit'

app/models/data_list.rb

+# frozen_string_literal: true
+
+class DataList
+  def initialize(batch, data_fields_hash, klass)
+    @batch = batch
+    @data_fields_hash = data_fields_hash
+    @klass = klass
+  end
+
+  def to_array
+    [klass, columns, values]
+  end
+
+  private
+
+  attr_reader :batch, :data_fields_hash, :klass
+
+  def columns
+    data_fields_hash.keys << 'service_id'
+  end
+
+  def values
+    batch.map { |row| data_fields_hash.values << row['id'] }
+  end
+end

app/models/service.rb

@@ -134,6 +134,14 @@ class Service < ApplicationRecord
     %w(active)
   end
 
+  def to_service_hash
+    as_json(methods: :type, except: %w[id template instance project_id])
+  end
+
+  def to_data_fields_hash
+    data_fields.as_json(only: data_fields.class.column_names).except('id', 'service_id')
+  end
+
   def test_data(project, user)
     Gitlab::DataBuilder::Push.build_sample(project, user)
   end

app/models/service_list.rb

+# frozen_string_literal: true
+
+class ServiceList
+  def initialize(batch, service_hash, extra_hash = {})
+    @batch = batch
+    @service_hash = service_hash
+    @extra_hash = extra_hash
+  end
+
+  def to_array
+    [Service, columns, values]
+  end
+
+  private
+
+  attr_reader :batch, :service_hash, :extra_hash
+
+  def columns
+    (service_hash.keys << 'project_id') + extra_hash.keys
+  end
+
+  def values
+    batch.map do |project_id|
+      (service_hash.values << project_id) + extra_hash.values
+    end
+  end
+end

app/services/admin/propagate_integration_service.rb

+# frozen_string_literal: true
+
+module Admin
+  class PropagateIntegrationService
+    BATCH_SIZE = 100
+
+    delegate :data_fields_present?, to: :integration
+
+    def self.propagate(integration:, overwrite:)
+      new(integration, overwrite).propagate
+    end
+
+    def initialize(integration, overwrite)
+      @integration = integration
+      @overwrite = overwrite
+    end
+
+    def propagate
+      if overwrite
+        update_integration_for_all_projects
+      else
+        update_integration_for_inherited_projects
+      end
+
+      create_integration_for_projects_without_integration
+    end
+
+    private
+
+    attr_reader :integration, :overwrite
+
+    # rubocop: disable Cop/InBatches
+    # rubocop: disable CodeReuse/ActiveRecord
+    def update_integration_for_inherited_projects
+      Service.where(type: integration.type, inherit_from_id: integration.id).in_batches(of: BATCH_SIZE) do |batch|
+        bulk_update_from_integration(batch)
+      end
+    end
+
+    def update_integration_for_all_projects
+      Service.where(type: integration.type).in_batches(of: BATCH_SIZE) do |batch|
+        bulk_update_from_integration(batch)
+      end
+    end
+    # rubocop: enable Cop/InBatches
+    # rubocop: enable CodeReuse/ActiveRecord
+
+    # rubocop: disable CodeReuse/ActiveRecord
+    def bulk_update_from_integration(batch)
+      # Retrieving the IDs instantiates the ActiveRecord relation (batch)
+      # into concrete models, otherwise update_all will clear the relation.
+      # https://stackoverflow.com/q/34811646/462015
+      batch_ids = batch.pluck(:id)
+
+      Service.transaction do
+        batch.update_all(service_hash)
+
+        if data_fields_present?
+          integration.data_fields.class.where(service_id: batch_ids).update_all(data_fields_hash)
+        end
+      end
+    end
+    # rubocop: enable CodeReuse/ActiveRecord
+
+    def create_integration_for_projects_without_integration
+      loop do
+        batch = Project.uncached { project_ids_without_integration }
+
+        bulk_create_from_integration(batch) unless batch.empty?
+
+        break if batch.size < BATCH_SIZE
+      end
+    end
+
+    def bulk_create_from_integration(batch)
+      service_list = ServiceList.new(batch, service_hash, { 'inherit_from_id' => integration.id }).to_array
+
+      Project.transaction do
+        results = bulk_insert(*service_list)
+
+        if data_fields_present?
+          data_list = DataList.new(results, data_fields_hash, integration.data_fields.class).to_array
+
+          bulk_insert(*data_list)
+        end
+
+        run_callbacks(batch)
+      end
+    end
+
+    def bulk_insert(klass, columns, values_array)
+      items_to_insert = values_array.map { |array| Hash[columns.zip(array)] }
+
+      klass.insert_all(items_to_insert, returning: [:id])
+    end
+
+    # rubocop: disable CodeReuse/ActiveRecord
+    def run_callbacks(batch)
+      if active_external_issue_tracker?
+        Project.where(id: batch).update_all(has_external_issue_tracker: true)
+      end
+
+      if active_external_wiki?
+        Project.where(id: batch).update_all(has_external_wiki: true)
+      end
+    end
+    # rubocop: enable CodeReuse/ActiveRecord
+
+    def active_external_issue_tracker?
+      integration.issue_tracker? && !integration.default
+    end
+
+    def active_external_wiki?
+      integration.type == 'ExternalWikiService'
+    end
+
+    def project_ids_without_integration
+      Project.connection.select_values(
+        <<-SQL
+          SELECT id
+          FROM projects
+          WHERE NOT EXISTS (
+            SELECT true
+            FROM services
+            WHERE services.project_id = projects.id
+            AND services.type = #{ActiveRecord::Base.connection.quote(integration.type)}
+          )
+          AND projects.pending_delete = false
+          AND projects.archived = false
+          LIMIT #{BATCH_SIZE}
+        SQL
+      )
+    end
+
+    def service_hash
+      @service_hash ||= integration.to_service_hash
+        .tap { |json| json['inherit_from_id'] = integration.id }
+    end
+
+    def data_fields_hash
+      @data_fields_hash ||= integration.to_data_fields_hash
+    end
+  end
+end

app/services/projects/propagate_service_template.rb

@@ -35,17 +35,15 @@ module Projects
     end
 
     def bulk_create_from_template(batch)
-      service_list = batch.map do |project_id|
-        service_hash.values << project_id
-      end
+      service_list = ServiceList.new(batch, service_hash).to_array
 
       Project.transaction do
-        results = bulk_insert(Service, service_hash.keys << 'project_id', service_list)
+        results = bulk_insert(*service_list)
 
         if data_fields_present?
-          data_list = results.map { |row| data_hash.values << row['id'] }
+          data_list = DataList.new(results, data_fields_hash, template.data_fields.class).to_array
 
-          bulk_insert(template.data_fields.class, data_hash.keys << 'service_id', data_list)
+          bulk_insert(*data_list)
         end
 
         run_callbacks(batch)
@@ -77,11 +75,11 @@ module Projects
     end
 
     def service_hash
-      @service_hash ||= template.as_json(methods: :type, except: %w[id template project_id])
+      @service_hash ||= template.to_service_hash
     end
 
-    def data_hash
-      @data_hash ||= template.data_fields.as_json(only: template.data_fields.class.column_names).except('id', 'service_id')
+    def data_fields_hash
+      @data_fields_hash ||= template.to_data_fields_hash
     end
 
     # rubocop: disable CodeReuse/ActiveRecord

app/workers/all_queues.yml

@@ -1291,6 +1291,13 @@
   :resource_boundary: :unknown
   :weight: 1
   :idempotent: true
+- :name: propagate_integration
+  :feature_category: :integrations
+  :has_external_dependencies: 
+  :urgency: :low
+  :resource_boundary: :unknown
+  :weight: 1
+  :idempotent: true
 - :name: propagate_service_template
   :feature_category: :source_code_management
   :has_external_dependencies: 

app/workers/propagate_integration_worker.rb

+# frozen_string_literal: true
+
+class PropagateIntegrationWorker
+  include ApplicationWorker
+
+  feature_category :integrations
+
+  idempotent!
+
+  def perform(integration_id, overwrite)
+    Admin::PropagateIntegrationService.propagate(
+      integration: Service.find(integration_id),
+      overwrite: overwrite
+    )
+  end
+end

config/sidekiq_queues.yml

@@ -210,6 +210,8 @@
   - 1
 - - prometheus_create_default_alerts
   - 1
+- - propagate_integration
+  - 1
 - - propagate_service_template
   - 1
 - - reactive_caching

spec/controllers/admin/integrations_controller_spec.rb

@@ -36,7 +36,9 @@ describe Admin::IntegrationsController do
     let(:integration) { create(:jira_service, :instance) }
 
     before do
-      put :update, params: { id: integration.class.to_param, service: { url: url } }
+      allow(PropagateIntegrationWorker).to receive(:perform_async)
+
+      put :update, params: { id: integration.class.to_param, overwrite: true, service: { url: url } }
     end
 
     context 'valid params' do
@@ -46,6 +48,10 @@ describe Admin::IntegrationsController do
         expect(response).to have_gitlab_http_status(:found)
         expect(integration.reload.url).to eq(url)
       end
+
+      it 'calls to PropagateIntegrationWorker' do
+        expect(PropagateIntegrationWorker).to have_received(:perform_async).with(integration.id, true)
+      end
     end
 
     context 'invalid params' do
@@ -56,6 +62,10 @@ describe Admin::IntegrationsController do
         expect(response).to render_template(:edit)
         expect(integration.reload.url).not_to eq(url)
       end
+
+      it 'does not call to PropagateIntegrationWorker' do
+        expect(PropagateIntegrationWorker).not_to have_received(:perform_async)
+      end
     end
   end
 end

spec/services/admin/propagate_integration_service_spec.rb

+# frozen_string_literal: true
+
+require 'spec_helper'
+
+RSpec.describe Admin::PropagateIntegrationService do
+  describe '.propagate' do
+    let(:excluded_attributes) { %w[id project_id inherit_from_id instance created_at updated_at title description] }
+    let!(:project) { create(:project) }
+    let!(:instance_integration) do
+      JiraService.create!(
+        instance: true,
+        active: true,
+        push_events: true,
+        url: 'http://update-jira.instance.com',
+        username: 'user',
+        password: 'secret'
+      )
+    end
+
+    let!(:inherited_integration) do
+      JiraService.create!(
+        project: create(:project),
+        inherit_from_id: instance_integration.id,
+        instance: false,
+        active: true,
+        push_events: false,
+        url: 'http://jira.instance.com',
+        username: 'user',
+        password: 'secret'
+      )
+    end
+
+    let!(:not_inherited_integration) do
+      JiraService.create!(
+        project: create(:project),
+        inherit_from_id: nil,
+        instance: false,
+        active: true,
+        push_events: false,
+        url: 'http://jira.instance.com',
+        username: 'user',
+        password: 'secret'
+      )
+    end
+
+    let!(:another_inherited_integration) do
+      BambooService.create!(
+        project: create(:project),
+        inherit_from_id: instance_integration.id,
+        instance: false,
+        active: true,
+        push_events: false,
+        bamboo_url: 'http://gitlab.com',
+        username: 'mic',
+        password: 'password',
+        build_key: 'build'
+      )
+    end
+
+    shared_examples 'inherits settings from integration' do
+      it 'updates the inherited integrations' do
+        described_class.propagate(integration: instance_integration, overwrite: overwrite)
+
+        expect(integration.reload.inherit_from_id).to eq(instance_integration.id)
+        expect(integration.attributes.except(*excluded_attributes))
+          .to eq(instance_integration.attributes.except(*excluded_attributes))
+      end
+
+      context 'integration with data fields' do
+        let(:excluded_attributes) { %w[id service_id created_at updated_at] }
+
+        it 'updates the data fields from inherited integrations' do
+          described_class.propagate(integration: instance_integration, overwrite: overwrite)
+
+          expect(integration.reload.data_fields.attributes.except(*excluded_attributes))
+            .to eq(instance_integration.data_fields.attributes.except(*excluded_attributes))
+        end
+      end
+    end
+
+    shared_examples 'does not inherit settings from integration' do
+      it 'does not update the not inherited integrations' do
+        described_class.propagate(integration: instance_integration, overwrite: overwrite)
+
+        expect(integration.reload.attributes.except(*excluded_attributes))
+          .not_to eq(instance_integration.attributes.except(*excluded_attributes))
+      end
+    end
+
+    context 'update only inherited integrations' do
+      let(:overwrite) { false }
+
+      it_behaves_like 'inherits settings from integration' do
+        let(:integration) { inherited_integration }
+      end
+
+      it_behaves_like 'does not inherit settings from integration' do
+        let(:integration) { not_inherited_integration }
+      end
+
+      it_behaves_like 'does not inherit settings from integration' do
+        let(:integration) { another_inherited_integration }
+      end
+
+      it_behaves_like 'inherits settings from integration' do
+        let(:integration) { project.jira_service }
+      end
+    end
+
+    context 'update all integrations' do
+      let(:overwrite) { true }
+
+      it_behaves_like 'inherits settings from integration' do
+        let(:integration) { inherited_integration }
+      end
+
+      it_behaves_like 'inherits settings from integration' do
+        let(:integration) { not_inherited_integration }
+      end
+
+      it_behaves_like 'does not inherit settings from integration' do
+        let(:integration) { another_inherited_integration }
+      end
+
+      it_behaves_like 'inherits settings from integration' do
+        let(:integration) { project.jira_service }
+      end
+    end
+
+    it 'updates project#has_external_issue_tracker for issue tracker services' do
+      described_class.propagate(integration: instance_integration, overwrite: true)
+
+      expect(project.reload.has_external_issue_tracker).to eq(true)
+    end
+
+    it 'updates project#has_external_wiki for external wiki services' do
+      instance_integration = ExternalWikiService.create!(
+        instance: true,
+        active: true,
+        push_events: false,
+        external_wiki_url: 'http://external-wiki-url.com'
+      )
+
+      described_class.propagate(integration: instance_integration, overwrite: true)
+
+      expect(project.reload.has_external_wiki).to eq(true)
+    end
+  end
+end

spec/workers/propagate_integration_worker_spec.rb

+# frozen_string_literal: true
+
+require 'spec_helper'
+
+describe PropagateIntegrationWorker do
+  describe '#perform' do
+    let(:integration) do
+      PushoverService.create(
+        template: true,
+        active: true,
+        device: 'MyDevice',
+        sound: 'mic',
+        priority: 4,
+        user_key: 'asdf',
+        api_key: '123456789'
+      )
+    end
+
+    it 'calls the propagate service with the integration' do
+      expect(Admin::PropagateIntegrationService).to receive(:propagate)
+        .with(integration: integration, overwrite: true)
+
+      subject.perform(integration.id, true)
+    end
+  end
+end