Commit b93691a6 authored by Arturo Herrero's avatar Arturo Herrero Committed by Adam Hegyi

Propagate instance-level integration to projects

When an admin saves an instance-level integration, they will have two
options that will propagate the settings differently.

- Apply changes to inherited integrations:
  - Updates integration for inherited projects
  - Creates integrations for projects without integrations
- Apply changes to all integrations:
  - Updates integration for all projects
  - Creates integrations for projects without integrations
parent 2e8d297b
...@@ -16,10 +16,12 @@ module IntegrationsActions ...@@ -16,10 +16,12 @@ module IntegrationsActions
def update def update
saved = integration.update(service_params[:service]) saved = integration.update(service_params[:service])
overwrite = ActiveRecord::Type::Boolean.new.cast(params[:overwrite])
respond_to do |format| respond_to do |format|
format.html do format.html do
if saved if saved
PropagateIntegrationWorker.perform_async(integration.id, overwrite)
redirect_to scoped_edit_integration_path(integration), notice: success_message redirect_to scoped_edit_integration_path(integration), notice: success_message
else else
render 'shared/integrations/edit' render 'shared/integrations/edit'
......
# frozen_string_literal: true
class DataList
def initialize(batch, data_fields_hash, klass)
@batch = batch
@data_fields_hash = data_fields_hash
@klass = klass
end
def to_array
[klass, columns, values]
end
private
attr_reader :batch, :data_fields_hash, :klass
def columns
data_fields_hash.keys << 'service_id'
end
def values
batch.map { |row| data_fields_hash.values << row['id'] }
end
end
...@@ -134,6 +134,14 @@ class Service < ApplicationRecord ...@@ -134,6 +134,14 @@ class Service < ApplicationRecord
%w(active) %w(active)
end end
def to_service_hash
as_json(methods: :type, except: %w[id template instance project_id])
end
def to_data_fields_hash
data_fields.as_json(only: data_fields.class.column_names).except('id', 'service_id')
end
def test_data(project, user) def test_data(project, user)
Gitlab::DataBuilder::Push.build_sample(project, user) Gitlab::DataBuilder::Push.build_sample(project, user)
end end
......
# frozen_string_literal: true
class ServiceList
def initialize(batch, service_hash, extra_hash = {})
@batch = batch
@service_hash = service_hash
@extra_hash = extra_hash
end
def to_array
[Service, columns, values]
end
private
attr_reader :batch, :service_hash, :extra_hash
def columns
(service_hash.keys << 'project_id') + extra_hash.keys
end
def values
batch.map do |project_id|
(service_hash.values << project_id) + extra_hash.values
end
end
end
# frozen_string_literal: true
module Admin
class PropagateIntegrationService
BATCH_SIZE = 100
delegate :data_fields_present?, to: :integration
def self.propagate(integration:, overwrite:)
new(integration, overwrite).propagate
end
def initialize(integration, overwrite)
@integration = integration
@overwrite = overwrite
end
def propagate
if overwrite
update_integration_for_all_projects
else
update_integration_for_inherited_projects
end
create_integration_for_projects_without_integration
end
private
attr_reader :integration, :overwrite
# rubocop: disable Cop/InBatches
# rubocop: disable CodeReuse/ActiveRecord
def update_integration_for_inherited_projects
Service.where(type: integration.type, inherit_from_id: integration.id).in_batches(of: BATCH_SIZE) do |batch|
bulk_update_from_integration(batch)
end
end
def update_integration_for_all_projects
Service.where(type: integration.type).in_batches(of: BATCH_SIZE) do |batch|
bulk_update_from_integration(batch)
end
end
# rubocop: enable Cop/InBatches
# rubocop: enable CodeReuse/ActiveRecord
# rubocop: disable CodeReuse/ActiveRecord
def bulk_update_from_integration(batch)
# Retrieving the IDs instantiates the ActiveRecord relation (batch)
# into concrete models, otherwise update_all will clear the relation.
# https://stackoverflow.com/q/34811646/462015
batch_ids = batch.pluck(:id)
Service.transaction do
batch.update_all(service_hash)
if data_fields_present?
integration.data_fields.class.where(service_id: batch_ids).update_all(data_fields_hash)
end
end
end
# rubocop: enable CodeReuse/ActiveRecord
def create_integration_for_projects_without_integration
loop do
batch = Project.uncached { project_ids_without_integration }
bulk_create_from_integration(batch) unless batch.empty?
break if batch.size < BATCH_SIZE
end
end
def bulk_create_from_integration(batch)
service_list = ServiceList.new(batch, service_hash, { 'inherit_from_id' => integration.id }).to_array
Project.transaction do
results = bulk_insert(*service_list)
if data_fields_present?
data_list = DataList.new(results, data_fields_hash, integration.data_fields.class).to_array
bulk_insert(*data_list)
end
run_callbacks(batch)
end
end
def bulk_insert(klass, columns, values_array)
items_to_insert = values_array.map { |array| Hash[columns.zip(array)] }
klass.insert_all(items_to_insert, returning: [:id])
end
# rubocop: disable CodeReuse/ActiveRecord
def run_callbacks(batch)
if active_external_issue_tracker?
Project.where(id: batch).update_all(has_external_issue_tracker: true)
end
if active_external_wiki?
Project.where(id: batch).update_all(has_external_wiki: true)
end
end
# rubocop: enable CodeReuse/ActiveRecord
def active_external_issue_tracker?
integration.issue_tracker? && !integration.default
end
def active_external_wiki?
integration.type == 'ExternalWikiService'
end
def project_ids_without_integration
Project.connection.select_values(
<<-SQL
SELECT id
FROM projects
WHERE NOT EXISTS (
SELECT true
FROM services
WHERE services.project_id = projects.id
AND services.type = #{ActiveRecord::Base.connection.quote(integration.type)}
)
AND projects.pending_delete = false
AND projects.archived = false
LIMIT #{BATCH_SIZE}
SQL
)
end
def service_hash
@service_hash ||= integration.to_service_hash
.tap { |json| json['inherit_from_id'] = integration.id }
end
def data_fields_hash
@data_fields_hash ||= integration.to_data_fields_hash
end
end
end
...@@ -35,17 +35,15 @@ module Projects ...@@ -35,17 +35,15 @@ module Projects
end end
def bulk_create_from_template(batch) def bulk_create_from_template(batch)
service_list = batch.map do |project_id| service_list = ServiceList.new(batch, service_hash).to_array
service_hash.values << project_id
end
Project.transaction do Project.transaction do
results = bulk_insert(Service, service_hash.keys << 'project_id', service_list) results = bulk_insert(*service_list)
if data_fields_present? if data_fields_present?
data_list = results.map { |row| data_hash.values << row['id'] } data_list = DataList.new(results, data_fields_hash, template.data_fields.class).to_array
bulk_insert(template.data_fields.class, data_hash.keys << 'service_id', data_list) bulk_insert(*data_list)
end end
run_callbacks(batch) run_callbacks(batch)
...@@ -77,11 +75,11 @@ module Projects ...@@ -77,11 +75,11 @@ module Projects
end end
def service_hash def service_hash
@service_hash ||= template.as_json(methods: :type, except: %w[id template project_id]) @service_hash ||= template.to_service_hash
end end
def data_hash def data_fields_hash
@data_hash ||= template.data_fields.as_json(only: template.data_fields.class.column_names).except('id', 'service_id') @data_fields_hash ||= template.to_data_fields_hash
end end
# rubocop: disable CodeReuse/ActiveRecord # rubocop: disable CodeReuse/ActiveRecord
......
...@@ -1291,6 +1291,13 @@ ...@@ -1291,6 +1291,13 @@
:resource_boundary: :unknown :resource_boundary: :unknown
:weight: 1 :weight: 1
:idempotent: true :idempotent: true
- :name: propagate_integration
:feature_category: :integrations
:has_external_dependencies:
:urgency: :low
:resource_boundary: :unknown
:weight: 1
:idempotent: true
- :name: propagate_service_template - :name: propagate_service_template
:feature_category: :source_code_management :feature_category: :source_code_management
:has_external_dependencies: :has_external_dependencies:
......
# frozen_string_literal: true
class PropagateIntegrationWorker
include ApplicationWorker
feature_category :integrations
idempotent!
def perform(integration_id, overwrite)
Admin::PropagateIntegrationService.propagate(
integration: Service.find(integration_id),
overwrite: overwrite
)
end
end
...@@ -210,6 +210,8 @@ ...@@ -210,6 +210,8 @@
- 1 - 1
- - prometheus_create_default_alerts - - prometheus_create_default_alerts
- 1 - 1
- - propagate_integration
- 1
- - propagate_service_template - - propagate_service_template
- 1 - 1
- - reactive_caching - - reactive_caching
......
...@@ -36,7 +36,9 @@ describe Admin::IntegrationsController do ...@@ -36,7 +36,9 @@ describe Admin::IntegrationsController do
let(:integration) { create(:jira_service, :instance) } let(:integration) { create(:jira_service, :instance) }
before do before do
put :update, params: { id: integration.class.to_param, service: { url: url } } allow(PropagateIntegrationWorker).to receive(:perform_async)
put :update, params: { id: integration.class.to_param, overwrite: true, service: { url: url } }
end end
context 'valid params' do context 'valid params' do
...@@ -46,6 +48,10 @@ describe Admin::IntegrationsController do ...@@ -46,6 +48,10 @@ describe Admin::IntegrationsController do
expect(response).to have_gitlab_http_status(:found) expect(response).to have_gitlab_http_status(:found)
expect(integration.reload.url).to eq(url) expect(integration.reload.url).to eq(url)
end end
it 'calls to PropagateIntegrationWorker' do
expect(PropagateIntegrationWorker).to have_received(:perform_async).with(integration.id, true)
end
end end
context 'invalid params' do context 'invalid params' do
...@@ -56,6 +62,10 @@ describe Admin::IntegrationsController do ...@@ -56,6 +62,10 @@ describe Admin::IntegrationsController do
expect(response).to render_template(:edit) expect(response).to render_template(:edit)
expect(integration.reload.url).not_to eq(url) expect(integration.reload.url).not_to eq(url)
end end
it 'does not call to PropagateIntegrationWorker' do
expect(PropagateIntegrationWorker).not_to have_received(:perform_async)
end
end end
end end
end end
# frozen_string_literal: true
require 'spec_helper'
RSpec.describe Admin::PropagateIntegrationService do
describe '.propagate' do
let(:excluded_attributes) { %w[id project_id inherit_from_id instance created_at updated_at title description] }
let!(:project) { create(:project) }
let!(:instance_integration) do
JiraService.create!(
instance: true,
active: true,
push_events: true,
url: 'http://update-jira.instance.com',
username: 'user',
password: 'secret'
)
end
let!(:inherited_integration) do
JiraService.create!(
project: create(:project),
inherit_from_id: instance_integration.id,
instance: false,
active: true,
push_events: false,
url: 'http://jira.instance.com',
username: 'user',
password: 'secret'
)
end
let!(:not_inherited_integration) do
JiraService.create!(
project: create(:project),
inherit_from_id: nil,
instance: false,
active: true,
push_events: false,
url: 'http://jira.instance.com',
username: 'user',
password: 'secret'
)
end
let!(:another_inherited_integration) do
BambooService.create!(
project: create(:project),
inherit_from_id: instance_integration.id,
instance: false,
active: true,
push_events: false,
bamboo_url: 'http://gitlab.com',
username: 'mic',
password: 'password',
build_key: 'build'
)
end
shared_examples 'inherits settings from integration' do
it 'updates the inherited integrations' do
described_class.propagate(integration: instance_integration, overwrite: overwrite)
expect(integration.reload.inherit_from_id).to eq(instance_integration.id)
expect(integration.attributes.except(*excluded_attributes))
.to eq(instance_integration.attributes.except(*excluded_attributes))
end
context 'integration with data fields' do
let(:excluded_attributes) { %w[id service_id created_at updated_at] }
it 'updates the data fields from inherited integrations' do
described_class.propagate(integration: instance_integration, overwrite: overwrite)
expect(integration.reload.data_fields.attributes.except(*excluded_attributes))
.to eq(instance_integration.data_fields.attributes.except(*excluded_attributes))
end
end
end
shared_examples 'does not inherit settings from integration' do
it 'does not update the not inherited integrations' do
described_class.propagate(integration: instance_integration, overwrite: overwrite)
expect(integration.reload.attributes.except(*excluded_attributes))
.not_to eq(instance_integration.attributes.except(*excluded_attributes))
end
end
context 'update only inherited integrations' do
let(:overwrite) { false }
it_behaves_like 'inherits settings from integration' do
let(:integration) { inherited_integration }
end
it_behaves_like 'does not inherit settings from integration' do
let(:integration) { not_inherited_integration }
end
it_behaves_like 'does not inherit settings from integration' do
let(:integration) { another_inherited_integration }
end
it_behaves_like 'inherits settings from integration' do
let(:integration) { project.jira_service }
end
end
context 'update all integrations' do
let(:overwrite) { true }
it_behaves_like 'inherits settings from integration' do
let(:integration) { inherited_integration }
end
it_behaves_like 'inherits settings from integration' do
let(:integration) { not_inherited_integration }
end
it_behaves_like 'does not inherit settings from integration' do
let(:integration) { another_inherited_integration }
end
it_behaves_like 'inherits settings from integration' do
let(:integration) { project.jira_service }
end
end
it 'updates project#has_external_issue_tracker for issue tracker services' do
described_class.propagate(integration: instance_integration, overwrite: true)
expect(project.reload.has_external_issue_tracker).to eq(true)
end
it 'updates project#has_external_wiki for external wiki services' do
instance_integration = ExternalWikiService.create!(
instance: true,
active: true,
push_events: false,
external_wiki_url: 'http://external-wiki-url.com'
)
described_class.propagate(integration: instance_integration, overwrite: true)
expect(project.reload.has_external_wiki).to eq(true)
end
end
end
# frozen_string_literal: true
require 'spec_helper'
describe PropagateIntegrationWorker do
describe '#perform' do
let(:integration) do
PushoverService.create(
template: true,
active: true,
device: 'MyDevice',
sound: 'mic',
priority: 4,
user_key: 'asdf',
api_key: '123456789'
)
end
it 'calls the propagate service with the integration' do
expect(Admin::PropagateIntegrationService).to receive(:propagate)
.with(integration: integration, overwrite: true)
subject.perform(integration.id, true)
end
end
end
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment