Commit 10feb38e authored by Alan (Maciej) Paruszewski's avatar Alan (Maciej) Paruszewski Committed by Bob Van Landuyt

Extend workers and services to support namespace for Security Policies

Changelog: added
EE: true
parent 4ea62832
......@@ -415,6 +415,8 @@
- 1
- - security_findings_delete_by_job_id
- 1
- - security_orchestration_policy_rule_schedule_namespace
- 1
- - security_scans
- 2
- - self_monitoring_project_create
......
......@@ -14,9 +14,10 @@ module Security
REQUIRE_APPROVAL = 'require_approval'
included do
delegate :approval_rules, to: :project
delegate :approval_rules, to: :project, allow_nil: true
def active_scan_result_policies
return [] if project.blank?
return [] unless ::Feature.enabled?(:scan_result_policy, project, default_enabled: :yaml)
scan_result_policies&.select { |config| config[:enabled] }&.first(LIMIT)
......@@ -27,6 +28,8 @@ module Security
end
def uniq_scanners
return [] if project.blank?
distinct_scanners = approval_rules.distinct_scanners
return [] if distinct_scanners.none?
......
......@@ -861,6 +861,16 @@ module EE
approval_rules.vulnerability_reports.first
end
def all_security_orchestration_policy_configurations
all_parent_groups = group&.self_and_ancestor_ids
return Array.wrap(security_orchestration_policy_configuration) if all_parent_groups.blank?
[
security_orchestration_policy_configuration,
*::Security::OrchestrationPolicyConfiguration.where(namespace_id: all_parent_groups)
].compact
end
private
def ci_minutes_usage
......
......@@ -81,6 +81,14 @@ module Security
security_policy_management_project.default_branch_or_main
end
def project?
!namespace?
end
def namespace?
namespace_id.present?
end
private
def policy_repo
......
......@@ -20,9 +20,9 @@ module Security
scope :runnable_schedules, -> { where("next_run_at < ?", Time.zone.now) }
scope :with_owner, -> { includes(:owner) }
scope :with_configuration_and_project, -> do
scope :with_configuration_and_project_or_namespace, -> do
includes(
security_orchestration_policy_configuration: [:project, :security_policy_management_project]
security_orchestration_policy_configuration: [:project, :namespace, :security_policy_management_project]
)
end
......@@ -32,17 +32,15 @@ module Security
end
end
def applicable_branches
strong_memoize(:applicable_branches) do
configured_branches = policy&.dig(:rules, rule_index, :branches)
next [] if configured_branches.blank?
def applicable_branches(project = security_orchestration_policy_configuration.project)
configured_branches = policy&.dig(:rules, rule_index, :branches)
return [] if configured_branches.blank? || project.blank?
branch_names = security_orchestration_policy_configuration.project.repository.branches
branch_names = project.repository.branches
configured_branches
.flat_map { |pattern| RefMatcher.new(pattern).matching(branch_names).map(&:name) }
.uniq
end
configured_branches
.flat_map { |pattern| RefMatcher.new(pattern).matching(branch_names).map(&:name) }
.uniq
end
def applicable_clusters
......
......@@ -4,9 +4,7 @@ module Security
module SecurityOrchestrationPolicies
class RuleScheduleService < BaseContainerService
def execute(schedule)
schedule.schedule_next_run!
branches = schedule.applicable_branches
branches = schedule.applicable_branches(container)
actions_for(schedule).each { |action| process_action(action, schedule, branches) }
end
......
......@@ -1326,6 +1326,15 @@
:weight: 1
:idempotent: true
:tags: []
- :name: security_orchestration_policy_rule_schedule_namespace
:worker_name: Security::OrchestrationPolicyRuleScheduleNamespaceWorker
:feature_category: :security_orchestration
:has_external_dependencies:
:urgency: :low
:resource_boundary: :unknown
:weight: 1
:idempotent: true
:tags: []
- :name: set_user_status_based_on_user_cap_setting
:worker_name: SetUserStatusBasedOnUserCapSettingWorker
:feature_category: :users
......
......@@ -26,12 +26,14 @@ module Security
.execute
end
configuration.transaction do
configuration.approval_rules.scan_finding.delete_all
configuration.active_scan_result_policies.each_with_index do |policy, policy_index|
Security::SecurityOrchestrationPolicies::ProcessScanResultPolicyService
.new(policy_configuration: configuration, policy: policy, policy_index: policy_index)
.execute
if configuration.project?
configuration.transaction do
configuration.approval_rules.scan_finding.delete_all
configuration.active_scan_result_policies.each_with_index do |policy, policy_index|
Security::SecurityOrchestrationPolicies::ProcessScanResultPolicyService
.new(policy_configuration: configuration, policy: policy, policy_index: policy_index)
.execute
end
end
end
......
# frozen_string_literal: true
module Security
class OrchestrationPolicyRuleScheduleNamespaceWorker
include ApplicationWorker
feature_category :security_orchestration
data_consistency :sticky
idempotent!
def perform(rule_schedule_id)
schedule = Security::OrchestrationPolicyRuleSchedule.find_by_id(rule_schedule_id)
return unless schedule
security_orchestration_policy_configuration = schedule.security_orchestration_policy_configuration
return if !security_orchestration_policy_configuration.namespace? || security_orchestration_policy_configuration.namespace.blank?
return if schedule.next_run_at.future?
schedule.schedule_next_run!
security_orchestration_policy_configuration.namespace.all_projects.find_in_batches.each do |projects|
projects.each do |project|
with_context(project: project, user: schedule.owner) do
Security::SecurityOrchestrationPolicies::RuleScheduleService
.new(container: project, current_user: schedule.owner)
.execute(schedule)
end
end
end
end
end
end
......@@ -13,15 +13,27 @@ module Security
feature_category :security_orchestration
def perform
Security::OrchestrationPolicyRuleSchedule.with_configuration_and_project.with_owner.runnable_schedules.find_in_batches do |schedules|
Security::OrchestrationPolicyRuleSchedule.with_configuration_and_project_or_namespace.with_owner.runnable_schedules.find_in_batches do |schedules|
schedules.each do |schedule|
with_context(project: schedule.security_orchestration_policy_configuration.project, user: schedule.owner) do
Security::SecurityOrchestrationPolicies::RuleScheduleService
.new(container: schedule.security_orchestration_policy_configuration.project, current_user: schedule.owner)
.execute(schedule)
if schedule.security_orchestration_policy_configuration.project?
schedule_rules(schedule)
else
Security::OrchestrationPolicyRuleScheduleNamespaceWorker.perform_async(schedule.id)
end
end
end
end
end
private
def schedule_rules(schedule)
schedule.schedule_next_run!
Security::SecurityOrchestrationPolicies::RuleScheduleService
.new(container: schedule.security_orchestration_policy_configuration.project, current_user: schedule.owner)
.execute(schedule)
end
end
end
......@@ -15,12 +15,13 @@ module Gitlab
def perform
return @config unless project&.feature_available?(:security_orchestration_policies)
return @config unless security_orchestration_policy_configuration&.policy_configuration_valid?
return @config if valid_security_orchestration_policy_configurations.blank?
return @config unless extend_configuration?
merged_config = @config
.deep_merge(on_demand_scans_template)
.deep_merge(pipeline_scan_template)
observe_processing_duration(Time.current - @start)
merged_config
......@@ -30,17 +31,40 @@ module Gitlab
attr_reader :project
delegate :security_orchestration_policy_configuration, to: :project, allow_nil: true
delegate :all_security_orchestration_policy_configurations, to: :project, allow_nil: true
def valid_security_orchestration_policy_configurations
@valid_security_orchestration_policy_configurations ||=
all_security_orchestration_policy_configurations&.select(&:policy_configuration_valid?)
end
def on_demand_scans_template
::Security::SecurityOrchestrationPolicies::OnDemandScanPipelineConfigurationService
.new(project)
.execute(security_orchestration_policy_configuration.on_demand_scan_actions(@ref))
.execute(on_demand_scan_actions)
end
def pipeline_scan_template
::Security::SecurityOrchestrationPolicies::ScanPipelineService
.new.execute(security_orchestration_policy_configuration.pipeline_scan_actions(@ref))
.new.execute(pipeline_scan_actions)
end
def on_demand_scan_actions
return [] if valid_security_orchestration_policy_configurations.blank?
valid_security_orchestration_policy_configurations
.flat_map { |security_orchestration_policy_configuration| security_orchestration_policy_configuration.on_demand_scan_actions(@ref) }
.compact
.uniq
end
def pipeline_scan_actions
return [] if valid_security_orchestration_policy_configurations.blank?
valid_security_orchestration_policy_configurations
.flat_map { |security_orchestration_policy_configuration| security_orchestration_policy_configuration.pipeline_scan_actions(@ref) }
.compact
.uniq
end
def observe_processing_duration(duration)
......
......@@ -12,23 +12,38 @@ RSpec.describe Gitlab::Ci::Config::SecurityOrchestrationPolicies::Processor do
let(:ref) { 'refs/heads/master' }
let(:source) { 'pipeline' }
let_it_be_with_refind(:project) { create(:project, :repository) }
let_it_be(:namespace) { create(:group) }
let_it_be(:namespace_policies_repository) { create(:project, :repository) }
let_it_be(:namespace_security_orchestration_policy_configuration) { create(:security_orchestration_policy_configuration, :namespace, namespace: namespace, security_policy_management_project: namespace_policies_repository) }
let_it_be(:namespace_policy) do
build(:scan_execution_policy, actions: [
{ scan: 'sast' },
{ scan: 'secret_detection' }
])
end
let_it_be(:policies_repository) { create(:project, :repository) }
let_it_be_with_refind(:project) { create(:project, :repository, group: namespace) }
let_it_be(:policies_repository) { create(:project, :repository, group: namespace) }
let_it_be(:security_orchestration_policy_configuration) { create(:security_orchestration_policy_configuration, project: project, security_policy_management_project: policies_repository) }
let_it_be(:policy) do
build(:scan_execution_policy, actions: [
{ scan: 'dast', site_profile: 'Site Profile', scanner_profile: 'Scanner Profile' },
{ scan: 'secret_detection' }
])
{ scan: 'dast', site_profile: 'Site Profile', scanner_profile: 'Scanner Profile' },
{ scan: 'secret_detection' }
])
end
let_it_be(:policy_yaml) { build(:orchestration_policy_yaml, scan_execution_policy: [policy]) }
let_it_be(:namespace_policy_yaml) { build(:orchestration_policy_yaml, scan_execution_policy: [namespace_policy]) }
before do
allow_next_instance_of(Repository) do |repository|
allow_next_instance_of(Repository, anything, anything, anything) do |repository|
allow(repository).to receive(:blob_data_at).and_return(policy_yaml)
end
allow_next_instance_of(Repository, anything, namespace_policies_repository, anything) do |repository|
allow(repository).to receive(:blob_data_at).and_return(namespace_policy_yaml)
end
end
shared_examples 'with pipeline source applicable for CI' do
......@@ -51,6 +66,11 @@ RSpec.describe Gitlab::Ci::Config::SecurityOrchestrationPolicies::Processor do
[build(:scan_execution_policy, rules: [{ type: 'pipeline', branches: 'production' }])])
end
let_it_be(:namespace_policy_yaml) do
build(:orchestration_policy_yaml, scan_execution_policy:
[build(:scan_execution_policy, rules: [{ type: 'pipeline', branches: 'production' }])])
end
it 'does not modify the config', :aggregate_failures do
expect(config).not_to receive(:deep_merge)
expect(subject).to eq(config)
......@@ -160,6 +180,19 @@ RSpec.describe Gitlab::Ci::Config::SecurityOrchestrationPolicies::Processor do
end
end
end
context 'when scan type is sast is configured for namespace policy project' do
it_behaves_like 'with different scan type' do
let(:expected_configuration) do
{
'sast-1': hash_including(
inherit: { variables: false },
trigger: { include: [{ template: "Security/SAST.gitlab-ci.yml" }] }
)
}
end
end
end
end
end
end
......@@ -3320,4 +3320,42 @@ RSpec.describe Project do
it { is_expected.not_to include(scan_finding_rule) }
end
describe '#all_security_orchestration_policy_configurations' do
subject { project.all_security_orchestration_policy_configurations }
context 'when security orchestration policy is configured for project only' do
let!(:project_security_orchestration_policy_configuration) do
create(:security_orchestration_policy_configuration, project: project)
end
it { is_expected.to match_array([project_security_orchestration_policy_configuration]) }
end
context 'when security orchestration policy is configured for namespaces and project' do
let!(:parent_group) { create(:group) }
let!(:child_group) { create(:group, parent: parent_group) }
let!(:child_group_2) { create(:group, parent: child_group) }
let!(:project) { create(:project, group: child_group_2) }
let!(:parent_security_orchestration_policy_configuration) { create(:security_orchestration_policy_configuration, :namespace, namespace: parent_group) }
let!(:child_security_orchestration_policy_configuration) { create(:security_orchestration_policy_configuration, :namespace, namespace: child_group) }
let!(:child_security_orchestration_policy_configuration_2) { create(:security_orchestration_policy_configuration, :namespace, namespace: child_group_2) }
let!(:project_security_orchestration_policy_configuration) do
create(:security_orchestration_policy_configuration, project: project)
end
it 'returns security policy configurations for all parent groups and project' do
expect(subject).to match_array(
[
parent_security_orchestration_policy_configuration,
child_security_orchestration_policy_configuration,
child_security_orchestration_policy_configuration_2,
project_security_orchestration_policy_configuration
]
)
end
end
end
end
......@@ -461,6 +461,16 @@ RSpec.describe Security::OrchestrationPolicyConfiguration do
expect(active_scan_result_policies.count).to be(5)
end
context 'when policy configuration is configured for namespace' do
let(:security_orchestration_policy_configuration) do
create(:security_orchestration_policy_configuration, :namespace, security_policy_management_project: security_policy_management_project)
end
it 'returns empty array' do
expect(active_scan_result_policies).to match_array([])
end
end
context 'when scan_result_policy feature flag is disabled' do
before do
stub_feature_flags(scan_result_policy: false)
......@@ -488,16 +498,54 @@ RSpec.describe Security::OrchestrationPolicyConfiguration do
subject { security_orchestration_policy_configuration.uniq_scanners }
context 'with approval rules' do
before do
create(:approval_project_rule, :scan_finding, scanners: %w(dast sast), project: project)
create(:approval_project_rule, :scan_finding, scanners: %w(dast container_scanning), project: project)
context 'when policy configuration is configured for project' do
before do
create(:approval_project_rule, :scan_finding, scanners: %w(dast sast), project: project)
create(:approval_project_rule, :scan_finding, scanners: %w(dast container_scanning), project: project)
end
it { is_expected.to contain_exactly('dast', 'sast', 'container_scanning') }
end
it { is_expected.to contain_exactly('dast', 'sast', 'container_scanning') }
context 'when policy configuration is configured for namespace' do
let(:security_orchestration_policy_configuration) do
create(:security_orchestration_policy_configuration, :namespace, security_policy_management_project: security_policy_management_project)
end
it { is_expected.to be_empty }
end
end
context 'without approval rules' do
it { is_expected.to be_empty }
end
end
describe '#project?' do
subject { security_orchestration_policy_configuration.project? }
context 'when project is assigned to policy configuration' do
it { is_expected.to eq true }
end
context 'when namespace is assigned to policy configuration' do
let(:security_orchestration_policy_configuration) { create(:security_orchestration_policy_configuration, :namespace) }
it { is_expected.to eq false }
end
end
describe '#namespace?' do
subject { security_orchestration_policy_configuration.namespace? }
context 'when project is assigned to policy configuration' do
it { is_expected.to eq false }
end
context 'when namespace is assigned to policy configuration' do
let(:security_orchestration_policy_configuration) { create(:security_orchestration_policy_configuration, :namespace) }
it { is_expected.to eq true }
end
end
end
......@@ -126,7 +126,9 @@ RSpec.describe Security::OrchestrationPolicyRuleSchedule do
}
end
subject { rule_schedule.applicable_branches }
let(:requested_project) { rule_schedule.security_orchestration_policy_configuration.project }
subject { rule_schedule.applicable_branches(requested_project) }
before do
allow(rule_schedule).to receive(:policy).and_return(policy)
......@@ -144,6 +146,13 @@ RSpec.describe Security::OrchestrationPolicyRuleSchedule do
it { is_expected.to be_empty }
end
context 'when provided project is not provided' do
let(:branches) { ['master'] }
let(:requested_project) { nil }
it { is_expected.to be_empty }
end
context 'when some of the branches exists' do
let(:branches) { %w[feature-a feature-b] }
......
......@@ -16,10 +16,8 @@ RSpec.describe Security::SecurityOrchestrationPolicies::RuleScheduleService do
subject(:service) { described_class.new(container: project, current_user: current_user) }
shared_examples 'does not execute scan' do
it 'does not create scan pipeline but updates next_run_at' do
it 'does not create scan pipeline' do
expect { service.execute(schedule) }.to change(Ci::Pipeline, :count).by(0)
expect(schedule.next_run_at).to be > Time.zone.now
end
end
......@@ -151,10 +149,8 @@ RSpec.describe Security::SecurityOrchestrationPolicies::RuleScheduleService do
end
context 'when policy actions exists and there are multiple matching branches' do
it 'creates multiple scan pipelines and updates next_run_at' do
it 'creates multiple scan pipelines' do
expect { service.execute(schedule) }.to change(Ci::Pipeline, :count).by(2)
expect(schedule.next_run_at).to be > Time.zone.now
end
end
......
......@@ -4,6 +4,7 @@ require 'spec_helper'
RSpec.describe Security::CreateOrchestrationPolicyWorker do
describe '#perform' do
let_it_be(:namespace) { create(:namespace) }
let_it_be(:configuration) { create(:security_orchestration_policy_configuration, configured_at: nil) }
let_it_be(:schedule) { create(:security_orchestration_policy_rule_schedule, security_orchestration_policy_configuration: configuration) }
......@@ -91,6 +92,25 @@ RSpec.describe Security::CreateOrchestrationPolicyWorker do
expect { worker.perform }.to change(configuration.approval_rules, :count).by(-1)
end
end
context 'with namespace associated with configuration' do
before do
configuration.update!(project: nil, namespace: namespace)
end
it 'executes process services for scan execution policies only' do
active_policies[:scan_execution_policy].each_with_index do |policy, policy_index|
expect_next_instance_of(Security::SecurityOrchestrationPolicies::ProcessRuleService,
policy_configuration: configuration, policy_index: policy_index, policy: policy) do |service|
expect(service).to receive(:execute)
end
end
expect(Security::SecurityOrchestrationPolicies::ProcessScanResultPolicyService).not_to receive(:new)
worker.perform
end
end
end
context 'when policy is invalid' do
......
# frozen_string_literal: true
require 'spec_helper'
RSpec.describe Security::OrchestrationPolicyRuleScheduleNamespaceWorker do
describe '#perform' do
let_it_be(:namespace) { create(:group) }
let_it_be(:project_1) { create(:project, namespace: namespace) }
let_it_be(:project_2) { create(:project, namespace: namespace) }
let_it_be(:security_orchestration_policy_configuration) { create(:security_orchestration_policy_configuration, :namespace, namespace: namespace) }
let_it_be(:schedule) { create(:security_orchestration_policy_rule_schedule, security_orchestration_policy_configuration: security_orchestration_policy_configuration) }
let(:schedule_id) { schedule.id }
let(:worker) { described_class.new }
context 'when schedule exists' do
context 'when schedule is created for security orchestration policy configuration in namespace' do
context 'when next_run_at is in future' do
before do
schedule.update_column(:next_run_at, 1.minute.from_now)
end
it 'does not execute the rule schedule service' do
expect(Security::SecurityOrchestrationPolicies::RuleScheduleService).not_to receive(:new)
worker.perform(schedule_id)
end
end
context 'when next_run_at is in the past' do
before do
schedule.update_column(:next_run_at, 1.minute.ago)
end
it 'executes the rule schedule service for all projects in the group' do
expect_next_instance_of(Security::SecurityOrchestrationPolicies::RuleScheduleService,
container: project_1, current_user: schedule.owner) do |service|
expect(service).to receive(:execute)
end
expect_next_instance_of(Security::SecurityOrchestrationPolicies::RuleScheduleService,
container: project_2, current_user: schedule.owner) do |service|
expect(service).to receive(:execute)
end
worker.perform(schedule_id)
end
it 'updates next run at value' do
worker.perform(schedule_id)
expect(schedule.reload.next_run_at).to be > Time.zone.now
end
end
end
context 'when schedule is created for security orchestration policy configuration in project' do
before do
security_orchestration_policy_configuration.update!(project: project_1, namespace: nil)
end
it 'does not execute the rule schedule service' do
expect(Security::SecurityOrchestrationPolicies::RuleScheduleService).not_to receive(:new)
worker.perform(schedule_id)
end
end
end
context 'when schedule does not exist' do
let(:schedule_id) { non_existing_record_id }
it 'does not execute the rule schedule service' do
expect(Security::SecurityOrchestrationPolicies::RuleScheduleService).not_to receive(:new)
worker.perform(schedule_id)
end
end
end
end
......@@ -4,7 +4,8 @@ require 'spec_helper'
RSpec.describe Security::OrchestrationPolicyRuleScheduleWorker do
describe '#perform' do
let_it_be(:schedule) { create(:security_orchestration_policy_rule_schedule) }
let_it_be(:security_orchestration_policy_configuration) { create(:security_orchestration_policy_configuration) }
let_it_be(:schedule) { create(:security_orchestration_policy_rule_schedule, security_orchestration_policy_configuration: security_orchestration_policy_configuration) }
subject(:worker) { described_class.new }
......@@ -13,13 +14,35 @@ RSpec.describe Security::OrchestrationPolicyRuleScheduleWorker do
schedule.update_column(:next_run_at, 1.minute.ago)
end
it 'executes the rule schedule service' do
expect_next_instance_of(Security::SecurityOrchestrationPolicies::RuleScheduleService,
container: schedule.security_orchestration_policy_configuration.project, current_user: schedule.owner) do |service|
expect(service).to receive(:execute)
context 'when schedule is created for security orchestration policy configuration in project' do
it 'executes the rule schedule service' do
expect_next_instance_of(Security::SecurityOrchestrationPolicies::RuleScheduleService,
container: schedule.security_orchestration_policy_configuration.project, current_user: schedule.owner) do |service|
expect(service).to receive(:execute)
end
worker.perform
end
worker.perform
it 'updates next run at value' do
worker.perform
expect(schedule.reload.next_run_at).to be > Time.zone.now
end
end
context 'when schedule is created for security orchestration policy configuration in namespace' do
let_it_be(:namespace) { create(:group) }
before do
security_orchestration_policy_configuration.update!(namespace: namespace, project: nil)
end
it 'schedules the OrchestrationPolicyRuleScheduleNamespaceWorker for namespace' do
expect(Security::OrchestrationPolicyRuleScheduleNamespaceWorker).to receive(:perform_async).with(schedule.id)
worker.perform
end
end
end
......@@ -28,7 +51,7 @@ RSpec.describe Security::OrchestrationPolicyRuleScheduleWorker do
schedule.update_column(:next_run_at, 1.minute.from_now)
end
it 'executes the rule schedule service' do
it 'does not execute the rule schedule service' do
expect(Security::SecurityOrchestrationPolicies::RuleScheduleService).not_to receive(:new)
worker.perform
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment