Commit 53771190 authored by Marius Bobin's avatar Marius Bobin Committed by Fabio Pitino

Throttle pipelines creation rate

parent 7121c173
...@@ -14,6 +14,7 @@ module Ci ...@@ -14,6 +14,7 @@ module Ci
Gitlab::Ci::Pipeline::Chain::Build::Associations, Gitlab::Ci::Pipeline::Chain::Build::Associations,
Gitlab::Ci::Pipeline::Chain::Validate::Abilities, Gitlab::Ci::Pipeline::Chain::Validate::Abilities,
Gitlab::Ci::Pipeline::Chain::Validate::Repository, Gitlab::Ci::Pipeline::Chain::Validate::Repository,
Gitlab::Ci::Pipeline::Chain::Limit::RateLimit,
Gitlab::Ci::Pipeline::Chain::Validate::SecurityOrchestrationPolicy, Gitlab::Ci::Pipeline::Chain::Validate::SecurityOrchestrationPolicy,
Gitlab::Ci::Pipeline::Chain::Skip, Gitlab::Ci::Pipeline::Chain::Skip,
Gitlab::Ci::Pipeline::Chain::Config::Content, Gitlab::Ci::Pipeline::Chain::Config::Content,
......
---
name: ci_throttle_pipelines_creation
introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/83969
rollout_issue_url: https://gitlab.com/gitlab-org/gitlab/-/issues/357520
milestone: '14.10'
type: development
group: group::pipeline execution
default_enabled: false
---
name: ci_throttle_pipelines_creation_dry_run
introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/83969
rollout_issue_url: https://gitlab.com/gitlab-org/gitlab/-/issues/357524
milestone: '14.10'
type: development
group: group::pipeline execution
default_enabled: true
...@@ -41,7 +41,8 @@ module Gitlab ...@@ -41,7 +41,8 @@ module Gitlab
auto_rollback_deployment: { threshold: 1, interval: 3.minutes }, auto_rollback_deployment: { threshold: 1, interval: 3.minutes },
search_rate_limit: { threshold: -> { application_settings.search_rate_limit }, interval: 1.minute }, search_rate_limit: { threshold: -> { application_settings.search_rate_limit }, interval: 1.minute },
search_rate_limit_unauthenticated: { threshold: -> { application_settings.search_rate_limit_unauthenticated }, interval: 1.minute }, search_rate_limit_unauthenticated: { threshold: -> { application_settings.search_rate_limit_unauthenticated }, interval: 1.minute },
gitlab_shell_operation: { threshold: 600, interval: 1.minute } gitlab_shell_operation: { threshold: 600, interval: 1.minute },
pipelines_create: { threshold: 4, interval: 1.minute }
}.freeze }.freeze
end end
......
# frozen_string_literal: true
module Gitlab
module Ci
module Pipeline
module Chain
module Limit
class RateLimit < Chain::Base
include Chain::Helpers
def perform!
return unless throttle_enabled?
# We exclude child-pipelines from the rate limit because they represent
# sub-pipelines that would otherwise hit the rate limit due to having the
# same scope (project, user, sha).
#
return if pipeline.parent_pipeline?
if rate_limit_throttled?
create_log_entry
error(throttle_message) unless dry_run?
end
end
def break?
@pipeline.errors.any?
end
private
def rate_limit_throttled?
::Gitlab::ApplicationRateLimiter.throttled?(
:pipelines_create, scope: [project, current_user, command.sha]
)
end
def create_log_entry
Gitlab::AppJsonLogger.info(
class: self.class.name,
namespace_id: project.namespace_id,
project_id: project.id,
commit_sha: command.sha,
current_user_id: current_user.id,
subscription_plan: project.actual_plan_name,
message: 'Activated pipeline creation rate limit'
)
end
def throttle_message
'Too many pipelines created in the last minute. Try again later.'
end
def throttle_enabled?
::Feature.enabled?(
:ci_throttle_pipelines_creation,
project,
default_enabled: :yaml)
end
def dry_run?
::Feature.enabled?(
:ci_throttle_pipelines_creation_dry_run,
project,
default_enabled: :yaml)
end
end
end
end
end
end
end
# frozen_string_literal: true
require 'spec_helper'
RSpec.describe ::Gitlab::Ci::Pipeline::Chain::Limit::RateLimit, :freeze_time, :clean_gitlab_redis_rate_limiting do
let_it_be(:user) { create(:user) }
let_it_be(:namespace) { create(:namespace) }
let_it_be(:project, reload: true) { create(:project, namespace: namespace) }
let(:save_incompleted) { false }
let(:throttle_message) do
'Too many pipelines created in the last minute. Try again later.'
end
let(:command) do
Gitlab::Ci::Pipeline::Chain::Command.new(
project: project,
current_user: user,
save_incompleted: save_incompleted
)
end
let(:pipeline) { build(:ci_pipeline, project: project, source: source) }
let(:source) { 'push' }
let(:step) { described_class.new(pipeline, command) }
def perform(count: 2)
count.times { step.perform! }
end
context 'when the limit is exceeded' do
before do
allow(Gitlab::ApplicationRateLimiter).to receive(:rate_limits)
.and_return(pipelines_create: { threshold: 1, interval: 1.minute })
stub_feature_flags(ci_throttle_pipelines_creation_dry_run: false)
end
it 'does not persist the pipeline' do
perform
expect(pipeline).not_to be_persisted
expect(pipeline.errors.added?(:base, throttle_message)).to be_truthy
end
it 'breaks the chain' do
perform
expect(step.break?).to be_truthy
end
it 'creates a log entry' do
expect(Gitlab::AppJsonLogger).to receive(:info).with(
a_hash_including(
class: described_class.name,
project_id: project.id,
subscription_plan: project.actual_plan_name,
commit_sha: command.sha
)
)
perform
end
context 'with child pipelines' do
let(:source) { 'parent_pipeline' }
it 'does not break the chain' do
perform
expect(step.break?).to be_falsey
end
it 'does not invalidate the pipeline' do
perform
expect(pipeline.errors).to be_empty
end
it 'does not log anything' do
expect(Gitlab::AppJsonLogger).not_to receive(:info)
perform
end
end
context 'when saving incompleted pipelines' do
let(:save_incompleted) { true }
it 'does not persist the pipeline' do
perform
expect(pipeline).not_to be_persisted
expect(pipeline.errors.added?(:base, throttle_message)).to be_truthy
end
it 'breaks the chain' do
perform
expect(step.break?).to be_truthy
end
end
context 'when ci_throttle_pipelines_creation is disabled' do
before do
stub_feature_flags(ci_throttle_pipelines_creation: false)
end
it 'does not break the chain' do
perform
expect(step.break?).to be_falsey
end
it 'does not invalidate the pipeline' do
perform
expect(pipeline.errors).to be_empty
end
it 'does not log anything' do
expect(Gitlab::AppJsonLogger).not_to receive(:info)
perform
end
end
context 'when ci_throttle_pipelines_creation_dry_run is enabled' do
before do
stub_feature_flags(ci_throttle_pipelines_creation_dry_run: true)
end
it 'does not break the chain' do
perform
expect(step.break?).to be_falsey
end
it 'does not invalidate the pipeline' do
perform
expect(pipeline.errors).to be_empty
end
it 'creates a log entry' do
expect(Gitlab::AppJsonLogger).to receive(:info).with(
a_hash_including(
class: described_class.name,
project_id: project.id,
subscription_plan: project.actual_plan_name,
commit_sha: command.sha
)
)
perform
end
end
end
context 'when the limit is not exceeded' do
it 'does not break the chain' do
perform
expect(step.break?).to be_falsey
end
it 'does not invalidate the pipeline' do
perform
expect(pipeline.errors).to be_empty
end
it 'does not log anything' do
expect(Gitlab::AppJsonLogger).not_to receive(:info)
perform
end
end
end
# frozen_string_literal: true
require 'spec_helper'
RSpec.describe Ci::CreatePipelineService, :freeze_time, :clean_gitlab_redis_rate_limiting do
describe 'rate limiting' do
let_it_be(:project) { create(:project, :repository) }
let_it_be(:user) { project.first_owner }
let(:ref) { 'refs/heads/master' }
before do
stub_ci_pipeline_yaml_file(gitlab_ci_yaml)
stub_feature_flags(ci_throttle_pipelines_creation_dry_run: false)
allow(Gitlab::ApplicationRateLimiter).to receive(:rate_limits)
.and_return(pipelines_create: { threshold: 1, interval: 1.minute })
end
context 'when user is under the limit' do
let(:pipeline) { create_pipelines(count: 1) }
it 'allows pipeline creation' do
expect(pipeline).to be_created_successfully
expect(pipeline.statuses).not_to be_empty
end
end
context 'when user is over the limit' do
let(:pipeline) { create_pipelines }
it 'blocks pipeline creation' do
throttle_message = 'Too many pipelines created in the last minute. Try again later.'
expect(pipeline).not_to be_persisted
expect(pipeline.statuses).to be_empty
expect(pipeline.errors.added?(:base, throttle_message)).to be_truthy
end
end
context 'with different users' do
let(:other_user) { create(:user) }
before do
project.add_maintainer(other_user)
end
it 'allows other members to create pipelines' do
blocked_pipeline = create_pipelines(user: user)
allowed_pipeline = create_pipelines(count: 1, user: other_user)
expect(blocked_pipeline).not_to be_persisted
expect(allowed_pipeline).to be_created_successfully
end
end
context 'with different commits' do
it 'allows user to create pipeline' do
blocked_pipeline = create_pipelines(ref: ref)
allowed_pipeline = create_pipelines(count: 1, ref: 'refs/heads/feature')
expect(blocked_pipeline).not_to be_persisted
expect(allowed_pipeline).to be_created_successfully
end
end
context 'with different projects' do
let_it_be(:other_project) { create(:project, :repository) }
before do
other_project.add_maintainer(user)
end
it 'allows user to create pipeline' do
blocked_pipeline = create_pipelines(project: project)
allowed_pipeline = create_pipelines(count: 1, project: other_project)
expect(blocked_pipeline).not_to be_persisted
expect(allowed_pipeline).to be_created_successfully
end
end
end
def create_pipelines(attrs = {})
attrs.reverse_merge!(user: user, ref: ref, project: project, count: 2)
service = described_class.new(attrs[:project], attrs[:user], { ref: attrs[:ref] })
attrs[:count].pred.times { service.execute(:push) }
service.execute(:push).payload
end
end
...@@ -12,6 +12,10 @@ RSpec.describe Ci::CreatePipelineService do ...@@ -12,6 +12,10 @@ RSpec.describe Ci::CreatePipelineService do
before do before do
stub_ci_pipeline_to_return_yaml_file stub_ci_pipeline_to_return_yaml_file
# Disable rate limiting for pipeline creation
allow(Gitlab::ApplicationRateLimiter).to receive(:rate_limits)
.and_return(pipelines_create: { threshold: 0, interval: 1.minute })
end end
describe '#execute' do describe '#execute' do
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment