Commit 9db81275 authored by Fabio Pitino's avatar Fabio Pitino Committed by Kamil Trzciński

Log when a pipeline limit is hit

Use existing Sentry exception logging for all
existing limits
parent 8bc9939f
...@@ -7,6 +7,8 @@ module EE ...@@ -7,6 +7,8 @@ module EE
# Abstract base class for CI/CD Quotas # Abstract base class for CI/CD Quotas
# #
class Limit class Limit
LimitExceededError = Class.new(StandardError)
def initialize(_context, _resource) def initialize(_context, _resource)
end end
...@@ -21,6 +23,13 @@ module EE ...@@ -21,6 +23,13 @@ module EE
def message def message
raise NotImplementedError raise NotImplementedError
end end
def log_error!(extra_context = {})
error = LimitExceededError.new(message)
# TODO: change this to Gitlab::Sentry.log_exception(error, extra_context)
# https://gitlab.com/gitlab-org/gitlab/issues/32906
::Gitlab::Sentry.track_acceptable_exception(error, extra: extra_context)
end
end end
end end
end end
......
...@@ -27,6 +27,7 @@ module EE ...@@ -27,6 +27,7 @@ module EE
retry_optimistic_lock(pipeline) do retry_optimistic_lock(pipeline) do
pipeline.drop!(:activity_limit_exceeded) pipeline.drop!(:activity_limit_exceeded)
limit.log_error!(project_id: project.id, plan: project.namespace.actual_plan_name)
end end
end end
......
...@@ -27,6 +27,7 @@ module EE ...@@ -27,6 +27,7 @@ module EE
retry_optimistic_lock(pipeline) do retry_optimistic_lock(pipeline) do
pipeline.drop!(:job_activity_limit_exceeded) pipeline.drop!(:job_activity_limit_exceeded)
limit.log_error!(project_id: project.id, plan: project.namespace.actual_plan_name)
end end
end end
......
...@@ -28,6 +28,7 @@ module EE ...@@ -28,6 +28,7 @@ module EE
pipeline.drop!(:size_limit_exceeded) pipeline.drop!(:size_limit_exceeded)
end end
limit.log_error!(project_id: project.id, plan: project.namespace.actual_plan_name)
error(limit.message) error(limit.message)
end end
......
...@@ -17,6 +17,8 @@ describe ::Gitlab::Ci::Pipeline::Chain::Limit::Activity do ...@@ -17,6 +17,8 @@ describe ::Gitlab::Ci::Pipeline::Chain::Limit::Activity do
let(:step) { described_class.new(pipeline, command) } let(:step) { described_class.new(pipeline, command) }
subject { step.perform! }
context 'when active pipelines limit is exceeded' do context 'when active pipelines limit is exceeded' do
before do before do
gold_plan = create(:gold_plan, active_pipelines_limit: 1) gold_plan = create(:gold_plan, active_pipelines_limit: 1)
...@@ -24,38 +26,59 @@ describe ::Gitlab::Ci::Pipeline::Chain::Limit::Activity do ...@@ -24,38 +26,59 @@ describe ::Gitlab::Ci::Pipeline::Chain::Limit::Activity do
create(:ci_pipeline, project: project, status: 'pending') create(:ci_pipeline, project: project, status: 'pending')
create(:ci_pipeline, project: project, status: 'running') create(:ci_pipeline, project: project, status: 'running')
step.perform!
end end
it 'drops the pipeline' do it 'drops the pipeline' do
subject
expect(pipeline.reload).to be_failed expect(pipeline.reload).to be_failed
end end
it 'persists the pipeline' do it 'persists the pipeline' do
subject
expect(pipeline).to be_persisted expect(pipeline).to be_persisted
end end
it 'breaks the chain' do it 'breaks the chain' do
subject
expect(step.break?).to be true expect(step.break?).to be true
end end
it 'sets a valid failure reason' do it 'sets a valid failure reason' do
subject
expect(pipeline.activity_limit_exceeded?).to be true expect(pipeline.activity_limit_exceeded?).to be true
end end
end
context 'when pipeline activity limit is not exceeded' do it 'logs the error' do
before do expect(Gitlab::Sentry).to receive(:track_acceptable_exception).with(
step.perform! instance_of(EE::Gitlab::Ci::Limit::LimitExceededError),
extra: { project_id: project.id, plan: namespace.actual_plan_name }
)
subject
end end
end
context 'when pipeline activity limit is not exceeded' do
it 'does not break the chain' do it 'does not break the chain' do
subject
expect(step.break?).to be false expect(step.break?).to be false
end end
it 'does not invalidate the pipeline' do it 'does not invalidate the pipeline' do
subject
expect(pipeline.errors).to be_empty expect(pipeline.errors).to be_empty
end end
it 'does not log any error' do
expect(Gitlab::Sentry).not_to receive(:track_acceptable_exception)
subject
end
end end
end end
...@@ -17,6 +17,8 @@ describe ::Gitlab::Ci::Pipeline::Chain::Limit::JobActivity do ...@@ -17,6 +17,8 @@ describe ::Gitlab::Ci::Pipeline::Chain::Limit::JobActivity do
let(:step) { described_class.new(pipeline, command) } let(:step) { described_class.new(pipeline, command) }
subject { step.perform! }
context 'when active jobs limit is exceeded' do context 'when active jobs limit is exceeded' do
before do before do
gold_plan = create(:gold_plan, active_jobs_limit: 2) gold_plan = create(:gold_plan, active_jobs_limit: 2)
...@@ -26,38 +28,59 @@ describe ::Gitlab::Ci::Pipeline::Chain::Limit::JobActivity do ...@@ -26,38 +28,59 @@ describe ::Gitlab::Ci::Pipeline::Chain::Limit::JobActivity do
create(:ci_build, pipeline: pipeline) create(:ci_build, pipeline: pipeline)
create(:ci_build, pipeline: pipeline) create(:ci_build, pipeline: pipeline)
create(:ci_build, pipeline: pipeline) create(:ci_build, pipeline: pipeline)
step.perform!
end end
it 'drops the pipeline' do it 'drops the pipeline' do
subject
expect(pipeline.reload).to be_failed expect(pipeline.reload).to be_failed
end end
it 'persists the pipeline' do it 'persists the pipeline' do
subject
expect(pipeline).to be_persisted expect(pipeline).to be_persisted
end end
it 'breaks the chain' do it 'breaks the chain' do
subject
expect(step.break?).to be true expect(step.break?).to be true
end end
it 'sets a valid failure reason' do it 'sets a valid failure reason' do
subject
expect(pipeline.job_activity_limit_exceeded?).to be true expect(pipeline.job_activity_limit_exceeded?).to be true
end end
end
context 'when job activity limit is not exceeded' do it 'logs the error' do
before do expect(Gitlab::Sentry).to receive(:track_acceptable_exception).with(
step.perform! instance_of(EE::Gitlab::Ci::Limit::LimitExceededError),
extra: { project_id: project.id, plan: namespace.actual_plan_name }
)
subject
end end
end
context 'when job activity limit is not exceeded' do
it 'does not break the chain' do it 'does not break the chain' do
subject
expect(step.break?).to be false expect(step.break?).to be false
end end
it 'does not invalidate the pipeline' do it 'does not invalidate the pipeline' do
subject
expect(pipeline.errors).to be_empty expect(pipeline.errors).to be_empty
end end
it 'does not log any error' do
expect(Gitlab::Sentry).not_to receive(:track_acceptable_exception)
subject
end
end end
end end
...@@ -19,12 +19,12 @@ describe ::Gitlab::Ci::Pipeline::Chain::Limit::Size do ...@@ -19,12 +19,12 @@ describe ::Gitlab::Ci::Pipeline::Chain::Limit::Size do
let(:step) { described_class.new(pipeline, command) } let(:step) { described_class.new(pipeline, command) }
subject { step.perform! }
context 'when pipeline size limit is exceeded' do context 'when pipeline size limit is exceeded' do
before do before do
gold_plan = create(:gold_plan, pipeline_size_limit: 1) gold_plan = create(:gold_plan, pipeline_size_limit: 1)
create(:gitlab_subscription, namespace: namespace, hosted_plan: gold_plan) create(:gitlab_subscription, namespace: namespace, hosted_plan: gold_plan)
step.perform!
end end
let(:pipeline) do let(:pipeline) do
...@@ -42,25 +42,44 @@ describe ::Gitlab::Ci::Pipeline::Chain::Limit::Size do ...@@ -42,25 +42,44 @@ describe ::Gitlab::Ci::Pipeline::Chain::Limit::Size do
end end
it 'drops the pipeline' do it 'drops the pipeline' do
subject
expect(pipeline.reload).to be_failed expect(pipeline.reload).to be_failed
end end
it 'persists the pipeline' do it 'persists the pipeline' do
subject
expect(pipeline).to be_persisted expect(pipeline).to be_persisted
end end
it 'breaks the chain' do it 'breaks the chain' do
subject
expect(step.break?).to be true expect(step.break?).to be true
end end
it 'sets a valid failure reason' do it 'sets a valid failure reason' do
subject
expect(pipeline.size_limit_exceeded?).to be true expect(pipeline.size_limit_exceeded?).to be true
end end
it 'appends validation error' do it 'appends validation error' do
subject
expect(pipeline.errors.to_a) expect(pipeline.errors.to_a)
.to include 'Pipeline size limit exceeded by 1 job!' .to include 'Pipeline size limit exceeded by 1 job!'
end end
it 'logs the error' do
expect(Gitlab::Sentry).to receive(:track_acceptable_exception).with(
instance_of(EE::Gitlab::Ci::Limit::LimitExceededError),
extra: { project_id: project.id, plan: namespace.actual_plan_name }
)
subject
end
end end
context 'when not saving incomplete pipelines' do context 'when not saving incomplete pipelines' do
...@@ -71,26 +90,36 @@ describe ::Gitlab::Ci::Pipeline::Chain::Limit::Size do ...@@ -71,26 +90,36 @@ describe ::Gitlab::Ci::Pipeline::Chain::Limit::Size do
end end
it 'does not drop the pipeline' do it 'does not drop the pipeline' do
subject
expect(pipeline).not_to be_failed expect(pipeline).not_to be_failed
end end
it 'breaks the chain' do it 'breaks the chain' do
subject
expect(step.break?).to be true expect(step.break?).to be true
end end
end end
end end
context 'when pipeline size limit is not exceeded' do context 'when pipeline size limit is not exceeded' do
before do
step.perform!
end
it 'does not break the chain' do it 'does not break the chain' do
subject
expect(step.break?).to be false expect(step.break?).to be false
end end
it 'does not persist the pipeline' do it 'does not persist the pipeline' do
subject
expect(pipeline).not_to be_persisted expect(pipeline).not_to be_persisted
end end
it 'does not log any error' do
expect(Gitlab::Sentry).not_to receive(:track_acceptable_exception)
subject
end
end end
end end
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment