Skip to content
Projects
Groups
Snippets
Help
Loading...
Help
Support
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in / Register
Toggle navigation
G
gitlab-ce
Project overview
Project overview
Details
Activity
Releases
Repository
Repository
Files
Commits
Branches
Tags
Contributors
Graph
Compare
Issues
0
Issues
0
List
Boards
Labels
Milestones
Merge Requests
1
Merge Requests
1
Analytics
Analytics
Repository
Value Stream
Wiki
Wiki
Snippets
Snippets
Members
Members
Collapse sidebar
Close sidebar
Activity
Graph
Create a new issue
Commits
Issue Boards
Open sidebar
nexedi
gitlab-ce
Commits
80b00a23
Commit
80b00a23
authored
3 years ago
by
Marius Bobin
Committed by
Fabio Pitino
3 years ago
Browse files
Options
Browse Files
Download
Email Patches
Plain Diff
Check CI quota when builds are retried [RUN ALL RSPEC] [RUN AS-IF-FOSS]
parent
2db78598
Changes
12
Hide whitespace changes
Inline
Side-by-side
Showing
12 changed files
with
324 additions
and
51 deletions
+324
-51
app/models/ci/build.rb
app/models/ci/build.rb
+9
-0
app/services/ci/retry_build_service.rb
app/services/ci/retry_build_service.rb
+5
-0
app/services/ci/retry_pipeline_service.rb
app/services/ci/retry_pipeline_service.rb
+14
-2
config/feature_flags/development/ci_quota_check_on_retries.yml
...g/feature_flags/development/ci_quota_check_on_retries.yml
+8
-0
ee/app/services/ci/pipeline_creation/drop_not_runnable_builds_service.rb
.../ci/pipeline_creation/drop_not_runnable_builds_service.rb
+3
-39
ee/app/services/ee/ci/retry_build_service.rb
ee/app/services/ee/ci/retry_build_service.rb
+10
-0
ee/app/services/ee/ci/retry_pipeline_service.rb
ee/app/services/ee/ci/retry_pipeline_service.rb
+33
-0
ee/lib/gitlab/ci/minutes/runners_availability.rb
ee/lib/gitlab/ci/minutes/runners_availability.rb
+59
-0
ee/spec/lib/gitlab/ci/minutes/runners_availability_spec.rb
ee/spec/lib/gitlab/ci/minutes/runners_availability_spec.rb
+48
-0
ee/spec/services/ci/retry_build_service_spec.rb
ee/spec/services/ci/retry_build_service_spec.rb
+54
-10
ee/spec/services/ci/retry_pipeline_service_spec.rb
ee/spec/services/ci/retry_pipeline_service_spec.rb
+64
-0
spec/models/ci/build_spec.rb
spec/models/ci/build_spec.rb
+17
-0
No files found.
app/models/ci/build.rb
View file @
80b00a23
...
...
@@ -136,6 +136,7 @@ module Ci
scope
:eager_load_job_artifacts
,
->
{
includes
(
:job_artifacts
)
}
scope
:eager_load_job_artifacts_archive
,
->
{
includes
(
:job_artifacts_archive
)
}
scope
:eager_load_tags
,
->
{
includes
(
:tags
)
}
scope
:eager_load_everything
,
->
do
includes
(
...
...
@@ -759,6 +760,14 @@ module Ci
self
.
token
&&
ActiveSupport
::
SecurityUtils
.
secure_compare
(
token
,
self
.
token
)
end
def
tag_list
if
tags
.
loaded?
tags
.
map
(
&
:name
)
else
super
end
end
def
has_tags?
tag_list
.
any?
end
...
...
This diff is collapsed.
Click to expand it.
app/services/ci/retry_build_service.rb
View file @
80b00a23
...
...
@@ -18,6 +18,9 @@ module Ci
build
.
ensure_scheduling_type!
reprocess!
(
build
).
tap
do
|
new_build
|
check_assignable_runners!
(
new_build
)
next
if
new_build
.
failed?
Gitlab
::
OptimisticLocking
.
retry_lock
(
new_build
,
name:
'retry_build'
,
&
:enqueue
)
AfterRequeueJobService
.
new
(
project
,
current_user
).
execute
(
build
)
...
...
@@ -54,6 +57,8 @@ module Ci
end
end
def
check_assignable_runners!
(
build
);
end
def
create_build!
(
attributes
)
build
=
project
.
builds
.
new
(
attributes
)
build
.
assign_attributes
(
::
Gitlab
::
Ci
::
Pipeline
::
Seed
::
Build
.
environment_attributes_for
(
build
))
...
...
This diff is collapsed.
Click to expand it.
app/services/ci/retry_pipeline_service.rb
View file @
80b00a23
...
...
@@ -13,8 +13,8 @@ module Ci
pipeline
.
ensure_scheduling_type!
pipeline
.
retryable_builds
.
preload_needs
.
find_each
do
|
build
|
next
unless
can
?
(
current_user
,
:update_build
,
build
)
builds_relation
(
pipeline
)
.
find_each
do
|
build
|
next
unless
can
_be_retried?
(
build
)
Ci
::
RetryBuildService
.
new
(
project
,
current_user
)
.
reprocess!
(
build
)
...
...
@@ -36,5 +36,17 @@ module Ci
.
new
(
pipeline
)
.
execute
end
private
def
builds_relation
(
pipeline
)
pipeline
.
retryable_builds
.
preload_needs
end
def
can_be_retried?
(
build
)
can?
(
current_user
,
:update_build
,
build
)
end
end
end
Ci
::
RetryPipelineService
.
prepend_mod_with
(
'Ci::RetryPipelineService'
)
This diff is collapsed.
Click to expand it.
config/feature_flags/development/ci_quota_check_on_retries.yml
0 → 100644
View file @
80b00a23
---
name
:
ci_quota_check_on_retries
introduced_by_url
:
https://gitlab.com/gitlab-org/gitlab/-/merge_requests/62702
rollout_issue_url
:
https://gitlab.com/gitlab-org/gitlab/-/issues/333765
milestone
:
'
14.0'
type
:
development
group
:
group::pipeline execution
default_enabled
:
false
This diff is collapsed.
Click to expand it.
ee/app/services/ci/pipeline_creation/drop_not_runnable_builds_service.rb
View file @
80b00a23
...
...
@@ -3,10 +3,9 @@
module
Ci
module
PipelineCreation
class
DropNotRunnableBuildsService
include
::
Gitlab
::
Utils
::
StrongMemoize
def
initialize
(
pipeline
)
@pipeline
=
pipeline
@runner_minutes
=
Gitlab
::
Ci
::
Minutes
::
RunnersAvailability
.
new
(
pipeline
.
project
)
end
##
...
...
@@ -16,8 +15,6 @@ module Ci
def
execute
return
unless
::
Feature
.
enabled?
(
:ci_drop_new_builds_when_ci_quota_exceeded
,
project
,
default_enabled: :yaml
)
return
unless
pipeline
.
created?
return
unless
project
.
shared_runners_enabled?
return
unless
project
.
ci_minutes_quota
.
minutes_used_up?
validate_build_matchers
end
...
...
@@ -25,51 +22,18 @@ module Ci
private
attr_reader
:pipeline
attr_reader
:runner_minutes
delegate
:project
,
to: :pipeline
def
validate_build_matchers
build_ids
=
pipeline
.
build_matchers
.
filter_map
{
|
matcher
|
matcher
.
build_ids
if
should_drop
?
(
matcher
)
}
.
filter_map
{
|
matcher
|
matcher
.
build_ids
unless
runner_minutes
.
available
?
(
matcher
)
}
.
flatten
drop_all_builds
(
build_ids
,
:ci_quota_exceeded
)
end
def
should_drop?
(
build_matcher
)
matches_instance_runners_and_quota_used_up?
(
build_matcher
)
&&
!
matches_private_runners?
(
build_matcher
)
end
def
matches_instance_runners_and_quota_used_up?
(
build_matcher
)
instance_runners
.
any?
do
|
matcher
|
matcher
.
matches?
(
build_matcher
)
&&
!
matcher
.
matches_quota?
(
build_matcher
)
end
end
def
matches_private_runners?
(
build_matcher
)
private_runners
.
any?
{
|
matcher
|
matcher
.
matches?
(
build_matcher
)
}
end
def
instance_runners
strong_memoize
(
:instance_runners
)
do
runner_matchers
.
select
(
&
:instance_type?
)
end
end
def
private_runners
strong_memoize
(
:private_runners
)
do
runner_matchers
.
reject
(
&
:instance_type?
)
end
end
def
runner_matchers
strong_memoize
(
:runner_matchers
)
do
project
.
all_runners
.
active
.
online
.
runner_matchers
end
end
##
# We skip pipeline processing until we drop all required builds. Otherwise
# as we drop the first build, the remaining builds to be dropped could
...
...
This diff is collapsed.
Click to expand it.
ee/app/services/ee/ci/retry_build_service.rb
View file @
80b00a23
...
...
@@ -37,6 +37,16 @@ module EE
raise
::
Gitlab
::
Access
::
AccessDeniedError
,
'Credit card required to be on file in order to retry a build'
end
end
override
:check_assignable_runners!
def
check_assignable_runners!
(
build
)
return
unless
::
Feature
.
enabled?
(
:ci_quota_check_on_retries
,
project
,
default_enabled: :yaml
)
runner_minutes
=
::
Gitlab
::
Ci
::
Minutes
::
RunnersAvailability
.
new
(
project
)
return
if
runner_minutes
.
available?
(
build
.
build_matcher
)
build
.
drop!
(
:ci_quota_exceeded
)
end
end
end
end
This diff is collapsed.
Click to expand it.
ee/app/services/ee/ci/retry_pipeline_service.rb
0 → 100644
View file @
80b00a23
# frozen_string_literal: true
module
EE
module
Ci
module
RetryPipelineService
extend
::
Gitlab
::
Utils
::
Override
include
::
Gitlab
::
Utils
::
StrongMemoize
private
override
:builds_relation
def
builds_relation
(
pipeline
)
super
.
eager_load_tags
end
override
:can_be_retried?
def
can_be_retried?
(
build
)
super
&&
!
ci_minutes_exceeded?
(
build
)
end
def
ci_minutes_exceeded?
(
build
)
::
Feature
.
enabled?
(
:ci_quota_check_on_retries
,
project
,
default_enabled: :yaml
)
&&
!
runner_minutes
.
available?
(
build
.
build_matcher
)
end
def
runner_minutes
strong_memoize
(
:runner_minutes
)
do
::
Gitlab
::
Ci
::
Minutes
::
RunnersAvailability
.
new
(
project
)
end
end
end
end
end
This diff is collapsed.
Click to expand it.
ee/lib/gitlab/ci/minutes/runners_availability.rb
0 → 100644
View file @
80b00a23
# frozen_string_literal: true
module
Gitlab
module
Ci
module
Minutes
class
RunnersAvailability
include
::
Gitlab
::
Utils
::
StrongMemoize
def
initialize
(
project
)
@project
=
project
end
def
available?
(
build_matcher
)
return
true
unless
project
.
shared_runners_enabled?
!
quota_exceeded?
(
build_matcher
)
end
private
attr_reader
:project
def
quota_exceeded?
(
build_matcher
)
matches_instance_runners_and_quota_used_up?
(
build_matcher
)
&&
!
matches_private_runners?
(
build_matcher
)
end
def
matches_instance_runners_and_quota_used_up?
(
build_matcher
)
instance_runners
.
any?
do
|
matcher
|
matcher
.
matches?
(
build_matcher
)
&&
!
matcher
.
matches_quota?
(
build_matcher
)
end
end
def
matches_private_runners?
(
build_matcher
)
private_runners
.
any?
{
|
matcher
|
matcher
.
matches?
(
build_matcher
)
}
end
def
instance_runners
strong_memoize
(
:instance_runners
)
do
runner_matchers
.
select
(
&
:instance_type?
)
end
end
def
private_runners
strong_memoize
(
:private_runners
)
do
runner_matchers
.
reject
(
&
:instance_type?
)
end
end
def
runner_matchers
strong_memoize
(
:runner_matchers
)
do
project
.
all_runners
.
active
.
online
.
runner_matchers
end
end
end
end
end
end
This diff is collapsed.
Click to expand it.
ee/spec/lib/gitlab/ci/minutes/runners_availability_spec.rb
0 → 100644
View file @
80b00a23
# frozen_string_literal: true
require
'spec_helper'
RSpec
.
describe
Gitlab
::
Ci
::
Minutes
::
RunnersAvailability
do
using
RSpec
::
Parameterized
::
TableSyntax
let_it_be
(
:instance_runner
)
{
create
(
:ci_runner
,
:instance
,
:online
)
}
let
(
:build
)
{
build_stubbed
(
:ci_build
,
project:
project
)
}
let
(
:minutes
)
{
described_class
.
new
(
project
)
}
describe
'#available?'
do
where
(
:shared_runners_enabled
,
:minutes_quota
,
:private_runner_available
,
:result
)
do
true
|
:with_not_used_build_minutes_limit
|
false
|
true
true
|
:with_not_used_build_minutes_limit
|
true
|
true
true
|
:with_used_build_minutes_limit
|
false
|
false
true
|
:with_used_build_minutes_limit
|
true
|
true
false
|
:with_used_build_minutes_limit
|
false
|
true
false
|
:with_used_build_minutes_limit
|
true
|
true
false
|
:with_not_used_build_minutes_limit
|
true
|
true
false
|
:with_not_used_build_minutes_limit
|
false
|
true
end
with_them
do
let!
(
:namespace
)
{
create
(
:namespace
,
minutes_quota
)
}
let!
(
:project
)
{
create
(
:project
,
namespace:
namespace
,
shared_runners_enabled:
shared_runners_enabled
)
}
let!
(
:private_runner
)
{
create
(
:ci_runner
,
:project
,
:online
,
projects:
[
project
],
active:
private_runner_available
)
}
subject
{
minutes
.
available?
(
build
.
build_matcher
)
}
it
{
is_expected
.
to
eq
(
result
)
}
end
end
context
'N+1 queries'
do
let_it_be
(
:project
)
{
create
(
:project
)
}
let_it_be
(
:private_runner
)
{
create
(
:ci_runner
,
:project
,
:online
,
projects:
[
project
])
}
it
'caches records loaded from database'
do
ActiveRecord
::
QueryRecorder
.
new
(
skip_cached:
false
)
do
minutes
.
available?
(
build
.
build_matcher
)
end
expect
{
minutes
.
available?
(
build
.
build_matcher
)
}.
not_to
exceed_all_query_limit
(
0
)
end
end
end
This diff is collapsed.
Click to expand it.
ee/spec/services/ci/retry_build_service_spec.rb
View file @
80b00a23
...
...
@@ -2,6 +2,18 @@
require
'spec_helper'
RSpec
.
describe
Ci
::
RetryBuildService
do
let_it_be
(
:user
)
{
create
(
:user
)
}
let
(
:build
)
{
create
(
:ci_build
,
project:
project
)
}
subject
(
:service
)
{
described_class
.
new
(
project
,
user
)
}
before
do
stub_not_protect_default_branch
project
.
add_developer
(
user
)
end
it_behaves_like
'restricts access to protected environments'
describe
'#reprocess'
do
...
...
@@ -9,12 +21,8 @@ RSpec.describe Ci::RetryBuildService do
let_it_be
(
:namespace
)
{
create
(
:namespace
)
}
let_it_be
(
:ultimate_plan
)
{
create
(
:ultimate_plan
)
}
let_it_be
(
:plan_limits
)
{
create
(
:plan_limits
,
plan:
ultimate_plan
)
}
let_it_be
(
:user
)
{
create
(
:user
)
}
let
(
:project
)
{
create
(
:project
,
namespace:
namespace
,
creator:
user
)
}
let
(
:build
)
{
create
(
:ci_build
,
project:
project
)
}
subject
(
:service
)
{
described_class
.
new
(
project
,
user
)
}
let
(
:new_build
)
do
travel_to
(
1
.
second
.
from_now
)
do
...
...
@@ -22,12 +30,6 @@ RSpec.describe Ci::RetryBuildService do
end
end
before
do
stub_not_protect_default_branch
project
.
add_developer
(
user
)
end
context
'dast'
do
let
(
:dast_site_profile
)
{
create
(
:dast_site_profile
,
project:
project
)
}
let
(
:dast_scanner_profile
)
{
create
(
:dast_scanner_profile
,
project:
project
)
}
...
...
@@ -120,4 +122,46 @@ RSpec.describe Ci::RetryBuildService do
end
end
end
describe
'#execute'
do
let
(
:new_build
)
do
travel_to
(
1
.
second
.
from_now
)
do
service
.
execute
(
build
)
end
end
context
'when the CI quota is exceeded'
do
let_it_be
(
:namespace
)
{
create
(
:namespace
,
:with_used_build_minutes_limit
)
}
let_it_be
(
:project
)
{
create
(
:project
,
namespace:
namespace
,
creator:
user
)
}
context
'when there are no runners available'
do
it
{
expect
(
new_build
).
not_to
be_failed
}
end
context
'when shared runners are available'
do
let_it_be
(
:runner
)
{
create
(
:ci_runner
,
:instance
,
:online
)
}
it
'fails the build'
do
expect
(
new_build
).
to
be_failed
expect
(
new_build
.
failure_reason
).
to
eq
(
'ci_quota_exceeded'
)
end
context
'with private runners'
do
let_it_be
(
:private_runner
)
do
create
(
:ci_runner
,
:project
,
:online
,
projects:
[
project
])
end
it
{
expect
(
new_build
).
not_to
be_failed
}
end
context
'when the feature is disabled'
do
before
do
stub_feature_flags
(
ci_quota_check_on_retries:
false
)
end
it
{
expect
(
new_build
).
not_to
be_failed
}
end
end
end
end
end
This diff is collapsed.
Click to expand it.
ee/spec/services/ci/retry_pipeline_service_spec.rb
0 → 100644
View file @
80b00a23
# frozen_string_literal: true
require
'spec_helper'
RSpec
.
describe
Ci
::
RetryPipelineService
do
let_it_be
(
:runner
)
{
create
(
:ci_runner
,
:instance
,
:online
)
}
let_it_be
(
:user
)
{
create
(
:user
)
}
let
(
:pipeline
)
{
create
(
:ci_pipeline
,
project:
project
)
}
let
(
:service
)
{
described_class
.
new
(
project
,
user
)
}
before
do
project
.
add_developer
(
user
)
create
(
:protected_branch
,
:developers_can_merge
,
name:
pipeline
.
ref
,
project:
project
)
end
context
'when the namespace is out of CI minutes'
do
let_it_be
(
:namespace
)
{
create
(
:namespace
,
:with_used_build_minutes_limit
)
}
let_it_be
(
:project
)
{
create
(
:project
,
namespace:
namespace
)
}
let_it_be
(
:private_runner
)
do
create
(
:ci_runner
,
:project
,
:online
,
projects:
[
project
],
tag_list:
[
'ruby'
],
run_untagged:
false
)
end
before
do
create_build
(
'rspec 1'
,
:failed
)
create_build
(
'rspec 2'
,
:canceled
,
tag_list:
[
'ruby'
])
end
it
'retries the builds with available runners'
do
service
.
execute
(
pipeline
)
expect
(
pipeline
.
statuses
.
count
).
to
eq
(
3
)
expect
(
build
(
'rspec 1'
)).
to
be_failed
expect
(
build
(
'rspec 2'
)).
to
be_pending
expect
(
pipeline
.
reload
).
to
be_running
end
context
'when the feature flag is disabled'
do
before
do
stub_feature_flags
(
ci_quota_check_on_retries:
false
)
end
it
'enqueues all builds'
do
service
.
execute
(
pipeline
)
expect
(
build
(
'rspec 1'
)).
to
be_pending
expect
(
build
(
'rspec 2'
)).
to
be_pending
expect
(
pipeline
.
reload
).
to
be_running
end
end
end
def
build
(
name
)
pipeline
.
reload
.
statuses
.
latest
.
find_by
(
name:
name
)
end
def
create_build
(
name
,
status
,
**
opts
)
create
(
:ci_build
,
name:
name
,
status:
status
,
pipeline:
pipeline
,
**
opts
)
do
|
build
|
::
Ci
::
ProcessPipelineService
.
new
(
pipeline
).
execute
end
end
end
This diff is collapsed.
Click to expand it.
spec/models/ci/build_spec.rb
View file @
80b00a23
...
...
@@ -1966,6 +1966,23 @@ RSpec.describe Ci::Build do
end
end
describe
'#tag_list'
do
let_it_be
(
:build
)
{
create
(
:ci_build
,
tag_list:
[
'tag'
])
}
context
'when tags are preloaded'
do
it
'does not trigger queries'
do
build_with_tags
=
described_class
.
eager_load_tags
.
id_in
([
build
]).
to_a
.
first
expect
{
build_with_tags
.
tag_list
}.
not_to
exceed_all_query_limit
(
0
)
expect
(
build_with_tags
.
tag_list
).
to
eq
([
'tag'
])
end
end
context
'when tags are not preloaded'
do
it
{
expect
(
described_class
.
find
(
build
.
id
).
tag_list
).
to
eq
([
'tag'
])
}
end
end
describe
'#has_tags?'
do
context
'when build has tags'
do
subject
{
create
(
:ci_build
,
tag_list:
[
'tag'
])
}
...
...
This diff is collapsed.
Click to expand it.
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment