Commit 6a821618 authored by Furkan Ayhan's avatar Furkan Ayhan Committed by Fabio Pitino

Add support of pipeline variables to include

parent 7e240aed
......@@ -422,22 +422,36 @@ configurations. Local configurations in the `.gitlab-ci.yml` file override inclu
> - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/284883) in GitLab 13.8.
> - [Feature flag removed](https://gitlab.com/gitlab-org/gitlab/-/issues/294294) in GitLab 13.9.
> - [Support for project, group, and instance variables added](https://gitlab.com/gitlab-org/gitlab/-/issues/219065) in GitLab 14.2.
> - [Support for pipeline variables added](https://gitlab.com/gitlab-org/gitlab/-/issues/337633) in GitLab 14.5.
In `include` sections in your `.gitlab-ci.yml` file, you can use:
- `$CI_COMMIT_REF_NAME` [predefined variable](../variables/predefined_variables.md) in GitLab 14.2
and later.
- When used in `include`, the `CI_COMMIT_REF_NAME` variable returns the full
ref path, like `refs/heads/branch-name`. In other cases, this variable returns only
the branch name, like `branch-name`.
To use `CI_COMMIT_REF_NAME` in `include:rules`, you might need to use `if: $CI_COMMIT_REF_NAME =~ /main/`
(not `== main`). [An issue exists](https://gitlab.com/gitlab-org/gitlab/-/issues/337633)
to align `CI_COMMIT_REF_NAME` behavior in all cases.
- [Project variables](../variables/index.md#add-a-cicd-variable-to-a-project)
- [Group variables](../variables/index.md#add-a-cicd-variable-to-a-group)
- [Instance variables](../variables/index.md#add-a-cicd-variable-to-an-instance)
- Project [predefined variables](../variables/predefined_variables.md).
- Project [predefined variables](../variables/predefined_variables.md)
- In GitLab 14.2 and later, the `$CI_COMMIT_REF_NAME` [predefined variable](../variables/predefined_variables.md).
When used in `include`, the `CI_COMMIT_REF_NAME` variable returns the full
ref path, like `refs/heads/branch-name`. In `include:rules`, you might need to use
`if: $CI_COMMIT_REF_NAME =~ /main/` (not `== main`). This behavior is resolved in GitLab 14.5.
In GitLab 14.5 and later, you can also use:
- [Trigger variables](../triggers/index.md#making-use-of-trigger-variables).
- [Scheduled pipeline variables](../pipelines/schedules.md#using-variables).
- [Manual pipeline run variables](../variables/index.md#override-a-variable-when-running-a-pipeline-manually).
- Pipeline [predefined variables](../variables/predefined_variables.md).
YAML files are parsed before the pipeline is created, so the following pipeline predefined variables
are **not** available:
- `CI_PIPELINE_ID`
- `CI_PIPELINE_URL`
- `CI_PIPELINE_IID`
- `CI_PIPELINE_CREATED_AT`
For example:
```yaml
include:
......@@ -448,9 +462,6 @@ include:
For an example of how you can include these predefined variables, and the variables' impact on CI/CD jobs,
see this [CI/CD variable demo](https://youtu.be/4XR8gw3Pkos).
There is a [related issue](https://gitlab.com/gitlab-org/gitlab/-/issues/337633)
that proposes expanding this feature to support more variables.
#### `rules` with `include`
> - Introduced in GitLab 14.2 [with a flag](../../administration/feature_flags.md) named `ci_include_rules`. Disabled by default.
......@@ -468,6 +479,9 @@ include:
- local: builds.yml
rules:
- if: '$INCLUDE_BUILDS == "true"'
- local: deploys.yml
rules:
- if: $CI_COMMIT_BRANCH == "main"
test:
stage: test
......
......@@ -40,14 +40,16 @@ RSpec.describe Gitlab::Ci::Config do
describe 'with security orchestration policy' do
let(:source) { 'push' }
let_it_be(:ref) { 'refs/heads/master' }
let_it_be(:ref) { 'master' }
let_it_be_with_refind(:project) { create(:project, :repository) }
let_it_be(:policies_repository) { create(:project, :repository) }
let_it_be(:security_orchestration_policy_configuration) { create(:security_orchestration_policy_configuration, project: project, security_policy_management_project: policies_repository) }
let_it_be(:policy_yaml) { build(:orchestration_policy_yaml, scan_execution_policy: [build(:scan_execution_policy)]) }
subject(:config) { described_class.new(ci_yml, source_ref_path: ref, project: project, source: source) }
let(:pipeline) { build(:ci_pipeline, project: project, ref: ref) }
subject(:config) { described_class.new(ci_yml, pipeline: pipeline, project: project, source: source) }
before do
allow_next_instance_of(Repository) do |repository|
......@@ -74,7 +76,7 @@ RSpec.describe Gitlab::Ci::Config do
end
context 'when policy is not applicable on branch from the pipeline' do
let_it_be(:ref) { 'refs/heads/production' }
let_it_be(:ref) { 'production' }
context 'when DAST profiles are not found' do
it 'adds a job with error message' do
......
......@@ -19,11 +19,12 @@ module Gitlab
attr_reader :root, :context, :source_ref_path, :source
def initialize(config, project: nil, sha: nil, user: nil, parent_pipeline: nil, source_ref_path: nil, source: nil)
@context = build_context(project: project, sha: sha, user: user, parent_pipeline: parent_pipeline, ref: source_ref_path)
def initialize(config, project: nil, pipeline: nil, sha: nil, user: nil, parent_pipeline: nil, source: nil)
@source_ref_path = pipeline&.source_ref_path
@context = build_context(project: project, pipeline: pipeline, sha: sha, user: user, parent_pipeline: parent_pipeline)
@context.set_deadline(TIMEOUT_SECONDS)
@source_ref_path = source_ref_path
@source = source
@config = expand_config(config)
......@@ -108,16 +109,16 @@ module Gitlab
end
end
def build_context(project:, sha:, user:, parent_pipeline:, ref:)
def build_context(project:, pipeline:, sha:, user:, parent_pipeline:)
Config::External::Context.new(
project: project,
sha: sha || find_sha(project),
user: user,
parent_pipeline: parent_pipeline,
variables: build_variables(project: project, ref: ref))
variables: build_variables(project: project, pipeline: pipeline))
end
def build_variables(project:, ref:)
def build_variables(project:, pipeline:)
Gitlab::Ci::Variables::Collection.new.tap do |variables|
break variables unless project
......@@ -126,18 +127,12 @@ module Gitlab
#
# See more detail in the docs: https://docs.gitlab.com/ee/ci/variables/#cicd-variable-precedence
variables.concat(project.predefined_variables)
variables.concat(pipeline_predefined_variables(ref: ref))
variables.concat(project.ci_instance_variables_for(ref: ref))
variables.concat(project.group.ci_variables_for(ref, project)) if project.group
variables.concat(project.ci_variables_for(ref: ref))
end
end
# https://gitlab.com/gitlab-org/gitlab/-/issues/337633 aims to add all predefined variables
# to this list, but only CI_COMMIT_REF_NAME is available right now to support compliance pipelines.
def pipeline_predefined_variables(ref:)
Gitlab::Ci::Variables::Collection.new.tap do |v|
v.append(key: 'CI_COMMIT_REF_NAME', value: ref)
variables.concat(pipeline.predefined_variables) if pipeline
variables.concat(project.ci_instance_variables_for(ref: source_ref_path))
variables.concat(project.group.ci_variables_for(source_ref_path, project)) if project.group
variables.concat(project.ci_variables_for(ref: source_ref_path))
variables.concat(pipeline.variables) if pipeline
variables.concat(pipeline.pipeline_schedule.job_variables) if pipeline&.pipeline_schedule
end
end
......
......@@ -14,7 +14,7 @@ module Gitlab
result = ::Gitlab::Ci::YamlProcessor.new(
@command.config_content, {
project: project,
source_ref_path: @pipeline.source_ref_path,
pipeline: @pipeline,
sha: @pipeline.sha,
source: @pipeline.source,
user: current_user,
......
......@@ -14,7 +14,7 @@ RSpec.describe Gitlab::Ci::Config do
end
let(:config) do
described_class.new(yml, project: nil, sha: nil, user: nil)
described_class.new(yml, project: nil, pipeline: nil, sha: nil, user: nil)
end
context 'when config is valid' do
......@@ -286,9 +286,12 @@ RSpec.describe Gitlab::Ci::Config do
end
context "when using 'include' directive" do
let(:group) { create(:group) }
let_it_be(:group) { create(:group) }
let(:project) { create(:project, :repository, group: group) }
let(:main_project) { create(:project, :repository, :public, group: group) }
let(:pipeline) { build(:ci_pipeline, project: project) }
let(:remote_location) { 'https://gitlab.com/gitlab-org/gitlab-foss/blob/1234/.gitlab-ci-1.yml' }
let(:local_location) { 'spec/fixtures/gitlab/ci/external_files/.gitlab-ci-template-1.yml' }
......@@ -327,7 +330,7 @@ RSpec.describe Gitlab::Ci::Config do
end
let(:config) do
described_class.new(gitlab_ci_yml, project: project, sha: '12345', user: user)
described_class.new(gitlab_ci_yml, project: project, pipeline: pipeline, sha: '12345', user: user)
end
before do
......@@ -724,7 +727,7 @@ RSpec.describe Gitlab::Ci::Config do
end
end
context "when an 'include' has rules" do
context "when an 'include' has rules with a project variable" do
let(:gitlab_ci_yml) do
<<~HEREDOC
include:
......@@ -751,5 +754,30 @@ RSpec.describe Gitlab::Ci::Config do
end
end
end
context "when an 'include' has rules with a pipeline variable" do
let(:gitlab_ci_yml) do
<<~HEREDOC
include:
- local: #{local_location}
rules:
- if: $CI_COMMIT_SHA == "#{project.commit.sha}"
HEREDOC
end
context 'when a pipeline is passed' do
it 'includes the file' do
expect(config.to_hash).to include(local_location_hash)
end
end
context 'when a pipeline is not passed' do
let(:pipeline) { nil }
it 'does not include the file' do
expect(config.to_hash).not_to include(local_location_hash)
end
end
end
end
end
......@@ -8,8 +8,10 @@ RSpec.describe Ci::CreatePipelineService do
let_it_be(:user) { project.owner }
let(:ref) { 'refs/heads/master' }
let(:variables_attributes) { [{ key: 'MYVAR', secret_value: 'hello' }] }
let(:source) { :push }
let(:service) { described_class.new(project, user, { ref: ref }) }
let(:service) { described_class.new(project, user, { ref: ref, variables_attributes: variables_attributes }) }
let(:pipeline) { service.execute(source).payload }
let(:file_location) { 'spec/fixtures/gitlab/ci/external_files/.gitlab-ci-template-1.yml' }
......@@ -24,6 +26,20 @@ RSpec.describe Ci::CreatePipelineService do
.and_return(File.read(Rails.root.join(file_location)))
end
shared_examples 'not including the file' do
it 'does not include the job in the file' do
expect(pipeline).to be_created_successfully
expect(pipeline.processables.pluck(:name)).to contain_exactly('job')
end
end
shared_examples 'including the file' do
it 'includes the job in the file' do
expect(pipeline).to be_created_successfully
expect(pipeline.processables.pluck(:name)).to contain_exactly('job', 'rspec')
end
end
context 'with a local file' do
let(:config) do
<<~EOY
......@@ -33,13 +49,10 @@ RSpec.describe Ci::CreatePipelineService do
EOY
end
it 'includes the job in the file' do
expect(pipeline).to be_created_successfully
expect(pipeline.processables.pluck(:name)).to contain_exactly('job', 'rspec')
end
it_behaves_like 'including the file'
end
context 'with a local file with rules' do
context 'with a local file with rules with a project variable' do
let(:config) do
<<~EOY
include:
......@@ -54,20 +67,64 @@ RSpec.describe Ci::CreatePipelineService do
context 'when the rules matches' do
let(:project_id) { project.id }
it 'includes the job in the file' do
expect(pipeline).to be_created_successfully
expect(pipeline.processables.pluck(:name)).to contain_exactly('job', 'rspec')
end
it_behaves_like 'including the file'
end
context 'when the rules does not match' do
let(:project_id) { non_existing_record_id }
it 'does not include the job in the file' do
expect(pipeline).to be_created_successfully
expect(pipeline.processables.pluck(:name)).to contain_exactly('job')
it_behaves_like 'not including the file'
end
end
context 'with a local file with rules with a predefined pipeline variable' do
let(:config) do
<<~EOY
include:
- local: #{file_location}
rules:
- if: $CI_PIPELINE_SOURCE == "#{pipeline_source}"
job:
script: exit 0
EOY
end
context 'when the rules matches' do
let(:pipeline_source) { 'push' }
it_behaves_like 'including the file'
end
context 'when the rules does not match' do
let(:pipeline_source) { 'web' }
it_behaves_like 'not including the file'
end
end
context 'with a local file with rules with a run pipeline variable' do
let(:config) do
<<~EOY
include:
- local: #{file_location}
rules:
- if: $MYVAR == "#{my_var}"
job:
script: exit 0
EOY
end
context 'when the rules matches' do
let(:my_var) { 'hello' }
it_behaves_like 'including the file'
end
context 'when the rules does not match' do
let(:my_var) { 'mello' }
it_behaves_like 'not including the file'
end
end
end
end
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment