Use non-predefined variables inside CI include blocks

Adds the ability to use project/group/instance CI
variables inside a .gitlab-ci.yml file include
block.

Changelog: added

doc/ci/yaml/index.md

@@ -419,9 +419,13 @@ configurations. Local configurations in the `.gitlab-ci.yml` file override inclu
 
 > - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/284883) in GitLab 13.8.
 > - [Feature flag removed](https://gitlab.com/gitlab-org/gitlab/-/issues/294294) in GitLab 13.9.
+> - [Custom variable support added](https://gitlab.com/gitlab-org/gitlab/-/issues/219065) in GitLab 14.2.
 
-You can [use some predefined variables in `include` sections](../variables/where_variables_can_be_used.md#gitlab-ciyml-file)
-in your `.gitlab-ci.yml` file:
+In `include` sections in your `.gitlab-ci.yml` file, you can use:
+
+- Project [predefined variables](../variables/predefined_variables.md).
+- [Custom instance, group, and project variables](../variables/index.md#custom-cicd-variables)
+  in GitLab 14.2 and later.
 
 ```yaml
 include:

ee/spec/lib/ee/gitlab/ci/pipeline/chain/validate/external_spec.rb

@@ -56,7 +56,7 @@ RSpec.describe Gitlab::Ci::Pipeline::Chain::Validate::External do
     it 'does not fire N+1 SQL queries' do
       stub_request(:post, validation_service_url)
 
-      expect { step.perform! }.not_to exceed_query_limit(4)
+      expect { step.perform! }.not_to exceed_query_limit(16)
     end
 
     context 'with a project in a subgroup' do

lib/gitlab/ci/config.rb

@@ -114,7 +114,22 @@ module Gitlab
           sha: sha || find_sha(project),
           user: user,
           parent_pipeline: parent_pipeline,
-          variables: project&.predefined_variables&.to_runner_variables)
+          variables: build_variables(project: project, ref: sha))
+      end
+
+      def build_variables(project:, ref:)
+        Gitlab::Ci::Variables::Collection.new.tap do |variables|
+          break variables unless project
+
+          # The order of the next 4 lines is important as priority of CI variables is
+          # defined globally within GitLab.
+          #
+          # See more detail in the docs: https://docs.gitlab.com/ee/ci/variables/#cicd-variable-precedence
+          variables.concat(project.predefined_variables)
+          variables.concat(project.ci_instance_variables_for(ref: ref))
+          variables.concat(project.group.ci_variables_for(ref, project)) if project.group
+          variables.concat(project.ci_variables_for(ref: ref))
+        end
       end
 
       def track_and_raise_for_dev_exception(error)

spec/lib/gitlab/ci/config_spec.rb

@@ -286,7 +286,9 @@ RSpec.describe Gitlab::Ci::Config do
   end
 
   context "when using 'include' directive" do
-    let(:project) { create(:project, :repository) }
+    let(:group) { create(:group) }
+    let(:project) { create(:project, :repository, group: group) }
+    let(:main_project) { create(:project, :repository, :public, group: group) }
     let(:remote_location) { 'https://gitlab.com/gitlab-org/gitlab-foss/blob/1234/.gitlab-ci-1.yml' }
     let(:local_location) { 'spec/fixtures/gitlab/ci/external_files/.gitlab-ci-template-1.yml' }
 
@@ -317,7 +319,9 @@ RSpec.describe Gitlab::Ci::Config do
       include:
         - #{local_location}
         - #{remote_location}
-
+        - project: '$MAIN_PROJECT'
+          ref: '$REF'
+          file: '$FILENAME'
       image: ruby:2.7
       HEREDOC
     end
@@ -331,6 +335,26 @@ RSpec.describe Gitlab::Ci::Config do
 
       allow(project.repository)
         .to receive(:blob_data_at).and_return(local_file_content)
+
+      main_project.repository.create_file(
+        main_project.creator,
+        '.gitlab-ci.yml',
+        local_file_content,
+        message: 'Add README.md',
+        branch_name: 'master'
+      )
+
+      main_project.repository.create_file(
+        main_project.creator,
+        '.another-ci-file.yml',
+        local_file_content,
+        message: 'Add README.md',
+        branch_name: 'master'
+      )
+
+      create(:ci_variable, project: project, key: "REF", value: "HEAD")
+      create(:ci_group_variable, group: group, key: "FILENAME", value: ".gitlab-ci.yml")
+      create(:ci_instance_variable, key: 'MAIN_PROJECT', value: main_project.full_path)
     end
 
     context "when gitlab_ci_yml has valid 'include' defined" do
@@ -344,6 +368,38 @@ RSpec.describe Gitlab::Ci::Config do
 
         expect(config.to_hash).to eq(composed_hash)
       end
+
+      context 'handling variables' do
+        it 'contains all project variables' do
+          ref = config.context.variables.find { |v| v[:key] == 'REF' }
+
+          expect(ref[:value]).to eq("HEAD")
+        end
+
+        it 'contains all group variables' do
+          filename = config.context.variables.find { |v| v[:key] == 'FILENAME' }
+
+          expect(filename[:value]).to eq(".gitlab-ci.yml")
+        end
+
+        it 'contains all instance variables' do
+          project = config.context.variables.find { |v| v[:key] == 'MAIN_PROJECT' }
+
+          expect(project[:value]).to eq(main_project.full_path)
+        end
+
+        context 'overriding a group variable at project level' do
+          before do
+            create(:ci_variable, project: project, key: "FILENAME", value: ".another-ci-file.yml")
+          end
+
+          it 'successfully overrides' do
+            filename = config.context.variables.to_hash[:FILENAME]
+
+            expect(filename).to eq('.another-ci-file.yml')
+          end
+        end
+      end
     end
 
     context "when gitlab_ci.yml has invalid 'include' defined" do

spec/lib/gitlab/ci/pipeline/chain/populate_spec.rb

@@ -107,6 +107,7 @@ RSpec.describe Gitlab::Ci::Pipeline::Chain::Populate do
     context 'when ref is protected' do
       before do
         allow(project).to receive(:protected_for?).with('master').and_return(true)
+        allow(project).to receive(:protected_for?).with('b83d6e391c22777fca1ed3012fce84f633d7fed0').and_return(true)
         allow(project).to receive(:protected_for?).with('refs/heads/master').and_return(true)
 
         dependencies.map(&:perform!)