Merge branch '25921-wildcard-paths-for-ci-includes' into 'master'

Implement wildcard support for pipeline include file paths [RUN ALL RSPEC] [RUN AS-IF-FOSS]

See merge request gitlab-org/gitlab!58999

config/feature_flags/development/ci_wildcard_file_paths.yml

+---
+name: ci_wildcard_file_paths
+introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/58999
+rollout_issue_url: https://gitlab.com/gitlab-org/gitlab/-/issues/327315
+milestone: '13.11'
+type: development
+group: group::pipeline authoring
+default_enabled: false

doc/ci/yaml/README.md

@@ -477,6 +477,41 @@ include: '.gitlab-ci-production.yml'
 
 Use local includes instead of symbolic links.
 
+##### `include:local` with wildcard file paths
+
+> - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/25921) in GitLab 13.11.
+> - It's [deployed behind a feature flag](../../user/feature_flags.md), disabled by default.
+> - It's disabled on GitLab.com.
+> - It's not recommended for production use.
+> - To use it in GitLab self-managed instances, ask a GitLab administrator to enable it. **(CORE ONLY)**
+
+You can use wildcard paths (`*`) with `include:local`.
+
+Example:
+
+```yaml
+include: 'configs/*.yml'
+```
+
+When the pipeline runs, it adds all `.yml` files in the `configs` folder into the pipeline configuration.
+
+The wildcard file paths feature is under development and not ready for production use. It is
+deployed behind a feature flag that is **disabled by default**.
+[GitLab administrators with access to the GitLab Rails console](../../administration/feature_flags.md)
+can enable it.
+
+To enable it:
+
+```ruby
+Feature.enable(:ci_wildcard_file_paths)
+```
+
+To disable it:
+
+```ruby
+Feature.disable(:ci_wildcard_file_paths)
+```
+
 #### `include:file`
 
 > [Introduced](https://gitlab.com/gitlab-org/gitlab-foss/-/issues/53903) in GitLab 11.7.

lib/gitlab/ci/config/external/mapper.rb

@@ -34,6 +34,7 @@ module Gitlab
               .compact
               .map(&method(:normalize_location))
               .flat_map(&method(:expand_project_files))
+              .flat_map(&method(:expand_wildcard_paths))
               .map(&method(:expand_variables))
               .each(&method(:verify_duplicates!))
               .map(&method(:select_first_matching))
@@ -63,6 +64,17 @@ module Gitlab
             end
           end
 
+          def expand_wildcard_paths(location)
+            return location unless ::Feature.enabled?(:ci_wildcard_file_paths, context.project, default_enabled: :yaml)
+
+            # We only support local files for wildcard paths
+            return location unless location[:local] && location[:local].include?('*')
+
+            context.project.repository.search_files_by_wildcard_path(location[:local], context.sha).map do |path|
+              { local: path }
+            end
+          end
+
           def normalize_location_string(location)
             if ::Gitlab::UrlSanitizer.valid?(location)
               { remote: location }

qa/qa/specs/features/browser_ui/4_verify/pipeline/include_local_config_file_paths_with_wildcard_spec.rb

+# frozen_string_literal: true
+
+module QA
+  RSpec.describe 'Verify', :requires_admin do
+    describe 'Include local config file paths with wildcard' do
+      let(:feature_flag) { :ci_wildcard_file_paths }
+
+      let(:project) do
+        Resource::Project.fabricate_via_api! do |project|
+          project.name = 'project-with-pipeline'
+        end
+      end
+
+      before do
+        Runtime::Feature.enable(feature_flag)
+        Flow::Login.sign_in
+        add_files_to_project
+        project.visit!
+        Flow::Pipeline.visit_latest_pipeline
+      end
+
+      after do
+        Runtime::Feature.disable(feature_flag)
+        project.remove_via_api!
+      end
+
+      it 'runs the pipeline with composed config', testcase: 'https://gitlab.com/gitlab-org/quality/testcases/-/issues/1757' do
+        Page::Project::Pipeline::Show.perform do |pipeline|
+          aggregate_failures 'pipeline has all expected jobs' do
+            expect(pipeline).to have_job('build')
+            expect(pipeline).to have_job('test')
+            expect(pipeline).not_to have_job('deploy')
+          end
+        end
+      end
+
+      private
+
+      def add_files_to_project
+        Resource::Repository::Commit.fabricate_via_api! do |commit|
+          commit.project = project
+          commit.commit_message = 'Add CI and local files'
+          commit.add_files([build_config_file, test_config_file, non_detectable_file, main_ci_file])
+        end
+      end
+
+      def main_ci_file
+        {
+          file_path: '.gitlab-ci.yml',
+          content: <<~YAML
+            include: 'configs/*.yml'
+          YAML
+        }
+      end
+
+      def build_config_file
+        {
+          file_path: 'configs/builds.yml',
+          content: <<~YAML
+            build:
+              stage: build
+              script: echo build
+          YAML
+        }
+      end
+
+      def test_config_file
+        {
+          file_path: 'configs/tests.yml',
+          content: <<~YAML
+            test:
+              stage: test
+              script: echo test
+          YAML
+        }
+      end
+
+      def non_detectable_file
+        {
+          file_path: 'configs/not_included.yaml', # we only include `*.yml` not `*.yaml`
+          content: <<~YAML
+            deploy:
+              stage: deploy
+              script: echo deploy
+          YAML
+        }
+      end
+    end
+  end
+end

spec/lib/gitlab/ci/config/external/mapper_spec.rb

@@ -324,5 +324,39 @@ RSpec.describe Gitlab::Ci::Config::External::Mapper do
         end
       end
     end
+
+    context 'when local file path has wildcard' do
+      let(:project) { create(:project, :repository) }
+
+      let(:values) do
+        { include: 'myfolder/*.yml' }
+      end
+
+      before do
+        allow_next_instance_of(Repository) do |repository|
+          allow(repository).to receive(:search_files_by_wildcard_path).with('myfolder/*.yml', '123456') do
+            ['myfolder/file1.yml', 'myfolder/file2.yml']
+          end
+        end
+      end
+
+      it 'includes the matched local files' do
+        expect(subject).to contain_exactly(an_instance_of(Gitlab::Ci::Config::External::File::Local),
+                                           an_instance_of(Gitlab::Ci::Config::External::File::Local))
+
+        expect(subject.map(&:location)).to contain_exactly('myfolder/file1.yml', 'myfolder/file2.yml')
+      end
+
+      context 'when the FF ci_wildcard_file_paths is disabled' do
+        before do
+          stub_feature_flags(ci_wildcard_file_paths: false)
+        end
+
+        it 'cannot find any file returns an error message' do
+          expect(subject).to contain_exactly(an_instance_of(Gitlab::Ci::Config::External::File::Local))
+          expect(subject[0].errors).to eq(['Local file `myfolder/*.yml` does not exist!'])
+        end
+      end
+    end
   end
 end

spec/lib/gitlab/ci/config/external/processor_spec.rb

@@ -366,5 +366,40 @@ RSpec.describe Gitlab::Ci::Config::External::Processor do
         expect(output.keys).to match_array([:image, :my_build, :my_test])
       end
     end
+
+    context 'when local file path has wildcard' do
+      let_it_be(:project) { create(:project, :repository) }
+
+      let(:values) do
+        { include: 'myfolder/*.yml', image: 'ruby:2.7' }
+      end
+
+      before do
+        allow_next_instance_of(Repository) do |repository|
+          allow(repository).to receive(:search_files_by_wildcard_path).with('myfolder/*.yml', sha) do
+            ['myfolder/file1.yml', 'myfolder/file2.yml']
+          end
+
+          allow(repository).to receive(:blob_data_at).with(sha, 'myfolder/file1.yml') do
+            <<~HEREDOC
+              my_build:
+                script: echo Hello World
+            HEREDOC
+          end
+
+          allow(repository).to receive(:blob_data_at).with(sha, 'myfolder/file2.yml') do
+            <<~HEREDOC
+              my_test:
+                script: echo Hello World
+            HEREDOC
+          end
+        end
+      end
+
+      it 'fetches the matched files' do
+        output = processor.perform
+        expect(output.keys).to match_array([:image, :my_build, :my_test])
+      end
+    end
   end
 end