Merge branch 'krakowski_master2' into 'master'

Add all pipeline sources as special keywords to 'only' and 'except'

Closes #32955

See merge request !11972

changelogs/unreleased/32955-special-keywords.yml

+---
+title: Add all pipeline sources as special keywords to 'only' and 'except'
+merge_request: 11844
+author: Filip Krakowski

doc/ci/yaml/README.md

@@ -393,7 +393,8 @@ There are a few rules that apply to the usage of refs policy:
 * `only` and `except` are inclusive. If both `only` and `except` are defined
    in a job specification, the ref is filtered by `only` and `except`.
 * `only` and `except` allow the use of regular expressions.
-* `only` and `except` allow the use of special keywords: `branches`, `tags`, and `triggers`.
+* `only` and `except` allow the use of special keywords:
+`api`, `branches`, `external`, `tags`, `pushes`, `schedules`, `triggers`, and `web`
 * `only` and `except` allow to specify a repository path to filter jobs for
    forks.
 
@@ -411,7 +412,7 @@ job:
 ```
 
 In this example, `job` will run only for refs that are tagged, or if a build is
-explicitly requested via an API trigger.
+explicitly requested via an API trigger or a [Pipeline Schedule](../../user/project/pipelines/schedules.md).
 
 ```yaml
 job:
@@ -419,6 +420,7 @@ job:
   only:
     - tags
     - triggers
+    - schedules
 ```
 
 The repository path can be used to have jobs executed only for the parent

lib/ci/gitlab_ci_yaml_processor.rb

@@ -20,26 +20,26 @@ module Ci
       raise ValidationError, e.message
     end
 
-    def jobs_for_ref(ref, tag = false, trigger_request = nil)
+    def jobs_for_ref(ref, tag = false, source = nil)
       @jobs.select do |_, job|
-        process?(job[:only], job[:except], ref, tag, trigger_request)
+        process?(job[:only], job[:except], ref, tag, source)
       end
     end
 
-    def jobs_for_stage_and_ref(stage, ref, tag = false, trigger_request = nil)
-      jobs_for_ref(ref, tag, trigger_request).select do |_, job|
+    def jobs_for_stage_and_ref(stage, ref, tag = false, source = nil)
+      jobs_for_ref(ref, tag, source).select do |_, job|
         job[:stage] == stage
       end
     end
 
-    def builds_for_ref(ref, tag = false, trigger_request = nil)
-      jobs_for_ref(ref, tag, trigger_request).map do |name, _|
+    def builds_for_ref(ref, tag = false, source = nil)
+      jobs_for_ref(ref, tag, source).map do |name, _|
         build_attributes(name)
       end
     end
 
-    def builds_for_stage_and_ref(stage, ref, tag = false, trigger_request = nil)
-      jobs_for_stage_and_ref(stage, ref, tag, trigger_request).map do |name, _|
+    def builds_for_stage_and_ref(stage, ref, tag = false, source = nil)
+      jobs_for_stage_and_ref(stage, ref, tag, source).map do |name, _|
         build_attributes(name)
       end
     end
@@ -51,11 +51,9 @@ module Ci
     end
 
     def stage_seeds(pipeline)
-      trigger_request = pipeline.trigger_requests.first
-
       seeds = @stages.uniq.map do |stage|
         builds = builds_for_stage_and_ref(
-          stage, pipeline.ref, pipeline.tag?, trigger_request)
+          stage, pipeline.ref, pipeline.tag?, pipeline.source)
 
         Gitlab::Ci::Stage::Seed.new(pipeline, stage, builds) if builds.any?
       end
@@ -193,30 +191,35 @@ module Ci
       end
     end
 
-    def process?(only_params, except_params, ref, tag, trigger_request)
+    def process?(only_params, except_params, ref, tag, source)
       if only_params.present?
-        return false unless matching?(only_params, ref, tag, trigger_request)
+        return false unless matching?(only_params, ref, tag, source)
       end
 
       if except_params.present?
-        return false if matching?(except_params, ref, tag, trigger_request)
+        return false if matching?(except_params, ref, tag, source)
       end
 
       true
     end
 
-    def matching?(patterns, ref, tag, trigger_request)
+    def matching?(patterns, ref, tag, source)
       patterns.any? do |pattern|
-        match_ref?(pattern, ref, tag, trigger_request)
+        pattern, path = pattern.split('@', 2)
+        matches_path?(path) && matches_pattern?(pattern, ref, tag, source)
       end
     end
 
-    def match_ref?(pattern, ref, tag, trigger_request)
-      pattern, path = pattern.split('@', 2)
-      return false if path && path != self.path
+    def matches_path?(path)
+      return true unless path
+
+      path == self.path
+    end
+
+    def matches_pattern?(pattern, ref, tag, source)
       return true if tag && pattern == 'tags'
       return true if !tag && pattern == 'branches'
-      return true if trigger_request.present? && pattern == 'triggers'
+      return true if source_to_pattern(source) == pattern
 
       if pattern.first == "/" && pattern.last == "/"
         Regexp.new(pattern[1...-1]) =~ ref
@@ -224,5 +227,13 @@ module Ci
         pattern == ref
       end
     end
+
+    def source_to_pattern(source)
+      if %w[api external web].include?(source)
+        source
+      else
+        source&.pluralize
+      end
+    end
   end
 end

spec/lib/ci/gitlab_ci_yaml_processor_spec.rb

@@ -123,6 +123,25 @@ module Ci
           expect(seeds.first.builds.dig(0, :name)).to eq 'spinach'
         end
       end
+
+      context 'when source policy is specified' do
+        let(:config) do
+          YAML.dump(production: { stage: 'deploy', script: 'cap prod', only: ['triggers'] },
+                    spinach: { stage: 'test', script: 'spinach', only: ['schedules'] })
+        end
+
+        let(:pipeline) do
+          create(:ci_empty_pipeline, source: :schedule)
+        end
+
+        it 'returns stage seeds only assigned to schedules' do
+          seeds = subject.stage_seeds(pipeline)
+
+          expect(seeds.size).to eq 1
+          expect(seeds.first.stage[:name]).to eq 'test'
+          expect(seeds.first.builds.dig(0, :name)).to eq 'spinach'
+        end
+      end
     end
 
     describe "#builds_for_ref" do
@@ -219,26 +238,44 @@ module Ci
           expect(config_processor.builds_for_stage_and_ref(type, "deploy").size).to eq(0)
         end
 
-        it "returns builds if only has a triggers keyword specified and a trigger is provided" do
-          config = YAML.dump({
-                               before_script: ["pwd"],
-                               rspec: { script: "rspec", type: type, only: ["triggers"] }
-                             })
+        it "returns builds if only has special keywords specified and source matches" do
+          possibilities = [{ keyword: 'pushes', source: 'push' },
+                           { keyword: 'web', source: 'web' },
+                           { keyword: 'triggers', source: 'trigger' },
+                           { keyword: 'schedules', source: 'schedule' },
+                           { keyword: 'api', source: 'api' },
+                           { keyword: 'external', source: 'external' }]
 
-          config_processor = GitlabCiYamlProcessor.new(config, path)
+          possibilities.each do |possibility|
+            config = YAML.dump({
+                                 before_script: ["pwd"],
+                                 rspec: { script: "rspec", type: type, only: [possibility[:keyword]] }
+                               })
 
-          expect(config_processor.builds_for_stage_and_ref(type, "deploy", false, true).size).to eq(1)
+            config_processor = GitlabCiYamlProcessor.new(config, path)
+
+            expect(config_processor.builds_for_stage_and_ref(type, "deploy", false, possibility[:source]).size).to eq(1)
+          end
         end
 
-        it "does not return builds if only has a triggers keyword specified and no trigger is provided" do
-          config = YAML.dump({
-                               before_script: ["pwd"],
-                               rspec: { script: "rspec", type: type, only: ["triggers"] }
-                             })
+        it "does not return builds if only has special keywords specified and source doesn't match" do
+          possibilities = [{ keyword: 'pushes', source: 'web' },
+                           { keyword: 'web', source: 'push' },
+                           { keyword: 'triggers', source: 'schedule' },
+                           { keyword: 'schedules', source: 'external' },
+                           { keyword: 'api', source: 'trigger' },
+                           { keyword: 'external', source: 'api' }]
 
-          config_processor = GitlabCiYamlProcessor.new(config, path)
+          possibilities.each do |possibility|
+            config = YAML.dump({
+                                 before_script: ["pwd"],
+                                 rspec: { script: "rspec", type: type, only: [possibility[:keyword]] }
+                               })
 
-          expect(config_processor.builds_for_stage_and_ref(type, "deploy").size).to eq(0)
+            config_processor = GitlabCiYamlProcessor.new(config, path)
+
+            expect(config_processor.builds_for_stage_and_ref(type, "deploy", false, possibility[:source]).size).to eq(0)
+          end
         end
 
         it "returns builds if only has current repository path" do
@@ -375,26 +412,44 @@ module Ci
           expect(config_processor.builds_for_stage_and_ref(type, "deploy").size).to eq(1)
         end
 
-        it "does not return builds if except has a triggers keyword specified and a trigger is provided" do
-          config = YAML.dump({
-                               before_script: ["pwd"],
-                               rspec: { script: "rspec", type: type, except: ["triggers"] }
-                             })
+        it "does not return builds if except has special keywords specified and source matches" do
+          possibilities = [{ keyword: 'pushes', source: 'push' },
+                           { keyword: 'web', source: 'web' },
+                           { keyword: 'triggers', source: 'trigger' },
+                           { keyword: 'schedules', source: 'schedule' },
+                           { keyword: 'api', source: 'api' },
+                           { keyword: 'external', source: 'external' }]
 
-          config_processor = GitlabCiYamlProcessor.new(config, path)
+          possibilities.each do |possibility|
+            config = YAML.dump({
+                                 before_script: ["pwd"],
+                                 rspec: { script: "rspec", type: type, except: [possibility[:keyword]] }
+                               })
+
+            config_processor = GitlabCiYamlProcessor.new(config, path)
 
-          expect(config_processor.builds_for_stage_and_ref(type, "deploy", false, true).size).to eq(0)
+            expect(config_processor.builds_for_stage_and_ref(type, "deploy", false, possibility[:source]).size).to eq(0)
+          end
         end
 
-        it "returns builds if except has a triggers keyword specified and no trigger is provided" do
-          config = YAML.dump({
-                               before_script: ["pwd"],
-                               rspec: { script: "rspec", type: type, except: ["triggers"] }
-                             })
+        it "returns builds if except has special keywords specified and source doesn't match" do
+          possibilities = [{ keyword: 'pushes', source: 'web' },
+                           { keyword: 'web', source: 'push' },
+                           { keyword: 'triggers', source: 'schedule' },
+                           { keyword: 'schedules', source: 'external' },
+                           { keyword: 'api', source: 'trigger' },
+                           { keyword: 'external', source: 'api' }]
 
-          config_processor = GitlabCiYamlProcessor.new(config, path)
+          possibilities.each do |possibility|
+            config = YAML.dump({
+                                 before_script: ["pwd"],
+                                 rspec: { script: "rspec", type: type, except: [possibility[:keyword]] }
+                               })
 
-          expect(config_processor.builds_for_stage_and_ref(type, "deploy").size).to eq(1)
+            config_processor = GitlabCiYamlProcessor.new(config, path)
+
+            expect(config_processor.builds_for_stage_and_ref(type, "deploy", false, possibility[:source]).size).to eq(1)
+          end
         end
 
         it "does not return builds if except has current repository path" do