Merge branch 'backstage/gb/jobs-triggering-policy-specifications' into 'master'

Implement job policy specifications

Closes #37280

See merge request gitlab-org/gitlab-ce!14265

app/models/ci/pipeline.rb

@@ -31,6 +31,7 @@ module Ci
     has_many :auto_canceled_jobs, class_name: 'CommitStatus', foreign_key: 'auto_canceled_by_id'
 
     delegate :id, to: :project, prefix: true
+    delegate :full_path, to: :project, prefix: true
 
     validates :source, exclusion: { in: %w(unknown), unless: :importing? }, on: :create
     validates :sha, presence: { unless: :importing? }
@@ -336,7 +337,7 @@ module Ci
       return @config_processor if defined?(@config_processor)
 
       @config_processor ||= begin
-        Gitlab::Ci::YamlProcessor.new(ci_yaml_file, project.full_path)
+        Gitlab::Ci::YamlProcessor.new(ci_yaml_file)
       rescue Gitlab::Ci::YamlProcessor::ValidationError, Psych::SyntaxError => e
         self.yaml_errors = e.message
         nil

lib/gitlab/ci/build/policy.rb

+module Gitlab
+  module Ci
+    module Build
+      module Policy
+        def self.fabricate(specs)
+          specifications = specs.to_h.map do |spec, value|
+            self.const_get(spec.to_s.camelize).new(value)
+          end
+
+          specifications.compact
+        end
+      end
+    end
+  end
+end

lib/gitlab/ci/build/policy/kubernetes.rb

+module Gitlab
+  module Ci
+    module Build
+      module Policy
+        class Kubernetes < Policy::Specification
+          def initialize(spec)
+            unless spec.to_sym == :active
+              raise UnknownPolicyError
+            end
+          end
+
+          def satisfied_by?(pipeline)
+            pipeline.has_kubernetes_active?
+          end
+        end
+      end
+    end
+  end
+end

lib/gitlab/ci/build/policy/refs.rb

+module Gitlab
+  module Ci
+    module Build
+      module Policy
+        class Refs < Policy::Specification
+          def initialize(refs)
+            @patterns = Array(refs)
+          end
+
+          def satisfied_by?(pipeline)
+            @patterns.any? do |pattern|
+              pattern, path = pattern.split('@', 2)
+
+              matches_path?(path, pipeline) &&
+                matches_pattern?(pattern, pipeline)
+            end
+          end
+
+          private
+
+          def matches_path?(path, pipeline)
+            return true unless path
+
+            pipeline.project_full_path == path
+          end
+
+          def matches_pattern?(pattern, pipeline)
+            return true if pipeline.tag? && pattern == 'tags'
+            return true if pipeline.branch? && pattern == 'branches'
+            return true if pipeline.source == pattern
+            return true if pipeline.source&.pluralize == pattern
+
+            if pattern.first == "/" && pattern.last == "/"
+              Regexp.new(pattern[1...-1]) =~ pipeline.ref
+            else
+              pattern == pipeline.ref
+            end
+          end
+        end
+      end
+    end
+  end
+end

lib/gitlab/ci/build/policy/specification.rb

+module Gitlab
+  module Ci
+    module Build
+      module Policy
+        ##
+        # Abstract class that defines an interface of job policy
+        # specification.
+        #
+        # Used for job's only/except policy configuration.
+        #
+        class Specification
+          UnknownPolicyError = Class.new(StandardError)
+
+          def initialize(spec)
+            @spec = spec
+          end
+
+          def satisfied_by?(pipeline)
+            raise NotImplementedError
+          end
+        end
+      end
+    end
+  end
+end

lib/gitlab/ci/yaml_processor.rb

@@ -5,12 +5,11 @@ module Gitlab
 
       include Gitlab::Ci::Config::Entry::LegacyValidationHelpers
 
-      attr_reader :path, :cache, :stages, :jobs
+      attr_reader :cache, :stages, :jobs
 
-      def initialize(config, path = nil)
+      def initialize(config)
         @ci_config = Gitlab::Ci::Config.new(config)
         @config = @ci_config.to_hash
-        @path = path
 
         unless @ci_config.valid?
           raise ValidationError, @ci_config.errors.first
@@ -21,28 +20,12 @@ module Gitlab
         raise ValidationError, e.message
       end
 
-      def builds_for_stage_and_ref(stage, ref, tag = false, source = nil)
-        jobs_for_stage_and_ref(stage, ref, tag, source).map do |name, _|
-          build_attributes(name)
-        end
-      end
-
       def builds
         @jobs.map do |name, _|
           build_attributes(name)
         end
       end
 
-      def stage_seeds(pipeline)
-        seeds = @stages.uniq.map do |stage|
-          builds = pipeline_stage_builds(stage, pipeline)
-
-          Gitlab::Ci::Stage::Seed.new(pipeline, stage, builds) if builds.any?
-        end
-
-        seeds.compact
-      end
-
       def build_attributes(name)
         job = @jobs[name.to_sym] || {}
 
@@ -70,6 +53,32 @@ module Gitlab
           }.compact }
       end
 
+      def pipeline_stage_builds(stage, pipeline)
+        selected_jobs = @jobs.select do |_, job|
+          next unless job[:stage] == stage
+
+          only_specs = Gitlab::Ci::Build::Policy
+            .fabricate(job.fetch(:only, {}))
+          except_specs = Gitlab::Ci::Build::Policy
+            .fabricate(job.fetch(:except, {}))
+
+          only_specs.all? { |spec| spec.satisfied_by?(pipeline) } &&
+            except_specs.none? { |spec| spec.satisfied_by?(pipeline) }
+        end
+
+        selected_jobs.map { |_, job| build_attributes(job[:name]) }
+      end
+
+      def stage_seeds(pipeline)
+        seeds = @stages.uniq.map do |stage|
+          builds = pipeline_stage_builds(stage, pipeline)
+
+          Gitlab::Ci::Stage::Seed.new(pipeline, stage, builds) if builds.any?
+        end
+
+        seeds.compact
+      end
+
       def self.validation_message(content)
         return 'Please provide content of .gitlab-ci.yml' if content.blank?
 
@@ -83,34 +92,6 @@ module Gitlab
 
       private
 
-      def pipeline_stage_builds(stage, pipeline)
-        builds = builds_for_stage_and_ref(
-          stage, pipeline.ref, pipeline.tag?, pipeline.source)
-
-        builds.select do |build|
-          job = @jobs[build.fetch(:name).to_sym]
-          has_kubernetes = pipeline.has_kubernetes_active?
-          only_kubernetes = job.dig(:only, :kubernetes)
-          except_kubernetes = job.dig(:except, :kubernetes)
-
-          [!only_kubernetes && !except_kubernetes,
-           only_kubernetes && has_kubernetes,
-           except_kubernetes && !has_kubernetes].any?
-        end
-      end
-
-      def jobs_for_ref(ref, tag = false, source = nil)
-        @jobs.select do |_, job|
-          process?(job.dig(:only, :refs), job.dig(:except, :refs), ref, tag, source)
-        end
-      end
-
-      def jobs_for_stage_and_ref(stage, ref, tag = false, source = nil)
-        jobs_for_ref(ref, tag, source).select do |_, job|
-          job[:stage] == stage
-        end
-      end
-
       def initial_parsing
         ##
         # Global config
@@ -203,51 +184,6 @@ module Gitlab
           raise ValidationError, "#{name} job: on_stop job #{on_stop} needs to have action stop defined"
         end
       end
-
-      def process?(only_params, except_params, ref, tag, source)
-        if only_params.present?
-          return false unless matching?(only_params, ref, tag, source)
-        end
-
-        if except_params.present?
-          return false if matching?(except_params, ref, tag, source)
-        end
-
-        true
-      end
-
-      def matching?(patterns, ref, tag, source)
-        patterns.any? do |pattern|
-          pattern, path = pattern.split('@', 2)
-          matches_path?(path) && matches_pattern?(pattern, ref, tag, source)
-        end
-      end
-
-      def matches_path?(path)
-        return true unless path
-
-        path == self.path
-      end
-
-      def matches_pattern?(pattern, ref, tag, source)
-        return true if tag && pattern == 'tags'
-        return true if !tag && pattern == 'branches'
-        return true if source_to_pattern(source) == pattern
-
-        if pattern.first == "/" && pattern.last == "/"
-          Regexp.new(pattern[1...-1]) =~ ref
-        else
-          pattern == ref
-        end
-      end
-
-      def source_to_pattern(source)
-        if %w[api external web].include?(source)
-          source
-        else
-          source&.pluralize
-        end
-      end
     end
   end
 end

spec/lib/gitlab/ci/build/policy/kubernetes_spec.rb

+require 'spec_helper'
+
+describe Gitlab::Ci::Build::Policy::Kubernetes do
+  let(:pipeline) { create(:ci_pipeline, project: project) }
+
+  context 'when kubernetes service is active' do
+    set(:project) { create(:kubernetes_project) }
+
+    it 'is satisfied by a kubernetes pipeline' do
+      expect(described_class.new('active'))
+        .to be_satisfied_by(pipeline)
+    end
+  end
+
+  context 'when kubernetes service is inactive' do
+    set(:project) { create(:project) }
+
+    it 'is not satisfied by a pipeline without kubernetes available' do
+      expect(described_class.new('active'))
+        .not_to be_satisfied_by(pipeline)
+    end
+  end
+
+  context 'when kubernetes policy is invalid' do
+    it 'raises an error' do
+      expect { described_class.new('unknown') }
+        .to raise_error(described_class::UnknownPolicyError)
+    end
+  end
+end

spec/lib/gitlab/ci/build/policy/refs_spec.rb

+require 'spec_helper'
+
+describe Gitlab::Ci::Build::Policy::Refs do
+  describe '#satisfied_by?' do
+    context 'when matching ref' do
+      let(:pipeline) { build_stubbed(:ci_pipeline, ref: 'master') }
+
+      it 'is satisfied when pipeline branch matches' do
+        expect(described_class.new(%w[master deploy]))
+          .to be_satisfied_by(pipeline)
+      end
+
+      it 'is not satisfied when pipeline branch does not match' do
+        expect(described_class.new(%w[feature fix]))
+          .not_to be_satisfied_by(pipeline)
+      end
+    end
+
+    context 'when maching tags' do
+      context 'when pipeline runs for a tag' do
+        let(:pipeline) do
+          build_stubbed(:ci_pipeline, ref: 'feature', tag: true)
+        end
+
+        it 'is satisfied when tags matcher is specified' do
+          expect(described_class.new(%w[master tags]))
+            .to be_satisfied_by(pipeline)
+        end
+      end
+
+      context 'when pipeline is not created for a tag' do
+        let(:pipeline) do
+          build_stubbed(:ci_pipeline, ref: 'feature', tag: false)
+        end
+
+        it 'is not satisfied when tag match is specified' do
+          expect(described_class.new(%w[master tags]))
+            .not_to be_satisfied_by(pipeline)
+        end
+      end
+    end
+
+    context 'when also matching a path' do
+      let(:pipeline) do
+        build_stubbed(:ci_pipeline, ref: 'master')
+      end
+
+      it 'is satisfied when provided patch matches specified one' do
+        expect(described_class.new(%W[master@#{pipeline.project_full_path}]))
+          .to be_satisfied_by(pipeline)
+      end
+
+      it 'is not satisfied when path differs' do
+        expect(described_class.new(%w[master@some/fork/repository]))
+          .not_to be_satisfied_by(pipeline)
+      end
+    end
+
+    context 'when maching a source' do
+      let(:pipeline) { build_stubbed(:ci_pipeline, source: :push) }
+
+      it 'is satisifed when provided source keyword matches' do
+        expect(described_class.new(%w[pushes]))
+          .to be_satisfied_by(pipeline)
+      end
+
+      it 'is not satisfied when provided source keyword does not match' do
+        expect(described_class.new(%w[triggers]))
+          .not_to be_satisfied_by(pipeline)
+      end
+    end
+
+    context 'when matching a ref by a regular expression' do
+      let(:pipeline) { build_stubbed(:ci_pipeline, ref: 'docs-something') }
+
+      it 'is satisfied when regexp matches pipeline ref' do
+        expect(described_class.new(['/docs-.*/']))
+          .to be_satisfied_by(pipeline)
+      end
+
+      it 'is not satisfied when regexp does not match pipeline ref' do
+        expect(described_class.new(['/fix-.*/']))
+          .not_to be_satisfied_by(pipeline)
+      end
+    end
+  end
+end

spec/lib/gitlab/ci/build/policy_spec.rb

+require 'spec_helper'
+
+describe Gitlab::Ci::Build::Policy do
+  let(:policy) { spy('policy specification') }
+
+  before do
+    stub_const("#{described_class}::Something", policy)
+  end
+
+  describe '.fabricate' do
+    context 'when policy exists' do
+      it 'fabricates and initializes relevant policy' do
+        specs = described_class.fabricate(something: 'some value')
+
+        expect(specs).to be_an Array
+        expect(specs).to be_one
+        expect(policy).to have_received(:new).with('some value')
+      end
+    end
+
+    context 'when some policies are not defined' do
+      it 'gracefully skips unknown policies' do
+        expect { described_class.fabricate(unknown: 'first') }
+          .to raise_error(NameError)
+      end
+    end
+
+    context 'when passing a nil value as specs' do
+      it 'returns an empty array' do
+        specs = described_class.fabricate(nil)
+
+        expect(specs).to be_an Array
+        expect(specs).to be_empty
+      end
+    end
+  end
+end

spec/models/ci/pipeline_spec.rb

@@ -26,6 +26,7 @@ describe Ci::Pipeline, :mailer do
   it { is_expected.to respond_to :git_author_name }
   it { is_expected.to respond_to :git_author_email }
   it { is_expected.to respond_to :short_sha }
+  it { is_expected.to delegate_method(:full_path).to(:project).with_prefix }
 
   describe '#source' do
     context 'when creating new pipeline' do