Merge branch 'control-dag-needs-limit-with-plan' into 'master'

Introduce `ci_needs_size_limit` to fine control DAG needs

See merge request gitlab-org/gitlab!37568

changelogs/unreleased/control-dag-needs-limit-with-plan.yml

+---
+title: Introduce `ci_needs_size_limit` to fine control needs
+merge_request: 37568
+author:
+type: changed

db/migrate/20200722075411_add_ci_needs_size_limit_to_plan_limit.rb

+# frozen_string_literal: true
+
+class AddCiNeedsSizeLimitToPlanLimit < ActiveRecord::Migration[6.0]
+  DOWNTIME = false
+
+  def change
+    add_column :plan_limits, :ci_needs_size_limit, :integer, default: 50, null: false
+  end
+end

db/schema_migrations/20200722075411

+5b8756999308abb47823c075128c07d62b899d6a39dff23242db6b12f18f239a
\ No newline at end of file

db/structure.sql

@@ -14029,7 +14029,8 @@ CREATE TABLE public.plan_limits (
     ci_max_artifact_size_requirements integer DEFAULT 0 NOT NULL,
     ci_max_artifact_size_coverage_fuzzing integer DEFAULT 0 NOT NULL,
     ci_max_artifact_size_browser_performance integer DEFAULT 0 NOT NULL,
-    ci_max_artifact_size_load_performance integer DEFAULT 0 NOT NULL
+    ci_max_artifact_size_load_performance integer DEFAULT 0 NOT NULL,
+    ci_needs_size_limit integer DEFAULT 50 NOT NULL
 );
 
 CREATE SEQUENCE public.plan_limits_id_seq

doc/ci/yaml/README.md

@@ -1977,8 +1977,8 @@ This example creates four paths of execution:
   - For GitLab.com, the limit is ten. For more information, see our
     [infrastructure issue](https://gitlab.com/gitlab-com/gl-infra/infrastructure/-/issues/7541).
   - For self-managed instances, the limit is:
-    - 10, if the `ci_dag_limit_needs` feature flag is enabled (default).
-    - 50, if the `ci_dag_limit_needs` feature flag is disabled.
+    - 10, if the `ci_plan_needs_size_limit` feature flag is disabled (default).
+    - 50, if the `ci_plan_needs_size_limit` feature flag is enabled. This limit [can be changed](#changing-the-needs-job-limit).
 - If `needs:` refers to a job that is marked as `parallel:`.
   the current job will depend on all parallel jobs created.
 - `needs:` is similar to `dependencies:` in that it needs to use jobs from prior stages,
@@ -1989,21 +1989,26 @@ This example creates four paths of execution:
 
 ##### Changing the `needs:` job limit
 
-The maximum number of jobs that can be defined within `needs:` defaults to 10, but
-can be changed to 50 via a feature flag. To change the limit to 50,
-[start a Rails console session](../../administration/troubleshooting/debug.md#starting-a-rails-console-session)
-and run:
+The maximum number of jobs that can be defined within `needs:` defaults to 10.
+
+To change this limit to 50 on a self-managed installation, a GitLab administrator
+with [access to the GitLab Rails console](../../administration/feature_flags.md)
+can enable the `:ci_plan_needs_size_limit` feature flag:
 
 ```ruby
-Feature::disable(:ci_dag_limit_needs)
+Feature::enable(:ci_plan_needs_size_limit)
 ```
 
-To set it back to 10, run the opposite command:
+After the feature flag is enabled, you can choose a custom limit. For example, to
+set the limit to 100:
 
 ```ruby
-Feature::enable(:ci_dag_limit_needs)
+Plan.default.actual_limits.update!(ci_needs_size_limit: 100)
 ```
 
+NOTE: **Note:**
+To disable the ability to use DAG, set the limit to `0`.
+
 #### Artifact downloads with `needs`
 
 > [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/14311) in GitLab v12.6.

lib/gitlab/ci/features.rb

@@ -74,6 +74,10 @@ module Gitlab
         ::Feature.enabled?(:ci_allow_to_create_merge_request_pipelines_in_target_project, target_project)
       end
 
+      def self.ci_plan_needs_size_limit?(project)
+        ::Feature.enabled?(:ci_plan_needs_size_limit, project)
+      end
+
       def self.job_entry_matches_all_keys?
         ::Feature.enabled?(:ci_job_entry_matches_all_keys)
       end

lib/gitlab/ci/pipeline/seed/build.rb

@@ -11,9 +11,7 @@ module Gitlab
 
           delegate :dig, to: :@seed_attributes
 
-          # When the `ci_dag_limit_needs` is enabled it uses the lower limit
-          LOW_NEEDS_LIMIT = 10
-          HARD_NEEDS_LIMIT = 50
+          DEFAULT_NEEDS_LIMIT = 10
 
           def initialize(pipeline, attributes, previous_stages)
             @pipeline = pipeline
@@ -142,10 +140,10 @@ module Gitlab
           end
 
           def max_needs_allowed
-            if Feature.enabled?(:ci_dag_limit_needs, @project, default_enabled: true)
-              LOW_NEEDS_LIMIT
+            if ::Gitlab::Ci::Features.ci_plan_needs_size_limit?(@pipeline.project)
+              @pipeline.project.actual_limits.ci_needs_size_limit
             else
-              HARD_NEEDS_LIMIT
+              DEFAULT_NEEDS_LIMIT
             end
           end
 

spec/lib/gitlab/ci/pipeline/seed/build_spec.rb

@@ -928,29 +928,51 @@ RSpec.describe Gitlab::Ci::Pipeline::Seed::Build do
       end
     end
 
-    context 'when lower limit of needs is reached' do
-      before do
-        stub_feature_flags(ci_dag_limit_needs: true)
-      end
+    context 'when using 101 needs' do
+      let(:needs_count) { 101 }
 
-      let(:needs_count) { described_class::LOW_NEEDS_LIMIT + 1 }
+      context 'when ci_plan_needs_size_limit is disabled' do
+        before do
+          stub_feature_flags(ci_plan_needs_size_limit: false)
+        end
 
-      it "returns an error" do
-        expect(subject.errors).to contain_exactly(
-          "rspec: one job can only need 10 others, but you have listed 11. See needs keyword documentation for more details")
+        it "returns an error" do
+          expect(subject.errors).to contain_exactly(
+            "rspec: one job can only need 10 others, but you have listed 101. See needs keyword documentation for more details")
+        end
       end
-    end
 
-    context 'when upper limit of needs is reached' do
-      before do
-        stub_feature_flags(ci_dag_limit_needs: false)
-      end
+      context 'when ci_plan_needs_size_limit is enabled' do
+        before do
+          stub_feature_flags(ci_plan_needs_size_limit: true)
+        end
 
-      let(:needs_count) { described_class::HARD_NEEDS_LIMIT + 1 }
+        it "returns an error" do
+          expect(subject.errors).to contain_exactly(
+            "rspec: one job can only need 50 others, but you have listed 101. See needs keyword documentation for more details")
+        end
 
-      it "returns an error" do
-        expect(subject.errors).to contain_exactly(
-          "rspec: one job can only need 50 others, but you have listed 51. See needs keyword documentation for more details")
+        context 'when ci_needs_size_limit is set to 100' do
+          before do
+            project.actual_limits.update!(ci_needs_size_limit: 100)
+          end
+
+          it "returns an error" do
+            expect(subject.errors).to contain_exactly(
+              "rspec: one job can only need 100 others, but you have listed 101. See needs keyword documentation for more details")
+          end
+        end
+
+        context 'when ci_needs_size_limit is set to 0' do
+          before do
+            project.actual_limits.update!(ci_needs_size_limit: 0)
+          end
+
+          it "returns an error" do
+            expect(subject.errors).to contain_exactly(
+              "rspec: one job can only need 0 others, but you have listed 101. See needs keyword documentation for more details")
+          end
+        end
       end
     end
   end