Automatic merge of gitlab-org/gitlab master

GITALY_SERVER_VERSION

-4ea88e921af65ba0577c40f8b54830c97adaa56c
+8752fc1097114316889560502579ddf477a20a2d

app/finders/concerns/packages/finder_helper.rb

@@ -13,7 +13,7 @@ module Packages
       return ::Packages::Package.none unless within_group
       return ::Packages::Package.none unless Ability.allowed?(user, :read_group, within_group)
 
-      projects = projects_visible_to_reporters(user, within_group.self_and_descendants.select(:id))
+      projects = projects_visible_to_reporters(user, within_group: within_group)
       ::Packages::Package.for_projects(projects.select(:id))
     end
 
@@ -21,13 +21,17 @@ module Packages
       return ::Project.none unless within_group
       return ::Project.none unless Ability.allowed?(user, :read_group, within_group)
 
-      projects_visible_to_reporters(user, within_group.self_and_descendants.select(:id))
+      projects_visible_to_reporters(user, within_group: within_group)
     end
 
-    def projects_visible_to_reporters(user, namespace_ids)
-      ::Project.in_namespace(namespace_ids)
+    def projects_visible_to_reporters(user, within_group:)
+      if user.is_a?(DeployToken) && Feature.enabled?(:packages_finder_helper_deploy_token)
+        user.accessible_projects
+      else
+        within_group.all_projects
                     .public_or_visible_to_user(user, ::Gitlab::Access::REPORTER)
       end
+    end
 
     def package_type
       params[:package_type].presence

app/finders/packages/maven/package_finder.rb

@@ -76,7 +76,7 @@ module Packages
 
       def group_level_improvements?
         strong_memoize(:group_level_improvements) do
-          Feature.enabled?(:maven_packages_group_level_improvements)
+          Feature.enabled?(:maven_packages_group_level_improvements, default_enabled: :yaml)
         end
       end
     end

app/models/packages/package.rb

@@ -138,7 +138,7 @@ class Packages::Package < ApplicationRecord
   after_commit :update_composer_cache, on: :destroy, if: -> { composer? }
 
   def self.for_projects(projects)
-    unless Feature.enabled?(:maven_packages_group_level_improvements)
+    unless Feature.enabled?(:maven_packages_group_level_improvements, default_enabled: :yaml)
       return none unless projects.any?
     end
 

app/services/merge_requests/base_service.rb

@@ -143,9 +143,13 @@ module MergeRequests
         merge_request, merge_request.project, current_user, old_reviewers)
     end
 
-    def create_pipeline_for(merge_request, user)
+    def create_pipeline_for(merge_request, user, async: false)
+      if async
+        MergeRequests::CreatePipelineWorker.perform_async(project.id, user.id, merge_request.id)
+      else
         MergeRequests::CreatePipelineService.new(project, user).execute(merge_request)
       end
+    end
 
     def abort_auto_merge(merge_request, reason)
       AutoMergeService.new(project, current_user).abort(merge_request, reason)

app/services/merge_requests/refresh_service.rb

@@ -162,10 +162,13 @@ module MergeRequests
     end
 
     def refresh_pipelines_on_merge_requests(merge_request)
-      create_pipeline_for(merge_request, current_user)
-
+      if Feature.enabled?(:code_review_async_pipeline_creation, project, default_enabled: :yaml)
+        create_pipeline_for(merge_request, current_user, async: true)
+      else
+        create_pipeline_for(merge_request, current_user, async: false)
         UpdateHeadPipelineForMergeRequestWorker.perform_async(merge_request.id)
       end
+    end
 
     def abort_auto_merges(merge_request)
       abort_auto_merge(merge_request, 'source branch was updated')

app/workers/all_queues.yml

@@ -1227,6 +1227,14 @@
   :weight: 4
   :idempotent:
   :tags: []
+- :name: pipeline_creation:merge_requests_create_pipeline
+  :feature_category: :continuous_integration
+  :has_external_dependencies:
+  :urgency: :high
+  :resource_boundary: :cpu
+  :weight: 4
+  :idempotent: true
+  :tags: []
 - :name: pipeline_creation:run_pipeline_schedule
   :feature_category: :continuous_integration
   :has_external_dependencies:

app/workers/merge_requests/create_pipeline_worker.rb

+# frozen_string_literal: true
+
+module MergeRequests
+  class CreatePipelineWorker
+    include ApplicationWorker
+    include PipelineQueue
+
+    queue_namespace :pipeline_creation
+    feature_category :continuous_integration
+    urgency :high
+    worker_resource_boundary :cpu
+    idempotent!
+
+    def perform(project_id, user_id, merge_request_id)
+      project = Project.find_by_id(project_id)
+      return unless project
+
+      user = User.find_by_id(user_id)
+      return unless user
+
+      merge_request = MergeRequest.find_by_id(merge_request_id)
+      return unless merge_request
+
+      MergeRequests::CreatePipelineService.new(project, user).execute(merge_request)
+      merge_request.update_head_pipeline
+    end
+  end
+end

changelogs/unreleased/220680-update-gatsby-project-template.yml

+---
+title: Update gatsby project template to address the pipeline failure
+merge_request: 37410
+author: Takuya Noguchi
+type: fixed

changelogs/unreleased/326099-enabled-by-default.yml

+---
+title: Reduce the number of SQL queries executed on Maven file API endpoints
+merge_request: 59136
+author:
+type: performance

changelogs/unreleased/move_pipeline_creation_async.yml

+---
+title: Create the pipelines asynchronously when refreshing merge requests
+merge_request: 58542
+author:
+type: performance

config/feature_flags/development/code_review_async_pipeline_creation.yml

+---
+name: code_review_async_pipeline_creation
+introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/58542
+rollout_issue_url: https://gitlab.com/gitlab-org/gitlab/-/issues/327559
+milestone: '13.11'
+type: development
+group: group::code review
+default_enabled: false

config/feature_flags/development/maven_packages_group_level_improvements.yml

@@ -5,4 +5,4 @@ rollout_issue_url: https://gitlab.com/gitlab-org/gitlab/-/issues/326099
 milestone: '13.11'
 type: development
 group: group::package
-default_enabled: false
+default_enabled: true

config/feature_flags/development/packages_finder_helper_deploy_token.yml

+---
+name: packages_finder_helper_deploy_token
+introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/58497
+rollout_issue_url: https://gitlab.com/gitlab-org/gitlab/-/issues/326808
+milestone: '13.11'
+type: development
+group: group::package
+default_enabled: false

ee/app/models/project_services/ee/jira_service.rb

@@ -89,6 +89,8 @@ module EE
     #
     # @return [Array] the array of IDs
     def project_issuetype_scheme_ids
+      raise NotImplementedError unless data_fields.deployment_cloud?
+
       query_url = Addressable::URI.join("#{client.options[:rest_base_path]}/", 'issuetypescheme/', 'project')
       query_url.query_values = { 'projectId' => jira_project_id }
 
@@ -103,6 +105,14 @@ module EE
     # @return [Array] the array of IDs
     def project_issuetype_ids
       strong_memoize(:project_issuetype_ids) do
+        if data_fields.deployment_server?
+          query_url = Addressable::URI.join("#{client.options[:rest_base_path]}/", 'project/', project_key)
+
+          client
+            .get(query_url.to_s)
+            .fetch('issueTypes', [])
+            .map { |issue_type| issue_type['id'] }
+        elsif data_fields.deployment_cloud?
           query_url = Addressable::URI.join("#{client.options[:rest_base_path]}/", 'issuetypescheme/', 'mapping')
           query_url.query_values = { 'issueTypeSchemeId' => project_issuetype_scheme_ids }
 
@@ -110,6 +120,9 @@ module EE
             .get(query_url.to_s)
             .fetch('values', [])
             .map { |schemes| schemes['issueTypeId'] }
+        else
+          raise NotImplementedError
+        end
       end
     end
 

ee/app/views/namespaces/pipelines_quota/_extra_shared_runners_minutes_quota.html.haml

 - return unless Gitlab.com?
 
 - minutes_quota = namespace.ci_minutes_quota
-- return unless minutes_quota.enabled?
+- return unless minutes_quota.enabled? && minutes_quota.purchased_minutes_report.limit > 0
 
 .row
   .col-sm-6

ee/changelogs/unreleased/327458-ci-minutes-ee.yml

+---
+title: Remove additional minutes when no minutes purchased
+merge_request: 59112
+author: Takuya Noguchi
+type: changed

ee/spec/models/project_services/jira_service_spec.rb

@@ -15,6 +15,11 @@ RSpec.describe JiraService do
     }
   end
 
+  before do
+    allow(jira_service.data_fields).to receive(:deployment_cloud?).and_return(true)
+    allow(jira_service.data_fields).to receive(:deployment_server?).and_return(false)
+  end
+
   describe 'validations' do
     it 'validates presence of project_key if issues_enabled' do
       jira_service.project_key = ''
@@ -111,6 +116,11 @@ RSpec.describe JiraService do
       end
 
       context 'when vulnerabilities integration is enabled' do
+        before do
+          allow(jira_service.project).to receive(:jira_vulnerabilities_integration_enabled?).and_return(true)
+        end
+
+        context 'when deployment type is cloud' do
           let(:project_info_result) { { 'id' => '10000' } }
 
           let(:issue_type_scheme_response) do
@@ -175,8 +185,6 @@ RSpec.describe JiraService do
           end
 
           before do
-          allow(jira_service.project).to receive(:jira_vulnerabilities_integration_enabled?).and_return(true)
-
             WebMock.stub_request(:get, /api\/2\/project\/GL/).with(basic_auth: %w(gitlab_jira_username gitlab_jira_password)).to_return(body: project_info_result.to_json )
             WebMock.stub_request(:get, /api\/2\/project\/GL\z/).with(basic_auth: %w(gitlab_jira_username gitlab_jira_password)).to_return(body: { 'id' => '10000' }.to_json, headers: headers)
             WebMock.stub_request(:get, /api\/2\/issuetype\z/).to_return(body: issue_types_response.to_json, headers: headers)
@@ -186,6 +194,73 @@ RSpec.describe JiraService do
 
           it { is_expected.to eq(success: true, result: { jira: true }, data: { issuetypes: [{ id: '10001', name: 'Bug', description: 'Jira Bug' }] }) }
         end
+
+        context 'when deployment type is server' do
+          let(:project_info_result) do
+            {
+              "id": "10000",
+              "issueTypes": issue_types_response
+            }
+          end
+
+          let(:issue_types_response) do
+            [
+              {
+                "avatarId": 10318,
+                "description": "A task that needs to be done.",
+                "iconUrl": "http://jira.reali.sh:8080/secure/viewavatar?size=xsmall&avatarId=10318&avatarType=issuetype",
+                "id": "10003",
+                "name": "Task",
+                "self": "http://jira.reali.sh:8080/rest/api/2/issuetype/10003",
+                "subtask": false
+              },
+              {
+                "description": "The sub-task of the issue",
+                "iconUrl": "http://jira.reali.sh:8080/images/icons/issuetypes/subtask_alternate.png",
+                "id": "10000",
+                "name": "Sub-task",
+                "self": "http://jira.reali.sh:8080/rest/api/2/issuetype/10000",
+                "subtask": true
+              },
+              {
+                "description": "Created by Jira Software - do not edit or delete. Issue type for a user story.",
+                "iconUrl": "http://jira.reali.sh:8080/images/icons/issuetypes/story.svg",
+                "id": "10002",
+                "name": "Story",
+                "self": "http://jira.reali.sh:8080/rest/api/2/issuetype/10002",
+                "subtask": false
+              },
+              {
+                "avatarId": 10303,
+                "description": "A problem which impairs or prevents the functions of the product.",
+                "iconUrl": "http://jira.reali.sh:8080/secure/viewavatar?size=xsmall&avatarId=10303&avatarType=issuetype",
+                "id": "10004",
+                "name": "Bug",
+                "self": "http://jira.reali.sh:8080/rest/api/2/issuetype/10004",
+                "subtask": false
+              },
+              {
+                "description": "Created by Jira Software - do not edit or delete. Issue type for a big user story that needs to be broken down.",
+                "iconUrl": "http://jira.reali.sh:8080/images/icons/issuetypes/epic.svg",
+                "id": "10001",
+                "name": "Epic",
+                "self": "http://jira.reali.sh:8080/rest/api/2/issuetype/10001",
+                "subtask": false
+              }
+            ]
+          end
+
+          before do
+            allow(jira_service.data_fields).to receive(:deployment_cloud?).and_return(false)
+            allow(jira_service.data_fields).to receive(:deployment_server?).and_return(true)
+
+            WebMock.stub_request(:get, /api\/2\/project\/GL/).with(basic_auth: %w(gitlab_jira_username gitlab_jira_password)).to_return(body: project_info_result.to_json, headers: headers)
+            WebMock.stub_request(:get, /api\/2\/issuetype\z/).to_return(body: issue_types_response.to_json, headers: headers)
+          end
+
+          it { is_expected.to eq(success: true, result: { jira: true }, data: { issuetypes: [{ description: "A task that needs to be done.", id: "10003", name: "Task" }, { description: "Created by Jira Software - do not edit or delete. Issue type for a user story.", id: "10002", name: "Story" }, { description: "A problem which impairs or prevents the functions of the product.", id: "10004", name: "Bug" }, { description: "Created by Jira Software - do not edit or delete. Issue type for a big user story that needs to be broken down.", id: "10001", name: "Epic" }] }) }
+        end
+      end
     end
   end
 

ee/spec/services/ee/merge_requests/refresh_service_spec.rb

@@ -249,7 +249,7 @@ RSpec.describe MergeRequests::RefreshService do
       end
     end
 
-    describe 'Pipelines for merge requests' do
+    describe 'Pipelines for merge requests', :sidekiq_inline do
       let(:service) { described_class.new(project, current_user) }
       let(:current_user) { merge_request.author }
 

lib/api/maven_packages.rb

@@ -80,7 +80,7 @@ module API
 
       def fetch_package(file_name:, project: nil, group: nil)
         order_by_package_file = false
-        if Feature.enabled?(:maven_packages_group_level_improvements)
+        if Feature.enabled?(:maven_packages_group_level_improvements, default_enabled: :yaml)
           order_by_package_file = file_name.include?(::Packages::Maven::Metadata.filename) &&
                                     !params[:path].include?(::Packages::Maven::FindOrCreatePackageService::SNAPSHOT_TERM)
         end

spec/factories/services.rb

@@ -62,6 +62,7 @@ FactoryBot.define do
       project_key { nil }
       vulnerabilities_enabled { false }
       vulnerabilities_issuetype { nil }
+      deployment_type { 'cloud' }
     end
 
     before(:create) do |service, evaluator|
@@ -72,7 +73,7 @@ FactoryBot.define do
                jira_issue_transition_id: evaluator.jira_issue_transition_id,
                username: evaluator.username, password: evaluator.password, issues_enabled: evaluator.issues_enabled,
                project_key: evaluator.project_key, vulnerabilities_enabled: evaluator.vulnerabilities_enabled,
-               vulnerabilities_issuetype: evaluator.vulnerabilities_issuetype
+               vulnerabilities_issuetype: evaluator.vulnerabilities_issuetype, deployment_type: evaluator.deployment_type
         )
       end
     end

spec/finders/concerns/packages/finder_helper_spec.rb

@@ -6,7 +6,6 @@ RSpec.describe ::Packages::FinderHelper do
   describe '#packages_visible_to_user' do
     using RSpec::Parameterized::TableSyntax
 
-    let_it_be(:user) { create(:user) }
     let_it_be_with_reload(:group) { create(:group) }
     let_it_be_with_reload(:project1) { create(:project, namespace: group) }
     let_it_be(:package1) { create(:package, project: project1) }
@@ -44,6 +43,9 @@ RSpec.describe ::Packages::FinderHelper do
       it { is_expected.to be_empty }
     end
 
+    context 'with a user' do
+      let_it_be(:user) { create(:user) }
+
       where(:group_visibility, :subgroup_visibility, :project2_visibility, :user_role, :shared_example_name) do
         'PUBLIC'  | 'PUBLIC'  | 'PUBLIC'  | :maintainer | 'returning both packages'
         'PUBLIC'  | 'PUBLIC'  | 'PUBLIC'  | :developer  | 'returning both packages'
@@ -82,6 +84,49 @@ RSpec.describe ::Packages::FinderHelper do
       end
     end
 
+    context 'with a group deploy token' do
+      let_it_be(:user) { create(:deploy_token, :group, read_package_registry: true) }
+      let_it_be(:group_deploy_token) { create(:group_deploy_token, deploy_token: user, group: group) }
+
+      shared_examples 'handling all conditions' do
+        where(:group_visibility, :subgroup_visibility, :project2_visibility, :shared_example_name) do
+          'PUBLIC'  | 'PUBLIC'  | 'PUBLIC'  | 'returning both packages'
+          'PUBLIC'  | 'PUBLIC'  | 'PRIVATE' | 'returning both packages'
+          'PUBLIC'  | 'PRIVATE' | 'PRIVATE' | 'returning both packages'
+          'PRIVATE' | 'PRIVATE' | 'PRIVATE' | 'returning both packages'
+        end
+
+        with_them do
+          before do
+            project2.update!(visibility_level: Gitlab::VisibilityLevel.const_get(project2_visibility, false))
+            subgroup.update!(visibility_level: Gitlab::VisibilityLevel.const_get(subgroup_visibility, false))
+            project1.update!(visibility_level: Gitlab::VisibilityLevel.const_get(group_visibility, false))
+            group.update!(visibility_level: Gitlab::VisibilityLevel.const_get(group_visibility, false))
+          end
+
+          it_behaves_like params[:shared_example_name]
+        end
+      end
+
+      context 'with packages_finder_helper_deploy_token enabled' do
+        before do
+          expect(group).not_to receive(:all_projects)
+        end
+
+        it_behaves_like 'handling all conditions'
+      end
+
+      context 'with packages_finder_helper_deploy_token disabled' do
+        before do
+          stub_feature_flags(packages_finder_helper_deploy_token: false)
+          expect(group).to receive(:all_projects).and_call_original
+        end
+
+        it_behaves_like 'handling all conditions'
+      end
+    end
+  end
+
   describe '#projects_visible_to_user' do
     using RSpec::Parameterized::TableSyntax
 
@@ -121,6 +166,9 @@ RSpec.describe ::Packages::FinderHelper do
       it { is_expected.to be_empty }
     end
 
+    context 'with a user' do
+      let_it_be(:user) { create(:user) }
+
       where(:group_visibility, :subgroup_visibility, :project2_visibility, :user_role, :shared_example_name) do
         'PUBLIC'  | 'PUBLIC'  | 'PUBLIC'  | :maintainer | 'returning both projects'
         'PUBLIC'  | 'PUBLIC'  | 'PUBLIC'  | :developer  | 'returning both projects'
@@ -158,4 +206,47 @@ RSpec.describe ::Packages::FinderHelper do
         it_behaves_like params[:shared_example_name]
       end
     end
+
+    context 'with a group deploy token' do
+      let_it_be(:user) { create(:deploy_token, :group, read_package_registry: true) }
+      let_it_be(:group_deploy_token) { create(:group_deploy_token, deploy_token: user, group: group) }
+
+      shared_examples 'handling all conditions' do
+        where(:group_visibility, :subgroup_visibility, :project2_visibility, :shared_example_name) do
+          'PUBLIC'  | 'PUBLIC'  | 'PUBLIC'  | 'returning both projects'
+          'PUBLIC'  | 'PUBLIC'  | 'PRIVATE' | 'returning both projects'
+          'PUBLIC'  | 'PRIVATE' | 'PRIVATE' | 'returning both projects'
+          'PRIVATE' | 'PRIVATE' | 'PRIVATE' | 'returning both projects'
+        end
+
+        with_them do
+          before do
+            project2.update!(visibility_level: Gitlab::VisibilityLevel.const_get(project2_visibility, false))
+            subgroup.update!(visibility_level: Gitlab::VisibilityLevel.const_get(subgroup_visibility, false))
+            project1.update!(visibility_level: Gitlab::VisibilityLevel.const_get(group_visibility, false))
+            group.update!(visibility_level: Gitlab::VisibilityLevel.const_get(group_visibility, false))
+          end
+
+          it_behaves_like params[:shared_example_name]
+        end
+      end
+
+      context 'with packages_finder_helper_deploy_token enabled' do
+        before do
+          expect(group).not_to receive(:all_projects)
+        end
+
+        it_behaves_like 'handling all conditions'
+      end
+
+      context 'with packages_finder_helper_deploy_token disabled' do
+        before do
+          stub_feature_flags(packages_finder_helper_deploy_token: false)
+          expect(group).to receive(:all_projects).and_call_original
+        end
+
+        it_behaves_like 'handling all conditions'
+      end
+    end
+  end
 end

spec/requests/api/maven_packages_spec.rb

@@ -387,7 +387,7 @@ RSpec.describe API::MavenPackages do
 
           subject
 
-          status = Feature.enabled?(:maven_packages_group_level_improvements) ? :not_found : :forbidden
+          status = Feature.enabled?(:maven_packages_group_level_improvements, default_enabled: :yaml) ? :not_found : :forbidden
           expect(response).to have_gitlab_http_status(status)
         end
 

spec/services/merge_requests/refresh_service_spec.rb

@@ -198,7 +198,7 @@ RSpec.describe MergeRequests::RefreshService do
       end
     end
 
-    describe 'Pipelines for merge requests' do
+    shared_examples 'Pipelines for merge requests' do
       before do
         stub_ci_pipeline_yaml_file(config)
       end
@@ -256,7 +256,7 @@ RSpec.describe MergeRequests::RefreshService do
             stub_feature_flags(ci_disallow_to_create_merge_request_pipelines_in_target_project: false)
           end
 
-          it 'creates detached merge request pipeline for fork merge request', :sidekiq_inline do
+          it 'creates detached merge request pipeline for fork merge request' do
             expect { subject }
               .to change { @fork_merge_request.pipelines_for_merge_request.count }.by(1)
 
@@ -364,6 +364,18 @@ RSpec.describe MergeRequests::RefreshService do
       end
     end
 
+    context 'when the code_review_async_pipeline_creation feature flag is on', :sidekiq_inline do
+      it_behaves_like 'Pipelines for merge requests'
+    end
+
+    context 'when the code_review_async_pipeline_creation feature flag is off', :sidekiq_inline do
+      before do
+        stub_feature_flags(code_review_async_pipeline_creation: false)
+      end
+
+      it_behaves_like 'Pipelines for merge requests'
+    end
+
     context 'push to origin repo source branch' do
       let(:refresh_service) { service.new(@project, @user) }
       let(:notification_service) { spy('notification_service') }

spec/workers/merge_requests/create_pipeline_worker_spec.rb

+# frozen_string_literal: true
+
+require 'spec_helper'
+
+RSpec.describe MergeRequests::CreatePipelineWorker do
+  subject(:worker) { described_class.new }
+
+  describe '#perform' do
+    let(:user) { create(:user) }
+    let(:project) { create(:project) }
+    let(:merge_request) { create(:merge_request) }
+
+    context 'when the objects exist' do
+      it 'calls the merge request create pipeline service and calls update head pipeline' do
+        aggregate_failures do
+          expect_next_instance_of(MergeRequests::CreatePipelineService, project, user) do |service|
+            expect(service).to receive(:execute).with(merge_request)
+          end
+
+          expect(MergeRequest).to receive(:find_by_id).with(merge_request.id).and_return(merge_request)
+          expect(merge_request).to receive(:update_head_pipeline)
+
+          subject.perform(project.id, user.id, merge_request.id)
+        end
+      end
+    end
+
+    shared_examples 'when object does not exist' do
+      it 'does not call the create pipeline service' do
+        expect(MergeRequests::CreatePipelineService).not_to receive(:new)
+
+        expect { subject.perform(project.id, user.id, merge_request.id) }
+          .not_to raise_exception
+      end
+    end
+
+    context 'when the project does not exist' do
+      before do
+        project.destroy!
+      end
+
+      it_behaves_like 'when object does not exist'
+    end
+
+    context 'when the user does not exist' do
+      before do
+        user.destroy!
+      end
+
+      it_behaves_like 'when object does not exist'
+    end
+
+    context 'when the merge request does not exist' do
+      before do
+        merge_request.destroy!
+      end
+
+      it_behaves_like 'when object does not exist'
+    end
+  end
+end