Automatic merge of gitlab-org/gitlab master

GITALY_SERVER_VERSION

-4ea88e921af65ba0577c40f8b54830c97adaa56c
+8752fc1097114316889560502579ddf477a20a2d

app/finders/concerns/packages/finder_helper.rb

@@ -13,7 +13,7 @@ module Packages
       return ::Packages::Package.none unless within_group
       return ::Packages::Package.none unless Ability.allowed?(user, :read_group, within_group)
 
-      projects = projects_visible_to_reporters(user, within_group.self_and_descendants.select(:id))
+      projects = projects_visible_to_reporters(user, within_group: within_group)
       ::Packages::Package.for_projects(projects.select(:id))
     end
 
@@ -21,12 +21,16 @@ module Packages
       return ::Project.none unless within_group
       return ::Project.none unless Ability.allowed?(user, :read_group, within_group)
 
-      projects_visible_to_reporters(user, within_group.self_and_descendants.select(:id))
+      projects_visible_to_reporters(user, within_group: within_group)
     end
 
-    def projects_visible_to_reporters(user, namespace_ids)
-      ::Project.in_namespace(namespace_ids)
-               .public_or_visible_to_user(user, ::Gitlab::Access::REPORTER)
+    def projects_visible_to_reporters(user, within_group:)
+      if user.is_a?(DeployToken) && Feature.enabled?(:packages_finder_helper_deploy_token)
+        user.accessible_projects
+      else
+        within_group.all_projects
+                    .public_or_visible_to_user(user, ::Gitlab::Access::REPORTER)
+      end
     end
 
     def package_type

app/finders/packages/maven/package_finder.rb

@@ -76,7 +76,7 @@ module Packages
 
       def group_level_improvements?
         strong_memoize(:group_level_improvements) do
-          Feature.enabled?(:maven_packages_group_level_improvements)
+          Feature.enabled?(:maven_packages_group_level_improvements, default_enabled: :yaml)
         end
       end
     end

app/models/packages/package.rb

@@ -138,7 +138,7 @@ class Packages::Package < ApplicationRecord
   after_commit :update_composer_cache, on: :destroy, if: -> { composer? }
 
   def self.for_projects(projects)
-    unless Feature.enabled?(:maven_packages_group_level_improvements)
+    unless Feature.enabled?(:maven_packages_group_level_improvements, default_enabled: :yaml)
       return none unless projects.any?
     end
 

app/services/merge_requests/base_service.rb

@@ -143,8 +143,12 @@ module MergeRequests
         merge_request, merge_request.project, current_user, old_reviewers)
     end
 
-    def create_pipeline_for(merge_request, user)
-      MergeRequests::CreatePipelineService.new(project, user).execute(merge_request)
+    def create_pipeline_for(merge_request, user, async: false)
+      if async
+        MergeRequests::CreatePipelineWorker.perform_async(project.id, user.id, merge_request.id)
+      else
+        MergeRequests::CreatePipelineService.new(project, user).execute(merge_request)
+      end
     end
 
     def abort_auto_merge(merge_request, reason)

app/services/merge_requests/refresh_service.rb

@@ -162,9 +162,12 @@ module MergeRequests
     end
 
     def refresh_pipelines_on_merge_requests(merge_request)
-      create_pipeline_for(merge_request, current_user)
-
-      UpdateHeadPipelineForMergeRequestWorker.perform_async(merge_request.id)
+      if Feature.enabled?(:code_review_async_pipeline_creation, project, default_enabled: :yaml)
+        create_pipeline_for(merge_request, current_user, async: true)
+      else
+        create_pipeline_for(merge_request, current_user, async: false)
+        UpdateHeadPipelineForMergeRequestWorker.perform_async(merge_request.id)
+      end
     end
 
     def abort_auto_merges(merge_request)

app/workers/all_queues.yml

@@ -1227,6 +1227,14 @@
   :weight: 4
   :idempotent:
   :tags: []
+- :name: pipeline_creation:merge_requests_create_pipeline
+  :feature_category: :continuous_integration
+  :has_external_dependencies:
+  :urgency: :high
+  :resource_boundary: :cpu
+  :weight: 4
+  :idempotent: true
+  :tags: []
 - :name: pipeline_creation:run_pipeline_schedule
   :feature_category: :continuous_integration
   :has_external_dependencies:

app/workers/merge_requests/create_pipeline_worker.rb

+# frozen_string_literal: true
+
+module MergeRequests
+  class CreatePipelineWorker
+    include ApplicationWorker
+    include PipelineQueue
+
+    queue_namespace :pipeline_creation
+    feature_category :continuous_integration
+    urgency :high
+    worker_resource_boundary :cpu
+    idempotent!
+
+    def perform(project_id, user_id, merge_request_id)
+      project = Project.find_by_id(project_id)
+      return unless project
+
+      user = User.find_by_id(user_id)
+      return unless user
+
+      merge_request = MergeRequest.find_by_id(merge_request_id)
+      return unless merge_request
+
+      MergeRequests::CreatePipelineService.new(project, user).execute(merge_request)
+      merge_request.update_head_pipeline
+    end
+  end
+end

changelogs/unreleased/220680-update-gatsby-project-template.yml

+---
+title: Update gatsby project template to address the pipeline failure
+merge_request: 37410
+author: Takuya Noguchi
+type: fixed

changelogs/unreleased/326099-enabled-by-default.yml

+---
+title: Reduce the number of SQL queries executed on Maven file API endpoints
+merge_request: 59136
+author:
+type: performance

changelogs/unreleased/move_pipeline_creation_async.yml

+---
+title: Create the pipelines asynchronously when refreshing merge requests
+merge_request: 58542
+author:
+type: performance

config/feature_flags/development/code_review_async_pipeline_creation.yml

+---
+name: code_review_async_pipeline_creation
+introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/58542
+rollout_issue_url: https://gitlab.com/gitlab-org/gitlab/-/issues/327559
+milestone: '13.11'
+type: development
+group: group::code review
+default_enabled: false

config/feature_flags/development/maven_packages_group_level_improvements.yml

@@ -5,4 +5,4 @@ rollout_issue_url: https://gitlab.com/gitlab-org/gitlab/-/issues/326099
 milestone: '13.11'
 type: development
 group: group::package
-default_enabled: false
+default_enabled: true

config/feature_flags/development/packages_finder_helper_deploy_token.yml

+---
+name: packages_finder_helper_deploy_token
+introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/58497
+rollout_issue_url: https://gitlab.com/gitlab-org/gitlab/-/issues/326808
+milestone: '13.11'
+type: development
+group: group::package
+default_enabled: false

ee/app/models/project_services/ee/jira_service.rb

@@ -89,6 +89,8 @@ module EE
     #
     # @return [Array] the array of IDs
     def project_issuetype_scheme_ids
+      raise NotImplementedError unless data_fields.deployment_cloud?
+
       query_url = Addressable::URI.join("#{client.options[:rest_base_path]}/", 'issuetypescheme/', 'project')
       query_url.query_values = { 'projectId' => jira_project_id }
 
@@ -103,13 +105,24 @@ module EE
     # @return [Array] the array of IDs
     def project_issuetype_ids
       strong_memoize(:project_issuetype_ids) do
-        query_url = Addressable::URI.join("#{client.options[:rest_base_path]}/", 'issuetypescheme/', 'mapping')
-        query_url.query_values = { 'issueTypeSchemeId' => project_issuetype_scheme_ids }
-
-        client
-          .get(query_url.to_s)
-          .fetch('values', [])
-          .map { |schemes| schemes['issueTypeId'] }
+        if data_fields.deployment_server?
+          query_url = Addressable::URI.join("#{client.options[:rest_base_path]}/", 'project/', project_key)
+
+          client
+            .get(query_url.to_s)
+            .fetch('issueTypes', [])
+            .map { |issue_type| issue_type['id'] }
+        elsif data_fields.deployment_cloud?
+          query_url = Addressable::URI.join("#{client.options[:rest_base_path]}/", 'issuetypescheme/', 'mapping')
+          query_url.query_values = { 'issueTypeSchemeId' => project_issuetype_scheme_ids }
+
+          client
+            .get(query_url.to_s)
+            .fetch('values', [])
+            .map { |schemes| schemes['issueTypeId'] }
+        else
+          raise NotImplementedError
+        end
       end
     end
 

ee/app/views/namespaces/pipelines_quota/_extra_shared_runners_minutes_quota.html.haml

 - return unless Gitlab.com?
 
 - minutes_quota = namespace.ci_minutes_quota
-- return unless minutes_quota.enabled?
+- return unless minutes_quota.enabled? && minutes_quota.purchased_minutes_report.limit > 0
 
 .row
   .col-sm-6

ee/changelogs/unreleased/327458-ci-minutes-ee.yml

+---
+title: Remove additional minutes when no minutes purchased
+merge_request: 59112
+author: Takuya Noguchi
+type: changed

ee/spec/models/project_services/jira_service_spec.rb

@@ -15,6 +15,11 @@ RSpec.describe JiraService do
     }
   end
 
+  before do
+    allow(jira_service.data_fields).to receive(:deployment_cloud?).and_return(true)
+    allow(jira_service.data_fields).to receive(:deployment_server?).and_return(false)
+  end
+
   describe 'validations' do
     it 'validates presence of project_key if issues_enabled' do
       jira_service.project_key = ''
@@ -111,80 +116,150 @@ RSpec.describe JiraService do
       end
 
       context 'when vulnerabilities integration is enabled' do
-        let(:project_info_result) { { 'id' => '10000' } }
+        before do
+          allow(jira_service.project).to receive(:jira_vulnerabilities_integration_enabled?).and_return(true)
+        end
 
-        let(:issue_type_scheme_response) do
-          {
-            values: [
-              {
-                issueTypeScheme: {
-                  id: '10126',
-                  name: 'GV: Software Development Issue Type Scheme',
-                  defaultIssueTypeId: '10001'
+        context 'when deployment type is cloud' do
+          let(:project_info_result) { { 'id' => '10000' } }
+
+          let(:issue_type_scheme_response) do
+            {
+              values: [
+                {
+                  issueTypeScheme: {
+                    id: '10126',
+                    name: 'GV: Software Development Issue Type Scheme',
+                    defaultIssueTypeId: '10001'
+                  },
+                  projectIds: [
+                    '10000'
+                  ]
+                }
+              ]
+            }
+          end
+
+          let(:issue_type_mapping_response) do
+            {
+              values: [
+                {
+                  issueTypeSchemeId: '10126',
+                  issueTypeId: '10003'
                 },
-                projectIds: [
-                  '10000'
-                ]
-              }
-            ]
-          }
-        end
+                {
+                  issueTypeSchemeId: '10126',
+                  issueTypeId: '10001'
+                }
+              ]
+            }
+          end
 
-        let(:issue_type_mapping_response) do
-          {
-            values: [
+          let(:issue_types_response) do
+            [
+              {
+                id: '10004',
+                description: 'A new feature of the product, which has yet to be developed.',
+                name: 'New Feature',
+                untranslatedName: 'New Feature',
+                subtask: false,
+                avatarId: 10311
+              },
               {
-                issueTypeSchemeId: '10126',
-                issueTypeId: '10003'
+                id: '10001',
+                description: 'Jira Bug',
+                name: 'Bug',
+                untranslatedName: 'Bug',
+                subtask: false,
+                avatarId: 10303
               },
               {
-                issueTypeSchemeId: '10126',
-                issueTypeId: '10001'
+                id: '10003',
+                description: 'A small piece of work thats part of a larger task.',
+                name: 'Sub-task',
+                untranslatedName: 'Sub-task',
+                subtask: true,
+                avatarId: 10316
               }
             ]
-          }
+          end
+
+          before do
+            WebMock.stub_request(:get, /api\/2\/project\/GL/).with(basic_auth: %w(gitlab_jira_username gitlab_jira_password)).to_return(body: project_info_result.to_json )
+            WebMock.stub_request(:get, /api\/2\/project\/GL\z/).with(basic_auth: %w(gitlab_jira_username gitlab_jira_password)).to_return(body: { 'id' => '10000' }.to_json, headers: headers)
+            WebMock.stub_request(:get, /api\/2\/issuetype\z/).to_return(body: issue_types_response.to_json, headers: headers)
+            WebMock.stub_request(:get, /api\/2\/issuetypescheme\/project\?projectId\=10000\z/).to_return(body: issue_type_scheme_response.to_json, headers: headers)
+            WebMock.stub_request(:get, /api\/2\/issuetypescheme\/mapping\?issueTypeSchemeId\=10126\z/).to_return(body: issue_type_mapping_response.to_json, headers: headers)
+          end
+
+          it { is_expected.to eq(success: true, result: { jira: true }, data: { issuetypes: [{ id: '10001', name: 'Bug', description: 'Jira Bug' }] }) }
         end
 
-        let(:issue_types_response) do
-          [
+        context 'when deployment type is server' do
+          let(:project_info_result) do
             {
-              id: '10004',
-              description: 'A new feature of the product, which has yet to be developed.',
-              name: 'New Feature',
-              untranslatedName: 'New Feature',
-              subtask: false,
-              avatarId: 10311
-            },
-            {
-              id: '10001',
-              description: 'Jira Bug',
-              name: 'Bug',
-              untranslatedName: 'Bug',
-              subtask: false,
-              avatarId: 10303
-            },
-            {
-              id: '10003',
-              description: 'A small piece of work thats part of a larger task.',
-              name: 'Sub-task',
-              untranslatedName: 'Sub-task',
-              subtask: true,
-              avatarId: 10316
+              "id": "10000",
+              "issueTypes": issue_types_response
             }
-          ]
-        end
+          end
 
-        before do
-          allow(jira_service.project).to receive(:jira_vulnerabilities_integration_enabled?).and_return(true)
+          let(:issue_types_response) do
+            [
+              {
+                "avatarId": 10318,
+                "description": "A task that needs to be done.",
+                "iconUrl": "http://jira.reali.sh:8080/secure/viewavatar?size=xsmall&avatarId=10318&avatarType=issuetype",
+                "id": "10003",
+                "name": "Task",
+                "self": "http://jira.reali.sh:8080/rest/api/2/issuetype/10003",
+                "subtask": false
+              },
+              {
+                "description": "The sub-task of the issue",
+                "iconUrl": "http://jira.reali.sh:8080/images/icons/issuetypes/subtask_alternate.png",
+                "id": "10000",
+                "name": "Sub-task",
+                "self": "http://jira.reali.sh:8080/rest/api/2/issuetype/10000",
+                "subtask": true
+              },
+              {
+                "description": "Created by Jira Software - do not edit or delete. Issue type for a user story.",
+                "iconUrl": "http://jira.reali.sh:8080/images/icons/issuetypes/story.svg",
+                "id": "10002",
+                "name": "Story",
+                "self": "http://jira.reali.sh:8080/rest/api/2/issuetype/10002",
+                "subtask": false
+              },
+              {
+                "avatarId": 10303,
+                "description": "A problem which impairs or prevents the functions of the product.",
+                "iconUrl": "http://jira.reali.sh:8080/secure/viewavatar?size=xsmall&avatarId=10303&avatarType=issuetype",
+                "id": "10004",
+                "name": "Bug",
+                "self": "http://jira.reali.sh:8080/rest/api/2/issuetype/10004",
+                "subtask": false
+              },
+              {
+                "description": "Created by Jira Software - do not edit or delete. Issue type for a big user story that needs to be broken down.",
+                "iconUrl": "http://jira.reali.sh:8080/images/icons/issuetypes/epic.svg",
+                "id": "10001",
+                "name": "Epic",
+                "self": "http://jira.reali.sh:8080/rest/api/2/issuetype/10001",
+                "subtask": false
+              }
+            ]
+          end
 
-          WebMock.stub_request(:get, /api\/2\/project\/GL/).with(basic_auth: %w(gitlab_jira_username gitlab_jira_password)).to_return(body: project_info_result.to_json )
-          WebMock.stub_request(:get, /api\/2\/project\/GL\z/).with(basic_auth: %w(gitlab_jira_username gitlab_jira_password)).to_return(body: { 'id' => '10000' }.to_json, headers: headers)
-          WebMock.stub_request(:get, /api\/2\/issuetype\z/).to_return(body: issue_types_response.to_json, headers: headers)
-          WebMock.stub_request(:get, /api\/2\/issuetypescheme\/project\?projectId\=10000\z/).to_return(body: issue_type_scheme_response.to_json, headers: headers)
-          WebMock.stub_request(:get, /api\/2\/issuetypescheme\/mapping\?issueTypeSchemeId\=10126\z/).to_return(body: issue_type_mapping_response.to_json, headers: headers)
-        end
+          before do
+            allow(jira_service.data_fields).to receive(:deployment_cloud?).and_return(false)
+            allow(jira_service.data_fields).to receive(:deployment_server?).and_return(true)
 
-        it { is_expected.to eq(success: true, result: { jira: true }, data: { issuetypes: [{ id: '10001', name: 'Bug', description: 'Jira Bug' }] }) }
+            WebMock.stub_request(:get, /api\/2\/project\/GL/).with(basic_auth: %w(gitlab_jira_username gitlab_jira_password)).to_return(body: project_info_result.to_json, headers: headers)
+            WebMock.stub_request(:get, /api\/2\/issuetype\z/).to_return(body: issue_types_response.to_json, headers: headers)
+          end
+
+          it { is_expected.to eq(success: true, result: { jira: true }, data: { issuetypes: [{ description: "A task that needs to be done.", id: "10003", name: "Task" }, { description: "Created by Jira Software - do not edit or delete. Issue type for a user story.", id: "10002", name: "Story" }, { description: "A problem which impairs or prevents the functions of the product.", id: "10004", name: "Bug" }, { description: "Created by Jira Software - do not edit or delete. Issue type for a big user story that needs to be broken down.", id: "10001", name: "Epic" }] }) }
+        end
       end
     end
   end

ee/spec/services/ee/merge_requests/refresh_service_spec.rb

@@ -249,7 +249,7 @@ RSpec.describe MergeRequests::RefreshService do
       end
     end
 
-    describe 'Pipelines for merge requests' do
+    describe 'Pipelines for merge requests', :sidekiq_inline do
       let(:service) { described_class.new(project, current_user) }
       let(:current_user) { merge_request.author }
 

lib/api/maven_packages.rb

@@ -80,7 +80,7 @@ module API
 
       def fetch_package(file_name:, project: nil, group: nil)
         order_by_package_file = false
-        if Feature.enabled?(:maven_packages_group_level_improvements)
+        if Feature.enabled?(:maven_packages_group_level_improvements, default_enabled: :yaml)
           order_by_package_file = file_name.include?(::Packages::Maven::Metadata.filename) &&
                                     !params[:path].include?(::Packages::Maven::FindOrCreatePackageService::SNAPSHOT_TERM)
         end

spec/factories/services.rb

@@ -62,6 +62,7 @@ FactoryBot.define do
       project_key { nil }
       vulnerabilities_enabled { false }
       vulnerabilities_issuetype { nil }
+      deployment_type { 'cloud' }
     end
 
     before(:create) do |service, evaluator|
@@ -72,7 +73,7 @@ FactoryBot.define do
                jira_issue_transition_id: evaluator.jira_issue_transition_id,
                username: evaluator.username, password: evaluator.password, issues_enabled: evaluator.issues_enabled,
                project_key: evaluator.project_key, vulnerabilities_enabled: evaluator.vulnerabilities_enabled,
-               vulnerabilities_issuetype: evaluator.vulnerabilities_issuetype
+               vulnerabilities_issuetype: evaluator.vulnerabilities_issuetype, deployment_type: evaluator.deployment_type
         )
       end
     end

spec/requests/api/maven_packages_spec.rb

@@ -387,7 +387,7 @@ RSpec.describe API::MavenPackages do
 
           subject
 
-          status = Feature.enabled?(:maven_packages_group_level_improvements) ? :not_found : :forbidden
+          status = Feature.enabled?(:maven_packages_group_level_improvements, default_enabled: :yaml) ? :not_found : :forbidden
           expect(response).to have_gitlab_http_status(status)
         end
 

spec/services/merge_requests/refresh_service_spec.rb

@@ -198,7 +198,7 @@ RSpec.describe MergeRequests::RefreshService do
       end
     end
 
-    describe 'Pipelines for merge requests' do
+    shared_examples 'Pipelines for merge requests' do
       before do
         stub_ci_pipeline_yaml_file(config)
       end
@@ -256,7 +256,7 @@ RSpec.describe MergeRequests::RefreshService do
             stub_feature_flags(ci_disallow_to_create_merge_request_pipelines_in_target_project: false)
           end
 
-          it 'creates detached merge request pipeline for fork merge request', :sidekiq_inline do
+          it 'creates detached merge request pipeline for fork merge request' do
             expect { subject }
               .to change { @fork_merge_request.pipelines_for_merge_request.count }.by(1)
 
@@ -364,6 +364,18 @@ RSpec.describe MergeRequests::RefreshService do
       end
     end
 
+    context 'when the code_review_async_pipeline_creation feature flag is on', :sidekiq_inline do
+      it_behaves_like 'Pipelines for merge requests'
+    end
+
+    context 'when the code_review_async_pipeline_creation feature flag is off', :sidekiq_inline do
+      before do
+        stub_feature_flags(code_review_async_pipeline_creation: false)
+      end
+
+      it_behaves_like 'Pipelines for merge requests'
+    end
+
     context 'push to origin repo source branch' do
       let(:refresh_service) { service.new(@project, @user) }
       let(:notification_service) { spy('notification_service') }

spec/workers/merge_requests/create_pipeline_worker_spec.rb

+# frozen_string_literal: true
+
+require 'spec_helper'
+
+RSpec.describe MergeRequests::CreatePipelineWorker do
+  subject(:worker) { described_class.new }
+
+  describe '#perform' do
+    let(:user) { create(:user) }
+    let(:project) { create(:project) }
+    let(:merge_request) { create(:merge_request) }
+
+    context 'when the objects exist' do
+      it 'calls the merge request create pipeline service and calls update head pipeline' do
+        aggregate_failures do
+          expect_next_instance_of(MergeRequests::CreatePipelineService, project, user) do |service|
+            expect(service).to receive(:execute).with(merge_request)
+          end
+
+          expect(MergeRequest).to receive(:find_by_id).with(merge_request.id).and_return(merge_request)
+          expect(merge_request).to receive(:update_head_pipeline)
+
+          subject.perform(project.id, user.id, merge_request.id)
+        end
+      end
+    end
+
+    shared_examples 'when object does not exist' do
+      it 'does not call the create pipeline service' do
+        expect(MergeRequests::CreatePipelineService).not_to receive(:new)
+
+        expect { subject.perform(project.id, user.id, merge_request.id) }
+          .not_to raise_exception
+      end
+    end
+
+    context 'when the project does not exist' do
+      before do
+        project.destroy!
+      end
+
+      it_behaves_like 'when object does not exist'
+    end
+
+    context 'when the user does not exist' do
+      before do
+        user.destroy!
+      end
+
+      it_behaves_like 'when object does not exist'
+    end
+
+    context 'when the merge request does not exist' do
+      before do
+        merge_request.destroy!
+      end
+
+      it_behaves_like 'when object does not exist'
+    end
+  end
+end