Merge branch '14-10-stable-ee-patch-2' into '14-10-stable-ee'

Prepare 14.10.2-ee release

See merge request gitlab-org/gitlab!86476

app/helpers/workhorse_helper.rb

@@ -38,8 +38,6 @@ module WorkhorseHelper
   # Send an entry from artifacts through Workhorse and set safe content type
   def send_artifacts_entry(file, entry)
     headers.store(*Gitlab::Workhorse.send_artifacts_entry(file, entry))
-    headers.store(*Gitlab::Workhorse.detect_content_type)
-
     head :ok
   end
 

app/policies/project_policy.rb

@@ -664,7 +664,7 @@ class ProjectPolicy < BasePolicy
     enable :read_security_configuration
   end
 
-  rule { can?(:guest_access) & can?(:read_commit_status) }.policy do
+  rule { can?(:guest_access) & can?(:download_code) }.policy do
     enable :create_merge_request_in
   end
 

app/views/notify/issue_due_email.html.haml

 %p.details
-  = sprintf(s_("Notify|%{author_link}'s issue %{issue_reference_link} is due soon."), { author_link: link_to(@issue.author_name, user_url(@issue.author)), issue_reference_link: issue_reference_link(@issue) })
+  = sprintf(s_("Notify|%{author_link}'s issue %{issue_reference_link} is due soon."), { author_link: link_to(@issue.author_name, user_url(@issue.author)), issue_reference_link: issue_reference_link(@issue) }).html_safe
 
 - if @issue.assignees.any?
   %p

data/whats_new/202204210001_14_10.yml

+- title: "Compliance report individual violation reporting"
+  body: |
+    The compliance report now reports every individual merge request violation for the projects within a group. This is a huge improvement over the previous version, which only showed the latest MR that had one or more violations. The new version allows you to see history and patterns of violations over time.
+  stage: manage
+  self-managed: true
+  gitlab-com: true
+  packages: [Ultimate]
+  url: 'https://docs.gitlab.com/ee/user/compliance/compliance_report/'
+  image_url: 'https://about.gitlab.com/images/14_10/manage_compliance_report_individual_violation.png'
+  published_at: 2022-04-22
+  release: 14.10
+- title: "Improved pipeline variables inheritance"
+  body: |
+    Previously, it was possible to pass some CI/CD variables to a downstream pipeline through a trigger job, but variables added in manual pipeline runs or by using the API could not be forwarded.
+
+    In this release we've added a new `trigger:forward` keyword to control what things you forward to downstream parent-child pipelines or multi-project pipelines, which provides a flexible way to handle variable inheritance in downstream pipelines.
+  stage: verify
+  self-managed: true
+  gitlab-com: true
+  packages: [Free, Premium, Ultimate]
+  url: 'https://docs.gitlab.com/ee/ci/yaml/#triggerforward'
+  image_url: 'https://about.gitlab.com/images/growth/verify.png'
+  published_at: 2022-04-22
+  release: 14.10
+- title: "Escalating manually created incidents"
+  body: |
+    In GitLab 13.10, we [released](https://gitlab.com/gitlab-org/gl-openshift/gitlab-runner-operator/-/issues/6) the GitLab Runner Operator for the Red Hat OpenShift container platform for Kubernetes. That release provided OpenShift users with the automation and management capabilities of the Operator Framework and simplified the ongoing management of runners in an OpenShift Kubernetes cluster. Available starting in 14.10 is a GitLab Runner Operator v1.7.0 that you can use in non-OpenShift Kubernetes clusters. This GitLab Runner Operator is available on [OperatorHub.io](https://operatorhub.io/operator/gitlab-runner-operator).
+  stage: monitor
+  self-managed: true
+  gitlab-com: true
+  packages: [Premium, Ultimate]
+  url: 'https://docs.gitlab.com/ee/operations/incident_management/paging.html#escalating-an-incident'
+  image_url: 'https://about.gitlab.com/images/14_10/manually_escalated_incident.png'
+  published_at: 2022-04-22
+  release: 14.10
+- title: "Expanded view of group runners"
+  body: |
+    Group runners are now displayed in an expanded view, where you can more easily administer and manage the runners associated with the namespace. To view the new UI, on the left sidebar, select **CI/CD**. This view includes the number of online, offline, and stale runners associated with the group and subgroups.
+  stage: verify
+  self-managed: true
+  gitlab-com: true
+  packages: [Free, Premium, Ultimate]
+  url: 'https://docs.gitlab.com/ee/ci/runners/runners_scope.html#group-runners'
+  image_url: 'https://about.gitlab.com/images/14_10/group-runners-view-new-3.png'
+  published_at: 2022-04-22
+  release: 14.10

doc/administration/audit_events.md

@@ -167,6 +167,17 @@ From there, you can see the following actions:
 - Users and groups allowed to merge and push to protected branch added or removed ([introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/338873) in GitLab 14.3)
 - Project deploy token was successfully created, revoked or deleted ([introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/353451) in GitLab 14.9)
 - Failed attempt to create a project deploy token ([introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/353451) in GitLab 14.9)
+- When merge method is updated ([introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/301124) in GitLab 14.9)
+- Merged results pipelines enabled or disabled ([introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/301124) in GitLab 14.9)
+- Merge trains enabled or disabled ([introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/301124) in GitLab 14.9)
+- Automatically resolve merge request diff discussions enabled or disabled ([introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/301124) in GitLab 14.9)
+- Show link to create or view a merge request when pushing from the command line enabled or disabled ([introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/301124) in GitLab 14.9)
+- Delete source branch option by default enabled or disabled ([introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/301124) in GitLab 14.9)
+- Squash commits when merging is updated ([introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/301124) in GitLab 14.9)
+- Pipelines must succeed enabled or disabled ([introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/301124) in GitLab 14.9)
+- Skipped pipelines are considered successful enabled or disabled ([introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/301124) in GitLab 14.9)
+- All discussions must be resolved enabled or disabled ([introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/301124) in GitLab 14.9)
+- Commit message suggestion is updated ([introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/301124) in GitLab 14.9)
 
 Project events can also be accessed via the [Project Audit Events API](../api/audit_events.md#project-audit-events).
 

doc/administration/geo/replication/datatypes.md

@@ -192,7 +192,7 @@ successfully, you must replicate their data using some other means.
 |[LFS objects](../../lfs/index.md)                                                                              | **Yes** (10.2)                                                          | **Yes** (14.6)                                                             | Via Object Storage provider if supported. Native Geo support (Beta).          | GitLab versions 11.11.x and 12.0.x are affected by [a bug that prevents any new LFS objects from replicating](https://gitlab.com/gitlab-org/gitlab/-/issues/32696).<br /><br />Replication is behind the feature flag `geo_lfs_object_replication`, enabled by default. Verification was behind the feature flag `geo_lfs_object_verification`, removed in 14.7. |
 |[Personal snippets](../../../user/snippets.md)                                                                 | **Yes** (10.2)                                                          | **Yes** (10.2)                                                             | No                                                                            |       |
 |[Project snippets](../../../user/snippets.md)                                                                  | **Yes** (10.2)                                                          | **Yes** (10.2)                                                             | No                                                                            |       |
-|[CI job artifacts](../../../ci/pipelines/job_artifacts.md)                                                     | **Yes** (10.4)                                                          | [No](https://gitlab.com/gitlab-org/gitlab/-/issues/8923)                   | Via Object Storage provider if supported. Native Geo support (Beta).          | Verified only manually using [Integrity Check Rake Task](../../raketasks/check.md) on both sites and comparing the output between them. Job logs also verified on transfer. |
+|[CI job artifacts](../../../ci/pipelines/job_artifacts.md)                                                     | **Yes** (10.4)                                                          | **Yes** (14.10)                                                            | Via Object Storage provider if supported. Native Geo support (Beta).          | Verification is behind the feature flag `geo_job_artifact_replication`, enabled by default in 14.10. |
 |[CI Pipeline Artifacts](https://gitlab.com/gitlab-org/gitlab/-/blob/master/app/models/ci/pipeline_artifact.rb) | [**Yes** (13.11)](https://gitlab.com/gitlab-org/gitlab/-/issues/238464) | [**Yes** (13.11)](https://gitlab.com/gitlab-org/gitlab/-/issues/238464)    | Via Object Storage provider if supported. Native Geo support (Beta).          | Persists additional artifacts after a pipeline completes. |
 |[Container Registry](../../packages/container_registry.md)                                                     | **Yes** (12.3)                                                          | No                                                                         | No                                                                            | Disabled by default. See [instructions](docker_registry.md) to enable. |
 |[Content in object storage (beta)](object_storage.md)                                                          | **Yes** (12.4)                                                          | [No](https://gitlab.com/gitlab-org/gitlab/-/issues/13845)                  | No                                                                            |       |

doc/api/group_clusters.md

@@ -315,7 +315,7 @@ Example response:
 
 ## Delete group cluster
 
-Deletes an existing group cluster.
+Deletes an existing group cluster. Does not remove existing resources within the connected Kubernetes cluster.
 
 ```plaintext
 DELETE /groups/:id/clusters/:cluster_id

doc/api/instance_clusters.md

@@ -290,7 +290,7 @@ Example response:
 
 ## Delete instance cluster
 
-Deletes an existing instance cluster.
+Deletes an existing instance cluster. Does not remove existing resources within the connected Kubernetes cluster.
 
 ```plaintext
 DELETE /admin/clusters/:cluster_id

doc/api/project_clusters.md

@@ -388,7 +388,7 @@ Example response:
 
 ## Delete project cluster
 
-Deletes an existing project cluster.
+Deletes an existing project cluster. Does not remove existing resources within the connected Kubernetes cluster.
 
 ```plaintext
 DELETE /projects/:id/clusters/:cluster_id

doc/update/index.md

@@ -192,9 +192,13 @@ pending_job_classes.each { |job_class| Gitlab::BackgroundMigration.steal(job_cla
 #### Background migrations stuck in 'pending' state
 
 GitLab 13.6 introduced an issue where a background migration named `BackfillJiraTrackerDeploymentType2` can be permanently stuck in a **pending** state across upgrades. To clean up this stuck migration, see the [13.6.0 version-specific instructions](#1360).
+
 GitLab 14.4 introduced an issue where a background migration named `PopulateTopicsTotalProjectsCountCache` can be permanently stuck in a **pending** state across upgrades when the instance lacks records that match the migration's target. To clean up this stuck migration, see the [14.4.0 version-specific instructions](#1440).
+
 GitLab 14.8 introduced an issue where a background migration named `PopulateTopicsNonPrivateProjectsCount` can be permanently stuck in a **pending** state across upgrades. To clean up this stuck migration, see the [14.8.0 version-specific instructions](#1480).
 
+GitLab 14.9 introduced an issue where a background migration named `ResetDuplicateCiRunnersTokenValuesOnProjects` can be permanently stuck in a **pending** state across upgrades when the instance lacks records that match the migration's target. To clean up this stuck migration, see the [14.9.0 version-specific instructions](#1490).
+
 For other background migrations stuck in pending, run the following check. If it returns non-zero and the count does not decrease over time, follow the rest of the steps in this section.
 
 ```shell
@@ -398,6 +402,35 @@ NOTE:
 Specific information that follow related to Ruby and Git versions do not apply to [Omnibus installations](https://docs.gitlab.com/omnibus/)
 and [Helm Chart deployments](https://docs.gitlab.com/charts/). They come with appropriate Ruby and Git versions and are not using system binaries for Ruby and Git. There is no need to install Ruby or Git when utilizing these two approaches.
 
+### 14.9.0
+
+- Database changes made by the upgrade to GitLab 14.9 can take hours or days to complete on larger GitLab instances. 
+  These [batched background migrations](#batched-background-migrations) update whole database tables to ensure corresponding
+  records in `namespaces` table for each record in `projects` table.
+
+  After you update to 14.9.0 or a later 14.9 patch version,
+  [batched background migrations need to finish](#batched-background-migrations)
+  before you update to a later version.
+
+  If the migrations are not finished and you try to update to a later version,
+  you'll see an error like:
+
+  ```plaintext
+  Expected batched background migration for the given configuration to be marked as 'finished', but it is 'active':
+  ```
+
+- GitLab 14.9.0 includes a
+  [background migration `ResetDuplicateCiRunnersTokenValuesOnProjects`](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/79140)
+  that may remain stuck permanently in a **pending** state.
+
+  To clean up this stuck job, run the following in the [GitLab Rails Console](../administration/operations/rails_console.md):
+
+  ```ruby
+  Gitlab::Database::BackgroundMigrationJob.pending.where(class_name: "ResetDuplicateCiRunnersTokenValuesOnProjects").find_each do |job|
+    puts Gitlab::Database::BackgroundMigrationJob.mark_all_as_succeeded("ResetDuplicateCiRunnersTokenValuesOnProjects", job.arguments)
+  end
+  ```
+
 ### 14.8.0
 
 - If upgrading from a version earlier than 14.6.5, 14.7.4, or 14.8.2, please review the [Critical Security Release: 14.8.2, 14.7.4, and 14.6.5](https://about.gitlab.com/releases/2022/02/25/critical-security-release-gitlab-14-8-2-released/) blog post.
@@ -455,7 +488,7 @@ that may remain stuck permanently in a **pending** state.
   can override the behavior of `tmpfiles.d` for the Gitaly files and avoid this issue:
 
   ```shell
-  sudo echo "x /tmp/gitaly-hooks-*" > /etc/tmpfiles.d/gitaly-workaround.conf
+  sudo printf "x /tmp/gitaly-%s-*\n" hooks git-exec-path >/etc/tmpfiles.d/gitaly-workaround.conf
   ```
 
 ### 14.6.0

ee/elastic/migrate/20210623081800_add_upvotes_to_issues.rb

@@ -46,11 +46,7 @@ class AddUpvotesToIssues < Elastic::Migration
   private
 
   def update_mappings!
-    client.indices.put_mapping index: index_name, body: {
-      properties: {
-        upvotes: { type: 'integer' }
-      }
-    }
+    helper.update_mapping(index_name: index_name, mappings: { properties: { upvotes: { type: 'integer' } } })
   end
 
   def process_batch!

ee/lib/gitlab/elastic/helper.rb

@@ -261,16 +261,28 @@ module Gitlab
 
       def get_mapping(index_name: nil)
         index = target_index_name(target: index_name)
-        mappings = client.indices.get_mapping(index: index)
+        mappings = client.indices.get_mapping({ index: index })
+
+        # The check for version 6 (and the spec testing this code) should be removed when support for
+        # Elasticsearch v6.8 is removed
+        if Gitlab::VersionInfo.parse(client.info['version']['number']).major == 6
+          mappings.dig(index, 'mappings', 'doc', 'properties')
+        else
           mappings.dig(index, 'mappings', 'properties')
         end
+      end
 
       def update_settings(index_name: nil, settings:)
         client.indices.put_settings(index: index_name || target_index_name, body: settings)
       end
 
       def update_mapping(index_name: nil, mappings:)
-        client.indices.put_mapping(index: index_name || target_index_name, body: mappings)
+        options = {
+          index: index_name || target_index_name,
+          body: mappings
+        }
+        options[:type] = 'doc' if Gitlab::VersionInfo.parse(client.info['version']['number']).major == 6
+        client.indices.put_mapping(options)
       end
 
       def get_meta(index_name: nil)

ee/spec/lib/ee/gitlab/elastic/helper_spec.rb

@@ -520,4 +520,34 @@ RSpec.describe Gitlab::Elastic::Helper, :request_store do
       end
     end
   end
+
+  describe '#get_mapping' do
+    let(:index_name) { Issue.__elasticsearch__.index_name }
+
+    subject { helper.get_mapping(index_name: index_name) }
+
+    it 'reads mappings from client', :elastic do
+      is_expected.not_to be_nil
+    end
+
+    context 'when using elasticsearch version 6.8' do
+      before do
+        info = {
+          'version' => {
+            'number' => '6.8.1',
+            'build_type' => 'docker',
+            'lucene_version' => '8.6.2'
+          }
+        }
+        mapping = { "#{index_name}": { mappings: { doc: { properties: { test: 1 } } } } }.with_indifferent_access
+
+        allow(Gitlab::Elastic::Helper.default.client).to receive(:info).and_return(info)
+        allow(helper.client.indices).to receive(:get_mapping).and_return(mapping)
+      end
+
+      it 'reads mappings from client' do
+        is_expected.not_to be_nil
+      end
+    end
+  end
 end

lib/api/helpers.rb

@@ -714,7 +714,6 @@ module API
 
     def send_artifacts_entry(file, entry)
       header(*Gitlab::Workhorse.send_artifacts_entry(file, entry))
-      header(*Gitlab::Workhorse.detect_content_type)
 
       body ''
     end

lib/gitlab/workhorse.rb

@@ -226,13 +226,6 @@ module Gitlab
         end
       end
 
-      def detect_content_type
-        [
-          Gitlab::Workhorse::DETECT_HEADER,
-          'true'
-        ]
-      end
-
       protected
 
       # This is the outermost encoding of a senddata: header. It is safe for

spec/controllers/projects/artifacts_controller_spec.rb

@@ -361,7 +361,6 @@ RSpec.describe Projects::ArtifactsController do
           subject
 
           expect(response).to have_gitlab_http_status(:ok)
-          expect(response.headers['Gitlab-Workhorse-Detect-Content-Type']).to eq('true')
           expect(send_data).to start_with('artifacts-entry:')
 
           expect(params.keys).to eq(%w(Archive Entry))

spec/lib/gitlab/workhorse_spec.rb

@@ -448,14 +448,6 @@ RSpec.describe Gitlab::Workhorse do
     end
   end
 
-  describe '.detect_content_type' do
-    subject { described_class.detect_content_type }
-
-    it 'returns array setting detect content type in workhorse' do
-      expect(subject).to eq(%w[Gitlab-Workhorse-Detect-Content-Type true])
-    end
-  end
-
   describe '.send_git_blob' do
     include FakeBlobHelpers
 

spec/policies/project_policy_spec.rb

@@ -103,6 +103,12 @@ RSpec.describe ProjectPolicy do
   end
 
   context 'creating_merge_request_in' do
+    context 'when the current_user can download_code' do
+      before do
+        expect(subject).to receive(:allowed?).with(:download_code).and_return(true)
+        allow(subject).to receive(:allowed?).with(any_args).and_call_original
+      end
+
       context 'when project is public' do
         let(:project) { public_project }
 
@@ -140,6 +146,50 @@ RSpec.describe ProjectPolicy do
       end
     end
 
+    context 'when the current_user can not download code' do
+      before do
+        expect(subject).to receive(:allowed?).with(:download_code).and_return(false)
+        allow(subject).to receive(:allowed?).with(any_args).and_call_original
+      end
+
+      context 'when project is public' do
+        let(:project) { public_project }
+
+        context 'when the current_user is guest' do
+          let(:current_user) { guest }
+
+          it { is_expected.not_to be_allowed(:create_merge_request_in) }
+        end
+      end
+
+      context 'when project is internal' do
+        let(:project) { internal_project }
+
+        context 'when the current_user is guest' do
+          let(:current_user) { guest }
+
+          it { is_expected.not_to be_allowed(:create_merge_request_in) }
+        end
+      end
+
+      context 'when project is private' do
+        let(:project) { private_project }
+
+        context 'when the current_user is guest' do
+          let(:current_user) { guest }
+
+          it { is_expected.not_to be_allowed(:create_merge_request_in) }
+        end
+
+        context 'when the current_user is reporter or above' do
+          let(:current_user) { reporter }
+
+          it { is_expected.not_to be_allowed(:create_merge_request_in) }
+        end
+      end
+    end
+  end
+
   context 'pipeline feature' do
     let(:project)      { private_project }
     let(:current_user) { developer }

spec/requests/api/ci/job_artifacts_spec.rb

@@ -558,8 +558,7 @@ RSpec.describe API::Ci::JobArtifacts do
             expect(response).to have_gitlab_http_status(:ok)
             expect(response.headers.to_h)
               .to include('Content-Type' => 'application/json',
-                          'Gitlab-Workhorse-Send-Data' => /artifacts-entry/,
-                          'Gitlab-Workhorse-Detect-Content-Type' => 'true')
+                          'Gitlab-Workhorse-Send-Data' => /artifacts-entry/)
           end
         end
 
@@ -629,8 +628,7 @@ RSpec.describe API::Ci::JobArtifacts do
           expect(response).to have_gitlab_http_status(:ok)
           expect(response.headers.to_h)
             .to include('Content-Type' => 'application/json',
-                        'Gitlab-Workhorse-Send-Data' => /artifacts-entry/,
-                        'Gitlab-Workhorse-Detect-Content-Type' => 'true')
+                        'Gitlab-Workhorse-Send-Data' => /artifacts-entry/)
           expect(response.parsed_body).to be_empty
         end
       end
@@ -648,8 +646,7 @@ RSpec.describe API::Ci::JobArtifacts do
           expect(response).to have_gitlab_http_status(:ok)
           expect(response.headers.to_h)
             .to include('Content-Type' => 'application/json',
-                        'Gitlab-Workhorse-Send-Data' => /artifacts-entry/,
-                        'Gitlab-Workhorse-Detect-Content-Type' => 'true')
+                        'Gitlab-Workhorse-Send-Data' => /artifacts-entry/)
         end
       end