Add latest changes from gitlab-org/gitlab@master

.gitlab/ci/rules.gitlab-ci.yml

@@ -287,6 +287,7 @@
 .frontend:rules:qa-frontend-node:
   rules:
     - <<: *if-master-refs
+      changes: *frontend-dependency-patterns
       when: on_success
     - <<: *if-merge-request
       changes: *frontend-dependency-patterns

.rubocop.yml

@@ -218,11 +218,9 @@ Gitlab/DuplicateSpecLocation:
     - ee/spec/lib/gitlab/gl_repository_spec.rb
     - ee/spec/models/namespace_spec.rb
     - ee/spec/services/merge_requests/refresh_service_spec.rb
-    - ee/spec/services/merge_requests/update_service_spec.rb
     - ee/spec/helpers/ee/auth_helper_spec.rb
     - ee/spec/models/ee/namespace_spec.rb
     - ee/spec/services/ee/merge_requests/refresh_service_spec.rb
-    - ee/spec/services/ee/merge_requests/update_service_spec.rb
 
 Cop/InjectEnterpriseEditionModule:
   Enabled: true
@@ -388,11 +386,6 @@ Performance/ChainArrayAllocation:
 # https://gitlab.com/gitlab-org/gitlab/-/issues/212541
 RSpec/RepeatedExample:
   Exclude:
-    - 'spec/features/issues/filtered_search/filter_issues_spec.rb'
     - 'spec/features/merge_request/user_posts_diff_notes_spec.rb'
     - 'spec/features/projects/files/template_type_dropdown_spec.rb'
-    - 'spec/finders/environments_finder_spec.rb'
-    - 'spec/helpers/users_helper_spec.rb'
-    - 'spec/lib/gitlab/import_export/project/relation_factory_spec.rb'
     - 'spec/services/notification_service_spec.rb'
-    - 'spec/services/web_hook_service_spec.rb'

Gemfile

@@ -319,7 +319,7 @@ gem 'peek', '~> 1.1'
 gem 'snowplow-tracker', '~> 0.6.1'
 
 # Memory benchmarks
-gem 'gitlab-derailed_benchmarks', require: false
+gem 'derailed_benchmarks', require: false
 
 # Metrics
 group :metrics do

Gemfile.lock

@@ -211,6 +211,17 @@ GEM
     declarative-option (0.1.0)
     default_value_for (3.3.0)
       activerecord (>= 3.2.0, < 6.1)
+    derailed_benchmarks (1.7.0)
+      benchmark-ips (~> 2)
+      get_process_mem (~> 0)
+      heapy (~> 0)
+      memory_profiler (~> 0)
+      mini_histogram (~> 0)
+      rack (>= 1)
+      rake (> 10, < 14)
+      ruby-statistics (>= 2.1)
+      thor (>= 0.19, < 2)
+      unicode_plot (>= 0.0.4, < 1.0.0)
     descendants_tracker (0.0.4)
       thread_safe (~> 0.3, >= 0.3.1)
     device_detector (1.0.0)
@@ -259,6 +270,7 @@ GEM
       launchy (~> 2.1)
       mail (~> 2.7)
     encryptor (3.0.0)
+    enumerable-statistics (2.0.1)
     equalizer (0.0.11)
     erubi (1.9.0)
     escape_utils (1.2.1)
@@ -290,7 +302,7 @@ GEM
     fast_blank (1.0.0)
     fast_gettext (1.6.0)
     ffaker (2.10.0)
-    ffi (1.11.3)
+    ffi (1.12.2)
     ffi-compiler (1.0.1)
       ffi (>= 1.0.0)
       rake
@@ -371,15 +383,6 @@ GEM
     github-markup (1.7.0)
     gitlab-chronic (0.10.5)
       numerizer (~> 0.2)
-    gitlab-derailed_benchmarks (1.6.1)
-      benchmark-ips (~> 2)
-      get_process_mem (~> 0)
-      heapy (~> 0)
-      memory_profiler (~> 0)
-      rack (>= 1)
-      rake (> 10, < 14)
-      ruby-statistics (>= 2.1)
-      thor (>= 0.19, < 2)
     gitlab-labkit (0.12.0)
       actionpack (>= 5.0.0, < 6.1.0)
       activesupport (>= 5.0.0, < 6.1.0)
@@ -632,6 +635,7 @@ GEM
       mime-types-data (~> 3.2015)
     mime-types-data (3.2019.0331)
     mimemagic (0.3.3)
+    mini_histogram (0.1.3)
     mini_magick (4.9.5)
     mini_mime (1.0.2)
     mini_portile2 (2.4.0)
@@ -1084,6 +1088,8 @@ GEM
       unf_ext
     unf_ext (0.0.7.5)
     unicode-display_width (1.6.0)
+    unicode_plot (0.0.4)
+      enumerable-statistics (>= 2.0.1)
     unicode_utils (1.4.0)
     unicorn (5.4.1)
       kgio (~> 2.6)
@@ -1189,6 +1195,7 @@ DEPENDENCIES
   database_cleaner (~> 1.7.0)
   deckar01-task_list (= 2.3.1)
   default_value_for (~> 3.3.0)
+  derailed_benchmarks
   device_detector
   devise (~> 4.6)
   devise-two-factor (~> 3.1.0)
@@ -1231,7 +1238,6 @@ DEPENDENCIES
   gitaly (~> 12.9.0.pre.rc4)
   github-markup (~> 1.7.0)
   gitlab-chronic (~> 0.10.5)
-  gitlab-derailed_benchmarks
   gitlab-labkit (= 0.12.0)
   gitlab-license (~> 1.0)
   gitlab-mail_room (~> 0.0.3)

app/models/concerns/import_state/sidekiq_job_tracker.rb

+# frozen_string_literal: true
+
+module ImportState
+  module SidekiqJobTracker
+    extend ActiveSupport::Concern
+
+    included do
+      # Refreshes the expiration time of the associated import job ID.
+      #
+      # This method can be used by asynchronous importers to refresh the status,
+      # preventing the StuckImportJobsWorker from marking the import as failed.
+      def refresh_jid_expiration
+        return unless jid
+
+        Gitlab::SidekiqStatus.set(jid, StuckImportJobsWorker::IMPORT_JOBS_EXPIRATION)
+      end
+
+      def self.jid_by(project_id:, status:)
+        select(:jid).with_status(status).find_by(project_id: project_id)
+      end
+    end
+  end
+end

app/models/jira_import_state.rb

@@ -2,6 +2,7 @@
 
 class JiraImportState < ApplicationRecord
   include AfterCommitQueue
+  include ImportState::SidekiqJobTracker
 
   self.table_name = 'jira_imports'
 
@@ -66,14 +67,4 @@ class JiraImportState < ApplicationRecord
   def in_progress?
     scheduled? || started?
   end
-
-  def refresh_jid_expiration
-    return unless jid
-
-    Gitlab::SidekiqStatus.set(jid, StuckImportJobsWorker::IMPORT_JOBS_EXPIRATION)
-  end
-
-  def self.jid_by(project_id:, status:)
-    select(:jid).with_status(status).find_by(project_id: project_id)
-  end
 end

app/models/merge_request.rb

@@ -1290,7 +1290,7 @@ class MergeRequest < ApplicationRecord
       variables.append(key: 'CI_MERGE_REQUEST_PROJECT_URL', value: project.web_url)
       variables.append(key: 'CI_MERGE_REQUEST_TARGET_BRANCH_NAME', value: target_branch.to_s)
       variables.append(key: 'CI_MERGE_REQUEST_TITLE', value: title)
-      variables.append(key: 'CI_MERGE_REQUEST_ASSIGNEES', value: assignee_username_list) if assignees.any?
+      variables.append(key: 'CI_MERGE_REQUEST_ASSIGNEES', value: assignee_username_list) if assignees.present?
       variables.append(key: 'CI_MERGE_REQUEST_MILESTONE', value: milestone.title) if milestone
       variables.append(key: 'CI_MERGE_REQUEST_LABELS', value: label_names.join(',')) if labels.present?
       variables.concat(source_project_variables)

app/models/project_import_state.rb

@@ -2,6 +2,7 @@
 
 class ProjectImportState < ApplicationRecord
   include AfterCommitQueue
+  include ImportState::SidekiqJobTracker
 
   self.table_name = "project_mirror_data"
 
@@ -88,20 +89,6 @@ class ProjectImportState < ApplicationRecord
     # import? does SQL work so only run it if it looks like there's an import running
     status == 'started' && project.import?
   end
-
-  # Refreshes the expiration time of the associated import job ID.
-  #
-  # This method can be used by asynchronous importers to refresh the status,
-  # preventing the StuckImportJobsWorker from marking the import as failed.
-  def refresh_jid_expiration
-    return unless jid
-
-    Gitlab::SidekiqStatus.set(jid, StuckImportJobsWorker::IMPORT_JOBS_EXPIRATION)
-  end
-
-  def self.jid_by(project_id:, status:)
-    select(:jid).with_status(status).find_by(project_id: project_id)
-  end
 end
 
 ProjectImportState.prepend_if_ee('EE::ProjectImportState')

app/policies/snippet_policy.rb

-# frozen_string_literal: true
-
-class SnippetPolicy < PersonalSnippetPolicy
-end

app/services/projects/hashed_storage/base_repository_service.rb

@@ -40,7 +40,13 @@ module Projects
           return true
         end
 
-        gitlab_shell.mv_repository(project.repository_storage, from_name, to_name)
+        gitlab_shell.mv_repository(project.repository_storage, from_name, to_name).tap do |moved|
+          if moved
+            logger.info("Repository moved from '#{from_name}' to '#{to_name}' (PROJECT_ID=#{project.id})")
+          else
+            logger.error("Repository cannot be moved from '#{from_name}' to '#{to_name}' (PROJECT_ID=#{project.id})")
+          end
+        end
       end
 
       def move_repositories

app/services/projects/hashed_storage/rollback_service.rb

@@ -5,6 +5,12 @@ module Projects
     class RollbackService < BaseService
       attr_reader :logger, :old_disk_path
 
+      def initialize(project, old_disk_path, logger: nil)
+        @project = project
+        @old_disk_path = old_disk_path
+        @logger = logger || Gitlab::AppLogger
+      end
+
       def execute
         # Rollback attachments from Hashed Storage to Legacy
         if project.hashed_storage?(:attachments)

changelogs/unreleased/198694-speed-up-new-merge-request-worker.yml

+---
+title: Resolve an N+1 in merge request CI variables
+merge_request: 28688
+author:
+type: performance

changelogs/unreleased/211452-rollback_to_legacy-causes-repos-to-404.yml

+---
+title: Fix storage rollback regression caused by previous refactor
+merge_request: 28496
+author:
+type: fixed

changelogs/unreleased/environment-finder-spec.yml

+---
+title: Fix duplicate spec in environment finder
+merge_request: 28857
+author: Rajendra Kadam
+type: added

changelogs/unreleased/filter-issues.yml

+---
+title: Fix duplicate spec in filter issues
+merge_request: 28860
+author: Rajendra Kadam
+type: added

changelogs/unreleased/relation-factory-spec.yml

+---
+title: Fix duplicate spec in factory relation spec
+merge_request: 28794
+author: Rajendra Kadam
+type: added

changelogs/unreleased/user-helpers-spec.yml

+---
+title: Fix duplicate spec from user helper spec
+merge_request: 28854
+author: Rajendra Kadam
+type: added

changelogs/unreleased/validate-dynamic-pipeline-dependencies.yml

----
-title: Validate dependency on job generating a CI config when using dynamic child pipelines
-merge_request: 27916
-author:
-type: added

changelogs/unreleased/webhook-service-spec.yml

+---
+title: Remove duplicate spec in web hook service spec
+merge_request: 28669
+author: Rajendra Kadam
+type: fixed

doc/administration/auth/ldap-ee.md

@@ -48,7 +48,7 @@ The process will execute the following access checks:
 NOTE: **Note:**
 In Active Directory, a user is marked as disabled/blocked if the user
 account control attribute (`userAccountControl:1.2.840.113556.1.4.803`)
-has bit 2 set. See <https://ctogonewild.com/2009/09/03/bitmask-searches-in-ldap/>
+has bit 2 set. See <https://ctovswild.com/2009/09/03/bitmask-searches-in-ldap/>
 for more information.
 
 The user will be set to `ldap_blocked` state in GitLab if the above conditions

doc/administration/gitaly/praefect.md

@@ -256,9 +256,9 @@ application server, or a Gitaly node.
 
    ```ruby
    # Name of storage hash must match storage name in git_data_dirs on GitLab
-   # server ('storage_1') and in git_data_dirs on Gitaly nodes ('gitaly-1')
+   # server ('praefect') and in git_data_dirs on Gitaly nodes ('gitaly-1')
    praefect['virtual_storages'] = {
-     'storage_1' => {
+     'praefect' => {
        'gitaly-1' => {
          'address' => 'tcp://GITALY_HOST:8075',
          'token'   => 'PRAEFECT_INTERNAL_TOKEN',

doc/administration/operations/extra_sidekiq_processes.md

@@ -228,20 +228,8 @@ When disabling `sidekiq_cluster`, you must copy your configuration for
 `sidekiq_cluster` will be overridden by the options for `sidekiq` when
 setting `sidekiq['cluster'] = true`.
 
-When using this feature, replace the `sidekiq` service with the
-`sidekiq-cluster`service for `gitlab-ctl` commands.
-
-For example, instead of:
-
-```shell
-sudo gitlab-ctl restart sidekiq
-```
-
-Use:
-
-```shell
-sudo gitlab-ctl restart sidekiq-cluster
-```
+When using this feature, the service called `sidekiq` will now be
+running `sidekiq-cluster`.
 
 The [concurrency](#managing-concurrency) and other options configured
 for Sidekiq will be respected.

doc/administration/troubleshooting/log_parsing.md

@@ -7,7 +7,7 @@ but if they are not available you can still quickly parse
 
 ## What is JQ?
 
-As noted in its [manual](https://stedolan.github.io/jq/manual), jq is a command-line JSON processor. The following examples
+As noted in its [manual](https://stedolan.github.io/jq/manual/), jq is a command-line JSON processor. The following examples
 include use cases targeted for parsing GitLab log files.
 
 ## Parsing Logs

doc/api/projects.md

@@ -162,6 +162,8 @@ When the user is authenticated and `simple` is not set this returns something li
     "merge_method": "merge",
     "autoclose_referenced_issues": true,
     "suggestion_commit_message": null,
+    "marked_for_deletion_at": "2020-04-03",
+    "marked_for_deletion_on": "2020-04-03",
     "statistics": {
       "commit_count": 37,
       "storage_size": 1038090,
@@ -406,6 +408,8 @@ This endpoint supports [keyset pagination](README.md#keyset-based-pagination) fo
     "merge_method": "merge",
     "autoclose_referenced_issues": true,
     "suggestion_commit_message": null,
+    "marked_for_deletion_at": "2020-04-03",
+    "marked_for_deletion_on": "2020-04-03",
     "statistics": {
       "commit_count": 37,
       "storage_size": 1038090,
@@ -870,6 +874,8 @@ GET /projects/:id
   "service_desk_address": null,
   "autoclose_referenced_issues": true,
   "suggestion_commit_message": null,
+  "marked_for_deletion_at": "2020-04-03",
+  "marked_for_deletion_on": "2020-04-03",
   "statistics": {
     "commit_count": 37,
     "storage_size": 1038090,

doc/api/search.md

 # Search API
 
-[Introduced][ce-41763] in GitLab 10.5
+> [Introduced](https://gitlab.com/gitlab-org/gitlab-foss/issues/41763) in GitLab 10.5.
 
 Every API call to search must be authenticated.
 
@@ -312,7 +312,7 @@ Example response:
 ]
 ```
 
-**Note:** `filename` is deprecated in favor of `path`. Both return the full path of the file inside the repository, but in the future `filename` will be only the file name and not the full path (see [this issue][gitlab-34521]).
+**Note:** `filename` is deprecated in favor of `path`. Both return the full path of the file inside the repository, but in the future `filename` will be only the file name and not the full path. For details, see [issue 34521](https://gitlab.com/gitlab-org/gitlab/issues/34521).
 
 ### Scope: commits **(STARTER)**
 
@@ -383,7 +383,7 @@ Example response:
 ]
 ```
 
-**Note:** `filename` is deprecated in favor of `path`. Both return the full path of the file inside the repository, but in the future `filename` will be only the file name and not the full path (see [this issue][gitlab-34521]).
+**Note:** `filename` is deprecated in favor of `path`. Both return the full path of the file inside the repository, but in the future `filename` will be only the file name and not the full path. For details, see [issue 34521](https://gitlab.com/gitlab-org/gitlab/issues/34521).
 
 ### Scope: users
 
@@ -653,7 +653,7 @@ Example response:
 ]
 ```
 
-**Note:** `filename` is deprecated in favor of `path`. Both return the full path of the file inside the repository, but in the future `filename` will be only the file name and not the full path (see [this issue][gitlab-34521]).
+**Note:** `filename` is deprecated in favor of `path`. Both return the full path of the file inside the repository, but in the future `filename` will be only the file name and not the full path. For details, see [issue 34521](https://gitlab.com/gitlab-org/gitlab/issues/34521).
 
 ### Scope: commits **(STARTER)**
 
@@ -724,7 +724,7 @@ Example response:
 ]
 ```
 
-**Note:** `filename` is deprecated in favor of `path`. Both return the full path of the file inside the repository, but in the future `filename` will be only the file name and not the full path (see [this issue][gitlab-34521]).
+**Note:** `filename` is deprecated in favor of `path`. Both return the full path of the file inside the repository, but in the future `filename` will be only the file name and not the full path. For details, see [issue 34521](https://gitlab.com/gitlab-org/gitlab/issues/34521).
 
 ### Scope: users
 
@@ -1009,7 +1009,7 @@ Example response:
 ]
 ```
 
-**Note:** `filename` is deprecated in favor of `path`. Both return the full path of the file inside the repository, but in the future `filename` will be only the file name and not the full path (see [this issue][gitlab-34521]).
+**Note:** `filename` is deprecated in favor of `path`. Both return the full path of the file inside the repository, but in the future `filename` will be only the file name and not the full path. For details, see [issue 34521](https://gitlab.com/gitlab-org/gitlab/issues/34521).
 
 ### Scope: commits
 
@@ -1082,7 +1082,7 @@ Example response:
 ]
 ```
 
-**Note:** `filename` is deprecated in favor of `path`. Both return the full path of the file inside the repository, but in the future `filename` will be only the file name and not the full path (see [this issue][gitlab-34521]).
+**Note:** `filename` is deprecated in favor of `path`. Both return the full path of the file inside the repository, but in the future `filename` will be only the file name and not the full path. For details, see [issue 34521](https://gitlab.com/gitlab-org/gitlab/issues/34521).
 
 ### Scope: users
 
@@ -1104,6 +1104,3 @@ Example response:
   }
 ]
 ```
-
-[ce-41763]: https://gitlab.com/gitlab-org/gitlab-foss/issues/41763
-[gitlab-34521]: https://gitlab.com/gitlab-org/gitlab/issues/34521

doc/ci/parent_child_pipelines.md

@@ -136,11 +136,12 @@ your own script to generate a YAML file, which is then [used to trigger a child
 This technique can be very powerful in generating pipelines targeting content that changed or to
 build a matrix of targets and architectures.
 
-In GitLab 12.9, the child pipeline could fail to be created in certain cases, causing the parent pipeline to fail.
-This is [resolved in GitLab 12.10](https://gitlab.com/gitlab-org/gitlab/-/issues/209070).
-
 ## Limitations
 
 A parent pipeline can trigger many child pipelines, but a child pipeline cannot trigger
 further child pipelines. See the [related issue](https://gitlab.com/gitlab-org/gitlab/issues/29651)
 for discussion on possible future improvements.
+
+When triggering dynamic child pipelines, if the job containing the CI config artifact is not a predecessor of the
+trigger job, the child pipeline will fail to be created, causing also the parent pipeline to fail.
+In the future we want to validate the trigger job's dependencies [at the time the parent pipeline is created](https://gitlab.com/gitlab-org/gitlab/-/issues/209070) rather than when the child pipeline is created.

doc/development/documentation/site_architecture/release_process.md

@@ -91,8 +91,9 @@ this needs to happen when the stable branches for all products have been created
    Once you push, the `image:docker-singe` job will create a new Docker image
    tagged with the branch name you created in the first step. In the end, the
    image will be uploaded in the [Container Registry](https://gitlab.com/gitlab-org/gitlab-docs/container_registry)
-   and it will be listed under the
-   [`registry` environment folder](https://gitlab.com/gitlab-org/gitlab-docs/-/environments/folders/registry).
+   and it will be listed under the `registry` environment folder at
+   `https://gitlab.com/gitlab-org/gitlab-docs/-/environments/folders/registry` (must
+   have developer access).
 
 Optionally, you can test locally by building the image and running it:
 

doc/integration/auth0.md

@@ -81,13 +81,10 @@ application.
 1. Change `YOUR_AUTH0_CLIENT_SECRET` to the client secret from the Auth0 Console
    page from step 5.
 
-1. [Reconfigure][] or [restart GitLab][] for the changes to take effect if you
+1. [Reconfigure](../administration/restart_gitlab.md#omnibus-gitlab-reconfigure) or [restart GitLab](../administration/restart_gitlab.md#installations-from-source) for the changes to take effect if you
    installed GitLab via Omnibus or from source respectively.
 
 On the sign in page there should now be an Auth0 icon below the regular sign in
 form. Click the icon to begin the authentication process. Auth0 will ask the
 user to sign in and authorize the GitLab application. If everything goes well
 the user will be returned to GitLab and will be signed in.
-
-[reconfigure]: ../administration/restart_gitlab.md#omnibus-gitlab-reconfigure
-[restart GitLab]: ../administration/restart_gitlab.md#installations-from-source

doc/integration/azure.md

@@ -81,10 +81,7 @@ To enable the Microsoft Azure OAuth2 OmniAuth provider you must register your ap
 
 1. Save the configuration file.
 
-1. [Reconfigure][] or [restart GitLab][] for the changes to take effect if you
+1. [Reconfigure](../administration/restart_gitlab.md#omnibus-gitlab-reconfigure) or [restart GitLab](../administration/restart_gitlab.md#installations-from-source) for the changes to take effect if you
    installed GitLab via Omnibus or from source respectively.
 
 On the sign in page there should now be a Microsoft icon below the regular sign in form. Click the icon to begin the authentication process. Microsoft will ask the user to sign in and authorize the GitLab application. If everything goes well the user will be returned to GitLab and will be signed in.
-
-[reconfigure]: ../administration/restart_gitlab.md#omnibus-gitlab-reconfigure
-[restart GitLab]: ../administration/restart_gitlab.md#installations-from-source

doc/integration/bitbucket.md

@@ -118,8 +118,8 @@ you to use.
    from the Bitbucket application page.
 
 1. Save the configuration file.
-1. For the changes to take effect, [reconfigure GitLab][] if you installed via
-   Omnibus, or [restart][] if installed from source.
+1. For the changes to take effect, [reconfigure GitLab](../administration/restart_gitlab.md#omnibus-gitlab-reconfigure) if you installed via
+   Omnibus, or [restart](../administration/restart_gitlab.md#installations-from-source) if installed from source.
 
 On the sign in page there should now be a Bitbucket icon below the regular sign
 in form. Click the icon to begin the authentication process. Bitbucket will ask
@@ -129,11 +129,7 @@ well, the user will be returned to GitLab and will be signed in.
 ## Bitbucket project import
 
 Once the above configuration is set up, you can use Bitbucket to sign into
-GitLab and [start importing your projects][bb-import].
+GitLab and [start importing your projects](../user/project/import/bitbucket.md).
 
 If you want to import projects from Bitbucket, but don't want to enable signing in,
 you can [disable Sign-Ins in the admin panel](omniauth.md#enable-or-disable-sign-in-with-an-omniauth-provider-without-disabling-import-sources).
-
-[bb-import]: ../user/project/import/bitbucket.md
-[reconfigure GitLab]: ../administration/restart_gitlab.md#omnibus-gitlab-reconfigure
-[restart]: ../administration/restart_gitlab.md#installations-from-source

doc/integration/cas.md

@@ -57,10 +57,8 @@ To enable the CAS OmniAuth provider you must register your application with your
 
 1. Save the configuration file.
 
-1. [Reconfigure][] or [restart GitLab][] for the changes to take effect if you
-   installed GitLab via Omnibus or from source respectively.
+1. [Reconfigure](../administration/restart_gitlab.md#omnibus-gitlab-reconfigure) or
+   [restart GitLab](../administration/restart_gitlab.md#installations-from-source) for the changes to
+   take effect if you installed GitLab via Omnibus or from source respectively.
 
 On the sign in page there should now be a CAS tab in the sign in form.
-
-[reconfigure]: ../administration/restart_gitlab.md#omnibus-gitlab-reconfigure
-[restart GitLab]: ../administration/restart_gitlab.md#installations-from-source

doc/integration/facebook.md

@@ -92,10 +92,7 @@ To enable the Facebook OmniAuth provider you must register your application with
 
 1. Save the configuration file.
 
-1. [Reconfigure][] or [restart GitLab][] for the changes to take effect if you
+1. [Reconfigure](../administration/restart_gitlab.md#omnibus-gitlab-reconfigure) or [restart GitLab](../administration/restart_gitlab.md#installations-from-source) for the changes to take effect if you
    installed GitLab via Omnibus or from source respectively.
 
 On the sign in page there should now be a Facebook icon below the regular sign in form. Click the icon to begin the authentication process. Facebook will ask the user to sign in and authorize the GitLab application. If everything goes well the user will be returned to GitLab and will be signed in.
-
-[reconfigure]: ../administration/restart_gitlab.md#omnibus-gitlab-reconfigure
-[restart GitLab]: ../administration/restart_gitlab.md#installations-from-source

doc/integration/gitlab.md

@@ -75,12 +75,9 @@ GitLab.com will generate an application ID and secret key for you to use.
 
 1. Save the configuration file.
 
-1. [Reconfigure][] or [restart GitLab][] for the changes to take effect if you
+1. [Reconfigure](../administration/restart_gitlab.md#omnibus-gitlab-reconfigure) or [restart GitLab](../administration/restart_gitlab.md#installations-from-source) for the changes to take effect if you
    installed GitLab via Omnibus or from source respectively.
 
 On the sign in page there should now be a GitLab.com icon below the regular sign in form.
 Click the icon to begin the authentication process. GitLab.com will ask the user to sign in and authorize the GitLab application.
 If everything goes well the user will be returned to your GitLab instance and will be signed in.
-
-[reconfigure]: ../administration/restart_gitlab.md#omnibus-gitlab-reconfigure
-[restart GitLab]: ../administration/restart_gitlab.md#installations-from-source

doc/integration/google.md

@@ -108,13 +108,10 @@ On your GitLab server:
    ```
 
 1. Save the configuration file.
-1. [Reconfigure][] or [restart GitLab][] for the changes to take effect if you
+1. [Reconfigure](../administration/restart_gitlab.md#omnibus-gitlab-reconfigure) or [restart GitLab](../administration/restart_gitlab.md#installations-from-source) for the changes to take effect if you
    installed GitLab via Omnibus or from source respectively.
 
 On the sign in page there should now be a Google icon below the regular sign in
 form. Click the icon to begin the authentication process. Google will ask the
 user to sign in and authorize the GitLab application. If everything goes well
 the user will be returned to GitLab and will be signed in.
-
-[reconfigure]: ../administration/restart_gitlab.md#omnibus-gitlab-reconfigure
-[restart GitLab]: ../administration/restart_gitlab.md#installations-from-source

doc/integration/jira_development_panel.md

 # GitLab Jira development panel integration **(PREMIUM)**
 
-> [Introduced][ee-2381] in [GitLab Premium][eep] 10.0.
+> [Introduced](https://gitlab.com/gitlab-org/gitlab/issues/2381) in [GitLab Premium](https://about.gitlab.com/pricing/) 10.0.
 
-Complementary to our [existing Jira][existing-jira] project integration, you're now able to integrate
-GitLab projects with [Jira Development Panel][jira-development-panel]. Both can be used
+Complementary to our [existing Jira](../user/project/integrations/jira.md) project integration, you're now able to integrate
+GitLab projects with [Jira Development Panel](https://confluence.atlassian.com/adminjiraserver070/). Both can be used
 simultaneously. This works with self-managed GitLab or GitLab.com integrated with:
 
 - Jira hosted by you.
@@ -17,7 +17,7 @@ as well as projects of the top-level group's subgroups nesting down, are connect
 a GitLab personal namespace in the Jira configuration, which will then connect the projects in that personal namespace to Jira.
 
 NOTE: **Note**:
-Note this is different from the [existing Jira][existing-jira] project integration, where the mapping
+Note this is different from the [existing Jira](../user/project/integrations/jira.md) project integration, where the mapping
 is one GitLab project to the entire Jira instance.
 
 We recommend that a GitLab group admin
@@ -164,7 +164,7 @@ Click the links to see your GitLab repository data.
 
 ## Limitations
 
-- This integration is currently not supported on GitLab instances under a [relative url][relative-url] (e.g. `http://example.com/gitlab`).
+- This integration is currently not supported on GitLab instances under a [relative url](https://docs.gitlab.com/omnibus/settings/configuration.html#configuring-a-relative-url-for-gitlab) (for example, `http://example.com/gitlab`).
 
 ## Changelog
 
@@ -175,9 +175,3 @@ Click the links to see your GitLab repository data.
 ### 11.1
 
 - [Support GitLab subgroups in Jira development panel](https://gitlab.com/gitlab-org/gitlab/issues/3561)
-
-[existing-jira]: ../user/project/integrations/jira.md
-[jira-development-panel]: https://confluence.atlassian.com/adminjiraserver070/integrating-with-development-tools-776637096.html#Integratingwithdevelopmenttools-Developmentpanelonissues
-[eep]: https://about.gitlab.com/pricing/
-[ee-2381]: https://gitlab.com/gitlab-org/gitlab/issues/2381
-[relative-url]: https://docs.gitlab.com/omnibus/settings/configuration.html#configuring-a-relative-url-for-gitlab

doc/integration/kerberos.md

 # Kerberos integration **(STARTER ONLY)**
 
-GitLab can integrate with [Kerberos][kerb] as an authentication mechanism.
+GitLab can integrate with [Kerberos](https://web.mit.edu/kerberos/) as an authentication mechanism.
 
 ## Overview
 
-[Kerberos][kerb] is a secure method for authenticating a request for a service in a
+[Kerberos](https://web.mit.edu/kerberos/) is a secure method for authenticating a request for a service in a
 computer network. Kerberos was developed in the Athena Project at the
-[Massachusetts Institute of Technology (MIT)][mit]. The name is taken from Greek
+[Massachusetts Institute of Technology (MIT)](http://web.mit.edu/). The name is taken from Greek
 mythology; Kerberos was a three-headed dog who guarded the gates of Hades.
 
 ## Use-cases
 
 - GitLab can be configured to allow your users to sign with their Kerberos credentials.
-- You can use Kerberos to [prevent][why-kerb] anyone from intercepting or eavesdropping on the transmitted password.
+- You can use Kerberos to [prevent](http://web.mit.edu/sipb/doc/working/guide/guide/node20.html) anyone from intercepting or eavesdropping on the transmitted password.
 
 ## Configuration
 
@@ -65,7 +65,7 @@ For source installations, make sure the `kerberos` gem group
      keytab: /etc/http.keytab
    ```
 
-1. [Restart GitLab] for the changes to take effect.
+1. [Restart GitLab](../administration/restart_gitlab.md#installations-from-source) for the changes to take effect.
 
 **Omnibus package installations**
 
@@ -78,7 +78,7 @@ For source installations, make sure the `kerberos` gem group
    gitlab_rails['kerberos_keytab'] = "/etc/http.keytab"
    ```
 
-1. [Reconfigure GitLab] for the changes to take effect.
+1. [Reconfigure GitLab](../administration/restart_gitlab.md#omnibus-gitlab-reconfigure) for the changes to take effect.
 
 GitLab will now offer the `negotiate` authentication method for signing in and
 HTTP Git access, enabling Git clients that support this authentication protocol
@@ -165,7 +165,7 @@ keep offering only `basic` authentication.
      https: true
    ```
 
-1. [Restart GitLab] and NGINX for the changes to take effect.
+1. [Restart GitLab](../administration/restart_gitlab.md#installations-from-source) and NGINX for the changes to take effect.
 
 **For Omnibus package installations**
 
@@ -177,7 +177,7 @@ keep offering only `basic` authentication.
    gitlab_rails['kerberos_https'] = true
    ```
 
-1. [Reconfigure GitLab] for the changes to take effect.
+1. [Reconfigure GitLab](../administration/restart_gitlab.md#omnibus-gitlab-reconfigure) for the changes to take effect.
 
 After this change, all Git remote URLs will have to be updated to
 `https://gitlab.example.com:8443/mygroup/myproject.git` in order to use
@@ -212,7 +212,7 @@ remove the OmniAuth provider named `kerberos` from your `gitlab.yml` /
        - { name: 'kerberos' } # <-- remove this line
    ```
 
-1. [Restart GitLab] for the changes to take effect.
+1. [Restart GitLab](../administration/restart_gitlab.md#installations-from-source) for the changes to take effect.
 
 **For Omnibus installations**
 
@@ -225,7 +225,7 @@ remove the OmniAuth provider named `kerberos` from your `gitlab.yml` /
    ]
    ```
 
-1. [Reconfigure GitLab] for the changes to take effect.
+1. [Reconfigure GitLab](../administration/restart_gitlab.md#omnibus-gitlab-reconfigure) for the changes to take effect.
 
 ## Support for Active Directory Kerberos environments
 
@@ -233,7 +233,7 @@ When using Kerberos ticket-based authentication in an Active Directory domain,
 it may be necessary to increase the maximum header size allowed by NGINX,
 as extensions to the Kerberos protocol may result in HTTP authentication headers
 larger than the default size of 8kB. Configure `large_client_header_buffers`
-to a larger value in [the NGINX configuration][nginx].
+to a larger value in [the NGINX configuration](http://nginx.org/en/docs/http/ngx_http_core_module.html#large_client_header_buffers).
 
 ## Troubleshooting
 
@@ -293,11 +293,3 @@ See also: [Git v2.11 release notes](https://github.com/git/git/blob/master/Docum
 - <https://help.ubuntu.com/community/Kerberos>
 - <http://blog.manula.org/2012/04/setting-up-kerberos-server-with-debian.html>
 - <https://www.roguelynn.com/words/explain-like-im-5-kerberos/>
-
-[restart gitlab]: ../administration/restart_gitlab.md#installations-from-source
-[reconfigure gitlab]: ../administration/restart_gitlab.md#omnibus-gitlab-reconfigure
-[nginx]: http://nginx.org/en/docs/http/ngx_http_core_module.html#large_client_header_buffers
-[kerb]: https://web.mit.edu/kerberos/
-[mit]: http://web.mit.edu/
-[why-kerb]: http://web.mit.edu/sipb/doc/working/guide/guide/node20.html
-[ee]: https://about.gitlab.com/pricing/

doc/integration/oauth2_generic.md

@@ -51,7 +51,7 @@ This strategy is designed to allow configuration of the simple OmniAuth SSO proc
 
 1. See [Initial OmniAuth Configuration](omniauth.md#initial-omniauth-configuration) for initial settings
 
-1. Add the provider-specific configuration for your provider, as [described in the gem's README][1]
+1. Add the provider-specific configuration for your provider, as [described in the gem's README](https://gitlab.com/satorix/omniauth-oauth2-generic#gitlab-config-example)
 
 1. Save the configuration file
 
@@ -61,5 +61,3 @@ On the sign in page there should now be a new button below the regular sign in f
 Click the button to begin your provider's authentication process. This will direct
 the browser to your OAuth2 Provider's authentication page. If everything goes well
 the user will be returned to your GitLab instance and will be signed in.
-
-[1]: https://gitlab.com/satorix/omniauth-oauth2-generic#gitlab-config-example

doc/integration/openid_connect_provider.md

@@ -16,13 +16,13 @@ mobile applications.
 On the client side, you can use [OmniAuth::OpenIDConnect](https://github.com/jjbohn/omniauth-openid-connect/) for Rails
 applications, or any of the other available [client implementations](https://openid.net/developers/libraries/#connect).
 
-GitLab's implementation uses the [doorkeeper-openid_connect] gem, refer
+GitLab's implementation uses the [doorkeeper-openid_connect](https://github.com/doorkeeper-gem/doorkeeper-openid_connect "Doorkeeper::OpenidConnect website") gem, refer
 to its README for more details about which parts of the specifications
 are supported.
 
 ## Enabling OpenID Connect for OAuth applications
 
-Refer to the [OAuth guide] for basic information on how to set up OAuth
+Refer to the [OAuth guide](oauth_provider.md) for basic information on how to set up OAuth
 applications in GitLab. To enable OIDC for an application, all you have to do
 is select the `openid` scope in the application settings.
 
@@ -45,6 +45,3 @@ Currently the following user information is shared with clients:
 | `groups`         | `array`   | Names of the groups the user is a member of
 
 Only the `sub` and `sub_legacy` claims are included in the ID token, all other claims are available from the `/oauth/userinfo` endpoint used by OIDC clients.
-
-[doorkeeper-openid_connect]: https://github.com/doorkeeper-gem/doorkeeper-openid_connect "Doorkeeper::OpenidConnect website"
-[OAuth guide]: oauth_provider.md "GitLab as OAuth2 authentication service provider"

doc/integration/saml.md

@@ -120,7 +120,7 @@ in your SAML IdP:
 1. Change the value of `issuer` to a unique name, which will identify the application
    to the IdP.
 
-1. For the changes to take effect, you must [reconfigure][] GitLab if you installed via Omnibus or [restart GitLab][] if you installed from source.
+1. For the changes to take effect, you must [reconfigure](../administration/restart_gitlab.md#omnibus-gitlab-reconfigure) GitLab if you installed via Omnibus or [restart GitLab](../administration/restart_gitlab.md#installations-from-source) if you installed from source.
 
 1. Register the GitLab SP in your SAML 2.0 IdP, using the application name specified
    in `issuer`.
@@ -294,7 +294,7 @@ If you want some SAML authentication methods to count as 2FA on a per session ba
    ]
    ```
 
-1. Save the file and [reconfigure][] GitLab for the changes to take effect.
+1. Save the file and [reconfigure](../administration/restart_gitlab.md#omnibus-gitlab-reconfigure) GitLab for the changes to take effect.
 
 ---
 
@@ -324,7 +324,7 @@ If you want some SAML authentication methods to count as 2FA on a per session ba
          }
    ```
 
-1. Save the file and [restart GitLab][] for the changes ot take effect
+1. Save the file and [restart GitLab](../administration/restart_gitlab.md#installations-from-source) for the changes ot take effect
 
 In addition to the changes in GitLab, make sure that your Idp is returning the
 `AuthnContext`. For example:
@@ -630,6 +630,3 @@ For this you need take the following into account:
 
 Make sure that one of the above described scenarios is valid, or the requests will
 fail with one of the mentioned errors.
-
-[reconfigure]: ../administration/restart_gitlab.md#omnibus-gitlab-reconfigure
-[restart GitLab]: ../administration/restart_gitlab.md#installations-from-source

doc/integration/trello_power_up.md

@@ -36,7 +36,5 @@ from Trello.
 your user **Settings** > **Access Tokens**.
 
 Learn more about generating a personal access token in the
-[Personal Access Token Documentation][personal-access-token-documentation].
+[Personal Access Token Documentation](../user/profile/personal_access_tokens.md).
 Don't forget to check the API scope checkbox!
-
-[personal-access-token-documentation]: ../user/profile/personal_access_tokens.md

doc/integration/twitter.md

@@ -75,10 +75,7 @@ To enable the Twitter OmniAuth provider you must register your application with
 
 1. Save the configuration file.
 
-1. [Reconfigure][] or [restart GitLab][] for the changes to take effect if you
+1. [Reconfigure](../administration/restart_gitlab.md#omnibus-gitlab-reconfigure) or [restart GitLab](../administration/restart_gitlab.md#installations-from-source) for the changes to take effect if you
    installed GitLab via Omnibus or from source respectively.
 
 On the sign in page there should now be a Twitter icon below the regular sign in form. Click the icon to begin the authentication process. Twitter will ask the user to sign in and authorize the GitLab application. If everything goes well the user will be returned to GitLab and will be signed in.
-
-[reconfigure]: ../administration/restart_gitlab.md#omnibus-gitlab-reconfigure
-[restart GitLab]: ../administration/restart_gitlab.md#installations-from-source

doc/user/application_security/dependency_scanning/index.md

@@ -62,7 +62,7 @@ The following languages and dependency managers are supported.
 | PHP ([Composer](https://getcomposer.org/))  | yes | [gemnasium](https://gitlab.com/gitlab-org/security-products/gemnasium) |
 | Python ([pip](https://pip.pypa.io/en/stable/)) | yes | [gemnasium](https://gitlab.com/gitlab-org/security-products/gemnasium) |
 | Python ([Pipfile](https://pipenv.kennethreitz.org/en/latest/basics/)) | not currently ([issue](https://gitlab.com/gitlab-org/gitlab/issues/11756 "Pipfile.lock support for Dependency Scanning"))| not available |
-| Python ([poetry](http://python-poetry.org/)) | not currently ([issue](https://gitlab.com/gitlab-org/gitlab/issues/7006 "Support Poetry in Dependency Scanning")) | not available |
+| Python ([poetry](https://python-poetry.org/)) | not currently ([issue](https://gitlab.com/gitlab-org/gitlab/issues/7006 "Support Poetry in Dependency Scanning")) | not available |
 | Ruby ([gem](https://rubygems.org/)) | yes | [gemnasium](https://gitlab.com/gitlab-org/security-products/gemnasium), [bundler-audit](https://github.com/rubysec/bundler-audit) |
 | Scala ([sbt](https://www.scala-sbt.org/)) | yes | [gemnasium](https://gitlab.com/gitlab-org/security-products/gemnasium) |
 | Go ([Go Modules](https://github.com/golang/go/wiki/Modules)) | yes ([alpha](https://gitlab.com/gitlab-org/gitlab/issues/7132)) | [gemnasium](https://gitlab.com/gitlab-org/security-products/gemnasium) |

doc/user/compliance/license_compliance/index.md

@@ -60,7 +60,7 @@ The following languages and package managers are supported.
 | Elixir     | [mix](https://elixir-lang.org/getting-started/mix-otp/introduction-to-mix.html) ([experimental support](https://github.com/pivotal/LicenseFinder#experimental-project-types)) |[License Finder](https://github.com/pivotal/LicenseFinder)|
 | C++/C      | [conan](https://conan.io/) ([experimental support](https://github.com/pivotal/LicenseFinder#experimental-project-types))|[License Finder](https://github.com/pivotal/LicenseFinder)|
 | Scala      | [sbt](https://www.scala-sbt.org/) ([experimental support](https://github.com/pivotal/LicenseFinder#experimental-project-types))|[License Finder](https://github.com/pivotal/LicenseFinder)|
-| Rust       | [cargo](https://crates.io) ([experimental support](https://github.com/pivotal/LicenseFinder#experimental-project-types))|[License Finder](https://github.com/pivotal/LicenseFinder)|
+| Rust       | [cargo](https://crates.io/) ([experimental support](https://github.com/pivotal/LicenseFinder#experimental-project-types))|[License Finder](https://github.com/pivotal/LicenseFinder)|
 | PHP        | [composer](https://getcomposer.org/) ([experimental support](https://github.com/pivotal/LicenseFinder#experimental-project-types))|[License Finder](https://github.com/pivotal/LicenseFinder)|
 
 ## Requirements

doc/user/markdown.md

@@ -995,7 +995,7 @@ Here's a sample audio clip:
 
 You can also use raw HTML in your Markdown, and it will usually work pretty well.
 
-See the documentation for HTML::Pipeline's [SanitizationFilter](https://www.rubydoc.info/gems/html-pipeline/1.11.0/HTML/Pipeline/SanitizationFilter#WHITELIST-constant)
+See the documentation for HTML::Pipeline's [SanitizationFilter](https://github.com/jch/html-pipeline/blob/v2.12.3/lib/html/pipeline/sanitization_filter.rb#L42)
 class for the list of allowed HTML tags and attributes. In addition to the default
 `SanitizationFilter` whitelist, GitLab allows `span`, `abbr`, `details` and `summary` elements.
 

doc/user/project/integrations/gitlab_slack_application.md

@@ -44,12 +44,12 @@ That's all! You can now start using the Slack slash commands.
 ## Usage
 
 After confirming the installation, you, and everyone else in your Slack team,
-can use all the [slash commands].
+can use all the [slash commands](../../../integration/slash_commands.md).
 
 When you perform your first slash command you will be asked to authorize your
 Slack user on GitLab.com.
 
-The only difference with the [manually configurable Slack slash commands][slack-manual]
+The only difference with the [manually configurable Slack slash commands](slack_slash_commands.md)
 is that all the commands should be prefixed with the `/gitlab` keyword.
 We are working on making this configurable in the future.
 
@@ -59,6 +59,3 @@ project, you would do:
 ```plaintext
 /gitlab gitlab-org/gitlab issue show 1001
 ```
-
-[slash commands]: ../../../integration/slash_commands.md
-[slack-manual]: slack_slash_commands.md

doc/user/project/integrations/index.md

@@ -2,7 +2,7 @@
 
 You can find the available integrations under your project's
 **Settings ➔ Integrations** page. You need to have at least
-[maintainer permission][permissions] on the project.
+[maintainer permission](../../permissions.md) on the project.
 
 ## Integrations
 
@@ -20,5 +20,3 @@ like pushes, issues or merge requests. GitLab will send a POST request with data
 to the webhook URL.
 
 [Learn more about webhooks.](webhooks.md)
-
-[permissions]: ../../permissions.md

doc/user/project/integrations/jira.md

@@ -37,7 +37,7 @@ configured. Therefore, you will not have to explicitly associate
 a GitLab project with any single Jira project.
 
 If you have one Jira instance, you can pre-fill the settings page with a default
-template. See the [Services Templates][services-templates] docs.
+template. See the [Services Templates](services_templates.md) docs.
 
 In order to enable the Jira service in GitLab, you need to first configure the project in Jira and then enter the correct values in GitLab.
 
@@ -55,7 +55,7 @@ In order to enable the Jira service in GitLab, you need to first configure the p
 
 > **Notes:**
 >
-> - The currently supported Jira versions are `v6.x` and `v7.x.`. GitLab 7.8 or
+> - The currently supported Jira versions are `v6.x, v7.x, v8.x` . GitLab 7.8 or
 >   higher is required.
 > - GitLab 8.14 introduced a new way to integrate with Jira which greatly simplified
 >   the configuration options you have to enter. If you are using an older version,
@@ -212,5 +212,3 @@ which may lead to a `401 unauthorized` error when testing your Jira integration.
 If CAPTCHA has been triggered, you will not be able to use Jira's REST API to
 authenticate with the Jira site. You will need to log in to your Jira instance
 and complete the CAPTCHA.
-
-[services-templates]: services_templates.md

doc/user/project/integrations/mattermost_slash_commands.md

@@ -15,7 +15,7 @@ Mattermost 3.4 and up is required.
 
 If you have the Omnibus GitLab package installed, Mattermost is already bundled
 in it. All you have to do is configure it. Read more in the
-[Omnibus GitLab Mattermost documentation][omnimmdocs].
+[Omnibus GitLab Mattermost documentation](https://docs.gitlab.com/omnibus/gitlab-mattermost/).
 
 ## Automated Configuration
 
@@ -133,7 +133,7 @@ The available slash commands are:
 | ------- | ----------- | ------- |
 | <kbd>/&lt;trigger&gt; issue new &lt;title&gt; <kbd>⇧ Shift</kbd>+<kbd>↵ Enter</kbd> &lt;description&gt;</kbd> | Create a new issue in the project that `<trigger>` is tied to. `<description>` is optional. | `/gitlab issue new We need to change the homepage` |
 | <kbd>/&lt;trigger&gt; issue show &lt;issue-number&gt;</kbd> | Show the issue with ID `<issue-number>` from the project that `<trigger>` is tied to. | `/gitlab issue show 42` |
-| <kbd>/&lt;trigger&gt; deploy &lt;environment&gt; to &lt;environment&gt;</kbd> | Start the CI job that deploys from one environment to another, for example `staging` to `production`. CI/CD must be [properly configured][ciyaml]. | `/gitlab deploy staging to production` |
+| <kbd>/&lt;trigger&gt; deploy &lt;environment&gt; to &lt;environment&gt;</kbd> | Start the CI job that deploys from one environment to another, for example `staging` to `production`. CI/CD must be [properly configured](../../../ci/yaml/README.md). | `/gitlab deploy staging to production` |
 
 To see a list of available commands to interact with GitLab, type the
 trigger word followed by <kbd>help</kbd>. Example: `/gitlab help`
@@ -147,9 +147,5 @@ the [permissions you have on the project](../../permissions.md#project-members-p
 
 ## Further reading
 
-- [Mattermost slash commands documentation][mmslashdocs]
-- [Omnibus GitLab Mattermost][omnimmdocs]
-
-[omnimmdocs]: https://docs.gitlab.com/omnibus/gitlab-mattermost/
-[mmslashdocs]: https://docs.mattermost.com/developer/slash-commands.html
-[ciyaml]: ../../../ci/yaml/README.md
+- [Mattermost slash commands documentation](https://docs.mattermost.com/developer/slash-commands.html)
+- [Omnibus GitLab Mattermost](https://docs.gitlab.com/omnibus/gitlab-mattermost/)

doc/user/project/integrations/prometheus.md

 # Prometheus integration
 
-> [Introduced][ce-8935] in GitLab 9.0.
+> [Introduced](https://gitlab.com/gitlab-org/gitlab-foss/-/merge_requests/8935) in GitLab 9.0.
 
-GitLab offers powerful integration with [Prometheus] for monitoring key metrics of your apps, directly within GitLab.
+GitLab offers powerful integration with [Prometheus](https://prometheus.io) for monitoring key metrics of your apps, directly within GitLab.
 Metrics for each environment are retrieved from Prometheus, and then displayed
 within the GitLab interface.
 
@@ -727,7 +727,7 @@ If the metric exceeds the threshold of the alert for over 5 minutes, an email wi
 
 ## Determining the performance impact of a merge
 
-> - [Introduced][ce-10408] in GitLab 9.2.
+> - [Introduced](https://gitlab.com/gitlab-org/gitlab-foss/-/merge_requests/10408) in GitLab 9.2.
 > - GitLab 9.3 added the [numeric comparison](https://gitlab.com/gitlab-org/gitlab-foss/issues/27439) of the 30 minute averages.
 
 Developers can view the performance impact of their changes within the merge
@@ -755,7 +755,7 @@ Prometheus server.
 
 ### Embedding GitLab-managed Kubernetes metrics
 
-> [Introduced][ce-29691] in GitLab 12.2.
+> [Introduced](https://gitlab.com/gitlab-org/gitlab-foss/-/merge_requests/29691) in GitLab 12.2.
 
 It is possible to display metrics charts within [GitLab Flavored Markdown](../../markdown.md#gitlab-flavored-markdown-gfm) fields such as issue or merge request descriptions. The maximum number of embedded charts allowed in a GitLab Flavored Markdown field is 100.
 
@@ -895,18 +895,3 @@ If the "No data found" screen continues to appear, it could be due to:
   [run a query](prometheus_library/kubernetes.md#metrics-supported), replacing `$CI_ENVIRONMENT_SLUG`
   with the name of your environment.
 - You may need to re-add the GitLab predefined common metrics. This can be done by running the [import common metrics Rake task](../../../administration/raketasks/maintenance.md#import-common-metrics).
-
-[autodeploy]: ../../../topics/autodevops/index.md#auto-deploy
-[kubernetes]: https://kubernetes.io
-[kube]: ./kubernetes.md
-[prometheus-k8s-sd]: https://prometheus.io/docs/operating/configuration/#<kubernetes_sd_config>
-[prometheus]: https://prometheus.io
-[gitlab-prometheus-k8s-monitor]: ../../../administration/monitoring/prometheus/index.md#configuring-prometheus-to-monitor-kubernetes
-[prometheus-docker-image]: https://hub.docker.com/r/prom/prometheus/
-[prometheus-yml]:samples/prometheus.yml
-[gitlab.com-ip-range]: https://gitlab.com/gitlab-com/infrastructure/issues/434
-[ci-environment-slug]: ../../../ci/variables/#predefined-environment-variables
-[ce-8935]: https://gitlab.com/gitlab-org/gitlab-foss/-/merge_requests/8935
-[ce-10408]: https://gitlab.com/gitlab-org/gitlab-foss/-/merge_requests/10408
-[ce-29691]: https://gitlab.com/gitlab-org/gitlab-foss/-/merge_requests/29691
-[promgldocs]: ../../../administration/monitoring/prometheus/index.md

doc/user/project/issues/design_management.md

@@ -126,6 +126,8 @@ to help summarize changes between versions.
 Designs can be explored in greater detail by zooming in and out of the image.
 Control the amount of zoom with the `+` and `-` buttons at the bottom of the image.
 While zoomed, you can still [start new discussions](#starting-discussions-on-designs) on the image, and see any existing ones.
+[Introduced](https://gitlab.com/gitlab-org/gitlab/issues/197324) in GitLab 12.10, while zoomed in,
+you can click-and-drag on the image to move around it.
 
 ![Design zooming](img/design_zooming_v12_7.png)
 

doc/user/project/merge_requests/test_coverage_visualization.md

@@ -34,7 +34,7 @@ for other languages have plugins to add support for it, like:
 Other coverage analysis frameworks support the format out of the box, for example:
 
 - [Istanbul](https://istanbul.js.org/docs/advanced/alternative-reporters/#cobertura) (JavaScript)
-- [Coverage.py](https://coverage.readthedocs.io/en/coverage-5.0/cmd.html#xml-reporting) (Python)
+- [Coverage.py](https://coverage.readthedocs.io/en/coverage-5.0.4/cmd.html#xml-reporting) (Python)
 
 Once configured, if you create a merge request that triggers a pipeline which collects
 coverage reports, the coverage will be shown in the diff view. This includes reports

doc/user/project/pages/custom_domains_ssl_tls_certification/lets_encrypt_integration.md

@@ -7,9 +7,6 @@ description: "Automatic Let's Encrypt SSL certificates for GitLab Pages."
 
 > [Introduced](https://gitlab.com/gitlab-org/gitlab-foss/issues/28996) in GitLab 12.1. For versions earlier than GitLab 12.1, see the [manual Let's Encrypt instructions](../lets_encrypt_for_gitlab_pages.md).
 
-NOTE: **Note:**
-This feature is in **beta** and may still have bugs. See all the related issues linked from this [issue's description](https://gitlab.com/gitlab-org/gitlab-foss/issues/28996) for more information.
-
 The GitLab Pages integration with Let's Encrypt (LE) allows you
 to use LE certificates for your Pages website with custom domains
 without the hassle of having to issue and update them yourself;
@@ -63,18 +60,35 @@ associated Pages domain. It also will be renewed automatically by GitLab.
 
 ## Troubleshooting
 
-### Error "Certificate misses intermediates"
+### Error "Something went wrong while obtaining Let's Encrypt certificate"
+
+> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/30146) in GitLab 13.0.
 
-If you get an error **Certificate misses intermediates** while trying to enable Let's Encrypt integration for your domain, follow the steps below:
+If you get an error **Something went wrong while obtaining Let's Encrypt certificate**, you can try obtaining the certificate again by following these steps:
 
 1. Go to your project's **Settings > Pages**.
-1. Turn off **Force HTTPS** if it's turned on.
-1. Click **Details** on your domain.
-1. Click the **Edit** button in the top right corner of domain details page.
-1. Enable Let's Encrypt integration.
-1. Click **Save**.
+1. Click **Edit** on your domain.
+1. Click **Retry**.
+1. If you're still seeing the same error:
+    1. Make sure you have properly set only one `CNAME` or `A` DNS record for your domain.
+    1. Make sure your domain **doesn't have** an `AAAA` DNS record.
+    1. If you have a `CAA` DNS record for your domain or any higher level domains, make sure [it includes `letsencrypt.org`](https://letsencrypt.org/docs/caa/).
+    1. Make sure [your domain is verified](index.md#1-add-a-custom-domain-to-pages).
+    1. Go to step 1.
+
+### Message "GitLab is obtaining a Let's Encrypt SSL certificate for this domain. This process can take some time. Please try again later." hangs for more than an hour
+
+If you've enabled Let's Encrypt integration, but a certificate is absent after an hour and you see the message, "GitLab is obtaining a Let's Encrypt SSL certificate for this domain. This process can take some time. Please try again later.", try to remove and add the domain for GitLab Pages again by following these steps:
+
 1. Go to your project's **Settings > Pages**.
-1. Turn on **Force HTTPS**.
+1. Click **Remove** on your domain.
+1. [Add the domain again and verify it](index.md#1-add-a-custom-domain-to-pages).
+1. [Enable Let's Encrypt integration for your domain](#enabling-lets-encrypt-integration-for-your-custom-domain).
+1. If you still see the same message after some time:
+    1. Make sure you have properly set only one `CNAME` or `A` DNS record for your domain.
+    1. Make sure your domain **doesn't have** an `AAAA` DNS record.
+    1. If you have a `CAA` DNS record for your domain or any higher level domains, make sure [it includes `letsencrypt.org`](https://letsencrypt.org/docs/caa/).
+    1. Go to step 1.
 
 <!-- Include any troubleshooting steps that you can foresee. If you know beforehand what issues
 one might have when setting this up, or when something is changed, or on upgrading, it's

doc/user/search/advanced_global_search.md

 # Advanced Global Search **(STARTER ONLY)**
 
-> - [Introduced][ee-109] in GitLab [Starter][ee] 8.4.
+> - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/109) in GitLab [Starter](https://about.gitlab.com/pricing/) 8.4.
 > - This is the user documentation. To install and configure Elasticsearch,
 >   visit the [administrator documentation](../../integration/elasticsearch.md).
 
@@ -16,7 +16,7 @@ The Advanced Global Search in GitLab is a powerful search service that saves
 you time. Instead of creating duplicate code and wasting time, you can
 now search for code within other teams that can help your own project.
 
-GitLab leverages the search capabilities of [Elasticsearch] and enables it when
+GitLab leverages the search capabilities of [Elasticsearch](https://www.elastic.co/elasticsearch/) and enables it when
 searching in:
 
 - GitLab application
@@ -56,20 +56,8 @@ project you have access to.
 You can also use the [Advanced Syntax Search](advanced_search_syntax.md) which
 provides some useful queries.
 
->**Note:**
+NOTE: **Note:**
 Elasticsearch has only data for the default branch. That means that if you go
 to the repository tree and switch the branch from the default to something else,
 then the "Code" tab in the search result page will be served by the regular
 search even if Elasticsearch is enabled.
-
-[ee-1305]: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/1305
-[aws-elastic]: https://docs.aws.amazon.com/elasticsearch-service/latest/developerguide/es-gsg.html
-[aws-iam]: http://docs.aws.amazon.com/IAM/latest/UserGuide/id_credentials_access-keys.html
-[aws-instance-profile]: http://docs.aws.amazon.com/codedeploy/latest/userguide/getting-started-create-iam-instance-profile.html#getting-started-create-iam-instance-profile-cli
-[ee-109]: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/109 "Elasticsearch Merge Request"
-[elasticsearch]: https://www.elastic.co/elasticsearch "Elasticsearch website"
-[install]: https://www.elastic.co/guide/en/elasticsearch/reference/current/_installation.html "Elasticsearch installation documentation"
-[pkg]: https://about.gitlab.com/downloads/ "Download Omnibus GitLab"
-[elastic-settings]: https://www.elastic.co/guide/en/elasticsearch/reference/current/setup-configuration.html#settings "Elasticsearch configuration settings"
-[ee]: https://about.gitlab.com/pricing/
-[es]: https://www.elastic.co/elasticsearch

doc/user/search/advanced_search_syntax.md

@@ -2,7 +2,7 @@
 
 > **Notes:**
 >
-> - Introduced in [GitLab Enterprise Starter][ee] 9.2
+> - Introduced in [GitLab Enterprise Starter](https://about.gitlab.com/pricing/) 9.2
 > - This is the user documentation. To install and configure Elasticsearch,
 >   visit the [administrator documentation](../../integration/elasticsearch.md).
 
@@ -37,7 +37,7 @@ better results.
 The Advanced Syntax Search supports fuzzy or exact search queries with prefixes,
 boolean operators, and much more.
 
-Full details can be found in the [Elasticsearch documentation][elastic], but
+Full details can be found in the [Elasticsearch documentation](https://www.elastic.co/guide/en/elasticsearch/reference/5.3/query-dsl-simple-query-string-query.html#_simple_query_string_syntax), but
 here's a quick guide:
 
 - Searches look for all the words in a query, in any order - e.g.: searching
@@ -67,6 +67,3 @@ Examples:
 - Finding the text 'def create' inside files with the `.rb` extension: `def create extension:rb`
 - Finding the text `sha` inside files in a folder called `encryption`: `sha path:encryption`
 - Finding any file starting with `hello` containing `world` and with the `.js` extension: `world filename:hello* extension:js`
-
-[ee]: https://about.gitlab.com/pricing/
-[elastic]: https://www.elastic.co/guide/en/elasticsearch/reference/5.3/query-dsl-simple-query-string-query.html#_simple_query_string_syntax

doc/user/search/index.md

@@ -171,5 +171,3 @@ GitLab instance.
 Use advanced queries for more targeted search results.
 
 [Learn how to use the Advanced Syntax Search.](advanced_search_syntax.md)
-
-[ee]: https://about.gitlab.com/pricing/

lib/gitlab/ci/config/entry/include.rb

@@ -15,11 +15,6 @@ module Gitlab
           validations do
             validates :config, hash_or_string: true
             validates :config, allowed_keys: ALLOWED_KEYS
-            validate do
-              if config[:artifact] && config[:job].blank?
-                errors.add(:config, "must specify the job where to fetch the artifact from")
-              end
-            end
           end
         end
       end

lib/gitlab/ci/templates/Security/DAST.gitlab-ci.yml

 # Read more about this feature here: https://docs.gitlab.com/ee/user/application_security/dast/
 
 # Configure the scanning tool through the environment variables.
-# List of the variables: https://gitlab.com/gitlab-org/security-products/dast#settings
+# List of the variables: https://docs.gitlab.com/ee/user/application_security/dast/#available-variables
 # How to set: https://docs.gitlab.com/ee/ci/yaml/#variables
 
 stages:
@@ -19,26 +19,10 @@ dast:
     name: "registry.gitlab.com/gitlab-org/security-products/dast:$DAST_VERSION"
   variables:
     GIT_STRATEGY: none
-  # URL to scan:
-  # DAST_WEBSITE: https://example.com/
-  #
-  # Time limit for target availability (scan is attempted even when timeout):
-  # DAST_TARGET_AVAILABILITY_TIMEOUT: 60
-  #
-  # Set these variables to scan with an authenticated user:
-  # DAST_AUTH_URL: https://example.com/sign-in
-  # DAST_USERNAME: john.doe@example.com
-  # DAST_PASSWORD: john-doe-password
-  # DAST_USERNAME_FIELD: session[user] # the name of username field at the sign-in HTML form
-  # DAST_PASSWORD_FIELD: session[password] # the name of password field at the sign-in HTML form
-  # DAST_AUTH_EXCLUDE_URLS: http://example.com/sign-out,http://example.com/sign-out-2 # optional: URLs to skip during the authenticated scan; comma-separated, no spaces in between
-  #
-  # Perform ZAP Full Scan, which includes both passive and active scanning:
-  # DAST_FULL_SCAN_ENABLED: "true"
   allow_failure: true
   script:
     - export DAST_WEBSITE=${DAST_WEBSITE:-$(cat environment_url.txt)}
-    - /analyze -t $DAST_WEBSITE
+    - /analyze
   artifacts:
     reports:
       dast: gl-dast-report.json

lib/gitlab/ci/yaml_processor.rb

@@ -142,7 +142,6 @@ module Gitlab
           validate_job_stage!(name, job)
           validate_job_dependencies!(name, job)
           validate_job_needs!(name, job)
-          validate_dynamic_child_pipeline_dependencies!(name, job)
           validate_job_environment!(name, job)
         end
       end
@@ -164,50 +163,35 @@ module Gitlab
       def validate_job_dependencies!(name, job)
         return unless job[:dependencies]
 
-        job[:dependencies].each do |dependency|
-          validate_job_dependency!(name, dependency)
-        end
-      end
+        stage_index = @stages.index(job[:stage])
 
-      def validate_dynamic_child_pipeline_dependencies!(name, job)
-        return unless includes = job.dig(:trigger, :include)
+        job[:dependencies].each do |dependency|
+          raise ValidationError, "#{name} job: undefined dependency: #{dependency}" unless @jobs[dependency.to_sym]
 
-        includes.each do |included|
-          next unless dependency = included[:job]
+          dependency_stage_index = @stages.index(@jobs[dependency.to_sym][:stage])
 
-          validate_job_dependency!(name, dependency)
+          unless dependency_stage_index.present? && dependency_stage_index < stage_index
+            raise ValidationError, "#{name} job: dependency #{dependency} is not defined in prior stages"
+          end
         end
       end
 
       def validate_job_needs!(name, job)
-        return unless needs = job.dig(:needs, :job)
+        return unless job.dig(:needs, :job)
 
-        needs.each do |need|
-          dependency = need[:name]
-          validate_job_dependency!(name, dependency, 'need')
-        end
-      end
+        stage_index = @stages.index(job[:stage])
 
-      def validate_job_dependency!(name, dependency, dependency_type = 'dependency')
-        unless @jobs[dependency.to_sym]
-          raise ValidationError, "#{name} job: undefined #{dependency_type}: #{dependency}"
-        end
+        job.dig(:needs, :job).each do |need|
+          need_job_name = need[:name]
 
-        job_stage_index = stage_index(name)
-        dependency_stage_index = stage_index(dependency)
+          raise ValidationError, "#{name} job: undefined need: #{need_job_name}" unless @jobs[need_job_name.to_sym]
 
-        # A dependency might be defined later in the configuration
-        # with a stage that does not exist
-        unless dependency_stage_index.present? && dependency_stage_index < job_stage_index
-          raise ValidationError, "#{name} job: #{dependency_type} #{dependency} is not defined in prior stages"
-        end
-      end
+          needs_stage_index = @stages.index(@jobs[need_job_name.to_sym][:stage])
 
-      def stage_index(name)
-        job = @jobs[name.to_sym]
-        return unless job
-
-        @stages.index(job[:stage])
+          unless needs_stage_index.present? && needs_stage_index < stage_index
+            raise ValidationError, "#{name} job: need #{need_job_name} is not defined in prior stages"
+          end
+        end
       end
 
       def validate_job_environment!(name, job)

lib/gitlab/import_export/json/ndjson_writer.rb

+# frozen_string_literal: true
+
+module Gitlab
+  module ImportExport
+    module JSON
+      class NdjsonWriter
+        include Gitlab::ImportExport::CommandLineUtil
+
+        def initialize(dir_path)
+          @dir_path = dir_path
+        end
+
+        def close
+        end
+
+        def write_attributes(exportable_path, hash)
+          # It will create:
+          # tree/project.json
+          with_file("#{exportable_path}.json") do |file|
+            file.write(hash.to_json)
+          end
+        end
+
+        def write_relation(exportable_path, relation, value)
+          # It will create:
+          # tree/project/ci_cd_setting.ndjson
+          with_file(exportable_path, "#{relation}.ndjson") do |file|
+            file.write(value.to_json)
+          end
+        end
+
+        def write_relation_array(exportable_path, relation, items)
+          # It will create:
+          # tree/project/merge_requests.ndjson
+          with_file(exportable_path, "#{relation}.ndjson") do |file|
+            items.each do |item|
+              file.write(item.to_json)
+              file.write("\n")
+            end
+          end
+        end
+
+        private
+
+        def with_file(*path)
+          file_path = File.join(@dir_path, *path)
+          raise ArgumentError, "The #{file_path} already exist" if File.exist?(file_path)
+
+          # ensure that path is created
+          mkdir_p(File.dirname(file_path))
+
+          File.open(file_path, "wb") do |file|
+            yield(file)
+          end
+        end
+      end
+    end
+  end
+end

lib/gitlab/import_export/project/tree_saver.rb

@@ -11,15 +11,9 @@ module Gitlab
           @project      = project
           @current_user = current_user
           @shared       = shared
-          @full_path    = File.join(@shared.export_path, ImportExport.project_filename)
         end
 
         def save
-          json_writer = ImportExport::JSON::LegacyWriter.new(
-            @full_path,
-            allowed_path: "project"
-          )
-
           ImportExport::JSON::StreamingSerializer.new(
             exportable,
             reader.project_tree,
@@ -57,6 +51,18 @@ module Gitlab
         def presenter_class
           Projects::ImportExport::ProjectExportPresenter
         end
+
+        def json_writer
+          @json_writer ||= begin
+            if ::Feature.enabled?(:project_export_as_ndjson, @project.namespace)
+              full_path = File.join(@shared.export_path, 'tree')
+              Gitlab::ImportExport::JSON::NdjsonWriter.new(full_path)
+            else
+              full_path = File.join(@shared.export_path, ImportExport.project_filename)
+              Gitlab::ImportExport::JSON::LegacyWriter.new(full_path, allowed_path: 'project')
+            end
+          end
+        end
       end
     end
   end

lib/gitlab/sidekiq_middleware.rb

@@ -29,12 +29,12 @@ module Gitlab
     # eg: `config.client_middleware(&Gitlab::SidekiqMiddleware.client_configurator)`
     def self.client_configurator
       lambda do |chain|
-        chain.add ::Gitlab::SidekiqStatus::ClientMiddleware
-        chain.add ::Gitlab::SidekiqMiddleware::ClientMetrics
         chain.add ::Gitlab::SidekiqMiddleware::WorkerContext::Client # needs to be before the Labkit middleware
         chain.add ::Labkit::Middleware::Sidekiq::Client
-        chain.add ::Gitlab::SidekiqMiddleware::AdminMode::Client
         chain.add ::Gitlab::SidekiqMiddleware::DuplicateJobs::Client
+        chain.add ::Gitlab::SidekiqStatus::ClientMiddleware
+        chain.add ::Gitlab::SidekiqMiddleware::AdminMode::Client
+        chain.add ::Gitlab::SidekiqMiddleware::ClientMetrics
       end
     end
   end

spec/features/issues/filtered_search/filter_issues_spec.rb

@@ -224,14 +224,6 @@ describe 'Filter issues', :js do
         expect_no_issues_list
         expect_filtered_search_input_empty
       end
-
-      it 'does show issues for bug label' do
-        input_filtered_search("label:!=~#{bug_label.title}")
-
-        expect_tokens([label_token(bug_label.title)])
-        expect_issues_list_count(6)
-        expect_filtered_search_input_empty
-      end
     end
 
     context 'label with multiple words' do

spec/features/projects/import_export/export_file_spec.rb

@@ -38,51 +38,109 @@ describe 'Import/Export - project export integration test', :js do
       sign_in(user)
     end
 
-    it 'exports a project successfully', :sidekiq_might_not_need_inline do
-      visit edit_project_path(project)
+    shared_examples 'export file without sensitive words' do
+      it 'exports a project successfully', :sidekiq_inline do
+        export_project_and_download_file(page, project)
 
-      expect(page).to have_content('Export project')
+        in_directory_with_expanded_export(project) do |exit_status, tmpdir|
+          expect(exit_status).to eq(0)
 
-      find(:link, 'Export project').send_keys(:return)
+          project_json_path = File.join(tmpdir, 'project.json')
+          expect(File).to exist(project_json_path)
 
-      visit edit_project_path(project)
+          project_hash = JSON.parse(IO.read(project_json_path))
 
-      expect(page).to have_content('Download export')
+          sensitive_words.each do |sensitive_word|
+            found = find_sensitive_attributes(sensitive_word, project_hash)
 
-      expect(project.export_status).to eq(:finished)
-      expect(project.export_file.path).to include('tar.gz')
+            expect(found).to be_nil, failure_message(found.try(:key_found), found.try(:parent), sensitive_word)
+          end
+        end
+      end
+    end
+
+    context "with legacy export" do
+      before do
+        stub_feature_flags(streaming_serializer: false)
+        stub_feature_flags(project_export_as_ndjson: false)
+      end
+
+      it_behaves_like "export file without sensitive words"
+    end
+
+    context "with streaming serializer" do
+      before do
+        stub_feature_flags(streaming_serializer: true)
+        stub_feature_flags(project_export_as_ndjson: false)
+      end
+
+      it_behaves_like "export file without sensitive words"
+    end
 
-      in_directory_with_expanded_export(project) do |exit_status, tmpdir|
-        expect(exit_status).to eq(0)
+    context "with ndjson" do
+      before do
+        stub_feature_flags(streaming_serializer: true)
+        stub_feature_flags(project_export_as_ndjson: true)
+      end
+
+      it 'exports a project successfully', :sidekiq_inline do
+        export_project_and_download_file(page, project)
+
+        in_directory_with_expanded_export(project) do |exit_status, tmpdir|
+          expect(exit_status).to eq(0)
 
-        project_json_path = File.join(tmpdir, 'project.json')
-        expect(File).to exist(project_json_path)
+          project_json_path = File.join(tmpdir, 'tree', 'project.json')
+          expect(File).to exist(project_json_path)
 
-        project_hash = JSON.parse(IO.read(project_json_path))
+          relations = []
+          relations << JSON.parse(IO.read(project_json_path))
+          Dir.glob(File.join(tmpdir, 'tree/project', '*.ndjson')) do |rb_filename|
+            File.foreach(rb_filename) do |line|
+              json = ActiveSupport::JSON.decode(line)
+              relations << json
+            end
+          end
 
-        sensitive_words.each do |sensitive_word|
-          found = find_sensitive_attributes(sensitive_word, project_hash)
+          relations.each do |relation_hash|
+            sensitive_words.each do |sensitive_word|
+              found = find_sensitive_attributes(sensitive_word, relation_hash)
 
-          expect(found).to be_nil, failure_message(found.try(:key_found), found.try(:parent), sensitive_word)
+              expect(found).to be_nil, failure_message(found.try(:key_found), found.try(:parent), sensitive_word)
+            end
+          end
         end
       end
     end
+  end
 
-    def failure_message(key_found, parent, sensitive_word)
-      <<-MSG
-        Found a new sensitive word <#{key_found}>, which is part of the hash #{parent.inspect}
+  def export_project_and_download_file(page, project)
+    visit edit_project_path(project)
 
-        If you think this information shouldn't get exported, please exclude the model or attribute in IMPORT_EXPORT_CONFIG.
+    expect(page).to have_content('Export project')
 
-        Otherwise, please add the exception to +safe_list+ in CURRENT_SPEC using #{sensitive_word} as the key and the
-        correspondent hash or model as the value.
+    find(:link, 'Export project').send_keys(:return)
 
-        Also, if the attribute is a generated unique token, please add it to RelationFactory::TOKEN_RESET_MODELS if it needs to be
-        reset (to prevent duplicate column problems while importing to the same instance).
+    visit edit_project_path(project)
 
-        IMPORT_EXPORT_CONFIG: #{Gitlab::ImportExport.config_file}
-        CURRENT_SPEC: #{__FILE__}
-      MSG
-    end
+    expect(page).to have_content('Download export')
+    expect(project.export_status).to eq(:finished)
+    expect(project.export_file.path).to include('tar.gz')
+  end
+
+  def failure_message(key_found, parent, sensitive_word)
+    <<-MSG
+      Found a new sensitive word <#{key_found}>, which is part of the hash #{parent.inspect}
+
+      If you think this information shouldn't get exported, please exclude the model or attribute in IMPORT_EXPORT_CONFIG.
+
+      Otherwise, please add the exception to +safe_list+ in CURRENT_SPEC using #{sensitive_word} as the key and the
+      correspondent hash or model as the value.
+
+      Also, if the attribute is a generated unique token, please add it to RelationFactory::TOKEN_RESET_MODELS if it needs to be
+      reset (to prevent duplicate column problems while importing to the same instance).
+
+      IMPORT_EXPORT_CONFIG: #{Gitlab::ImportExport.config_file}
+      CURRENT_SPEC: #{__FILE__}
+    MSG
   end
 end

spec/finders/environments_finder_spec.rb

@@ -37,11 +37,6 @@ describe EnvironmentsFinder do
           .to be_empty
       end
 
-      it 'returns environment when with_tags is set' do
-        expect(described_class.new(project, user, ref: 'master', commit: commit, with_tags: true).execute)
-          .to contain_exactly(environment, environment_two)
-      end
-
       # We expect two Gitaly calls: FindCommit, CommitIsAncestor
       # This tests to ensure we don't call one CommitIsAncestor per environment
       it 'only calls Gitaly twice when multiple environments are present', :request_store do

spec/helpers/users_helper_spec.rb

@@ -95,9 +95,9 @@ describe UsersHelper do
     end
 
     it 'includes the settings tab if the user can update themself' do
-      expect(helper).to receive(:can?).with(user, :read_user, user) { true }
+      expect(helper).to receive(:can?).with(user, :update_user, user) { true }
 
-      expect(items).to include(:profile)
+      expect(items).to include(:settings)
     end
 
     context 'when terms are enforced' do

spec/lib/gitlab/ci/yaml_processor_spec.rb

@@ -1647,48 +1647,6 @@ module Gitlab
 
           it { expect { subject }.to raise_error(Gitlab::Ci::YamlProcessor::ValidationError, /is not defined in prior stages/) }
         end
-
-        context 'when trigger job includes artifact generated by a dependency' do
-          context 'when dependency is defined in previous stages' do
-            let(:config) do
-              {
-                build1: { stage: 'build', script: 'test' },
-                test1: { stage: 'test', trigger: {
-                  include: [{ job: 'build1', artifact: 'generated.yml' }]
-                } }
-              }
-            end
-
-            it { expect { subject }.not_to raise_error }
-          end
-
-          context 'when dependency is defined in later stages' do
-            let(:config) do
-              {
-                build1: { stage: 'build', script: 'test' },
-                test1: { stage: 'test', trigger: {
-                  include: [{ job: 'deploy1', artifact: 'generated.yml' }]
-                } },
-                deploy1: { stage: 'deploy', script: 'test' }
-              }
-            end
-
-            it { expect { subject }.to raise_error(Gitlab::Ci::YamlProcessor::ValidationError, /is not defined in prior stages/) }
-          end
-
-          context 'when dependency is not defined' do
-            let(:config) do
-              {
-                build1: { stage: 'build', script: 'test' },
-                test1: { stage: 'test', trigger: {
-                  include: [{ job: 'non-existent', artifact: 'generated.yml' }]
-                } }
-              }
-            end
-
-            it { expect { subject }.to raise_error(Gitlab::Ci::YamlProcessor::ValidationError, /undefined dependency: non-existent/) }
-          end
-        end
       end
 
       describe "Job Needs" do
@@ -2094,34 +2052,6 @@ module Gitlab
         end
       end
 
-      describe 'with trigger:include' do
-        context 'when artifact and job are specified' do
-          let(:config) do
-            YAML.dump({
-              build1: { stage: 'build', script: 'test' },
-              test1: { stage: 'test', trigger: {
-                include: [{ artifact: 'generated.yml', job: 'build1' }]
-              } }
-            })
-          end
-
-          it { expect { subject }.not_to raise_error }
-        end
-
-        context 'when artifact is specified without job' do
-          let(:config) do
-            YAML.dump({
-              build1: { stage: 'build', script: 'test' },
-              test1: { stage: 'test', trigger: {
-                include: [{ artifact: 'generated.yml' }]
-              } }
-            })
-          end
-
-          it { expect { subject }.to raise_error(Gitlab::Ci::YamlProcessor::ValidationError, /must specify the job where to fetch the artifact from/) }
-        end
-      end
-
       describe "Error handling" do
         it "fails to parse YAML" do
           expect do

spec/lib/gitlab/import_export/fork_spec.rb

@@ -32,6 +32,8 @@ describe 'forked project import' do
   end
 
   before do
+    stub_feature_flags(project_export_as_ndjson: false)
+
     allow_next_instance_of(Gitlab::ImportExport) do |instance|
       allow(instance).to receive(:storage_path).and_return(export_path)
     end

spec/lib/gitlab/import_export/import_export_equivalence_spec.rb

@@ -20,6 +20,10 @@ describe Gitlab::ImportExport do
 
   let(:json_fixture) { 'complex' }
 
+  before do
+    stub_feature_flags(project_export_as_ndjson: false)
+  end
+
   it 'yields the initial tree when importing and exporting it again' do
     project = create(:project, creator: create(:user, :admin))
 

spec/lib/gitlab/import_export/json/ndjson_writer_spec.rb

+# frozen_string_literal: true
+
+require "spec_helper"
+
+describe Gitlab::ImportExport::JSON::NdjsonWriter do
+  include ImportExport::CommonUtil
+
+  let(:path) { "#{Dir.tmpdir}/ndjson_writer_spec/tree" }
+  let(:exportable_path) { 'projects' }
+
+  subject { described_class.new(path) }
+
+  after do
+    FileUtils.rm_rf(path)
+  end
+
+  describe "#write_attributes" do
+    it "writes correct json to root" do
+      expected_hash = { "key" => "value_1", "key_1" => "value_2" }
+      subject.write_attributes(exportable_path, expected_hash)
+
+      expect(consume_attributes(path, exportable_path)).to eq(expected_hash)
+    end
+  end
+
+  describe "#write_relation" do
+    context "when single relation is serialized" do
+      it "appends json in correct file " do
+        relation = "relation"
+        value =  { "key" => "value_1", "key_1" => "value_1" }
+        subject.write_relation(exportable_path, relation, value)
+
+        expect(consume_relations(path, exportable_path, relation)).to eq([value])
+      end
+    end
+
+    context "when single relation is already serialized" do
+      it "raise exception" do
+        values = [{ "key" => "value_1", "key_1" => "value_1" }, { "key" => "value_2", "key_1" => "value_2" }]
+        relation = "relation"
+        file_path = File.join(path, exportable_path, "#{relation}.ndjson")
+        subject.write_relation(exportable_path, relation, values[0])
+
+        expect {subject.write_relation(exportable_path, relation, values[1])}.to raise_exception("The #{file_path} already exist")
+      end
+    end
+  end
+
+  describe "#write_relation_array" do
+    it "writes json in correct files" do
+      values =  [{ "key" => "value_1", "key_1" => "value_1" }, { "key" => "value_2", "key_1" => "value_2" }]
+      relations = %w(relation1 relation2)
+      relations.each do |relation|
+        subject.write_relation_array(exportable_path, relation, values.to_enum)
+      end
+      subject.close
+
+      relations.each do |relation|
+        expect(consume_relations(path, exportable_path, relation)).to eq(values)
+      end
+    end
+  end
+end

spec/lib/gitlab/import_export/project/relation_factory_spec.rb

@@ -160,7 +160,7 @@ describe Gitlab::ImportExport::Project::RelationFactory do
     end
 
     it 'has preloaded target project' do
-      expect(created_object.source_project).to equal(project)
+      expect(created_object.target_project).to equal(project)
     end
   end
 

spec/lib/gitlab/sidekiq_middleware_spec.rb

@@ -134,12 +134,12 @@ describe Gitlab::SidekiqMiddleware do
     let(:middleware_expected_args) { [worker_class_arg, job, queue, redis_pool] }
     let(:expected_middlewares) do
       [
-        Gitlab::SidekiqStatus::ClientMiddleware,
-        Gitlab::SidekiqMiddleware::ClientMetrics,
-        Gitlab::SidekiqMiddleware::WorkerContext::Client,
-        Labkit::Middleware::Sidekiq::Client,
-        Gitlab::SidekiqMiddleware::AdminMode::Client,
-        Gitlab::SidekiqMiddleware::DuplicateJobs::Client
+         ::Gitlab::SidekiqMiddleware::WorkerContext::Client,
+         ::Labkit::Middleware::Sidekiq::Client,
+         ::Gitlab::SidekiqMiddleware::DuplicateJobs::Client,
+         ::Gitlab::SidekiqStatus::ClientMiddleware,
+         ::Gitlab::SidekiqMiddleware::AdminMode::Client,
+         ::Gitlab::SidekiqMiddleware::ClientMetrics
       ]
     end
 

spec/models/merge_request_spec.rb

@@ -3701,4 +3701,18 @@ describe MergeRequest do
       end
     end
   end
+
+  describe '#predefined_variables' do
+    let(:merge_request) { create(:merge_request) }
+
+    it 'caches all SQL-sourced data on the first call' do
+      control = ActiveRecord::QueryRecorder.new { merge_request.predefined_variables }.count
+
+      expect(control).to be > 0
+
+      count = ActiveRecord::QueryRecorder.new { merge_request.predefined_variables }.count
+
+      expect(count).to eq(0)
+    end
+  end
 end

spec/services/projects/hashed_storage/migration_service_spec.rb

@@ -5,6 +5,11 @@ require 'spec_helper'
 describe Projects::HashedStorage::MigrationService do
   let(:project) { create(:project, :empty_repo, :wiki_repo, :legacy_storage) }
   let(:logger) { double }
+  let!(:project_attachment) { build(:file_uploader, project: project) }
+  let(:project_hashed_path) { Storage::Hashed.new(project).disk_path }
+  let(:project_legacy_path) { Storage::LegacyProject.new(project).disk_path }
+  let(:wiki_hashed_path) { "#{project_hashed_path}.wiki" }
+  let(:wiki_legacy_path) { "#{project_legacy_path}.wiki" }
 
   subject(:service) { described_class.new(project, project.full_path, logger: logger) }
 
@@ -29,9 +34,24 @@ describe Projects::HashedStorage::MigrationService do
 
         service.execute
       end
+
+      it 'migrates legacy repositories to hashed storage' do
+        legacy_attachments_path = FileUploader.absolute_base_dir(project)
+        hashed_project = project.dup.tap { |p| p.id = project.id }
+        hashed_project.storage_version = ::Project::HASHED_STORAGE_FEATURES[:attachments]
+        hashed_attachments_path = FileUploader.absolute_base_dir(hashed_project)
+
+        expect(logger).to receive(:info).with(/Repository moved from '#{project_legacy_path}' to '#{project_hashed_path}'/)
+        expect(logger).to receive(:info).with(/Repository moved from '#{wiki_legacy_path}' to '#{wiki_hashed_path}'/)
+        expect(logger).to receive(:info).with(/Project attachments moved from '#{legacy_attachments_path}' to '#{hashed_attachments_path}'/)
+
+        expect { service.execute }.to change { project.storage_version }.from(nil).to(2)
+      end
     end
 
     context 'attachments migration' do
+      let(:project) { create(:project, :empty_repo, :wiki_repo, storage_version: ::Project::HASHED_STORAGE_FEATURES[:repository]) }
+
       let(:attachments_service) do
         Projects::HashedStorage::MigrateAttachmentsService.new(project: project,
                                                                old_disk_path: project.full_path,
@@ -51,6 +71,17 @@ describe Projects::HashedStorage::MigrationService do
 
         service.execute
       end
+
+      it 'migrates legacy attachments to hashed storage' do
+        legacy_attachments_path = FileUploader.absolute_base_dir(project)
+        hashed_project = project.dup.tap { |p| p.id = project.id }
+        hashed_project.storage_version = ::Project::HASHED_STORAGE_FEATURES[:attachments]
+        hashed_attachments_path = FileUploader.absolute_base_dir(hashed_project)
+
+        expect(logger).to receive(:info).with(/Project attachments moved from '#{legacy_attachments_path}' to '#{hashed_attachments_path}'/)
+
+        expect { service.execute }.to change { project.storage_version }.from(1).to(2)
+      end
     end
   end
 end

spec/services/projects/hashed_storage/rollback_service_spec.rb

@@ -5,6 +5,11 @@ require 'spec_helper'
 describe Projects::HashedStorage::RollbackService do
   let(:project) { create(:project, :empty_repo, :wiki_repo) }
   let(:logger) { double }
+  let!(:project_attachment) { build(:file_uploader, project: project) }
+  let(:project_hashed_path) { Storage::Hashed.new(project).disk_path }
+  let(:project_legacy_path) { Storage::LegacyProject.new(project).disk_path }
+  let(:wiki_hashed_path) { "#{project_hashed_path}.wiki" }
+  let(:wiki_legacy_path) { "#{project_legacy_path}.wiki" }
 
   subject(:service) { described_class.new(project, project.disk_path, logger: logger) }
 
@@ -26,6 +31,20 @@ describe Projects::HashedStorage::RollbackService do
 
         service.execute
       end
+
+      it 'rollbacks to legacy storage' do
+        hashed_attachments_path = FileUploader.absolute_base_dir(project)
+        legacy_project = project.dup
+        legacy_project.storage_version = nil
+        legacy_attachments_path = FileUploader.absolute_base_dir(legacy_project)
+
+        expect(logger).to receive(:info).with(/Project attachments moved from '#{hashed_attachments_path}' to '#{legacy_attachments_path}'/)
+
+        expect(logger).to receive(:info).with(/Repository moved from '#{project_hashed_path}' to '#{project_legacy_path}'/)
+        expect(logger).to receive(:info).with(/Repository moved from '#{wiki_hashed_path}' to '#{wiki_legacy_path}'/)
+
+        expect { service.execute }.to change { project.storage_version }.from(2).to(nil)
+      end
     end
 
     context 'repository rollback' do
@@ -47,6 +66,13 @@ describe Projects::HashedStorage::RollbackService do
 
         service.execute
       end
+
+      it 'rollbacks to legacy storage' do
+        expect(logger).to receive(:info).with(/Repository moved from '#{project_hashed_path}' to '#{project_legacy_path}'/)
+        expect(logger).to receive(:info).with(/Repository moved from '#{wiki_hashed_path}' to '#{wiki_legacy_path}'/)
+
+        expect { service.execute }.to change { project.storage_version }.from(1).to(nil)
+      end
     end
   end
 end

spec/services/web_hook_service_spec.rb

@@ -71,16 +71,6 @@ describe WebHookService do
       end
     end
 
-    it 'POSTs to the webhook URL' do
-      stub_full_request(project_hook.url, method: :post)
-
-      service_instance.execute
-
-      expect(WebMock).to have_requested(:post, stubbed_hostname(project_hook.url)).with(
-        headers: headers
-      ).once
-    end
-
     it 'POSTs the data as JSON' do
       stub_full_request(project_hook.url, method: :post)
 

spec/support/import_export/common_util.rb

@@ -26,6 +26,21 @@ module ImportExport
       "tmp/tests/gitlab-test/import_export"
     end
 
+    def get_json(path, exportable_path, key, ndjson_enabled)
+      if ndjson_enabled
+        json = if key == :projects
+                 consume_attributes(path, exportable_path)
+               else
+                 consume_relations(path, exportable_path, key)
+               end
+      else
+        json = project_json(path)
+        json = json[key.to_s] unless key == :projects
+      end
+
+      json
+    end
+
     def restore_then_save_project(project, import_path:, export_path:)
       project_restorer = get_project_restorer(project, import_path)
       project_saver = get_project_saver(project, export_path)
@@ -50,5 +65,30 @@ module ImportExport
         allow(shared).to receive(:export_path).and_return(path)
       end
     end
+
+    def consume_attributes(dir_path, exportable_path)
+      path = File.join(dir_path, "#{exportable_path}.json")
+      return unless File.exist?(path)
+
+      ActiveSupport::JSON.decode(IO.read(path))
+    end
+
+    def consume_relations(dir_path, exportable_path, key)
+      path = File.join(dir_path, exportable_path, "#{key}.ndjson")
+      return unless File.exist?(path)
+
+      relations = []
+
+      File.foreach(path) do |line|
+        json = ActiveSupport::JSON.decode(line)
+        relations << json
+      end
+
+      key == :project_feature ? relations.first : relations.flatten
+    end
+
+    def project_json(filename)
+      ActiveSupport::JSON.decode(IO.read(filename))
+    end
   end
 end