Add latest changes from gitlab-org/gitlab@master

app/graphql/gitlab_schema.rb

@@ -10,11 +10,12 @@ class GitlabSchema < GraphQL::Schema
   DEFAULT_MAX_DEPTH = 15
   AUTHENTICATED_MAX_DEPTH = 20
 
+  use GraphQL::Pagination::Connections
   use BatchLoader::GraphQL
   use Gitlab::Graphql::Authorize
   use Gitlab::Graphql::Present
   use Gitlab::Graphql::CallsGitaly
-  use Gitlab::Graphql::Connections
+  use Gitlab::Graphql::Pagination::Connections
   use Gitlab::Graphql::GenericTracing
   use Gitlab::Graphql::Timeout, max_seconds: Gitlab.config.gitlab.graphql_timeout
 

changelogs/unreleased/drop-unused-indexes.yml

+---
+title: Remove unused index for vulnerability severity levels
+merge_request: 29023
+author:
+type: other

db/post_migrate/20200407094005_drop_vulnerability_severity_index.rb

+# frozen_string_literal: true
+
+class DropVulnerabilitySeverityIndex < ActiveRecord::Migration[6.0]
+  include Gitlab::Database::MigrationHelpers
+
+  disable_ddl_transaction!
+
+  DOWNTIME = false
+  INDEX_NAME = 'undefined_vulnerabilities'
+
+  def up
+    remove_concurrent_index_by_name :vulnerability_occurrences, INDEX_NAME
+  end
+
+  def down
+    add_concurrent_index(:vulnerability_occurrences, :id, where: 'severity = 0', name: INDEX_NAME)
+  end
+end

db/post_migrate/20200407094923_drop_vulnerabilities_severity_index.rb

+# frozen_string_literal: true
+
+class DropVulnerabilitiesSeverityIndex < ActiveRecord::Migration[6.0]
+  include Gitlab::Database::MigrationHelpers
+
+  disable_ddl_transaction!
+
+  DOWNTIME = false
+  INDEX_NAME = 'undefined_vulnerability'
+
+  def up
+    remove_concurrent_index_by_name :vulnerabilities, INDEX_NAME
+  end
+
+  def down
+    add_concurrent_index(:vulnerabilities, :id, where: 'severity = 0', name: INDEX_NAME)
+  end
+end

db/structure.sql

@@ -10391,10 +10391,6 @@ CREATE INDEX tmp_build_stage_position_index ON public.ci_builds USING btree (sta
 
 CREATE INDEX tmp_idx_on_user_id_where_bio_is_filled ON public.users USING btree (id) WHERE ((COALESCE(bio, ''::character varying))::text IS DISTINCT FROM ''::text);
 
-CREATE INDEX undefined_vulnerabilities ON public.vulnerability_occurrences USING btree (id) WHERE (severity = 0);
-
-CREATE INDEX undefined_vulnerability ON public.vulnerabilities USING btree (id) WHERE (severity = 0);
-
 CREATE UNIQUE INDEX users_security_dashboard_projects_unique_index ON public.users_security_dashboard_projects USING btree (project_id, user_id);
 
 CREATE UNIQUE INDEX vulnerability_feedback_unique_idx ON public.vulnerability_feedback USING btree (project_id, category, feedback_type, project_fingerprint);
@@ -13012,5 +13008,7 @@ COPY "schema_migrations" (version) FROM STDIN;
 20200403184110
 20200403185127
 20200403185422
+20200407094005
+20200407094923
 \.
 

doc/.vale/gitlab/Substitutions.yml

@@ -14,4 +14,3 @@ swap:
   postgres: PostgreSQL
   self hosted: self-managed
   self-hosted: self-managed
-  

doc/administration/auth/how_to_configure_ldap_gitlab_ee/index.md

@@ -44,7 +44,7 @@ _The reporter permission allows support staff to manage issues, labels, and revi
 
 > Guest permissions allows people ops staff to review and lodge new issues while allowing no read or write access to project code or [confidential issues](../../../user/project/issues/confidential_issues.md#permissions-and-access-to-confidential-issues) created by other users.
 
-See the [permission list](../../../user/permissions.md) for complementary info.
+See the [permission list](../../../user/permissions.md) for complementary information.
 
 #### Group permissions - example
 

doc/administration/high_availability/README.md

@@ -273,7 +273,7 @@ On different cloud vendors a best effort like for like can be used.
 
 [^8]: The architectures were built and tested with the [Intel Xeon E5 v3 (Haswell)](https://cloud.google.com/compute/docs/cpu-platforms)
       CPU platform on GCP. On different hardware you may find that adjustments, either lower
-      or higher, are required for your CPU or Node counts accordingly. For more info a
+      or higher, are required for your CPU or Node counts accordingly. For more information, a
       [Sysbench](https://github.com/akopytov/sysbench) benchmark of the CPU can be found
       [here](https://gitlab.com/gitlab-org/quality/performance/-/wikis/Reference-Architectures/GCP-CPU-Benchmarks).
 

doc/administration/troubleshooting/diagnostics_tools.md

@@ -13,7 +13,7 @@ before attempting to use these tools.
 ## gitlabsos
 
 The [gitlabsos](https://gitlab.com/gitlab-com/support/toolbox/gitlabsos/) utility
-provides a unified method of gathering info and logs from GitLab and the system it's
+provides a unified method of gathering information and logs from GitLab and the system it's
 running on.
 
 ## strace-parser

doc/administration/troubleshooting/gitlab_rails_cheat_sheet.md

@@ -751,7 +751,7 @@ This content has been moved to the [Troubleshooting Sidekiq docs](./sidekiq.md).
 
 ## LFS
 
-### Get info about LFS objects and associated project
+### Get information about LFS objects and associated project
 
 ```ruby
 o=LfsObject.find_by(oid: "<oid>")

doc/administration/troubleshooting/kubernetes_cheat_sheet.md

@@ -168,7 +168,7 @@ and they will assist you with any issues you are having.
   kubectl exec -it <task-runner-pod-name> -- /srv/gitlab/bin/rails dbconsole -p
   ```
 
-- How to get info about Helm installation status:
+- How to get information about Helm installation status:
 
   ```shell
   helm status name-of-installation
@@ -199,7 +199,7 @@ and they will assist you with any issues you are having.
     helm upgrade <release name> <chart path> -f gitlab.yaml
     ```
 
-- How to get the manifest for a release. It can be useful because it contains the info about
+- How to get the manifest for a release. It can be useful because it contains the information about
 all Kubernetes resources and dependent charts:
 
   ```shell

doc/api/graphql/index.md

@@ -72,7 +72,7 @@ Root-level queries are defined in
 
 GitLab supports batching queries into a single request using
 [apollo-link-batch-http](https://www.apollographql.com/docs/link/links/batch-http/). More
-info about multiplexed queries is also available for
+information about multiplexed queries is also available for
 [graphql-ruby](https://graphql-ruby.org/queries/multiplex.html) the
 library GitLab uses on the backend.
 

doc/api/oauth2.md

@@ -102,7 +102,7 @@ CAUTION: **Important:**
 Avoid using this flow for applications that store data outside of the GitLab
 instance. If you do, make sure to verify `application id` associated with the
 access token before granting access to the data
-(see [`/oauth/token/info`](#retrieving-the-token-info)).
+(see [`/oauth/token/info`](#retrieving-the-token-information)).
 
 Unlike the web flow, the client receives an `access token` immediately as a
 result of the authorization request. The flow does not use the client secret
@@ -213,7 +213,7 @@ or you can put the token to the Authorization header:
 curl --header "Authorization: Bearer OAUTH-TOKEN" https://gitlab.example.com/api/v4/user
 ```
 
-## Retrieving the Token Info
+## Retrieving the token information
 
 To verify the details of a token, use the `token/info` endpoint provided by the Doorkeeper gem.
 For more information, see [`/oauth/token/info`](https://github.com/doorkeeper-gem/doorkeeper/wiki/API-endpoint-descriptions-and-examples#get----oauthtokeninfo).

doc/api/repository_files.md

@@ -82,7 +82,7 @@ X-Gitlab-Size: 1476
 
 ## Get file blame from repository
 
-Allows you to receive blame information. Each blame range contains lines and corresponding commit info.
+Allows you to receive blame information. Each blame range contains lines and corresponding commit information.
 
 ```plaintext
 GET /projects/:id/repository/files/:file_path/blame

doc/api/v3_to_v4.md

@@ -67,7 +67,7 @@ Below are the changes made between V3 and V4.
 - Remove `GET /groups/owned`. Use `GET /groups?owned=true` instead [!9505](https://gitlab.com/gitlab-org/gitlab-foss/-/merge_requests/9505)
 - Return 202 with JSON body on async removals on V4 API (`DELETE /projects/:id/repository/merged_branches` and `DELETE /projects/:id`) [!9449](https://gitlab.com/gitlab-org/gitlab-foss/-/merge_requests/9449)
 - `GET /projects/:id/milestones?iid[]=x&iid[]=y` array filter has been renamed to `iids` [!9096](https://gitlab.com/gitlab-org/gitlab-foss/-/merge_requests/9096)
-- Return basic info about pipeline in `GET /projects/:id/pipelines` [!8875](https://gitlab.com/gitlab-org/gitlab-foss/-/merge_requests/8875)
+- Return basic information about pipeline in `GET /projects/:id/pipelines` [!8875](https://gitlab.com/gitlab-org/gitlab-foss/-/merge_requests/8875)
 - Renamed all `build` references to `job` [!9463](https://gitlab.com/gitlab-org/gitlab-foss/-/merge_requests/9463)
 - Drop `GET /projects/:id/repository/commits/:sha/jobs` [!9463](https://gitlab.com/gitlab-org/gitlab-foss/-/merge_requests/9463)
 - Rename Build Triggers to be Pipeline Triggers API [!9713](https://gitlab.com/gitlab-org/gitlab-foss/-/merge_requests/9713)

doc/ci/docker/using_docker_images.md

@@ -532,7 +532,7 @@ runtime.
 
 There are two approaches that you can take in order to access a
 private registry. Both require setting the environment variable
-`DOCKER_AUTH_CONFIG` with appropriate authentication info.
+`DOCKER_AUTH_CONFIG` with appropriate authentication information.
 
 1. Per-job: To configure one job to access a private registry, add
    `DOCKER_AUTH_CONFIG` as a job variable.

doc/ci/variables/README.md

@@ -453,7 +453,7 @@ pass CI variables to the running application by prefixing the key of the
 variable with `K8S_SECRET_`.
 
 These [prefixed
-variables](../../topics/autodevops/index.md#application-secret-variables) will
+variables](../../topics/autodevops/customize.md#application-secret-variables) will
 then be available as environment variables on the running application
 container.
 

doc/development/README.md

@@ -222,3 +222,4 @@ Complementary reads:
 
 - [Run full Auto DevOps cycle in a GDK instance](https://gitlab.com/gitlab-org/gitlab-development-kit/blob/master/doc/howto/auto_devops.md)
 - [Using GitLab Runner with GDK](https://gitlab.com/gitlab-org/gitlab-development-kit/blob/master/doc/howto/runner.md)
+- [Using the Web IDE terminal with GDK](https://gitlab.com/gitlab-org/gitlab-development-kit/-/blob/master/doc/howto/web_ide_terminal_gdk_setup.md)

doc/development/api_graphql_styleguide.md

@@ -150,7 +150,7 @@ query($project_path: ID!) {
 ```
 
 This would return the first 2 pipelines of a project and related
-pagination info., ordered by descending ID. The returned data would
+pagination information, ordered by descending ID. The returned data would
 look like this:
 
 ```json

doc/development/documentation/site_architecture/global_nav.md

@@ -272,7 +272,7 @@ and the following syntax rules.
 - For `index.html` files, use the clean (canonical) URL: `path/to/`.
 - For EE-only docs, use the same relative path, but add the attribute `ee_only: true` below
   the `doc_url` or `category_url`, as explained above. This displays
-  an "info" icon on the nav to make the user aware that the feature is
+  an "information" icon on the nav to make the user aware that the feature is
   EE-only.
 
 DANGER: **Important!**

doc/development/documentation/styleguide.md

@@ -583,7 +583,7 @@ For other punctuation rules, please refer to the
 - [Avoid using symbols and special chars](https://gitlab.com/gitlab-org/gitlab-docs/issues/84)
   in headers. Whenever possible, they should be plain and short text.
 - Avoid adding things that show ephemeral statuses. For example, if a feature is
-  considered beta or experimental, put this info in a note, not in the heading.
+  considered beta or experimental, put this information in a note, not in the heading.
 - When introducing a new document, be careful for the headings to be
   grammatically and syntactically correct. Mention an [assigned technical writer (TW)](https://about.gitlab.com/handbook/product/categories/)
   for review.

doc/development/emails.md

@@ -17,7 +17,7 @@ dummy data.
 The previews live in [`app/mailers/previews`][previews] and can be viewed at
 [`/rails/mailers`](http://localhost:3000/rails/mailers).
 
-See the [Rails guides](https://guides.rubyonrails.org/action_mailer_basics.html#previewing-emails) for more info.
+See the [Rails guides](https://guides.rubyonrails.org/action_mailer_basics.html#previewing-emails) for more information.
 
 [previews]: https://gitlab.com/gitlab-org/gitlab-foss/tree/master/app/mailers/previews
 

doc/development/fe_guide/style/scss.md

@@ -264,7 +264,7 @@ Note that this won't fix every problem, but it should fix a majority.
 ### Ignoring issues
 
 If you want a line or set of lines to be ignored by the linter, you can use
-`// scss-lint:disable RuleName` ([more info](https://github.com/sds/scss-lint#disabling-linters-via-source)):
+`// scss-lint:disable RuleName` ([more information](https://github.com/sds/scss-lint#disabling-linters-via-source)):
 
 ```scss
 // This lint rule is disabled because it is supported only in Chrome/Safari

doc/development/feature_flags/controls.md

@@ -98,6 +98,7 @@ Slack:
 
 This will enable the feature for GitLab.com, with `new_navigation_bar` being the
 name of the feature.
+This command does *not* enable the feature for 25% of the total users. Instead, when the feature is checked with `enabled?`, it will return `true` 25% of the time.
 
 If you are not certain what percentages to use, simply use the following steps:
 
@@ -136,9 +137,8 @@ you run these 2 commands:
 /chatops run feature set some_feature 25
 ```
 
-Then `some_feature` will be enabled for 25% of the time the users are interacting with
-`gitlab-org/gitlab`. Note that the the feature is not enabled to 25%
-of the users, rather a simple randomization is made each time the `enabled?` is checked.
+Then `some_feature` will be enabled for both 25% of users and all users interacting with
+`gitlab-org/gitlab`.
 
 NOTE: **Note:**
 **Percentage of time** rollout is not a good idea if what you want is to make sure a feature

doc/development/integrations/secure.md

@@ -80,7 +80,7 @@ See [GitLab CI/CD predefined variables](../../ci/variables/predefined_variables.
 
 Also, scanning jobs should be skipped when the corresponding variable prefixed with `_DISABLED` is present.
 See `DEPENDENCY_SCANNING_DISABLED`, `CONTAINER_SCANNING_DISABLED`, `SAST_DISABLED`, and `DAST_DISABLED`
-in [Auto DevOps documentation](../../topics/autodevops/index.md#disable-jobs).
+in [Auto DevOps documentation](../../topics/autodevops/customize.md#disable-jobs).
 
 Finally, SAST and Dependency Scanning job definitions should use
 `CI_PROJECT_REPOSITORY_LANGUAGES` (comma-separated list of values)

doc/integration/saml.md

@@ -370,14 +370,14 @@ You may also bypass the auto signin feature by browsing to
 >**Note:**
 This setting is only available on GitLab 8.6 and above.
 This setting should only be used to map attributes that are part of the
-OmniAuth info hash schema.
+OmniAuth `info` hash schema.
 
 `attribute_statements` is used to map Attribute Names in a SAMLResponse to entries
-in the OmniAuth [info hash](https://github.com/omniauth/omniauth/wiki/Auth-Hash-Schema#schema-10-and-later).
+in the OmniAuth [`info` hash](https://github.com/omniauth/omniauth/wiki/Auth-Hash-Schema#schema-10-and-later).
 
 For example, if your SAMLResponse contains an Attribute called 'EmailAddress',
 specify `{ email: ['EmailAddress'] }` to map the Attribute to the
-corresponding key in the info hash. URI-named Attributes are also supported, e.g.
+corresponding key in the `info` hash. URI-named Attributes are also supported, e.g.
 `{ email: ['http://schemas.xmlsoap.org/ws/2005/05/identity/claims/emailaddress'] }`.
 
 This setting allows you tell GitLab where to look for certain attributes required

doc/integration/shibboleth.md

@@ -8,7 +8,7 @@ In order to enable Shibboleth support in GitLab we need to use Apache instead of
 
 To enable the Shibboleth OmniAuth provider you must configure Apache Shibboleth module.
 The installation and configuration of the module itself is out of the scope of this document.
-Check <https://wiki.shibboleth.net/confluence/display/SP3/Apache> for more info.
+Check <https://wiki.shibboleth.net/confluence/display/SP3/Apache> for more information.
 
 You can find Apache config in [GitLab Recipes](https://gitlab.com/gitlab-org/gitlab-recipes/tree/master/web-server/apache).
 

doc/security/crime_vulnerability.md

@@ -4,12 +4,11 @@ type: reference
 
 # How we manage the TLS protocol CRIME vulnerability
 
-> CRIME ("Compression Ratio Info-leak Made Easy") is a security exploit against
+[CRIME](https://en.wikipedia.org/w/index.php?title=CRIME&oldid=692423806) is a security exploit against
 secret web cookies over connections using the HTTPS and SPDY protocols that also
 use data compression. When used to recover the content of secret
 authentication cookies, it allows an attacker to perform session hijacking on an
 authenticated web session, allowing the launching of further attacks.
-([CRIME](https://en.wikipedia.org/w/index.php?title=CRIME&oldid=692423806))
 
 ## Description
 

doc/topics/application_development_platform/index.md

@@ -37,7 +37,7 @@ with various cloud providers.
 In order to provide modern DevOps workflows, our Application Development Platform will rely on
 [Auto DevOps](../autodevops/index.md) to provide those workflows. Auto DevOps works with
 any Kubernetes cluster; you're not limited to running on GitLab's infrastructure. Additionally, Auto DevOps offers
-an incremental consumption path. Because it is [composable](../autodevops/index.md#using-components-of-auto-devops),
+an incremental consumption path. Because it is [composable](../autodevops/customize.md#using-components-of-auto-devops),
 you can use as much or as little of the default pipeline as you'd like, and deeply customize without having to integrate a completely different platform.
 
 ### Security

doc/topics/autodevops/quick_start_guide.md

@@ -207,7 +207,7 @@ deployment and clicking a square will take you to the pod's logs page.
 
 TIP: **Tip:**
 There is only one pod hosting the application at the moment, but you can add
-more pods by defining the [`REPLICAS` variable](index.md#environment-variables)
+more pods by defining the [`REPLICAS` variable](customize.md#environment-variables)
 under **Settings > CI/CD > Environment variables**.
 
 ### Working with branches
@@ -278,8 +278,8 @@ and customized to fit your workflow. Here are some helpful resources for further
 
 1. [Auto DevOps](index.md)
 1. [Multiple Kubernetes clusters](index.md#using-multiple-kubernetes-clusters-premium) **(PREMIUM)**
-1. [Incremental rollout to production](index.md#incremental-rollout-to-production-premium) **(PREMIUM)**
-1. [Disable jobs you don't need with environment variables](index.md#environment-variables)
+1. [Incremental rollout to production](customize.md#incremental-rollout-to-production-premium) **(PREMIUM)**
+1. [Disable jobs you don't need with environment variables](customize.md#environment-variables)
 1. [Use a static IP for your cluster](../../user/clusters/applications.md#using-a-static-ip)
-1. [Use your own buildpacks to build your application](index.md#custom-buildpacks)
+1. [Use your own buildpacks to build your application](customize.md#custom-buildpacks)
 1. [Prometheus monitoring](../../user/project/integrations/prometheus.md)

doc/topics/autodevops/stages.md

@@ -19,7 +19,7 @@ If a project's repository contains a `Dockerfile` at its root, Auto Build will u
 If you are also using Auto Review Apps and Auto Deploy and choose to provide
 your own `Dockerfile`, make sure you expose your application to port
 `5000` as this is the port assumed by the
-[default Helm chart](https://gitlab.com/gitlab-org/charts/auto-deploy-app). Alternatively you can override the default values by [customizing the Auto Deploy Helm chart](index.md#custom-helm-chart)
+[default Helm chart](https://gitlab.com/gitlab-org/charts/auto-deploy-app). Alternatively you can override the default values by [customizing the Auto Deploy Helm chart](customize.md#custom-helm-chart)
 
 ### Auto Build using Heroku buildpacks
 
@@ -78,7 +78,7 @@ Auto Test automatically runs the appropriate tests for your application using
 buildpacks](https://devcenter.heroku.com/articles/buildpacks) by analyzing
 your project to detect the language and framework. Several languages and
 frameworks are detected automatically, but if your language is not detected,
-you may succeed with a [custom buildpack](index.md#custom-buildpacks). Check the
+you may succeed with a [custom buildpack](customize.md#custom-buildpacks). Check the
 [currently supported languages](index.md#currently-supported-languages).
 
 Auto Test uses tests you already have in your application. If there are no
@@ -182,7 +182,7 @@ be deleted.
 
 Review apps are deployed using the
 [auto-deploy-app](https://gitlab.com/gitlab-org/charts/auto-deploy-app) chart with
-Helm, which can be [customized](index.md#custom-helm-chart). The app will be deployed into the [Kubernetes
+Helm, which can be [customized](customize.md#custom-helm-chart). The app will be deployed into the [Kubernetes
 namespace](../../user/project/clusters/index.md#deployment-variables)
 for the environment.
 
@@ -265,8 +265,8 @@ Auto Deploy doesn't include deployments to staging or canary by default, but the
 [Auto DevOps template](https://gitlab.com/gitlab-org/gitlab/blob/master/lib/gitlab/ci/templates/Auto-DevOps.gitlab-ci.yml) contains job definitions for these tasks if you want to
 enable them.
 
-You can make use of [environment variables](index.md#environment-variables) to automatically
-scale your pod replicas and to apply custom arguments to the Auto DevOps `helm upgrade` commands. This is an easy way to [customize the Auto Deploy Helm chart](index.md#custom-helm-chart).
+You can make use of [environment variables](customize.md#environment-variables) to automatically
+scale your pod replicas and to apply custom arguments to the Auto DevOps `helm upgrade` commands. This is an easy way to [customize the Auto Deploy Helm chart](customize.md#custom-helm-chart).
 
 Apps are deployed using the
 [auto-deploy-app](https://gitlab.com/gitlab-org/charts/auto-deploy-app) chart with
@@ -315,7 +315,7 @@ including support for `Deployment` in the `extensions/v1beta1` version.
 
 To use Auto Deploy on a Kubernetes 1.16+ cluster, you must:
 
-1. Set the following in the [`.gitlab/auto-deploy-values.yaml` file](index.md#customize-values-for-helm-chart):
+1. Set the following in the [`.gitlab/auto-deploy-values.yaml` file](customize.md#customize-values-for-helm-chart):
 
    ```yml
    deploymentApiVersion: apps/v1
@@ -391,7 +391,7 @@ need to:
 
 Once you have configured your worker to respond to health checks, run a Sidekiq
 worker for your Rails application. You can enable workers by setting the
-following in the [`.gitlab/auto-deploy-values.yaml` file](index.md#customize-values-for-helm-chart):
+following in the [`.gitlab/auto-deploy-values.yaml` file](customize.md#customize-values-for-helm-chart):
 
 ```yml
 workers:

doc/topics/autodevops/upgrading_postgresql.md

 # Upgrading PostgreSQL for Auto DevOps
 
-Auto DevOps provides an [in-cluster PostgreSQL database](index.md#postgresql-database-support)
+Auto DevOps provides an [in-cluster PostgreSQL database](customize.md#postgresql-database-support)
 for your application.
 
 The version of the chart used to provision PostgreSQL:

doc/topics/git/numerous_undo_possibilities_in_git/index.md

@@ -14,7 +14,7 @@ In this tutorial, we will show you different ways of undoing your work in Git, f
 we will assume you have a basic working knowledge of. Check GitLab's
 [Git documentation](../index.md) for reference.
 
-Also, we will only provide some general info of the commands, which is enough
+Also, we will only provide some general information of the commands, which is enough
 to get you started for the easy cases/examples, but for anything more advanced
 please refer to the [Git book](https://git-scm.com/book/en/v2).
 

doc/user/admin_area/settings/continuous_integration.md

@@ -116,7 +116,7 @@ Once saved, you can see the build quota in the group admin view.
 The quota can also be viewed in the project admin view if shared Runners
 are enabled.
 
-![Project admin info](img/admin_project_quota_view.png)
+![Project admin information](img/admin_project_quota_view.png)
 
 You can see an overview of the pipeline minutes quota of all projects of
 a group in the **Usage Quotas** page available to the group page settings list.

doc/user/application_security/container_scanning/index.md

@@ -367,8 +367,8 @@ the report JSON unless stated otherwise. Presence of optional fields depends on
 | `vulnerabilities[].message`                          | A short text that describes the vulnerability, it may include occurrence's specific information. Optional.                                                                                                                                                                                                                                                                                 |
 | `vulnerabilities[].description`                      | A long text that describes the vulnerability. Optional.                                                                                                                                                                                                                                                                                                                                    |
 | `vulnerabilities[].cve`                              | A fingerprint string value that represents a concrete occurrence of the vulnerability. It's used to determine whether two vulnerability occurrences are same or different. May not be 100% accurate. **This is NOT a [CVE](https://cve.mitre.org/)**.                                                                                                                                      |
-| `vulnerabilities[].severity`                         | How much the vulnerability impacts the software. Possible values: `Undefined` (an analyzer has not provided this info), `Info`, `Unknown`, `Low`, `Medium`, `High`, `Critical`.  **Note:** Our current container scanning tool based on [klar](https://github.com/optiopay/klar) only provides the following levels: `Unknown`, `Low`, `Medium`, `High`, `Critical`.                       |
-| `vulnerabilities[].confidence`                       | How reliable the vulnerability's assessment is. Possible values: `Undefined` (an analyzer has not provided this info), `Ignore`, `Unknown`, `Experimental`, `Low`, `Medium`, `High`, `Confirmed`.  **Note:** Our current container scanning tool based on [klar](https://github.com/optiopay/klar) does not provide a confidence level, so this value is currently hardcoded to `Unknown`. |
+| `vulnerabilities[].severity`                         | How much the vulnerability impacts the software. Possible values: `Undefined` (an analyzer has not provided this information), `Info`, `Unknown`, `Low`, `Medium`, `High`, `Critical`.  **Note:** Our current container scanning tool based on [klar](https://github.com/optiopay/klar) only provides the following levels: `Unknown`, `Low`, `Medium`, `High`, `Critical`.                       |
+| `vulnerabilities[].confidence`                       | How reliable the vulnerability's assessment is. Possible values: `Undefined` (an analyzer has not provided this information), `Ignore`, `Unknown`, `Experimental`, `Low`, `Medium`, `High`, `Confirmed`.  **Note:** Our current container scanning tool based on [klar](https://github.com/optiopay/klar) does not provide a confidence level, so this value is currently hardcoded to `Unknown`. |
 | `vulnerabilities[].solution`                         | Explanation of how to fix the vulnerability. Optional.                                                                                                                                                                                                                                                                                                                                     |
 | `vulnerabilities[].scanner`                          | A node that describes the analyzer used to find this vulnerability.                                                                                                                                                                                                                                                                                                                        |
 | `vulnerabilities[].scanner.id`                       | Id of the scanner as a snake_case string.                                                                                                                                                                                                                                                                                                                                                  |

doc/user/application_security/dependency_scanning/index.md

@@ -365,8 +365,8 @@ the report JSON unless stated otherwise. Presence of optional fields depends on
 | `vulnerabilities[].message`                          | A short text that describes the vulnerability, it may include occurrence's specific information. Optional. |
 | `vulnerabilities[].description`                      | A long text that describes the vulnerability. Optional. |
 | `vulnerabilities[].cve`                              | A fingerprint string value that represents a concrete occurrence of the vulnerability. It's used to determine whether two vulnerability occurrences are same or different. May not be 100% accurate. **This is NOT a [CVE](https://cve.mitre.org/)**. |
-| `vulnerabilities[].severity`                         | How much the vulnerability impacts the software. Possible values: `Undefined` (an analyzer has not provided this info), `Info`, `Unknown`, `Low`, `Medium`, `High`, `Critical`. |
-| `vulnerabilities[].confidence`                       | How reliable the vulnerability's assessment is. Possible values: `Undefined` (an analyzer has not provided this info), `Ignore`, `Unknown`, `Experimental`, `Low`, `Medium`, `High`, `Confirmed`. |
+| `vulnerabilities[].severity`                         | How much the vulnerability impacts the software. Possible values: `Undefined` (an analyzer has not provided this information), `Info`, `Unknown`, `Low`, `Medium`, `High`, `Critical`. |
+| `vulnerabilities[].confidence`                       | How reliable the vulnerability's assessment is. Possible values: `Undefined` (an analyzer has not provided this information), `Ignore`, `Unknown`, `Experimental`, `Low`, `Medium`, `High`, `Confirmed`. |
 | `vulnerabilities[].solution`                         | Explanation of how to fix the vulnerability. Optional. |
 | `vulnerabilities[].scanner`                          | A node that describes the analyzer used to find this vulnerability. |
 | `vulnerabilities[].scanner.id`                       | Id of the scanner as a snake_case string. |

doc/user/application_security/offline_deployments/index.md

@@ -73,7 +73,7 @@ hosted within your network.
 ## Specific scanner instructions
 
 Each individual scanner may be slightly different than the steps described
-above. You can find more info at each of the pages below:
+above. You can find more information at each of the pages below:
 
 - [Container scanning offline directions](../container_scanning/index.md#running-container-scanning-in-an-offline-environment)
 - [SAST offline directions](../sast/index.md#gitlab-sast-in-an-offline-environment)

doc/user/application_security/sast/index.md

@@ -443,8 +443,8 @@ the report JSON unless stated otherwise. Presence of optional fields depends on
 | `vulnerabilities[].message`             | A short text that describes the vulnerability, it may include the occurrence's specific information. Optional. |
 | `vulnerabilities[].description`         | A long text that describes the vulnerability. Optional. |
 | `vulnerabilities[].cve`                 | A fingerprint string value that represents a concrete occurrence of the vulnerability. Is used to determine whether two vulnerability occurrences are same or different. May not be 100% accurate. **This is NOT a [CVE](https://cve.mitre.org/)**. |
-| `vulnerabilities[].severity`            | How much the vulnerability impacts the software. Possible values: `Undefined` (an analyzer has not provided this info), `Info`, `Unknown`, `Low`, `Medium`, `High`, `Critical`. |
-| `vulnerabilities[].confidence`          | How reliable the vulnerability's assessment is. Possible values: `Undefined` (an analyzer has not provided this info), `Ignore`, `Unknown`, `Experimental`, `Low`, `Medium`, `High`, `Confirmed`. |
+| `vulnerabilities[].severity`            | How much the vulnerability impacts the software. Possible values: `Undefined` (an analyzer has not provided this information), `Info`, `Unknown`, `Low`, `Medium`, `High`, `Critical`. |
+| `vulnerabilities[].confidence`          | How reliable the vulnerability's assessment is. Possible values: `Undefined` (an analyzer has not provided this information), `Ignore`, `Unknown`, `Experimental`, `Low`, `Medium`, `High`, `Confirmed`. |
 | `vulnerabilities[].solution`            | Explanation of how to fix the vulnerability. Optional. |
 | `vulnerabilities[].scanner`             | A node that describes the analyzer used to find this vulnerability. |
 | `vulnerabilities[].scanner.id`          | Id of the scanner as a snake_case string. |

doc/user/application_security/security_dashboard/index.md

@@ -156,7 +156,7 @@ That way, reports are created even if no code change happens.
 ## Security scans using Auto DevOps
 
 When using [Auto DevOps](../../../topics/autodevops/index.md), use
-[special environment variables](../../../topics/autodevops/index.md#environment-variables)
+[special environment variables](../../../topics/autodevops/customize.md#environment-variables)
 to configure daily security scans.
 
 <!-- ## Troubleshooting

doc/user/group/index.md

@@ -99,7 +99,7 @@ To create a new Group, either:
 
 Add the following information:
 
-![new group info](img/create_new_group_info.png)
+![new group information](img/create_new_group_info.png)
 
 1. The **Group name** will automatically populate the URL. Optionally, you can change it.
    This is the name that displays in group views.

doc/user/index.md

@@ -100,7 +100,7 @@ directly from GitLab. No third-party integrations needed.
 There is a lot you can customize and configure
 to enjoy the best of GitLab.
 
-- [Settings](profile/index.md): Manage your user settings to change your personal info,
+- [Settings](profile/index.md): Manage your user settings to change your personal information,
   personal access tokens, authorized applications, etc.
 - [Authentication](../topics/authentication/index.md): Read through the authentication
   methods available in GitLab.

doc/user/markdown.md

@@ -110,7 +110,7 @@ changing how standard Markdown is used:
 | [emphasis](#emphasis)                 | [multiple underscores in words](#multiple-underscores-in-words-and-mid-word-emphasis)
 | [headers](#headers)                   | [linkable Header IDs](#header-ids-and-links) |
 | [images](#images)                     | [embedded videos](#videos) and [audio](#audio) |
-| [linebreaks](#line-breaks)            | [more linebreak control](#newlines) |
+| [line breaks](#line-breaks)           | [more line break control](#newlines) |
 | [links](#links)                       | [automatically linking URLs](#url-auto-linking) |
 
 ## New GFM Markdown extensions
@@ -256,7 +256,7 @@ when rendered within GitLab, may appear different depending on the OS and browse
 Most emoji are natively supported on macOS, Windows, iOS, Android and will fallback to image-based emoji where there is lack of support.
 
 NOTE: **Note:** On Linux, you can download [Noto Color Emoji](https://www.google.com/get/noto/help/emoji/)
-to get full native emoji support. Ubuntu 18.04 (like many modern Linux distros) has
+to get full native emoji support. Ubuntu 18.04 (like many modern Linux distributions) has
 this font installed by default.
 
 ### Front matter
@@ -1148,7 +1148,7 @@ GFM adheres to the Markdown specification in how [paragraphs and line breaks are
 A paragraph is one or more consecutive lines of text, separated by one or
 more blank lines (two newlines at the end of the first paragraph), as [explained above](#line-breaks).
 
-If you need more control over line-breaks or soft returns, you can add a single line-break
+If you need more control over line breaks or soft returns, you can add a single line break
 by ending a line with a backslash, or two or more spaces. Two newlines in a row will create a new
 paragraph, with a blank line in between:
 
@@ -1163,21 +1163,6 @@ Another line, this time ending with a backslash.\
 A new line due to the previous backslash.
 ```
 
-<!-- (Do *NOT* remove the two ending whitespaces in the third line) -->
-<!-- (They are needed for the Markdown text to render correctly) -->
-
-First paragraph.
-Another line in the same paragraph.
-A third line in the same paragraph, but this time ending with two spaces.  
-A new line directly under the first paragraph.
-
-<!-- (Do *NOT* remove the two ending whitespaces in the second line) -->
-<!-- (They are needed for the Markdown text to render correctly on docs.gitlab.com, the backslash works fine inside GitLab itself) -->
-
-Second paragraph.
-Another line, this time ending with a backslash.  
-A new line due to the previous backslash.
-
 ### Links
 
 There are two ways to create links, inline-style and reference-style:
@@ -1406,7 +1391,7 @@ Example:
 | header 1 | header 2 | header 3 |
 | ---      |  ------  |---------:|
 | cell 1   | cell 2   | cell 3   |
-| cell 4 | cell 5 is longer | cell 6 is much longer than the others, but that's ok. It will eventually wrap the text when the cell is too large for the display size. |
+| cell 4 | cell 5 is longer | cell 6 is much longer than the others, but that's okay. It will eventually wrap the text when the cell is too large for the display size. |
 | cell 7   |          | cell <br> 9 |
 
 Additionally, you can choose the alignment of text within columns by adding colons (`:`)

doc/user/packages/conan_repository/index.md

@@ -246,9 +246,9 @@ conan search Hello/0.1@my-group+my-project/beta --all --remote=gitlab
 
 The scope of your search will include all projects you have permission to access, this includes your private projects as well as all public projects.
 
-## Fetching Conan package info from the GitLab Package Registry
+## Fetching Conan package information from the GitLab Package Registry
 
-The `conan info` command will return info about a given package:
+The `conan info` command will return information about a given package:
 
 ```shell
 conan info Hello/0.1@my-group+my-project/beta
@@ -261,7 +261,7 @@ The GitLab Conan repository supports the following Conan CLI commands:
 - `conan upload`: Upload your recipe and package files to the GitLab Package Registry.
 - `conan install`: Install a conan package from the GitLab Package Registry, this includes using the `conanfile.txt` file.
 - `conan search`: Search the GitLab Package Registry for public packages, and private packages you have permission to view.
-- `conan info`: View the info on a given package from the GitLab Package Registry.
+- `conan info`: View the information on a given package from the GitLab Package Registry.
 - `conan remove`: Delete the package from the GitLab Package Registry.
 
 ## Using GitLab CI with Conan packages

doc/user/project/clusters/add_eks_clusters.md

@@ -279,7 +279,7 @@ If a default Storage Class doesn't already exist and is desired, follow Amazon's
 to create one.
 
 Alternatively, disable PostgreSQL by setting the project variable
-[`POSTGRES_ENABLED`](../../../topics/autodevops/#environment-variables) to `false`.
+[`POSTGRES_ENABLED`](../../../topics/autodevops/customize.md#environment-variables) to `false`.
 
 ### Deploy the app to EKS
 

doc/user/project/integrations/irker.md

@@ -4,7 +4,7 @@ GitLab provides a way to push update messages to an Irker server. When
 configured, pushes to a project will trigger the service to send data directly
 to the Irker server.
 
-See the project homepage for further info: <https://gitlab.com/esr/irker>
+See the project homepage for further information: <https://gitlab.com/esr/irker>
 
 ## Needed setup
 

doc/user/project/integrations/prometheus.md

@@ -775,7 +775,7 @@ GitLab unfurls the link as an embedded metrics panel:
 ![Embedded Metrics Rendered](img/embedded_metrics_rendered_v12_8.png)
 
 You can also embed a single chart. To get a link to a chart, click the
-**{ellipsis_v}** **More info** menu in the upper right corner of the chart,
+**{ellipsis_v}** **More actions** menu in the upper right corner of the chart,
 and select **Copy link to chart**, as shown in this example:
 
 ![Copy Link To Chart](img/copy_link_to_chart_v12_10.png)

doc/user/project/merge_requests/reviewing_and_managing_merge_requests.md

@@ -116,7 +116,7 @@ environment.
 Deployments that are ongoing will be shown, as well as the deploying/deployed state
 for environments. If it's the first time the branch is deployed, the link
 will return a `404` error until done. During the deployment, the stop button will
-be disabled. If the pipeline fails to deploy, the deployment info will be hidden.
+be disabled. If the pipeline fails to deploy, the deployment information will be hidden.
 
 ![Merge request pipeline](img/merge_request_pipeline.png)
 

doc/user/project/web_ide/index.md

@@ -270,7 +270,7 @@ terminal:
 - The `webide-file-sync` executable must start **after** the project
   directory is available. This is why we need to add `sleep 5` to the `command`.
   See [this issue](https://gitlab.com/gitlab-org/webide-file-sync/issues/7) for
-  more info.
+  more information.
 - `$CI_PROJECT_DIR` is a
   [predefined environment variable](../../../ci/variables/predefined_variables.md)
   for GitLab Runners. This is where your project's repository will be.

lib/gitlab/graphql/connections.rb

-# frozen_string_literal: true
-
-module Gitlab
-  module Graphql
-    module Connections
-      def self.use(_schema)
-        GraphQL::Relay::BaseConnection.register_connection_implementation(
-          ActiveRecord::Relation,
-          Gitlab::Graphql::Connections::Keyset::Connection
-        )
-        GraphQL::Relay::BaseConnection.register_connection_implementation(
-          Gitlab::Graphql::FilterableArray,
-          Gitlab::Graphql::Connections::FilterableArrayConnection
-        )
-        GraphQL::Relay::BaseConnection.register_connection_implementation(
-          Gitlab::Graphql::ExternallyPaginatedArray,
-          Gitlab::Graphql::Connections::ExternallyPaginatedArrayConnection
-        )
-      end
-    end
-  end
-end

lib/gitlab/graphql/pagination/connections.rb

+# frozen_string_literal: true
+
+module Gitlab
+  module Graphql
+    module Pagination
+      module Connections
+        def self.use(schema)
+          schema.connections.add(
+            ActiveRecord::Relation,
+            Gitlab::Graphql::Pagination::Keyset::Connection)
+
+          schema.connections.add(
+            Gitlab::Graphql::FilterableArray,
+            Gitlab::Graphql::Pagination::FilterableArrayConnection)
+
+          schema.connections.add(
+            Gitlab::Graphql::ExternallyPaginatedArray,
+            Gitlab::Graphql::Pagination::ExternallyPaginatedArrayConnection)
+        end
+      end
+    end
+  end
+end

lib/gitlab/graphql/connections/externally_paginated_array_connection.rb → lib/gitlab/graphql/pagination/externally_paginated_array_connection.rb

@@ -3,20 +3,14 @@
 # Make a customized connection type
 module Gitlab
   module Graphql
-    module Connections
-      class ExternallyPaginatedArrayConnection < GraphQL::Relay::ArrayConnection
-        # As the pagination happens externally
-        # we just return all the nodes here.
-        def sliced_nodes
-          @nodes
-        end
-
+    module Pagination
+      class ExternallyPaginatedArrayConnection < GraphQL::Pagination::ArrayConnection
         def start_cursor
-          nodes.previous_cursor
+          items.previous_cursor
         end
 
         def end_cursor
-          nodes.next_cursor
+          items.next_cursor
         end
 
         def next_page?

lib/gitlab/graphql/connections/filterable_array_connection.rb → lib/gitlab/graphql/pagination/filterable_array_connection.rb

@@ -2,14 +2,14 @@
 
 module Gitlab
   module Graphql
-    module Connections
+    module Pagination
       # FilterableArrayConnection is useful especially for lazy-loaded values.
       # It allows us to call a callback only on the slice of array being
       # rendered in the "after loaded" phase.  For example we can check
       # permissions only on a small subset of items.
-      class FilterableArrayConnection < GraphQL::Relay::ArrayConnection
-        def paged_nodes
-          @filtered_nodes ||= nodes.filter_callback.call(super)
+      class FilterableArrayConnection < GraphQL::Pagination::ArrayConnection
+        def nodes
+          @nodes ||= items.filter_callback.call(super)
         end
       end
     end

lib/gitlab/graphql/connections/keyset/conditions/base_condition.rb → lib/gitlab/graphql/pagination/keyset/conditions/base_condition.rb

@@ -2,7 +2,7 @@
 
 module Gitlab
   module Graphql
-    module Connections
+    module Pagination
       module Keyset
         module Conditions
           class BaseCondition

lib/gitlab/graphql/connections/keyset/conditions/not_null_condition.rb → lib/gitlab/graphql/pagination/keyset/conditions/not_null_condition.rb

@@ -2,7 +2,7 @@
 
 module Gitlab
   module Graphql
-    module Connections
+    module Pagination
       module Keyset
         module Conditions
           class NotNullCondition < BaseCondition

lib/gitlab/graphql/connections/keyset/conditions/null_condition.rb → lib/gitlab/graphql/pagination/keyset/conditions/null_condition.rb

@@ -2,7 +2,7 @@
 
 module Gitlab
   module Graphql
-    module Connections
+    module Pagination
       module Keyset
         module Conditions
           class NullCondition < BaseCondition

lib/gitlab/graphql/connections/keyset/connection.rb → lib/gitlab/graphql/pagination/keyset/connection.rb

@@ -27,21 +27,21 @@
 #
 module Gitlab
   module Graphql
-    module Connections
+    module Pagination
       module Keyset
-        class Connection < GraphQL::Relay::BaseConnection
+        class Connection < GraphQL::Pagination::ActiveRecordRelationConnection
           include Gitlab::Utils::StrongMemoize
 
-          def cursor_from_node(node)
+          def cursor_for(node)
             encoded_json_from_ordering(node)
           end
 
           def sliced_nodes
             @sliced_nodes ||=
               begin
-                OrderInfo.validate_ordering(ordered_nodes, order_list)
+                OrderInfo.validate_ordering(ordered_items, order_list)
 
-                sliced = ordered_nodes
+                sliced = ordered_items
                 sliced = slice_nodes(sliced, before, :before) if before.present?
                 sliced = slice_nodes(sliced, after, :after) if after.present?
 
@@ -49,12 +49,12 @@ module Gitlab
               end
           end
 
-          def paged_nodes
+          def nodes
             # These are the nodes that will be loaded into memory for rendering
             # So we're ok loading them into memory here as that's bound to happen
             # anyway. Having them ready means we can modify the result while
             # rendering the fields.
-            @paged_nodes ||= load_paged_nodes.to_a
+            @nodes ||= load_paged_nodes.to_a
           end
 
           private
@@ -85,31 +85,31 @@ module Gitlab
             @limit_value ||= [first, last, max_page_size].compact.min
           end
 
-          def ordered_nodes
-            strong_memoize(:order_nodes) do
-              unless nodes.primary_key.present?
+          def ordered_items
+            strong_memoize(:ordered_items) do
+              unless items.primary_key.present?
                 raise ArgumentError.new('Relation must have a primary key')
               end
 
-              list = OrderInfo.build_order_list(nodes)
+              list = OrderInfo.build_order_list(items)
 
               # ensure there is a primary key ordering
-              if list&.last&.attribute_name != nodes.primary_key
-                nodes.order(arel_table[nodes.primary_key].desc) # rubocop: disable CodeReuse/ActiveRecord
+              if list&.last&.attribute_name != items.primary_key
+                items.order(arel_table[items.primary_key].desc) # rubocop: disable CodeReuse/ActiveRecord
               else
-                nodes
+                items
               end
             end
           end
 
           def order_list
             strong_memoize(:order_list) do
-              OrderInfo.build_order_list(ordered_nodes)
+              OrderInfo.build_order_list(ordered_items)
             end
           end
 
           def arel_table
-            nodes.arel_table
+            items.arel_table
           end
 
           # Storing the current order values in the cursor allows us to

lib/gitlab/graphql/connections/keyset/order_info.rb → lib/gitlab/graphql/pagination/keyset/order_info.rb

@@ -2,7 +2,7 @@
 
 module Gitlab
   module Graphql
-    module Connections
+    module Pagination
       module Keyset
         class OrderInfo
           attr_reader :attribute_name, :sort_direction, :named_function

lib/gitlab/graphql/connections/keyset/query_builder.rb → lib/gitlab/graphql/pagination/keyset/query_builder.rb

@@ -2,7 +2,7 @@
 
 module Gitlab
   module Graphql
-    module Connections
+    module Pagination
       module Keyset
         class QueryBuilder
           def initialize(arel_table, order_list, decoded_cursor, before_or_after)

lib/gitlab/import_export/group/tree_restorer.rb

@@ -20,6 +20,7 @@ module Gitlab
         def restore
           @group_attributes = relation_reader.consume_attributes(nil)
           @group_members = relation_reader.consume_relation(nil, 'members')
+            .map(&:first)
 
           # We need to remove `name` and `path` as we did consume it in previous pass
           @group_attributes.delete('name')

lib/gitlab/import_export/json/legacy_reader.rb

@@ -53,6 +53,7 @@ module Gitlab
 
         def initialize(relation_names:, allowed_path:)
           @relation_names = relation_names.map(&:to_s)
+          @consumed_relations = Set.new
 
           # This is legacy reader, to be used in transition
           # period before `.ndjson`,
@@ -81,17 +82,19 @@ module Gitlab
             raise ArgumentError, "Invalid #{importable_name} passed to `consume_relation`. Use #{@allowed_path} instead."
           end
 
-          value = relations.delete(key)
+          Enumerator.new do |documents|
+            next unless @consumed_relations.add?("#{importable_path}/#{key}")
 
-          return value unless block_given?
-          return if value.nil?
+            value = relations.delete(key)
+            next if value.nil?
 
             if value.is_a?(Array)
               value.each.with_index do |item, idx|
-              yield(item, idx)
+                documents << [item, idx]
               end
             else
-            yield(value, 0)
+              documents << [value, 0]
+            end
           end
         end
 

lib/gitlab/import_export/json/ndjson_reader.rb

+# frozen_string_literal: true
+
+module Gitlab
+  module ImportExport
+    module JSON
+      class NdjsonReader
+        MAX_JSON_DOCUMENT_SIZE = 50.megabytes
+
+        attr_reader :dir_path
+
+        def initialize(dir_path)
+          @dir_path = dir_path
+          @consumed_relations = Set.new
+        end
+
+        def exist?
+          Dir.exist?(@dir_path)
+        end
+
+        # This can be removed once legacy_reader is deprecated.
+        def legacy?
+          false
+        end
+
+        def consume_attributes(importable_path)
+          # This reads from `tree/project.json`
+          path = file_path("#{importable_path}.json")
+          data = File.read(path, MAX_JSON_DOCUMENT_SIZE)
+          json_decode(data)
+        end
+
+        def consume_relation(importable_path, key)
+          Enumerator.new do |documents|
+            next unless @consumed_relations.add?("#{importable_path}/#{key}")
+
+            # This reads from `tree/project/merge_requests.ndjson`
+            path = file_path(importable_path, "#{key}.ndjson")
+            next unless File.exist?(path)
+
+            File.foreach(path, MAX_JSON_DOCUMENT_SIZE).with_index do |line, line_num|
+              documents << [json_decode(line), line_num]
+            end
+          end
+        end
+
+        private
+
+        def json_decode(string)
+          ActiveSupport::JSON.decode(string)
+        rescue ActiveSupport::JSON.parse_error => e
+          Gitlab::ErrorTracking.log_exception(e)
+          raise Gitlab::ImportExport::Error, 'Incorrect JSON format'
+        end
+
+        def file_path(*path)
+          File.join(dir_path, *path)
+        end
+      end
+    end
+  end
+end

lib/gitlab/import_export/project/tree_restorer.rb

@@ -17,8 +17,13 @@ module Gitlab
         end
 
         def restore
+          unless relation_reader
+            raise Gitlab::ImportExport::Error, 'invalid import format'
+          end
+
           @project_attributes = relation_reader.consume_attributes(importable_path)
           @project_members = relation_reader.consume_relation(importable_path, 'project_members')
+            .map(&:first)
 
           if relation_tree_restorer.restore
             import_failure_service.with_retry(action: 'set_latest_merge_request_diff_ids!') do
@@ -38,13 +43,26 @@ module Gitlab
 
         def relation_reader
           strong_memoize(:relation_reader) do
+            [ndjson_relation_reader, legacy_relation_reader]
+              .compact.find(&:exist?)
+          end
+        end
+
+        def ndjson_relation_reader
+          return unless Feature.enabled?(:project_import_ndjson, project.namespace)
+
+          ImportExport::JSON::NdjsonReader.new(
+            File.join(shared.export_path, 'tree')
+          )
+        end
+
+        def legacy_relation_reader
           ImportExport::JSON::LegacyReader::File.new(
             File.join(shared.export_path, 'project.json'),
             relation_names: reader.project_relation_names,
             allowed_path: importable_path
           )
         end
-        end
 
         def relation_tree_restorer
           @relation_tree_restorer ||= RelationTreeRestorer.new(

lib/gitlab/import_export/relation_tree_restorer.rb

@@ -67,7 +67,7 @@ module Gitlab
       end
 
       def process_relation!(relation_key, relation_definition)
-        @relation_reader.consume_relation(@importable_path, relation_key) do |data_hash, relation_index|
+        @relation_reader.consume_relation(@importable_path, relation_key).each do |data_hash, relation_index|
           process_relation_item!(relation_key, relation_definition, relation_index, data_hash)
         end
       end

spec/fixtures/lib/gitlab/import_export/light/project.json

@@ -186,5 +186,23 @@
     }
   ],
   "snippets": [],
-  "hooks": []
+  "hooks": [],
+  "custom_attributes": [
+    {
+      "id": 201,
+      "project_id": 5,
+      "created_at": "2016-06-14T15:01:51.315Z",
+      "updated_at": "2016-06-14T15:01:51.315Z",
+      "key": "color",
+      "value": "red"
+    },
+    {
+      "id": 202,
+      "project_id": 5,
+      "created_at": "2016-06-14T15:01:51.315Z",
+      "updated_at": "2016-06-14T15:01:51.315Z",
+      "key": "size",
+      "value": "small"
+    }
+  ]
 }

spec/graphql/gitlab_schema_spec.rb

@@ -3,7 +3,7 @@
 require 'spec_helper'
 
 describe GitlabSchema do
-  let_it_be(:implementations) { GraphQL::Relay::BaseConnection::CONNECTION_IMPLEMENTATIONS }
+  let_it_be(:connections) { GitlabSchema.connections.all_wrappers }
   let(:user) { build :user }
 
   it 'uses batch loading' do
@@ -34,22 +34,22 @@ describe GitlabSchema do
     expect(described_class.query).to eq(::Types::QueryType)
   end
 
-  it 'paginates active record relations using `Connections::Keyset::Connection`' do
-    connection = implementations[ActiveRecord::Relation.name]
+  it 'paginates active record relations using `Pagination::Keyset::Connection`' do
+    connection = connections[ActiveRecord::Relation]
 
-    expect(connection).to eq(Gitlab::Graphql::Connections::Keyset::Connection)
+    expect(connection).to eq(Gitlab::Graphql::Pagination::Keyset::Connection)
   end
 
-  it 'paginates ExternallyPaginatedArray using `Connections::ExternallyPaginatedArrayConnection`' do
-    connection = implementations[Gitlab::Graphql::ExternallyPaginatedArray.name]
+  it 'paginates ExternallyPaginatedArray using `Pagination::ExternallyPaginatedArrayConnection`' do
+    connection = connections[Gitlab::Graphql::ExternallyPaginatedArray]
 
-    expect(connection).to eq(Gitlab::Graphql::Connections::ExternallyPaginatedArrayConnection)
+    expect(connection).to eq(Gitlab::Graphql::Pagination::ExternallyPaginatedArrayConnection)
   end
 
-  it 'paginates FilterableArray using `Connections::FilterableArrayConnection`' do
-    connection = implementations[Gitlab::Graphql::FilterableArray.name]
+  it 'paginates FilterableArray using `Pagination::FilterableArrayConnection`' do
+    connection = connections[Gitlab::Graphql::FilterableArray]
 
-    expect(connection).to eq(Gitlab::Graphql::Connections::FilterableArrayConnection)
+    expect(connection).to eq(Gitlab::Graphql::Pagination::FilterableArrayConnection)
   end
 
   describe '.execute' do

spec/lib/gitlab/graphql/connections/externally_paginated_array_connection_spec.rb → spec/lib/gitlab/graphql/pagination/externally_paginated_array_connection_spec.rb

@@ -2,7 +2,7 @@
 
 require 'spec_helper'
 
-describe Gitlab::Graphql::Connections::ExternallyPaginatedArrayConnection do
+describe Gitlab::Graphql::Pagination::ExternallyPaginatedArrayConnection do
   let(:prev_cursor) { 1 }
   let(:next_cursor) { 6 }
   let(:values) { [2, 3, 4, 5] }
@@ -10,21 +10,13 @@ describe Gitlab::Graphql::Connections::ExternallyPaginatedArrayConnection do
   let(:arguments) { {} }
 
   subject(:connection) do
-    described_class.new(all_nodes, arguments)
+    described_class.new(all_nodes, { max_page_size: values.size }.merge(arguments))
   end
 
-  describe '#sliced_nodes' do
-    let(:sliced_nodes) { connection.sliced_nodes }
+  describe '#nodes' do
+    let(:paged_nodes) { connection.nodes }
 
-    it 'returns all the nodes' do
-      expect(connection.sliced_nodes).to eq(values)
-    end
-  end
-
-  describe '#paged_nodes' do
-    let(:paged_nodes) { connection.send(:paged_nodes) }
-
-    it_behaves_like "connection with paged nodes" do
+    it_behaves_like 'connection with paged nodes' do
       let(:paged_nodes_size) { values.size }
     end
   end

spec/lib/gitlab/graphql/connections/filterable_array_connection_spec.rb → spec/lib/gitlab/graphql/pagination/filterable_array_connection_spec.rb

@@ -2,19 +2,19 @@
 
 require 'spec_helper'
 
-describe Gitlab::Graphql::Connections::FilterableArrayConnection do
+describe Gitlab::Graphql::Pagination::FilterableArrayConnection do
   let(:callback) { proc { |nodes| nodes } }
   let(:all_nodes) { Gitlab::Graphql::FilterableArray.new(callback, 1, 2, 3, 4, 5) }
   let(:arguments) { {} }
 
   subject(:connection) do
-    described_class.new(all_nodes, arguments, max_page_size: 3)
+    described_class.new(all_nodes, { max_page_size: 3 }.merge(arguments))
   end
 
-  describe '#paged_nodes' do
-    let(:paged_nodes) { subject.paged_nodes }
+  describe '#nodes' do
+    let(:paged_nodes) { subject.nodes }
 
-    it_behaves_like "connection with paged nodes" do
+    it_behaves_like 'connection with paged nodes' do
       let(:paged_nodes_size) { 3 }
     end
 
@@ -22,7 +22,7 @@ describe Gitlab::Graphql::Connections::FilterableArrayConnection do
       let(:callback) { proc { |nodes| nodes[1..-1] } }
 
       it 'does not return filtered elements' do
-        expect(subject.paged_nodes).to contain_exactly(all_nodes[1], all_nodes[2])
+        expect(subject.nodes).to contain_exactly(all_nodes[1], all_nodes[2])
       end
     end
   end

spec/lib/gitlab/graphql/connections/keyset/conditions/not_null_condition_spec.rb → spec/lib/gitlab/graphql/pagination/keyset/conditions/not_null_condition_spec.rb

@@ -2,7 +2,7 @@
 
 require 'spec_helper'
 
-describe Gitlab::Graphql::Connections::Keyset::Conditions::NotNullCondition do
+describe Gitlab::Graphql::Pagination::Keyset::Conditions::NotNullCondition do
   describe '#build' do
     let(:operators) { ['>', '>'] }
     let(:before_or_after) { :after }
@@ -75,7 +75,7 @@ describe Gitlab::Graphql::Connections::Keyset::Conditions::NotNullCondition do
     context 'when ordering by LOWER' do
       let(:arel_table) { Project.arel_table }
       let(:relation)   { Project.order(arel_table['name'].lower.asc).order(:id) }
-      let(:order_list) { Gitlab::Graphql::Connections::Keyset::OrderInfo.build_order_list(relation) }
+      let(:order_list) { Gitlab::Graphql::Pagination::Keyset::OrderInfo.build_order_list(relation) }
       let(:values)     { ['Test', 500] }
 
       context 'when :after' do

spec/lib/gitlab/graphql/connections/keyset/conditions/null_condition_spec.rb → spec/lib/gitlab/graphql/pagination/keyset/conditions/null_condition_spec.rb

@@ -2,7 +2,7 @@
 
 require 'spec_helper'
 
-describe Gitlab::Graphql::Connections::Keyset::Conditions::NullCondition do
+describe Gitlab::Graphql::Pagination::Keyset::Conditions::NullCondition do
   describe '#build' do
     let(:values) { [nil, 500] }
     let(:operators) { [nil, '>'] }
@@ -58,7 +58,7 @@ describe Gitlab::Graphql::Connections::Keyset::Conditions::NullCondition do
     context 'when ordering by LOWER' do
       let(:arel_table) { Project.arel_table }
       let(:relation)   { Project.order(arel_table['name'].lower.asc).order(:id) }
-      let(:order_list) { Gitlab::Graphql::Connections::Keyset::OrderInfo.build_order_list(relation) }
+      let(:order_list) { Gitlab::Graphql::Pagination::Keyset::OrderInfo.build_order_list(relation) }
 
       context 'when :after' do
         it 'generates sql' do

spec/lib/gitlab/graphql/connections/keyset/connection_spec.rb → spec/lib/gitlab/graphql/pagination/keyset/connection_spec.rb

@@ -2,25 +2,28 @@
 
 require 'spec_helper'
 
-describe Gitlab::Graphql::Connections::Keyset::Connection do
+describe Gitlab::Graphql::Pagination::Keyset::Connection do
   let(:nodes) { Project.all.order(id: :asc) }
   let(:arguments) { {} }
+  let(:query_type) { GraphQL::ObjectType.new }
+  let(:schema) { GraphQL::Schema.define(query: query_type, mutation: nil)}
+  let(:context) { GraphQL::Query::Context.new(query: OpenStruct.new(schema: schema), values: nil, object: nil) }
 
   subject(:connection) do
-    described_class.new(nodes, arguments, max_page_size: 3)
+    described_class.new(nodes, { context: context, max_page_size: 3 }.merge(arguments))
   end
 
   def encoded_cursor(node)
-    described_class.new(nodes, {}).cursor_from_node(node)
+    described_class.new(nodes, { context: context }).cursor_for(node)
   end
 
   def decoded_cursor(cursor)
     JSON.parse(Base64Bp.urlsafe_decode64(cursor))
   end
 
-  describe '#cursor_from_nodes' do
+  describe '#cursor_for' do
     let(:project) { create(:project) }
-    let(:cursor)  { connection.cursor_from_node(project) }
+    let(:cursor)  { connection.cursor_for(project) }
 
     it 'returns an encoded ID' do
       expect(decoded_cursor(cursor)).to eq('id' => project.id.to_s)
@@ -264,11 +267,11 @@ describe Gitlab::Graphql::Connections::Keyset::Connection do
     end
   end
 
-  describe '#paged_nodes' do
+  describe '#nodes' do
     let_it_be(:all_nodes) { create_list(:project, 5) }
-    let(:paged_nodes) { subject.paged_nodes }
+    let(:paged_nodes) { subject.nodes }
 
-    it_behaves_like "connection with paged nodes" do
+    it_behaves_like 'connection with paged nodes' do
       let(:paged_nodes_size) { 3 }
     end
 

spec/lib/gitlab/graphql/connections/keyset/order_info_spec.rb → spec/lib/gitlab/graphql/pagination/keyset/order_info_spec.rb

@@ -2,7 +2,7 @@
 
 require 'spec_helper'
 
-describe Gitlab::Graphql::Connections::Keyset::OrderInfo do
+describe Gitlab::Graphql::Pagination::Keyset::OrderInfo do
   describe '#build_order_list' do
     let(:order_list) { described_class.build_order_list(relation) }
 

spec/lib/gitlab/graphql/connections/keyset/query_builder_spec.rb → spec/lib/gitlab/graphql/pagination/keyset/query_builder_spec.rb

@@ -2,7 +2,7 @@
 
 require 'spec_helper'
 
-describe Gitlab::Graphql::Connections::Keyset::QueryBuilder do
+describe Gitlab::Graphql::Pagination::Keyset::QueryBuilder do
   context 'when number of ordering fields is 0' do
     it 'raises an error' do
       expect { described_class.new(Issue.arel_table, [], {}, :after) }
@@ -12,7 +12,7 @@ describe Gitlab::Graphql::Connections::Keyset::QueryBuilder do
 
   describe '#conditions' do
     let(:relation) { Issue.order(relative_position: :desc).order(:id) }
-    let(:order_list) { Gitlab::Graphql::Connections::Keyset::OrderInfo.build_order_list(relation) }
+    let(:order_list) { Gitlab::Graphql::Pagination::Keyset::OrderInfo.build_order_list(relation) }
     let(:arel_table) { Issue.arel_table }
     let(:builder) { described_class.new(arel_table, order_list, decoded_cursor, before_or_after) }
     let(:before_or_after) { :after }

spec/lib/gitlab/import_export/json/legacy_reader/shared_example.rb

@@ -15,7 +15,6 @@ RSpec.shared_examples 'import/export json legacy reader' do
       subject { legacy_reader.consume_attributes("project") }
 
       context 'no excluded attributes' do
-        let(:excluded_attributes) { [] }
         let(:relation_names) { [] }
 
         it 'returns the whole tree from parsed JSON' do
@@ -42,60 +41,53 @@ RSpec.shared_examples 'import/export json legacy reader' do
 
   describe '#consume_relation' do
     context 'when valid path is passed' do
-      let(:key) { 'description' }
+      let(:key) { 'labels' }
 
-      context 'block not given' do
-        it 'returns value of the key' do
-          expect(legacy_reader).to receive(:relations).and_return({ key => 'test value' })
-          expect(legacy_reader.consume_relation("project", key)).to eq('test value')
-        end
+      subject { legacy_reader.consume_relation("project", key) }
+
+      context 'key has not been consumed' do
+        it 'returns an Enumerator' do
+          expect(subject).to be_an_instance_of(Enumerator)
         end
 
-      context 'key has been consumed' do
+        context 'value is nil' do
           before do
-          legacy_reader.consume_relation("project", key)
+            expect(legacy_reader).to receive(:relations).and_return({ key => nil })
           end
 
-        it 'does not yield' do
-          expect do |blk|
-            legacy_reader.consume_relation("project", key, &blk)
-          end.not_to yield_control
+          it 'yields nothing to the Enumerator' do
+            expect(subject.to_a).to eq([])
           end
         end
 
-      context 'value is nil' do
+        context 'value is an array' do
           before do
-          expect(legacy_reader).to receive(:relations).and_return({ key => nil })
+            expect(legacy_reader).to receive(:relations).and_return({ key => %w[label1 label2] })
           end
 
-        it 'does not yield' do
-          expect do |blk|
-            legacy_reader.consume_relation("project", key, &blk)
-          end.not_to yield_control
+          it 'yields every relation value to the Enumerator' do
+            expect(subject.to_a).to eq([['label1', 0], ['label2', 1]])
           end
         end
 
         context 'value is not array' do
           before do
-          expect(legacy_reader).to receive(:relations).and_return({ key => 'value' })
+            expect(legacy_reader).to receive(:relations).and_return({ key => 'non-array value' })
           end
 
-        it 'yield the value with index 0' do
-          expect do |blk|
-            legacy_reader.consume_relation("project", key, &blk)
-          end.to yield_with_args('value', 0)
+          it 'yields the value with index 0 to the Enumerator' do
+            expect(subject.to_a).to eq([['non-array value', 0]])
+          end
         end
       end
 
-      context 'value is an array' do
+      context 'key has been consumed' do
         before do
-          expect(legacy_reader).to receive(:relations).and_return({ key => %w[item1 item2 item3] })
+          legacy_reader.consume_relation("project", key).first
         end
 
-        it 'yield each array element with index' do
-          expect do |blk|
-            legacy_reader.consume_relation("project", key, &blk)
-          end.to yield_successive_args(['item1', 0], ['item2', 1], ['item3', 2])
+        it 'yields nothing to the Enumerator' do
+          expect(subject.to_a).to eq([])
         end
       end
     end

spec/lib/gitlab/import_export/json/ndjson_reader_spec.rb

+# frozen_string_literal: true
+
+require 'spec_helper'
+
+describe Gitlab::ImportExport::JSON::NdjsonReader do
+  include ImportExport::CommonUtil
+
+  let(:fixture) { 'spec/fixtures/lib/gitlab/import_export/light/tree' }
+  let(:root_tree) { JSON.parse(File.read(File.join(fixture, 'project.json'))) }
+  let(:ndjson_reader) { described_class.new(dir_path) }
+  let(:importable_path) { 'project' }
+
+  before :all do
+    extract_archive('spec/fixtures/lib/gitlab/import_export/light', 'tree.tar.gz')
+  end
+
+  after :all do
+    cleanup_artifacts_from_extract_archive('light')
+  end
+
+  describe '#exist?' do
+    subject { ndjson_reader.exist? }
+
+    context 'given valid dir_path' do
+      let(:dir_path) { fixture }
+
+      it { is_expected.to be true }
+    end
+
+    context 'given invalid dir_path' do
+      let(:dir_path) { 'invalid-dir-path' }
+
+      it { is_expected.to be false }
+    end
+  end
+
+  describe '#legacy?' do
+    let(:dir_path) { fixture }
+
+    subject { ndjson_reader.legacy? }
+
+    it { is_expected.to be false }
+  end
+
+  describe '#consume_attributes' do
+    let(:dir_path) { fixture }
+
+    subject { ndjson_reader.consume_attributes(importable_path) }
+
+    it 'returns the whole root tree from parsed JSON' do
+      expect(subject).to eq(root_tree)
+    end
+  end
+
+  describe '#consume_relation' do
+    let(:dir_path) { fixture }
+
+    subject { ndjson_reader.consume_relation(importable_path, key) }
+
+    context 'given any key' do
+      let(:key) { 'any-key' }
+
+      it 'returns an Enumerator' do
+        expect(subject).to be_an_instance_of(Enumerator)
+      end
+    end
+
+    context 'key has been consumed' do
+      let(:key) { 'issues' }
+
+      before do
+        ndjson_reader.consume_relation(importable_path, key).first
+      end
+
+      it 'yields nothing to the Enumerator' do
+        expect(subject.to_a).to eq([])
+      end
+    end
+
+    context 'key has not been consumed' do
+      context 'relation file does not exist' do
+        let(:key) { 'non-exist-relation-file-name' }
+
+        before do
+          relation_file_path = File.join(dir_path, importable_path, "#{key}.ndjson")
+          expect(File).to receive(:exist?).with(relation_file_path).and_return(false)
+        end
+
+        it 'yields nothing to the Enumerator' do
+          expect(subject.to_a).to eq([])
+        end
+      end
+
+      context 'relation file is empty' do
+        let(:key) { 'empty' }
+
+        it 'yields nothing to the Enumerator' do
+          expect(subject.to_a).to eq([])
+        end
+      end
+
+      context 'relation file contains multiple lines' do
+        let(:key) { 'custom_attributes' }
+        let(:attr_1) { JSON.parse('{"id":201,"project_id":5,"created_at":"2016-06-14T15:01:51.315Z","updated_at":"2016-06-14T15:01:51.315Z","key":"color","value":"red"}') }
+        let(:attr_2) { JSON.parse('{"id":202,"project_id":5,"created_at":"2016-06-14T15:01:51.315Z","updated_at":"2016-06-14T15:01:51.315Z","key":"size","value":"small"}') }
+
+        it 'yields every relation value to the Enumerator' do
+          expect(subject.to_a).to eq([[attr_1, 0], [attr_2, 1]])
+        end
+      end
+    end
+  end
+end

spec/lib/gitlab/import_export/project/tree_restorer_spec.rb

@@ -11,6 +11,7 @@ describe Gitlab::ImportExport::Project::TreeRestorer do
 
   let(:shared) { project.import_export_shared }
 
+  RSpec.shared_examples 'project tree restorer work properly' do |reader|
     describe 'restore project tree' do
       before_all do
         # Using an admin for import, so we can check assignment of existing members
@@ -25,6 +26,7 @@ describe Gitlab::ImportExport::Project::TreeRestorer do
           @shared = @project.import_export_shared
 
           setup_import_export_config('complex')
+          setup_reader(reader)
 
           allow_any_instance_of(Repository).to receive(:fetch_source_branch!).and_return(true)
           allow_any_instance_of(Gitlab::Git::Repository).to receive(:branch_exists?).and_return(false)
@@ -38,6 +40,10 @@ describe Gitlab::ImportExport::Project::TreeRestorer do
         end
       end
 
+      after(:context) do
+        cleanup_artifacts_from_extract_archive('complex')
+      end
+
       context 'JSON' do
         it 'restores models based on JSON' do
           expect(@restored_project_json).to be_truthy
@@ -82,6 +88,7 @@ describe Gitlab::ImportExport::Project::TreeRestorer do
 
               expect(merge_request_note.note_html).to match(/#{note_content}/)
             end
+          end
 
           context 'merge request system note metadata' do
             it 'restores title action for unmark wip' do
@@ -103,7 +110,6 @@ describe Gitlab::ImportExport::Project::TreeRestorer do
             end
           end
         end
-      end
 
         it 'creates a valid pipeline note' do
           expect(Ci::Pipeline.find_by_sha('sha-notes').notes).not_to be_empty
@@ -507,7 +513,7 @@ describe Gitlab::ImportExport::Project::TreeRestorer do
           allow(shared).to receive(:export_path).and_call_original
 
           expect(project_tree_restorer.restore).to eq(false)
-        expect(shared.errors).to include('Incorrect JSON format')
+          expect(shared.errors).to include('invalid import format')
         end
       end
     end
@@ -521,16 +527,14 @@ describe Gitlab::ImportExport::Project::TreeRestorer do
       context 'with a simple project' do
         before do
           setup_import_export_config('light')
+          setup_reader(reader)
+
           expect(restored_project_json).to eq(true)
         end
 
-      it_behaves_like 'restores project successfully',
-                      issues: 1,
-                      labels: 2,
-                      label_with_priorities: 'A project label',
-                      milestones: 1,
-                      first_issue_labels: 1,
-                      services: 1
+        after do
+          cleanup_artifacts_from_extract_archive('light')
+        end
 
         it 'issue system note metadata restored successfully' do
           note_content = 'created merge request !1 to address this issue'
@@ -542,6 +546,20 @@ describe Gitlab::ImportExport::Project::TreeRestorer do
           expect(note.system_note_metadata.commit_count).to be_nil
         end
 
+        context 'when there is an existing build with build token' do
+          before do
+            create(:ci_build, token: 'abcd')
+          end
+
+          it_behaves_like 'restores project successfully',
+            issues: 1,
+            labels: 2,
+            label_with_priorities: 'A project label',
+            milestones: 1,
+            first_issue_labels: 1,
+            services: 1
+        end
+
         context 'when there is an existing build with build token' do
           before do
             create(:ci_build, token: 'abcd')
@@ -559,9 +577,15 @@ describe Gitlab::ImportExport::Project::TreeRestorer do
       context 'multiple pipelines reference the same external pull request' do
         before do
           setup_import_export_config('multi_pipeline_ref_one_external_pr')
+          setup_reader(reader)
+
           expect(restored_project_json).to eq(true)
         end
 
+        after do
+          cleanup_artifacts_from_extract_archive('multi_pipeline_ref_one_external_pr')
+        end
+
         it_behaves_like 'restores project successfully',
           issues: 0,
           labels: 0,
@@ -585,11 +609,17 @@ describe Gitlab::ImportExport::Project::TreeRestorer do
 
         before do
           setup_import_export_config('light')
+          setup_reader(reader)
+
           expect(project)
             .to receive(:merge_requests)
             .and_raise(exception)
         end
 
+        after do
+          cleanup_artifacts_from_extract_archive('light')
+        end
+
         it 'report post import error' do
           expect(restored_project_json).to eq(false)
           expect(shared.errors).to include('post_import_error')
@@ -601,6 +631,8 @@ describe Gitlab::ImportExport::Project::TreeRestorer do
 
         before do
           setup_import_export_config('light')
+          setup_reader(reader)
+
           expect(project)
             .to receive(:merge_requests)
             .and_raise(exception)
@@ -610,6 +642,10 @@ describe Gitlab::ImportExport::Project::TreeRestorer do
           expect(restored_project_json).to eq(true)
         end
 
+        after do
+          cleanup_artifacts_from_extract_archive('light')
+        end
+
         it_behaves_like 'restores project successfully',
           issues: 1,
           labels: 2,
@@ -635,6 +671,11 @@ describe Gitlab::ImportExport::Project::TreeRestorer do
       context 'when the project has overridden params in import data' do
         before do
           setup_import_export_config('light')
+          setup_reader(reader)
+        end
+
+        after do
+          cleanup_artifacts_from_extract_archive('light')
         end
 
         it 'handles string versions of visibility_level' do
@@ -697,9 +738,15 @@ describe Gitlab::ImportExport::Project::TreeRestorer do
 
         before do
           setup_import_export_config('group')
+          setup_reader(reader)
+
           expect(restored_project_json).to eq(true)
         end
 
+        after do
+          cleanup_artifacts_from_extract_archive('group')
+        end
+
         it_behaves_like 'restores project successfully',
           issues: 3,
           labels: 2,
@@ -730,6 +777,11 @@ describe Gitlab::ImportExport::Project::TreeRestorer do
 
         before do
           setup_import_export_config('light')
+          setup_reader(reader)
+        end
+
+        after do
+          cleanup_artifacts_from_extract_archive('light')
         end
 
         it 'does not import any templated services' do
@@ -776,6 +828,11 @@ describe Gitlab::ImportExport::Project::TreeRestorer do
 
         before do
           setup_import_export_config('milestone-iid')
+          setup_reader(reader)
+        end
+
+        after do
+          cleanup_artifacts_from_extract_archive('milestone-iid')
         end
 
         it 'preserves the project milestone IID' do
@@ -791,6 +848,11 @@ describe Gitlab::ImportExport::Project::TreeRestorer do
       context 'with external authorization classification labels' do
         before do
           setup_import_export_config('light')
+          setup_reader(reader)
+        end
+
+        after do
+          cleanup_artifacts_from_extract_archive('light')
         end
 
         it 'converts empty external classification authorization labels to nil' do
@@ -818,7 +880,8 @@ describe Gitlab::ImportExport::Project::TreeRestorer do
       end
 
       before do
-      allow_any_instance_of(Gitlab::ImportExport::JSON::LegacyReader::File).to receive(:valid?).and_return(true)
+        allow_any_instance_of(Gitlab::ImportExport::JSON::LegacyReader::File).to receive(:exist?).and_return(true)
+        allow_any_instance_of(Gitlab::ImportExport::JSON::NdjsonReader).to receive(:exist?).and_return(false)
         allow_any_instance_of(Gitlab::ImportExport::JSON::LegacyReader::File).to receive(:tree_hash) { tree_hash }
       end
 
@@ -901,10 +964,15 @@ describe Gitlab::ImportExport::Project::TreeRestorer do
 
       before do
         setup_import_export_config('with_invalid_records')
+        setup_reader(reader)
 
         subject
       end
 
+      after do
+        cleanup_artifacts_from_extract_archive('with_invalid_records')
+      end
+
       context 'when failures occur because a relation fails to be processed' do
         it_behaves_like 'restores project successfully',
           issues: 0,
@@ -928,4 +996,26 @@ describe Gitlab::ImportExport::Project::TreeRestorer do
         end
       end
     end
+  end
+
+  context 'enable ndjson import' do
+    before_all do
+      # Test suite `restore project tree` run `project_tree_restorer.restore` in `before_all`.
+      # `Enable all features by default for testing` happens in `before(:each)`
+      # So it requires manually enable feature flag to allow ndjson_reader
+      Feature.enable(:project_import_ndjson)
+    end
+
+    it_behaves_like 'project tree restorer work properly', :legacy_reader
+
+    it_behaves_like 'project tree restorer work properly', :ndjson_reader
+  end
+
+  context 'disable ndjson import' do
+    before do
+      stub_feature_flags(project_import_ndjson: false)
+    end
+
+    it_behaves_like 'project tree restorer work properly', :legacy_reader
+  end
 end

spec/lib/gitlab/import_export/relation_tree_restorer_spec.rb

@@ -14,7 +14,7 @@ describe Gitlab::ImportExport::RelationTreeRestorer do
 
   let(:user) { create(:user) }
   let(:shared) { Gitlab::ImportExport::Shared.new(importable) }
-  let(:attributes) { {} }
+  let(:attributes) { relation_reader.consume_attributes(importable_name) }
 
   let(:members_mapper) do
     Gitlab::ImportExport::MembersMapper.new(exported_members: {}, user: user, importable: importable)
@@ -30,7 +30,7 @@ describe Gitlab::ImportExport::RelationTreeRestorer do
       relation_factory:      relation_factory,
       reader:                reader,
       importable:            importable,
-      importable_path:       nil,
+      importable_path:       importable_path,
       importable_attributes: attributes
     )
   end
@@ -94,21 +94,24 @@ describe Gitlab::ImportExport::RelationTreeRestorer do
   end
 
   context 'when restoring a project' do
-    let(:path) { 'spec/fixtures/lib/gitlab/import_export/complex/project.json' }
     let(:importable) { create(:project, :builds_enabled, :issues_disabled, name: 'project', path: 'project') }
+    let(:importable_name) { 'project' }
+    let(:importable_path) { 'project' }
     let(:object_builder) { Gitlab::ImportExport::Project::ObjectBuilder }
     let(:relation_factory) { Gitlab::ImportExport::Project::RelationFactory }
     let(:reader) { Gitlab::ImportExport::Reader.new(shared: shared) }
 
     context 'using legacy reader' do
+      let(:path) { 'spec/fixtures/lib/gitlab/import_export/complex/project.json' }
       let(:relation_reader) do
         Gitlab::ImportExport::JSON::LegacyReader::File.new(
           path,
-          relation_names: reader.project_relation_names
+          relation_names: reader.project_relation_names,
+          allowed_path: 'project'
         )
       end
 
-      let(:attributes) { relation_reader.consume_attributes(nil) }
+      let(:attributes) { relation_reader.consume_attributes('project') }
 
       it_behaves_like 'import project successfully'
 
@@ -118,6 +121,21 @@ describe Gitlab::ImportExport::RelationTreeRestorer do
 
         include_examples 'logging of relations creation'
       end
+
+      context 'using ndjson reader' do
+        let(:path) { 'spec/fixtures/lib/gitlab/import_export/complex/tree' }
+        let(:relation_reader) { Gitlab::ImportExport::JSON::NdjsonReader.new(path) }
+
+        before :all do
+          extract_archive('spec/fixtures/lib/gitlab/import_export/complex', 'tree.tar.gz')
+        end
+
+        after :all do
+          cleanup_artifacts_from_extract_archive('complex')
+        end
+
+        it_behaves_like 'import project successfully'
+      end
     end
   end
 
@@ -125,9 +143,16 @@ describe Gitlab::ImportExport::RelationTreeRestorer do
     let(:path) { 'spec/fixtures/lib/gitlab/import_export/group_exports/no_children/group.json' }
     let(:group) { create(:group) }
     let(:importable) { create(:group, parent: group) }
+    let(:importable_name) { nil }
+    let(:importable_path) { nil }
     let(:object_builder) { Gitlab::ImportExport::Group::ObjectBuilder }
     let(:relation_factory) { Gitlab::ImportExport::Group::RelationFactory }
-    let(:relation_reader) { Gitlab::ImportExport::JSON::LegacyReader::File.new(path, relation_names: reader.group_relation_names) }
+    let(:relation_reader) do
+      Gitlab::ImportExport::JSON::LegacyReader::File.new(
+        path,
+        relation_names: reader.group_relation_names)
+    end
+
     let(:reader) do
       Gitlab::ImportExport::Reader.new(
         shared: shared,
@@ -135,6 +160,10 @@ describe Gitlab::ImportExport::RelationTreeRestorer do
       )
     end
 
+    it 'restores group tree' do
+      expect(subject).to eq(true)
+    end
+
     include_examples 'logging of relations creation'
   end
 end

spec/support/helpers/graphql_helpers.rb

@@ -378,8 +378,9 @@ module GraphqlHelpers
 
   def execute_query(query_type)
     schema = Class.new(GraphQL::Schema) do
+      use GraphQL::Pagination::Connections
       use Gitlab::Graphql::Authorize
-      use Gitlab::Graphql::Connections
+      use Gitlab::Graphql::Pagination::Connections
 
       query(query_type)
     end

spec/support/import_export/common_util.rb

@@ -15,9 +15,39 @@ module ImportExport
       export_path = [prefix, 'spec', 'fixtures', 'lib', 'gitlab', 'import_export', name].compact
       export_path = File.join(*export_path)
 
+      extract_archive(export_path, 'tree.tar.gz')
+
       allow_any_instance_of(Gitlab::ImportExport).to receive(:export_path) { export_path }
     end
 
+    def extract_archive(path, archive)
+      if File.exist?(File.join(path, archive))
+        system("cd #{path}; tar xzvf #{archive} &> /dev/null")
+      end
+    end
+
+    def cleanup_artifacts_from_extract_archive(name, prefix = nil)
+      export_path = [prefix, 'spec', 'fixtures', 'lib', 'gitlab', 'import_export', name].compact
+      export_path = File.join(*export_path)
+
+      if File.exist?(File.join(export_path, 'tree.tar.gz'))
+        system("cd #{export_path}; rm -fr tree &> /dev/null")
+      end
+    end
+
+    def setup_reader(reader)
+      case reader
+      when :legacy_reader
+        allow_any_instance_of(Gitlab::ImportExport::JSON::LegacyReader::File).to receive(:exist?).and_return(true)
+        allow_any_instance_of(Gitlab::ImportExport::JSON::NdjsonReader).to receive(:exist?).and_return(false)
+      when :ndjson_reader
+        allow_any_instance_of(Gitlab::ImportExport::JSON::LegacyReader::File).to receive(:exist?).and_return(false)
+        allow_any_instance_of(Gitlab::ImportExport::JSON::NdjsonReader).to receive(:exist?).and_return(true)
+      else
+        raise "invalid reader #{reader}. Supported readers: :legacy_reader, :ndjson_reader"
+      end
+    end
+
     def fixtures_path
       "spec/fixtures/lib/gitlab/import_export"
     end