Commit e2ee1eec authored by GitLab Bot's avatar GitLab Bot

Add latest changes from gitlab-org/gitlab@master

parent 060c8424
...@@ -10,11 +10,12 @@ class GitlabSchema < GraphQL::Schema ...@@ -10,11 +10,12 @@ class GitlabSchema < GraphQL::Schema
DEFAULT_MAX_DEPTH = 15 DEFAULT_MAX_DEPTH = 15
AUTHENTICATED_MAX_DEPTH = 20 AUTHENTICATED_MAX_DEPTH = 20
use GraphQL::Pagination::Connections
use BatchLoader::GraphQL use BatchLoader::GraphQL
use Gitlab::Graphql::Authorize use Gitlab::Graphql::Authorize
use Gitlab::Graphql::Present use Gitlab::Graphql::Present
use Gitlab::Graphql::CallsGitaly use Gitlab::Graphql::CallsGitaly
use Gitlab::Graphql::Connections use Gitlab::Graphql::Pagination::Connections
use Gitlab::Graphql::GenericTracing use Gitlab::Graphql::GenericTracing
use Gitlab::Graphql::Timeout, max_seconds: Gitlab.config.gitlab.graphql_timeout use Gitlab::Graphql::Timeout, max_seconds: Gitlab.config.gitlab.graphql_timeout
......
---
title: Remove unused index for vulnerability severity levels
merge_request: 29023
author:
type: other
# frozen_string_literal: true
class DropVulnerabilitySeverityIndex < ActiveRecord::Migration[6.0]
include Gitlab::Database::MigrationHelpers
disable_ddl_transaction!
DOWNTIME = false
INDEX_NAME = 'undefined_vulnerabilities'
def up
remove_concurrent_index_by_name :vulnerability_occurrences, INDEX_NAME
end
def down
add_concurrent_index(:vulnerability_occurrences, :id, where: 'severity = 0', name: INDEX_NAME)
end
end
# frozen_string_literal: true
class DropVulnerabilitiesSeverityIndex < ActiveRecord::Migration[6.0]
include Gitlab::Database::MigrationHelpers
disable_ddl_transaction!
DOWNTIME = false
INDEX_NAME = 'undefined_vulnerability'
def up
remove_concurrent_index_by_name :vulnerabilities, INDEX_NAME
end
def down
add_concurrent_index(:vulnerabilities, :id, where: 'severity = 0', name: INDEX_NAME)
end
end
...@@ -10391,10 +10391,6 @@ CREATE INDEX tmp_build_stage_position_index ON public.ci_builds USING btree (sta ...@@ -10391,10 +10391,6 @@ CREATE INDEX tmp_build_stage_position_index ON public.ci_builds USING btree (sta
CREATE INDEX tmp_idx_on_user_id_where_bio_is_filled ON public.users USING btree (id) WHERE ((COALESCE(bio, ''::character varying))::text IS DISTINCT FROM ''::text); CREATE INDEX tmp_idx_on_user_id_where_bio_is_filled ON public.users USING btree (id) WHERE ((COALESCE(bio, ''::character varying))::text IS DISTINCT FROM ''::text);
CREATE INDEX undefined_vulnerabilities ON public.vulnerability_occurrences USING btree (id) WHERE (severity = 0);
CREATE INDEX undefined_vulnerability ON public.vulnerabilities USING btree (id) WHERE (severity = 0);
CREATE UNIQUE INDEX users_security_dashboard_projects_unique_index ON public.users_security_dashboard_projects USING btree (project_id, user_id); CREATE UNIQUE INDEX users_security_dashboard_projects_unique_index ON public.users_security_dashboard_projects USING btree (project_id, user_id);
CREATE UNIQUE INDEX vulnerability_feedback_unique_idx ON public.vulnerability_feedback USING btree (project_id, category, feedback_type, project_fingerprint); CREATE UNIQUE INDEX vulnerability_feedback_unique_idx ON public.vulnerability_feedback USING btree (project_id, category, feedback_type, project_fingerprint);
...@@ -13012,5 +13008,7 @@ COPY "schema_migrations" (version) FROM STDIN; ...@@ -13012,5 +13008,7 @@ COPY "schema_migrations" (version) FROM STDIN;
20200403184110 20200403184110
20200403185127 20200403185127
20200403185422 20200403185422
20200407094005
20200407094923
\. \.
...@@ -14,4 +14,3 @@ swap: ...@@ -14,4 +14,3 @@ swap:
postgres: PostgreSQL postgres: PostgreSQL
self hosted: self-managed self hosted: self-managed
self-hosted: self-managed self-hosted: self-managed
...@@ -44,7 +44,7 @@ _The reporter permission allows support staff to manage issues, labels, and revi ...@@ -44,7 +44,7 @@ _The reporter permission allows support staff to manage issues, labels, and revi
> Guest permissions allows people ops staff to review and lodge new issues while allowing no read or write access to project code or [confidential issues](../../../user/project/issues/confidential_issues.md#permissions-and-access-to-confidential-issues) created by other users. > Guest permissions allows people ops staff to review and lodge new issues while allowing no read or write access to project code or [confidential issues](../../../user/project/issues/confidential_issues.md#permissions-and-access-to-confidential-issues) created by other users.
See the [permission list](../../../user/permissions.md) for complementary info. See the [permission list](../../../user/permissions.md) for complementary information.
#### Group permissions - example #### Group permissions - example
......
...@@ -273,7 +273,7 @@ On different cloud vendors a best effort like for like can be used. ...@@ -273,7 +273,7 @@ On different cloud vendors a best effort like for like can be used.
[^8]: The architectures were built and tested with the [Intel Xeon E5 v3 (Haswell)](https://cloud.google.com/compute/docs/cpu-platforms) [^8]: The architectures were built and tested with the [Intel Xeon E5 v3 (Haswell)](https://cloud.google.com/compute/docs/cpu-platforms)
CPU platform on GCP. On different hardware you may find that adjustments, either lower CPU platform on GCP. On different hardware you may find that adjustments, either lower
or higher, are required for your CPU or Node counts accordingly. For more info a or higher, are required for your CPU or Node counts accordingly. For more information, a
[Sysbench](https://github.com/akopytov/sysbench) benchmark of the CPU can be found [Sysbench](https://github.com/akopytov/sysbench) benchmark of the CPU can be found
[here](https://gitlab.com/gitlab-org/quality/performance/-/wikis/Reference-Architectures/GCP-CPU-Benchmarks). [here](https://gitlab.com/gitlab-org/quality/performance/-/wikis/Reference-Architectures/GCP-CPU-Benchmarks).
......
...@@ -13,7 +13,7 @@ before attempting to use these tools. ...@@ -13,7 +13,7 @@ before attempting to use these tools.
## gitlabsos ## gitlabsos
The [gitlabsos](https://gitlab.com/gitlab-com/support/toolbox/gitlabsos/) utility The [gitlabsos](https://gitlab.com/gitlab-com/support/toolbox/gitlabsos/) utility
provides a unified method of gathering info and logs from GitLab and the system it's provides a unified method of gathering information and logs from GitLab and the system it's
running on. running on.
## strace-parser ## strace-parser
......
...@@ -751,7 +751,7 @@ This content has been moved to the [Troubleshooting Sidekiq docs](./sidekiq.md). ...@@ -751,7 +751,7 @@ This content has been moved to the [Troubleshooting Sidekiq docs](./sidekiq.md).
## LFS ## LFS
### Get info about LFS objects and associated project ### Get information about LFS objects and associated project
```ruby ```ruby
o=LfsObject.find_by(oid: "<oid>") o=LfsObject.find_by(oid: "<oid>")
......
...@@ -168,7 +168,7 @@ and they will assist you with any issues you are having. ...@@ -168,7 +168,7 @@ and they will assist you with any issues you are having.
kubectl exec -it <task-runner-pod-name> -- /srv/gitlab/bin/rails dbconsole -p kubectl exec -it <task-runner-pod-name> -- /srv/gitlab/bin/rails dbconsole -p
``` ```
- How to get info about Helm installation status: - How to get information about Helm installation status:
```shell ```shell
helm status name-of-installation helm status name-of-installation
...@@ -199,7 +199,7 @@ and they will assist you with any issues you are having. ...@@ -199,7 +199,7 @@ and they will assist you with any issues you are having.
helm upgrade <release name> <chart path> -f gitlab.yaml helm upgrade <release name> <chart path> -f gitlab.yaml
``` ```
- How to get the manifest for a release. It can be useful because it contains the info about - How to get the manifest for a release. It can be useful because it contains the information about
all Kubernetes resources and dependent charts: all Kubernetes resources and dependent charts:
```shell ```shell
......
...@@ -72,7 +72,7 @@ Root-level queries are defined in ...@@ -72,7 +72,7 @@ Root-level queries are defined in
GitLab supports batching queries into a single request using GitLab supports batching queries into a single request using
[apollo-link-batch-http](https://www.apollographql.com/docs/link/links/batch-http/). More [apollo-link-batch-http](https://www.apollographql.com/docs/link/links/batch-http/). More
info about multiplexed queries is also available for information about multiplexed queries is also available for
[graphql-ruby](https://graphql-ruby.org/queries/multiplex.html) the [graphql-ruby](https://graphql-ruby.org/queries/multiplex.html) the
library GitLab uses on the backend. library GitLab uses on the backend.
......
...@@ -102,7 +102,7 @@ CAUTION: **Important:** ...@@ -102,7 +102,7 @@ CAUTION: **Important:**
Avoid using this flow for applications that store data outside of the GitLab Avoid using this flow for applications that store data outside of the GitLab
instance. If you do, make sure to verify `application id` associated with the instance. If you do, make sure to verify `application id` associated with the
access token before granting access to the data access token before granting access to the data
(see [`/oauth/token/info`](#retrieving-the-token-info)). (see [`/oauth/token/info`](#retrieving-the-token-information)).
Unlike the web flow, the client receives an `access token` immediately as a Unlike the web flow, the client receives an `access token` immediately as a
result of the authorization request. The flow does not use the client secret result of the authorization request. The flow does not use the client secret
...@@ -213,7 +213,7 @@ or you can put the token to the Authorization header: ...@@ -213,7 +213,7 @@ or you can put the token to the Authorization header:
curl --header "Authorization: Bearer OAUTH-TOKEN" https://gitlab.example.com/api/v4/user curl --header "Authorization: Bearer OAUTH-TOKEN" https://gitlab.example.com/api/v4/user
``` ```
## Retrieving the Token Info ## Retrieving the token information
To verify the details of a token, use the `token/info` endpoint provided by the Doorkeeper gem. To verify the details of a token, use the `token/info` endpoint provided by the Doorkeeper gem.
For more information, see [`/oauth/token/info`](https://github.com/doorkeeper-gem/doorkeeper/wiki/API-endpoint-descriptions-and-examples#get----oauthtokeninfo). For more information, see [`/oauth/token/info`](https://github.com/doorkeeper-gem/doorkeeper/wiki/API-endpoint-descriptions-and-examples#get----oauthtokeninfo).
......
...@@ -82,7 +82,7 @@ X-Gitlab-Size: 1476 ...@@ -82,7 +82,7 @@ X-Gitlab-Size: 1476
## Get file blame from repository ## Get file blame from repository
Allows you to receive blame information. Each blame range contains lines and corresponding commit info. Allows you to receive blame information. Each blame range contains lines and corresponding commit information.
```plaintext ```plaintext
GET /projects/:id/repository/files/:file_path/blame GET /projects/:id/repository/files/:file_path/blame
......
...@@ -67,7 +67,7 @@ Below are the changes made between V3 and V4. ...@@ -67,7 +67,7 @@ Below are the changes made between V3 and V4.
- Remove `GET /groups/owned`. Use `GET /groups?owned=true` instead [!9505](https://gitlab.com/gitlab-org/gitlab-foss/-/merge_requests/9505) - Remove `GET /groups/owned`. Use `GET /groups?owned=true` instead [!9505](https://gitlab.com/gitlab-org/gitlab-foss/-/merge_requests/9505)
- Return 202 with JSON body on async removals on V4 API (`DELETE /projects/:id/repository/merged_branches` and `DELETE /projects/:id`) [!9449](https://gitlab.com/gitlab-org/gitlab-foss/-/merge_requests/9449) - Return 202 with JSON body on async removals on V4 API (`DELETE /projects/:id/repository/merged_branches` and `DELETE /projects/:id`) [!9449](https://gitlab.com/gitlab-org/gitlab-foss/-/merge_requests/9449)
- `GET /projects/:id/milestones?iid[]=x&iid[]=y` array filter has been renamed to `iids` [!9096](https://gitlab.com/gitlab-org/gitlab-foss/-/merge_requests/9096) - `GET /projects/:id/milestones?iid[]=x&iid[]=y` array filter has been renamed to `iids` [!9096](https://gitlab.com/gitlab-org/gitlab-foss/-/merge_requests/9096)
- Return basic info about pipeline in `GET /projects/:id/pipelines` [!8875](https://gitlab.com/gitlab-org/gitlab-foss/-/merge_requests/8875) - Return basic information about pipeline in `GET /projects/:id/pipelines` [!8875](https://gitlab.com/gitlab-org/gitlab-foss/-/merge_requests/8875)
- Renamed all `build` references to `job` [!9463](https://gitlab.com/gitlab-org/gitlab-foss/-/merge_requests/9463) - Renamed all `build` references to `job` [!9463](https://gitlab.com/gitlab-org/gitlab-foss/-/merge_requests/9463)
- Drop `GET /projects/:id/repository/commits/:sha/jobs` [!9463](https://gitlab.com/gitlab-org/gitlab-foss/-/merge_requests/9463) - Drop `GET /projects/:id/repository/commits/:sha/jobs` [!9463](https://gitlab.com/gitlab-org/gitlab-foss/-/merge_requests/9463)
- Rename Build Triggers to be Pipeline Triggers API [!9713](https://gitlab.com/gitlab-org/gitlab-foss/-/merge_requests/9713) - Rename Build Triggers to be Pipeline Triggers API [!9713](https://gitlab.com/gitlab-org/gitlab-foss/-/merge_requests/9713)
......
...@@ -532,7 +532,7 @@ runtime. ...@@ -532,7 +532,7 @@ runtime.
There are two approaches that you can take in order to access a There are two approaches that you can take in order to access a
private registry. Both require setting the environment variable private registry. Both require setting the environment variable
`DOCKER_AUTH_CONFIG` with appropriate authentication info. `DOCKER_AUTH_CONFIG` with appropriate authentication information.
1. Per-job: To configure one job to access a private registry, add 1. Per-job: To configure one job to access a private registry, add
`DOCKER_AUTH_CONFIG` as a job variable. `DOCKER_AUTH_CONFIG` as a job variable.
......
...@@ -453,7 +453,7 @@ pass CI variables to the running application by prefixing the key of the ...@@ -453,7 +453,7 @@ pass CI variables to the running application by prefixing the key of the
variable with `K8S_SECRET_`. variable with `K8S_SECRET_`.
These [prefixed These [prefixed
variables](../../topics/autodevops/index.md#application-secret-variables) will variables](../../topics/autodevops/customize.md#application-secret-variables) will
then be available as environment variables on the running application then be available as environment variables on the running application
container. container.
......
...@@ -222,3 +222,4 @@ Complementary reads: ...@@ -222,3 +222,4 @@ Complementary reads:
- [Run full Auto DevOps cycle in a GDK instance](https://gitlab.com/gitlab-org/gitlab-development-kit/blob/master/doc/howto/auto_devops.md) - [Run full Auto DevOps cycle in a GDK instance](https://gitlab.com/gitlab-org/gitlab-development-kit/blob/master/doc/howto/auto_devops.md)
- [Using GitLab Runner with GDK](https://gitlab.com/gitlab-org/gitlab-development-kit/blob/master/doc/howto/runner.md) - [Using GitLab Runner with GDK](https://gitlab.com/gitlab-org/gitlab-development-kit/blob/master/doc/howto/runner.md)
- [Using the Web IDE terminal with GDK](https://gitlab.com/gitlab-org/gitlab-development-kit/-/blob/master/doc/howto/web_ide_terminal_gdk_setup.md)
...@@ -150,7 +150,7 @@ query($project_path: ID!) { ...@@ -150,7 +150,7 @@ query($project_path: ID!) {
``` ```
This would return the first 2 pipelines of a project and related This would return the first 2 pipelines of a project and related
pagination info., ordered by descending ID. The returned data would pagination information, ordered by descending ID. The returned data would
look like this: look like this:
```json ```json
......
...@@ -272,7 +272,7 @@ and the following syntax rules. ...@@ -272,7 +272,7 @@ and the following syntax rules.
- For `index.html` files, use the clean (canonical) URL: `path/to/`. - For `index.html` files, use the clean (canonical) URL: `path/to/`.
- For EE-only docs, use the same relative path, but add the attribute `ee_only: true` below - For EE-only docs, use the same relative path, but add the attribute `ee_only: true` below
the `doc_url` or `category_url`, as explained above. This displays the `doc_url` or `category_url`, as explained above. This displays
an "info" icon on the nav to make the user aware that the feature is an "information" icon on the nav to make the user aware that the feature is
EE-only. EE-only.
DANGER: **Important!** DANGER: **Important!**
......
...@@ -583,7 +583,7 @@ For other punctuation rules, please refer to the ...@@ -583,7 +583,7 @@ For other punctuation rules, please refer to the
- [Avoid using symbols and special chars](https://gitlab.com/gitlab-org/gitlab-docs/issues/84) - [Avoid using symbols and special chars](https://gitlab.com/gitlab-org/gitlab-docs/issues/84)
in headers. Whenever possible, they should be plain and short text. in headers. Whenever possible, they should be plain and short text.
- Avoid adding things that show ephemeral statuses. For example, if a feature is - Avoid adding things that show ephemeral statuses. For example, if a feature is
considered beta or experimental, put this info in a note, not in the heading. considered beta or experimental, put this information in a note, not in the heading.
- When introducing a new document, be careful for the headings to be - When introducing a new document, be careful for the headings to be
grammatically and syntactically correct. Mention an [assigned technical writer (TW)](https://about.gitlab.com/handbook/product/categories/) grammatically and syntactically correct. Mention an [assigned technical writer (TW)](https://about.gitlab.com/handbook/product/categories/)
for review. for review.
......
...@@ -17,7 +17,7 @@ dummy data. ...@@ -17,7 +17,7 @@ dummy data.
The previews live in [`app/mailers/previews`][previews] and can be viewed at The previews live in [`app/mailers/previews`][previews] and can be viewed at
[`/rails/mailers`](http://localhost:3000/rails/mailers). [`/rails/mailers`](http://localhost:3000/rails/mailers).
See the [Rails guides](https://guides.rubyonrails.org/action_mailer_basics.html#previewing-emails) for more info. See the [Rails guides](https://guides.rubyonrails.org/action_mailer_basics.html#previewing-emails) for more information.
[previews]: https://gitlab.com/gitlab-org/gitlab-foss/tree/master/app/mailers/previews [previews]: https://gitlab.com/gitlab-org/gitlab-foss/tree/master/app/mailers/previews
......
...@@ -264,7 +264,7 @@ Note that this won't fix every problem, but it should fix a majority. ...@@ -264,7 +264,7 @@ Note that this won't fix every problem, but it should fix a majority.
### Ignoring issues ### Ignoring issues
If you want a line or set of lines to be ignored by the linter, you can use If you want a line or set of lines to be ignored by the linter, you can use
`// scss-lint:disable RuleName` ([more info](https://github.com/sds/scss-lint#disabling-linters-via-source)): `// scss-lint:disable RuleName` ([more information](https://github.com/sds/scss-lint#disabling-linters-via-source)):
```scss ```scss
// This lint rule is disabled because it is supported only in Chrome/Safari // This lint rule is disabled because it is supported only in Chrome/Safari
......
...@@ -98,6 +98,7 @@ Slack: ...@@ -98,6 +98,7 @@ Slack:
This will enable the feature for GitLab.com, with `new_navigation_bar` being the This will enable the feature for GitLab.com, with `new_navigation_bar` being the
name of the feature. name of the feature.
This command does *not* enable the feature for 25% of the total users. Instead, when the feature is checked with `enabled?`, it will return `true` 25% of the time.
If you are not certain what percentages to use, simply use the following steps: If you are not certain what percentages to use, simply use the following steps:
...@@ -136,9 +137,8 @@ you run these 2 commands: ...@@ -136,9 +137,8 @@ you run these 2 commands:
/chatops run feature set some_feature 25 /chatops run feature set some_feature 25
``` ```
Then `some_feature` will be enabled for 25% of the time the users are interacting with Then `some_feature` will be enabled for both 25% of users and all users interacting with
`gitlab-org/gitlab`. Note that the the feature is not enabled to 25% `gitlab-org/gitlab`.
of the users, rather a simple randomization is made each time the `enabled?` is checked.
NOTE: **Note:** NOTE: **Note:**
**Percentage of time** rollout is not a good idea if what you want is to make sure a feature **Percentage of time** rollout is not a good idea if what you want is to make sure a feature
......
...@@ -80,7 +80,7 @@ See [GitLab CI/CD predefined variables](../../ci/variables/predefined_variables. ...@@ -80,7 +80,7 @@ See [GitLab CI/CD predefined variables](../../ci/variables/predefined_variables.
Also, scanning jobs should be skipped when the corresponding variable prefixed with `_DISABLED` is present. Also, scanning jobs should be skipped when the corresponding variable prefixed with `_DISABLED` is present.
See `DEPENDENCY_SCANNING_DISABLED`, `CONTAINER_SCANNING_DISABLED`, `SAST_DISABLED`, and `DAST_DISABLED` See `DEPENDENCY_SCANNING_DISABLED`, `CONTAINER_SCANNING_DISABLED`, `SAST_DISABLED`, and `DAST_DISABLED`
in [Auto DevOps documentation](../../topics/autodevops/index.md#disable-jobs). in [Auto DevOps documentation](../../topics/autodevops/customize.md#disable-jobs).
Finally, SAST and Dependency Scanning job definitions should use Finally, SAST and Dependency Scanning job definitions should use
`CI_PROJECT_REPOSITORY_LANGUAGES` (comma-separated list of values) `CI_PROJECT_REPOSITORY_LANGUAGES` (comma-separated list of values)
......
...@@ -370,14 +370,14 @@ You may also bypass the auto signin feature by browsing to ...@@ -370,14 +370,14 @@ You may also bypass the auto signin feature by browsing to
>**Note:** >**Note:**
This setting is only available on GitLab 8.6 and above. This setting is only available on GitLab 8.6 and above.
This setting should only be used to map attributes that are part of the This setting should only be used to map attributes that are part of the
OmniAuth info hash schema. OmniAuth `info` hash schema.
`attribute_statements` is used to map Attribute Names in a SAMLResponse to entries `attribute_statements` is used to map Attribute Names in a SAMLResponse to entries
in the OmniAuth [info hash](https://github.com/omniauth/omniauth/wiki/Auth-Hash-Schema#schema-10-and-later). in the OmniAuth [`info` hash](https://github.com/omniauth/omniauth/wiki/Auth-Hash-Schema#schema-10-and-later).
For example, if your SAMLResponse contains an Attribute called 'EmailAddress', For example, if your SAMLResponse contains an Attribute called 'EmailAddress',
specify `{ email: ['EmailAddress'] }` to map the Attribute to the specify `{ email: ['EmailAddress'] }` to map the Attribute to the
corresponding key in the info hash. URI-named Attributes are also supported, e.g. corresponding key in the `info` hash. URI-named Attributes are also supported, e.g.
`{ email: ['http://schemas.xmlsoap.org/ws/2005/05/identity/claims/emailaddress'] }`. `{ email: ['http://schemas.xmlsoap.org/ws/2005/05/identity/claims/emailaddress'] }`.
This setting allows you tell GitLab where to look for certain attributes required This setting allows you tell GitLab where to look for certain attributes required
......
...@@ -8,7 +8,7 @@ In order to enable Shibboleth support in GitLab we need to use Apache instead of ...@@ -8,7 +8,7 @@ In order to enable Shibboleth support in GitLab we need to use Apache instead of
To enable the Shibboleth OmniAuth provider you must configure Apache Shibboleth module. To enable the Shibboleth OmniAuth provider you must configure Apache Shibboleth module.
The installation and configuration of the module itself is out of the scope of this document. The installation and configuration of the module itself is out of the scope of this document.
Check <https://wiki.shibboleth.net/confluence/display/SP3/Apache> for more info. Check <https://wiki.shibboleth.net/confluence/display/SP3/Apache> for more information.
You can find Apache config in [GitLab Recipes](https://gitlab.com/gitlab-org/gitlab-recipes/tree/master/web-server/apache). You can find Apache config in [GitLab Recipes](https://gitlab.com/gitlab-org/gitlab-recipes/tree/master/web-server/apache).
......
...@@ -4,12 +4,11 @@ type: reference ...@@ -4,12 +4,11 @@ type: reference
# How we manage the TLS protocol CRIME vulnerability # How we manage the TLS protocol CRIME vulnerability
> CRIME ("Compression Ratio Info-leak Made Easy") is a security exploit against [CRIME](https://en.wikipedia.org/w/index.php?title=CRIME&oldid=692423806) is a security exploit against
secret web cookies over connections using the HTTPS and SPDY protocols that also secret web cookies over connections using the HTTPS and SPDY protocols that also
use data compression. When used to recover the content of secret use data compression. When used to recover the content of secret
authentication cookies, it allows an attacker to perform session hijacking on an authentication cookies, it allows an attacker to perform session hijacking on an
authenticated web session, allowing the launching of further attacks. authenticated web session, allowing the launching of further attacks.
([CRIME](https://en.wikipedia.org/w/index.php?title=CRIME&oldid=692423806))
## Description ## Description
......
...@@ -37,7 +37,7 @@ with various cloud providers. ...@@ -37,7 +37,7 @@ with various cloud providers.
In order to provide modern DevOps workflows, our Application Development Platform will rely on In order to provide modern DevOps workflows, our Application Development Platform will rely on
[Auto DevOps](../autodevops/index.md) to provide those workflows. Auto DevOps works with [Auto DevOps](../autodevops/index.md) to provide those workflows. Auto DevOps works with
any Kubernetes cluster; you're not limited to running on GitLab's infrastructure. Additionally, Auto DevOps offers any Kubernetes cluster; you're not limited to running on GitLab's infrastructure. Additionally, Auto DevOps offers
an incremental consumption path. Because it is [composable](../autodevops/index.md#using-components-of-auto-devops), an incremental consumption path. Because it is [composable](../autodevops/customize.md#using-components-of-auto-devops),
you can use as much or as little of the default pipeline as you'd like, and deeply customize without having to integrate a completely different platform. you can use as much or as little of the default pipeline as you'd like, and deeply customize without having to integrate a completely different platform.
### Security ### Security
......
# Customizing Auto DevOps
While Auto DevOps provides great defaults to get you started, you can customize
almost everything to fit your needs; from custom [buildpacks](#custom-buildpacks),
to [`Dockerfile`s](#custom-dockerfile), [Helm charts](#custom-helm-chart), or
even copying the complete [CI/CD configuration](#customizing-gitlab-ciyml)
into your project to enable staging and canary deployments, and more.
## Custom buildpacks
If the automatic buildpack detection fails for your project, or if you want to
use a custom buildpack, you can override the buildpack(s) using a project variable
or a `.buildpacks` file in your project:
- **Project variable** - Create a project variable `BUILDPACK_URL` with the URL
of the buildpack to use.
- **`.buildpacks` file** - Add a file in your project's repo called `.buildpacks`
and add the URL of the buildpack to use on a line in the file. If you want to
use multiple buildpacks, you can enter them in, one on each line.
### Multiple buildpacks
Using multiple buildpacks isn't fully supported by Auto DevOps because, when using the `.buildpacks`
file, Auto Test will not work.
The buildpack [heroku-buildpack-multi](https://github.com/heroku/heroku-buildpack-multi/),
which is used under the hood to parse the `.buildpacks` file, doesn't provide the necessary commands
`bin/test-compile` and `bin/test`.
If your goal is to use only a single custom buildpack, you should provide the project variable
`BUILDPACK_URL` instead.
## Custom `Dockerfile`
If your project has a `Dockerfile` in the root of the project repo, Auto DevOps
will build a Docker image based on the Dockerfile rather than using buildpacks.
This can be much faster and result in smaller images, especially if your
Dockerfile is based on [Alpine](https://hub.docker.com/_/alpine/).
## Passing arguments to `docker build`
Arguments can be passed to the `docker build` command using the
`AUTO_DEVOPS_BUILD_IMAGE_EXTRA_ARGS` project variable.
For example, to build a Docker image based on based on the `ruby:alpine`
instead of the default `ruby:latest`:
1. Set `AUTO_DEVOPS_BUILD_IMAGE_EXTRA_ARGS` to `--build-arg=RUBY_VERSION=alpine`.
1. Add the following to a custom `Dockerfile`:
```dockerfile
ARG RUBY_VERSION=latest
FROM ruby:$RUBY_VERSION
# ... put your stuff here
```
NOTE: **Note:**
Passing in complex values (newlines and spaces, for example) will likely
cause escaping issues due to the way this argument is used in Auto DevOps.
Consider using Base64 encoding of such values to avoid this problem.
CAUTION: **Warning:**
Avoid passing secrets as Docker build arguments if possible, as they may be
persisted in your image. See
[this discussion](https://github.com/moby/moby/issues/13490) for details.
## Passing secrets to `docker build`
> [Introduced](https://gitlab.com/gitlab-org/gitlab/issues/25514) in GitLab 12.3, but available in versions 11.9 and above.
CI environment variables can be passed as [build
secrets](https://docs.docker.com/develop/develop-images/build_enhancements/#new-docker-build-secret-information) to the `docker build` command by listing them comma separated by name in the
`AUTO_DEVOPS_BUILD_IMAGE_FORWARDED_CI_VARIABLES` variable. For example, in order to forward the variables `CI_COMMIT_SHA` and `CI_ENVIRONMENT_NAME`, one would set `AUTO_DEVOPS_BUILD_IMAGE_FORWARDED_CI_VARIABLES` to `CI_COMMIT_SHA,CI_ENVIRONMENT_NAME`.
Unlike build arguments, these are not persisted by Docker in the final image
(though you can still persist them yourself, so **be careful**).
In projects:
- Without a `Dockerfile`, these are available automatically as environment
variables.
- With a `Dockerfile`, the following is required:
1. Activate the experimental `Dockerfile` syntax by adding the following
to the top of the file:
```dockerfile
# syntax = docker/dockerfile:experimental
```
1. To make secrets available in any `RUN $COMMAND` in the `Dockerfile`, mount
the secret file and source it prior to running `$COMMAND`:
```dockerfile
RUN --mount=type=secret,id=auto-devops-build-secrets . /run/secrets/auto-devops-build-secrets && $COMMAND
```
NOTE: **Note:**
When `AUTO_DEVOPS_BUILD_IMAGE_FORWARDED_CI_VARIABLES` is set, Auto DevOps
enables the experimental [Docker BuildKit](https://docs.docker.com/develop/develop-images/build_enhancements/)
feature to use the `--secret` flag.
## Custom Helm Chart
Auto DevOps uses [Helm](https://helm.sh/) to deploy your application to Kubernetes.
You can override the Helm chart used by bundling up a chart into your project
repo or by specifying a project variable:
- **Bundled chart** - If your project has a `./chart` directory with a `Chart.yaml`
file in it, Auto DevOps will detect the chart and use it instead of the [default
one](https://gitlab.com/gitlab-org/charts/auto-deploy-app).
This can be a great way to control exactly how your application is deployed.
- **Project variable** - Create a [project variable](../../ci/variables/README.md#gitlab-cicd-environment-variables)
`AUTO_DEVOPS_CHART` with the URL of a custom chart to use or create two project variables `AUTO_DEVOPS_CHART_REPOSITORY` with the URL of a custom chart repository and `AUTO_DEVOPS_CHART` with the path to the chart.
## Customize values for Helm Chart
> [Introduced](https://gitlab.com/gitlab-org/gitlab/issues/30628) in GitLab 12.6, `.gitlab/auto-deploy-values.yaml` will be used by default for Helm upgrades.
You can override the default values in the `values.yaml` file in the [default Helm chart](https://gitlab.com/gitlab-org/charts/auto-deploy-app).
This can be achieved by either:
- Adding a file named `.gitlab/auto-deploy-values.yaml` to your repository. It will
be automatically used if found.
- Adding a file with a different name or path to the repository, and set the
`HELM_UPGRADE_VALUES_FILE` [environment variable](#environment-variables) with the path and name.
NOTE: **Note:**
For GitLab 12.5 and earlier, the `HELM_UPGRADE_EXTRA_ARGS` environment variable can be used to override the default chart values.
To do so, set `HELM_UPGRADE_EXTRA_ARGS` to `--values my-values.yaml`.
## Custom Helm chart per environment
You can specify the use of a custom Helm chart per environment by scoping the environment variable
to the desired environment. See [Limiting environment scopes of variables](../../ci/variables/README.md#limiting-environment-scopes-of-environment-variables).
## Customizing `.gitlab-ci.yml`
Auto DevOps is completely customizable because the [Auto DevOps template](https://gitlab.com/gitlab-org/gitlab/blob/master/lib/gitlab/ci/templates/Auto-DevOps.gitlab-ci.yml):
- Is just an implementation of a [`.gitlab-ci.yml`](../../ci/yaml/README.md) file.
- Uses only features available to any implementation of `.gitlab-ci.yml`.
If you want to modify the CI/CD pipeline used by Auto DevOps, you can [`include`
the template](../../ci/yaml/README.md#includetemplate) and customize as
needed. To do this, add a `.gitlab-ci.yml` file to the root of your repository
containing the following:
```yml
include:
- template: Auto-DevOps.gitlab-ci.yml
```
Then add any extra changes you want. Your additions will be merged with the
[Auto DevOps template](https://gitlab.com/gitlab-org/gitlab/blob/master/lib/gitlab/ci/templates/Auto-DevOps.gitlab-ci.yml) using the behaviour described for
[`include`](../../ci/yaml/README.md#include).
It is also possible to copy and paste the contents of the [Auto DevOps template](https://gitlab.com/gitlab-org/gitlab/blob/master/lib/gitlab/ci/templates/Auto-DevOps.gitlab-ci.yml)
into your project and edit this as needed. You may prefer to do it
that way if you want to specifically remove any part of it.
## Customizing the Kubernetes namespace
> [Introduced](https://gitlab.com/gitlab-org/gitlab/issues/27630) in GitLab 12.6.
For **non**-GitLab-managed clusters, the namespace can be customized using
`.gitlab-ci.yml` by specifying
[`environment:kubernetes:namespace`](../../ci/environments.md#configuring-kubernetes-deployments).
For example, the following configuration overrides the namespace used for
`production` deployments:
```yaml
include:
- template: Auto-DevOps.gitlab-ci.yml
production:
environment:
kubernetes:
namespace: production
```
When deploying to a custom namespace with Auto DevOps, the service account
provided with the cluster needs at least the `edit` role within the namespace.
- If the service account can create namespaces, then the namespace can be created on-demand.
- Otherwise, the namespace must exist prior to deployment.
## Using components of Auto DevOps
If you only require a subset of the features offered by Auto DevOps, you can include
individual Auto DevOps jobs into your own `.gitlab-ci.yml`. Each component job relies
on a stage that should be defined in the `.gitlab-ci.yml` that includes the template.
For example, to make use of [Auto Build](stages.md#auto-build), you can add the following to
your `.gitlab-ci.yml`:
```yaml
stages:
- build
include:
- template: Jobs/Build.gitlab-ci.yml
```
Consult the [Auto DevOps template](https://gitlab.com/gitlab-org/gitlab/blob/master/lib/gitlab/ci/templates/Auto-DevOps.gitlab-ci.yml) for information on available jobs.
## PostgreSQL database support
In order to support applications that require a database,
[PostgreSQL](https://www.postgresql.org/) is provisioned by default. The credentials to access
the database are preconfigured, but can be customized by setting the associated
[variables](#environment-variables). These credentials can be used for defining a
`DATABASE_URL` of the format:
```yaml
postgres://user:password@postgres-host:postgres-port/postgres-database
```
### Upgrading PostgresSQL
CAUTION: **Deprecation**
The variable `AUTO_DEVOPS_POSTGRES_CHANNEL` that controls default provisioned
PostgreSQL currently defaults to `1`. This is scheduled to change to `2` in
[GitLab 13.0](https://gitlab.com/gitlab-org/gitlab/-/issues/210499).
The version of the chart used to provision PostgreSQL:
- Is 0.7.1 in GitLab 12.8 and earlier.
- Can be set to from 0.7.1 to 8.2.1 in GitLab 12.9 and later.
GitLab encourages users to [migrate their database](upgrading_postgresql.md)
to the newer PostgreSQL.
To use the new PostgreSQL:
- New projects can set the `AUTO_DEVOPS_POSTGRES_CHANNEL` variable to `2`.
- Old projects can be upgraded by following the guide to
[upgrading PostgresSQL](upgrading_postgresql.md).
### Using external PostgreSQL database providers
While Auto DevOps provides out-of-the-box support for a PostgreSQL container for
production environments, for some use-cases it may not be sufficiently secure or
resilient and you may wish to use an external managed provider for PostgreSQL.
For example, AWS Relational Database Service.
You will need to define environment-scoped variables for `POSTGRES_ENABLED` and `DATABASE_URL` in your project's CI/CD settings.
To achieve this:
1. Disable the built-in PostgreSQL installation for the required environments using
scoped [environment variables](../../ci/environments.md#scoping-environments-with-specs).
For this use case, it's likely that only `production` will need to be added to this
list as the builtin PostgreSQL setup for Review Apps and staging will be sufficient
as a high availability setup is not required.
![Auto Metrics](img/disable_postgres.png)
1. Define the `DATABASE_URL` CI variable as a scoped environment variable that will be
available to your application. This should be a URL in the following format:
```yaml
postgres://user:password@postgres-host:postgres-port/postgres-database
```
You will need to ensure that your Kubernetes cluster has network access to wherever
PostgreSQL is hosted.
## Environment variables
The following variables can be used for setting up the Auto DevOps domain,
providing a custom Helm chart, or scaling your application. PostgreSQL can
also be customized, and you can easily use a [custom buildpack](#custom-buildpacks).
### Build and deployment
The following table lists variables related to building and deploying
applications.
| **Variable** | **Description** |
|-----------------------------------------|------------------------------------|
| `ADDITIONAL_HOSTS` | Fully qualified domain names specified as a comma-separated list that are added to the Ingress hosts. |
| `<ENVIRONMENT>_ADDITIONAL_HOSTS` | For a specific environment, the fully qualified domain names specified as a comma-separated list that are added to the Ingress hosts. This takes precedence over `ADDITIONAL_HOSTS`. |
| `AUTO_DEVOPS_BUILD_IMAGE_CNB_ENABLED` | When set to a non-empty value and no `Dockerfile` is present, Auto Build builds your application using Cloud Native Buildpacks instead of Herokuish. [More details](stages.md#auto-build-using-cloud-native-buildpacks-beta). |
| `AUTO_DEVOPS_BUILD_IMAGE_EXTRA_ARGS` | Extra arguments to be passed to the `docker build` command. Note that using quotes will not prevent word splitting. [More details](#passing-arguments-to-docker-build). |
| `AUTO_DEVOPS_BUILD_IMAGE_FORWARDED_CI_VARIABLES` | A [comma-separated list of CI variable names](#passing-secrets-to-docker-build) to be passed to the `docker build` command as secrets. |
| `AUTO_DEVOPS_CHART` | Helm Chart used to deploy your apps. Defaults to the one [provided by GitLab](https://gitlab.com/gitlab-org/charts/auto-deploy-app). |
| `AUTO_DEVOPS_CHART_REPOSITORY` | Helm Chart repository used to search for charts. Defaults to `https://charts.gitlab.io`. |
| `AUTO_DEVOPS_CHART_REPOSITORY_NAME` | From GitLab 11.11, used to set the name of the Helm repository. Defaults to `gitlab`. |
| `AUTO_DEVOPS_CHART_REPOSITORY_USERNAME` | From GitLab 11.11, used to set a username to connect to the Helm repository. Defaults to no credentials. Also set `AUTO_DEVOPS_CHART_REPOSITORY_PASSWORD`. |
| `AUTO_DEVOPS_CHART_REPOSITORY_PASSWORD` | From GitLab 11.11, used to set a password to connect to the Helm repository. Defaults to no credentials. Also set `AUTO_DEVOPS_CHART_REPOSITORY_USERNAME`. |
| `AUTO_DEVOPS_MODSECURITY_SEC_RULE_ENGINE` | From GitLab 12.5, used in combination with [Modsecurity feature flag](../../user/clusters/applications.md#web-application-firewall-modsecurity) to toggle [Modsecurity's `SecRuleEngine`](https://github.com/SpiderLabs/ModSecurity/wiki/Reference-Manual-(v2.x)#SecRuleEngine) behavior. Defaults to `DetectionOnly`. |
| `BUILDPACK_URL` | Buildpack's full URL. Can point to either Git repositories or a tarball URL. For Git repositories, it is possible to point to a specific `ref`. For example `https://github.com/heroku/heroku-buildpack-ruby.git#v142`. |
| `CANARY_ENABLED` | From GitLab 11.0, used to define a [deploy policy for canary environments](#deploy-policy-for-canary-environments-premium). |
| `CANARY_PRODUCTION_REPLICAS` | Number of canary replicas to deploy for [Canary Deployments](../../user/project/canary_deployments.md) in the production environment. Takes precedence over `CANARY_REPLICAS`. Defaults to 1. |
| `CANARY_REPLICAS` | Number of canary replicas to deploy for [Canary Deployments](../../user/project/canary_deployments.md). Defaults to 1. |
| `HELM_RELEASE_NAME` | From GitLab 12.1, allows the `helm` release name to be overridden. Can be used to assign unique release names when deploying multiple projects to a single namespace. |
| `HELM_UPGRADE_VALUES_FILE` | From GitLab 12.6, allows the `helm upgrade` values file to be overridden. Defaults to `.gitlab/auto-deploy-values.yaml`. |
| `HELM_UPGRADE_EXTRA_ARGS` | From GitLab 11.11, allows extra arguments in `helm` commands when deploying the application. Note that using quotes will not prevent word splitting. **Tip:** you can use this variable to [customize the Auto Deploy Helm chart](#custom-helm-chart) by applying custom override values with `--values my-values.yaml`. |
| `INCREMENTAL_ROLLOUT_MODE` | From GitLab 11.4, if present, can be used to enable an [incremental rollout](#incremental-rollout-to-production-premium) of your application for the production environment. Set to `manual` for manual deployment jobs or `timed` for automatic rollout deployments with a 5 minute delay each one. |
| `K8S_SECRET_*` | From GitLab 11.7, any variable prefixed with [`K8S_SECRET_`](#application-secret-variables) will be made available by Auto DevOps as environment variables to the deployed application. |
| `KUBE_INGRESS_BASE_DOMAIN` | From GitLab 11.8, can be used to set a domain per cluster. See [cluster domains](../../user/project/clusters/index.md#base-domain) for more information. |
| `PRODUCTION_REPLICAS` | Number of replicas to deploy in the production environment. Takes precedence over `REPLICAS` and defaults to 1. For zero downtime upgrades, set to 2 or greater. |
| `REPLICAS` | Number of replicas to deploy. Defaults to 1. |
| `ROLLOUT_RESOURCE_TYPE` | From GitLab 11.9, allows specification of the resource type being deployed when using a custom Helm chart. Default value is `deployment`. |
| `ROLLOUT_STATUS_DISABLED` | From GitLab 12.0, used to disable rollout status check because it doesn't support all resource types, for example, `cronjob`. |
| `STAGING_ENABLED` | From GitLab 10.8, used to define a [deploy policy for staging and production environments](#deploy-policy-for-staging-and-production-environments). |
TIP: **Tip:**
Set up the replica variables using a
[project variable](../../ci/variables/README.md#gitlab-cicd-environment-variables)
and scale your application by just redeploying it!
CAUTION: **Caution:**
You should *not* scale your application using Kubernetes directly. This can
cause confusion with Helm not detecting the change, and subsequent deploys with
Auto DevOps can undo your changes.
### Database
The following table lists variables related to the database.
| **Variable** | **Description** |
|-----------------------------------------|------------------------------------|
| `DB_INITIALIZE` | From GitLab 11.4, used to specify the command to run to initialize the application's PostgreSQL database. Runs inside the application pod. |
| `DB_MIGRATE` | From GitLab 11.4, used to specify the command to run to migrate the application's PostgreSQL database. Runs inside the application pod. |
| `POSTGRES_ENABLED` | Whether PostgreSQL is enabled. Defaults to `"true"`. Set to `false` to disable the automatic deployment of PostgreSQL. |
| `POSTGRES_USER` | The PostgreSQL user. Defaults to `user`. Set it to use a custom username. |
| `POSTGRES_PASSWORD` | The PostgreSQL password. Defaults to `testing-password`. Set it to use a custom password. |
| `POSTGRES_DB` | The PostgreSQL database name. Defaults to the value of [`$CI_ENVIRONMENT_SLUG`](../../ci/variables/README.md#predefined-environment-variables). Set it to use a custom database name. |
| `POSTGRES_VERSION` | Tag for the [`postgres` Docker image](https://hub.docker.com/_/postgres) to use. Defaults to `9.6.2`. |
### Security tools
The following table lists variables related to security tools.
| **Variable** | **Description** |
|-----------------------------------------|------------------------------------|
| `SAST_CONFIDENCE_LEVEL` | Minimum confidence level of security issues you want to be reported; `1` for Low, `2` for Medium, `3` for High. Defaults to `3`. |
### Disable jobs
The following table lists variables used to disable jobs.
| **Variable** | **Description** |
|-----------------------------------------|------------------------------------|
| `CODE_QUALITY_DISABLED` | From GitLab 11.0, used to disable the `codequality` job. If the variable is present, the job will not be created. |
| `CONTAINER_SCANNING_DISABLED` | From GitLab 11.0, used to disable the `sast:container` job. If the variable is present, the job will not be created. |
| `DAST_DISABLED` | From GitLab 11.0, used to disable the `dast` job. If the variable is present, the job will not be created. |
| `DEPENDENCY_SCANNING_DISABLED` | From GitLab 11.0, used to disable the `dependency_scanning` job. If the variable is present, the job will not be created. |
| `LICENSE_MANAGEMENT_DISABLED` | From GitLab 11.0, used to disable the `license_management` job. If the variable is present, the job will not be created. |
| `PERFORMANCE_DISABLED` | From GitLab 11.0, used to disable the `performance` job. If the variable is present, the job will not be created. |
| `REVIEW_DISABLED` | From GitLab 11.0, used to disable the `review` and the manual `review:stop` job. If the variable is present, these jobs will not be created. |
| `SAST_DISABLED` | From GitLab 11.0, used to disable the `sast` job. If the variable is present, the job will not be created. |
| `TEST_DISABLED` | From GitLab 11.0, used to disable the `test` job. If the variable is present, the job will not be created. |
### Application secret variables
> [Introduced](https://gitlab.com/gitlab-org/gitlab-foss/issues/49056) in GitLab 11.7.
Some applications need to define secret variables that are
accessible by the deployed application. Auto DevOps detects variables where the key starts with
`K8S_SECRET_` and make these prefixed variables available to the
deployed application, as environment variables.
To configure your application variables:
1. Go to your project's **Settings > CI/CD**, then expand the section
called **Variables**.
1. Create a CI Variable, ensuring the key is prefixed with
`K8S_SECRET_`. For example, you can create a variable with key
`K8S_SECRET_RAILS_MASTER_KEY`.
1. Run an Auto Devops pipeline either by manually creating a new
pipeline or by pushing a code change to GitLab.
Auto DevOps pipelines will take your application secret variables to
populate a Kubernetes secret. This secret is unique per environment.
When deploying your application, the secret is loaded as environment
variables in the container running the application. Following the
example above, you can see the secret below containing the
`RAILS_MASTER_KEY` variable.
```shell
$ kubectl get secret production-secret -n minimal-ruby-app-54 -o yaml
apiVersion: v1
data:
RAILS_MASTER_KEY: MTIzNC10ZXN0
kind: Secret
metadata:
creationTimestamp: 2018-12-20T01:48:26Z
name: production-secret
namespace: minimal-ruby-app-54
resourceVersion: "429422"
selfLink: /api/v1/namespaces/minimal-ruby-app-54/secrets/production-secret
uid: 57ac2bfd-03f9-11e9-b812-42010a9400e4
type: Opaque
```
Environment variables are generally considered immutable in a Kubernetes
pod. Therefore, if you update an application secret without changing any
code then manually create a new pipeline, you will find that any running
application pods will not have the updated secrets. In this case, you
can either push a code update to GitLab to force the Kubernetes
Deployment to recreate pods or manually delete running pods to
cause Kubernetes to create new pods with updated secrets.
NOTE: **Note:**
Variables with multiline values are not currently supported due to
limitations with the current Auto DevOps scripting environment.
### Advanced replica variables setup
Apart from the two replica-related variables for production mentioned above,
you can also use others for different environments.
There's a very specific mapping between Kubernetes' label named `track`,
GitLab CI/CD environment names, and the replicas environment variable.
The general rule is: `TRACK_ENV_REPLICAS`. Where:
- `TRACK`: The capitalized value of the `track`
[Kubernetes label](https://kubernetes.io/docs/concepts/overview/working-with-objects/labels/)
in the Helm Chart app definition. If not set, it will not be taken into account
to the variable name.
- `ENV`: The capitalized environment name of the deploy job that is set in
`.gitlab-ci.yml`.
That way, you can define your own `TRACK_ENV_REPLICAS` variables with which
you will be able to scale the pod's replicas easily.
In the example below, the environment's name is `qa` and it deploys the track
`foo` which would result in looking for the `FOO_QA_REPLICAS` environment
variable:
```yaml
QA testing:
stage: deploy
environment:
name: qa
script:
- deploy foo
```
The track `foo` being referenced would also need to be defined in the
application's Helm chart, like:
```yaml
replicaCount: 1
image:
repository: gitlab.example.com/group/project
tag: stable
pullPolicy: Always
secrets:
- name: gitlab-registry
application:
track: foo
tier: web
service:
enabled: true
name: web
type: ClusterIP
url: http://my.host.com/
externalPort: 5000
internalPort: 5000
```
### Deploy policy for staging and production environments
> [Introduced](https://gitlab.com/gitlab-org/gitlab-ci-yml/-/merge_requests/160) in GitLab 10.8.
TIP: **Tip:**
You can also set this inside your [project's settings](index.md#deployment-strategy).
The normal behavior of Auto DevOps is to use Continuous Deployment, pushing
automatically to the `production` environment every time a new pipeline is run
on the default branch. However, there are cases where you might want to use a
staging environment and deploy to production manually. For this scenario, the
`STAGING_ENABLED` environment variable was introduced.
If `STAGING_ENABLED` is defined in your project (e.g., set `STAGING_ENABLED` to
`1` as a CI/CD variable), then the application will be automatically deployed
to a `staging` environment, and a `production_manual` job will be created for
you when you're ready to manually deploy to production.
### Deploy policy for canary environments **(PREMIUM)**
> [Introduced](https://gitlab.com/gitlab-org/gitlab-ci-yml/-/merge_requests/171) in GitLab 11.0.
A [canary environment](../../user/project/canary_deployments.md) can be used
before any changes are deployed to production.
If `CANARY_ENABLED` is defined in your project (e.g., set `CANARY_ENABLED` to
`1` as a CI/CD variable) then two manual jobs will be created:
- `canary` which will deploy the application to the canary environment
- `production_manual` which is to be used by you when you're ready to manually
deploy to production.
### Incremental rollout to production **(PREMIUM)**
> [Introduced](https://gitlab.com/gitlab-org/gitlab/issues/5415) in GitLab 10.8.
TIP: **Tip:**
You can also set this inside your [project's settings](index.md#deployment-strategy).
When you have a new version of your app to deploy in production, you may want
to use an incremental rollout to replace just a few pods with the latest code.
This will allow you to first check how the app is behaving, and later manually
increasing the rollout up to 100%.
If `INCREMENTAL_ROLLOUT_MODE` is set to `manual` in your project, then instead
of the standard `production` job, 4 different
[manual jobs](../../ci/pipelines/index.md#manual-actions-from-pipeline-graphs)
will be created:
1. `rollout 10%`
1. `rollout 25%`
1. `rollout 50%`
1. `rollout 100%`
The percentage is based on the `REPLICAS` variable and defines the number of
pods you want to have for your deployment. If you say `10`, and then you run
the `10%` rollout job, there will be `1` new pod + `9` old ones.
To start a job, click on the play icon next to the job's name. You are not
required to go from `10%` to `100%`, you can jump to whatever job you want.
You can also scale down by running a lower percentage job, just before hitting
`100%`. Once you get to `100%`, you cannot scale down, and you'd have to roll
back by redeploying the old version using the
[rollback button](../../ci/environments.md#retrying-and-rolling-back) in the
environment page.
Below, you can see how the pipeline will look if the rollout or staging
variables are defined.
Without `INCREMENTAL_ROLLOUT_MODE` and without `STAGING_ENABLED`:
![Staging and rollout disabled](img/rollout_staging_disabled.png)
Without `INCREMENTAL_ROLLOUT_MODE` and with `STAGING_ENABLED`:
![Staging enabled](img/staging_enabled.png)
With `INCREMENTAL_ROLLOUT_MODE` set to `manual` and without `STAGING_ENABLED`:
![Rollout enabled](img/rollout_enabled.png)
With `INCREMENTAL_ROLLOUT_MODE` set to `manual` and with `STAGING_ENABLED`
![Rollout and staging enabled](img/rollout_staging_enabled.png)
CAUTION: **Caution:**
Before GitLab 11.4 this feature was enabled by the presence of the
`INCREMENTAL_ROLLOUT_ENABLED` environment variable.
This configuration is deprecated and will be removed in the future.
### Timed incremental rollout to production **(PREMIUM)**
> [Introduced](https://gitlab.com/gitlab-org/gitlab/issues/7545) in GitLab 11.4.
TIP: **Tip:**
You can also set this inside your [project's settings](index.md#deployment-strategy).
This configuration is based on
[incremental rollout to production](#incremental-rollout-to-production-premium).
Everything behaves the same way, except:
- It's enabled by setting the `INCREMENTAL_ROLLOUT_MODE` variable to `timed`.
- Instead of the standard `production` job, the following jobs are created with a 5 minute delay between each :
1. `timed rollout 10%`
1. `timed rollout 25%`
1. `timed rollout 50%`
1. `timed rollout 100%`
## Auto DevOps banner
The following Auto DevOps banner will show for maintainers+ on new projects when Auto DevOps is not
enabled:
![Auto DevOps banner](img/autodevops_banner_v12_6.png)
The banner can be disabled for:
- A user when they dismiss it themselves.
- A project by explicitly [disabling Auto DevOps](index.md#enablingdisabling-auto-devops).
- An entire GitLab instance:
- By an administrator running the following in a Rails console:
```ruby
Feature.get(:auto_devops_banner_disabled).enable
```
- Through the REST API with an admin access token:
```shell
curl --data "value=true" --header "PRIVATE-TOKEN: <personal_access_token>" https://gitlab.example.com/api/v4/features/auto_devops_banner_disabled
```
...@@ -61,7 +61,7 @@ in multiple ways: ...@@ -61,7 +61,7 @@ in multiple ways:
and code quality testing. and code quality testing.
- Auto DevOps offers an incremental graduation path. If you need advanced customizations, - Auto DevOps offers an incremental graduation path. If you need advanced customizations,
you can start modifying the templates without having to start over on a you can start modifying the templates without having to start over on a
completely different platform. Review the [customizing](#customizing) section for more information. completely different platform. Review the [customizing](customize.md) documentation for more information.
## Features ## Features
...@@ -91,7 +91,7 @@ knowledge of the following: ...@@ -91,7 +91,7 @@ knowledge of the following:
- [Prometheus](https://prometheus.io/docs/introduction/overview/) - [Prometheus](https://prometheus.io/docs/introduction/overview/)
Auto DevOps provides great defaults for all the stages; you can, however, Auto DevOps provides great defaults for all the stages; you can, however,
[customize](#customizing) almost everything to your needs. [customize](customize.md) almost everything to your needs.
For an overview on the creation of Auto DevOps, read more For an overview on the creation of Auto DevOps, read more
[in this blog post](https://about.gitlab.com/blog/2017/06/29/whats-next-for-gitlab-ci/). [in this blog post](https://about.gitlab.com/blog/2017/06/29/whats-next-for-gitlab-ci/).
...@@ -300,12 +300,12 @@ The available options are: ...@@ -300,12 +300,12 @@ The available options are:
- **Continuous deployment to production**: Enables [Auto Deploy](stages.md#auto-deploy) - **Continuous deployment to production**: Enables [Auto Deploy](stages.md#auto-deploy)
with `master` branch directly deployed to production. with `master` branch directly deployed to production.
- **Continuous deployment to production using timed incremental rollout**: Sets the - **Continuous deployment to production using timed incremental rollout**: Sets the
[`INCREMENTAL_ROLLOUT_MODE`](#timed-incremental-rollout-to-production-premium) variable [`INCREMENTAL_ROLLOUT_MODE`](customize.md#timed-incremental-rollout-to-production-premium) variable
to `timed`, and production deployment will be executed with a 5 minute delay between to `timed`, and production deployment will be executed with a 5 minute delay between
each increment in rollout. each increment in rollout.
- **Automatic deployment to staging, manual deployment to production**: Sets the - **Automatic deployment to staging, manual deployment to production**: Sets the
[`STAGING_ENABLED`](#deploy-policy-for-staging-and-production-environments) and [`STAGING_ENABLED`](customize.md#deploy-policy-for-staging-and-production-environments) and
[`INCREMENTAL_ROLLOUT_MODE`](#incremental-rollout-to-production-premium) variables [`INCREMENTAL_ROLLOUT_MODE`](customize.md#incremental-rollout-to-production-premium) variables
to `1` and `manual`. This means: to `1` and `manual`. This means:
- `master` branch is directly deployed to staging. - `master` branch is directly deployed to staging.
...@@ -336,8 +336,8 @@ be configured. ...@@ -336,8 +336,8 @@ be configured.
| Cluster name | Cluster environment scope | `KUBE_INGRESS_BASE_DOMAIN` variable value | Variable environment scope | Notes | | Cluster name | Cluster environment scope | `KUBE_INGRESS_BASE_DOMAIN` variable value | Variable environment scope | Notes |
|--------------|---------------------------|-------------------------------------------|----------------------------|---| |--------------|---------------------------|-------------------------------------------|----------------------------|---|
| review | `review/*` | `review.example.com` | `review/*` | The review cluster which will run all [Review Apps](../../ci/review_apps/index.md). `*` is a wildcard, which means it will be used by every environment name starting with `review/`. | | review | `review/*` | `review.example.com` | `review/*` | The review cluster which will run all [Review Apps](../../ci/review_apps/index.md). `*` is a wildcard, which means it will be used by every environment name starting with `review/`. |
| staging | `staging` | `staging.example.com` | `staging` | (Optional) The staging cluster which will run the deployments of the staging environments. You need to [enable it first](#deploy-policy-for-staging-and-production-environments). | | staging | `staging` | `staging.example.com` | `staging` | (Optional) The staging cluster which will run the deployments of the staging environments. You need to [enable it first](customize.md#deploy-policy-for-staging-and-production-environments). |
| production | `production` | `example.com` | `production` | The production cluster which will run the deployments of the production environment. You can use [incremental rollouts](#incremental-rollout-to-production-premium). | | production | `production` | `example.com` | `production` | The production cluster which will run the deployments of the production environment. You can use [incremental rollouts](customize.md#incremental-rollout-to-production-premium). |
To add a different cluster for each environment: To add a different cluster for each environment:
...@@ -358,607 +358,6 @@ and verifying that your app is deployed as a review app in the Kubernetes ...@@ -358,607 +358,6 @@ and verifying that your app is deployed as a review app in the Kubernetes
cluster with the `review/*` environment scope. Similarly, you can check the cluster with the `review/*` environment scope. Similarly, you can check the
other environments. other environments.
## Customizing
While Auto DevOps provides great defaults to get you started, you can customize
almost everything to fit your needs; from custom [buildpacks](#custom-buildpacks),
to [`Dockerfile`s](#custom-dockerfile), [Helm charts](#custom-helm-chart), or
even copying the complete [CI/CD configuration](#customizing-gitlab-ciyml)
into your project to enable staging and canary deployments, and more.
### Custom buildpacks
If the automatic buildpack detection fails for your project, or if you want to
use a custom buildpack, you can override the buildpack(s) using a project variable
or a `.buildpacks` file in your project:
- **Project variable** - Create a project variable `BUILDPACK_URL` with the URL
of the buildpack to use.
- **`.buildpacks` file** - Add a file in your project's repo called `.buildpacks`
and add the URL of the buildpack to use on a line in the file. If you want to
use multiple buildpacks, you can enter them in, one on each line.
#### Multiple buildpacks
Using multiple buildpacks isn't fully supported by Auto DevOps because, when using the `.buildpacks`
file, Auto Test will not work.
The buildpack [heroku-buildpack-multi](https://github.com/heroku/heroku-buildpack-multi/),
which is used under the hood to parse the `.buildpacks` file, doesn't provide the necessary commands
`bin/test-compile` and `bin/test`.
If your goal is to use only a single custom buildpack, you should provide the project variable
`BUILDPACK_URL` instead.
### Custom `Dockerfile`
If your project has a `Dockerfile` in the root of the project repo, Auto DevOps
will build a Docker image based on the Dockerfile rather than using buildpacks.
This can be much faster and result in smaller images, especially if your
Dockerfile is based on [Alpine](https://hub.docker.com/_/alpine/).
### Passing arguments to `docker build`
Arguments can be passed to the `docker build` command using the
`AUTO_DEVOPS_BUILD_IMAGE_EXTRA_ARGS` project variable.
For example, to build a Docker image based on based on the `ruby:alpine`
instead of the default `ruby:latest`:
1. Set `AUTO_DEVOPS_BUILD_IMAGE_EXTRA_ARGS` to `--build-arg=RUBY_VERSION=alpine`.
1. Add the following to a custom `Dockerfile`:
```dockerfile
ARG RUBY_VERSION=latest
FROM ruby:$RUBY_VERSION
# ... put your stuff here
```
NOTE: **Note:**
Passing in complex values (newlines and spaces, for example) will likely
cause escaping issues due to the way this argument is used in Auto DevOps.
Consider using Base64 encoding of such values to avoid this problem.
CAUTION: **Warning:**
Avoid passing secrets as Docker build arguments if possible, as they may be
persisted in your image. See
[this discussion](https://github.com/moby/moby/issues/13490) for details.
### Passing secrets to `docker build`
> [Introduced](https://gitlab.com/gitlab-org/gitlab/issues/25514) in GitLab 12.3, but available in versions 11.9 and above.
CI environment variables can be passed as [build
secrets](https://docs.docker.com/develop/develop-images/build_enhancements/#new-docker-build-secret-information) to the `docker build` command by listing them comma separated by name in the
`AUTO_DEVOPS_BUILD_IMAGE_FORWARDED_CI_VARIABLES` variable. For example, in order to forward the variables `CI_COMMIT_SHA` and `CI_ENVIRONMENT_NAME`, one would set `AUTO_DEVOPS_BUILD_IMAGE_FORWARDED_CI_VARIABLES` to `CI_COMMIT_SHA,CI_ENVIRONMENT_NAME`.
Unlike build arguments, these are not persisted by Docker in the final image
(though you can still persist them yourself, so **be careful**).
In projects:
- Without a `Dockerfile`, these are available automatically as environment
variables.
- With a `Dockerfile`, the following is required:
1. Activate the experimental `Dockerfile` syntax by adding the following
to the top of the file:
```dockerfile
# syntax = docker/dockerfile:experimental
```
1. To make secrets available in any `RUN $COMMAND` in the `Dockerfile`, mount
the secret file and source it prior to running `$COMMAND`:
```dockerfile
RUN --mount=type=secret,id=auto-devops-build-secrets . /run/secrets/auto-devops-build-secrets && $COMMAND
```
NOTE: **Note:**
When `AUTO_DEVOPS_BUILD_IMAGE_FORWARDED_CI_VARIABLES` is set, Auto DevOps
enables the experimental [Docker BuildKit](https://docs.docker.com/develop/develop-images/build_enhancements/)
feature to use the `--secret` flag.
### Custom Helm Chart
Auto DevOps uses [Helm](https://helm.sh/) to deploy your application to Kubernetes.
You can override the Helm chart used by bundling up a chart into your project
repo or by specifying a project variable:
- **Bundled chart** - If your project has a `./chart` directory with a `Chart.yaml`
file in it, Auto DevOps will detect the chart and use it instead of the [default
one](https://gitlab.com/gitlab-org/charts/auto-deploy-app).
This can be a great way to control exactly how your application is deployed.
- **Project variable** - Create a [project variable](../../ci/variables/README.md#gitlab-cicd-environment-variables)
`AUTO_DEVOPS_CHART` with the URL of a custom chart to use or create two project variables `AUTO_DEVOPS_CHART_REPOSITORY` with the URL of a custom chart repository and `AUTO_DEVOPS_CHART` with the path to the chart.
### Customize values for Helm Chart
> [Introduced](https://gitlab.com/gitlab-org/gitlab/issues/30628) in GitLab 12.6, `.gitlab/auto-deploy-values.yaml` will be used by default for Helm upgrades.
You can override the default values in the `values.yaml` file in the [default Helm chart](https://gitlab.com/gitlab-org/charts/auto-deploy-app).
This can be achieved by either:
- Adding a file named `.gitlab/auto-deploy-values.yaml` to your repository. It will
be automatically used if found.
- Adding a file with a different name or path to the repository, and set the
`HELM_UPGRADE_VALUES_FILE` [environment variable](#environment-variables) with the path and name.
NOTE: **Note:**
For GitLab 12.5 and earlier, the `HELM_UPGRADE_EXTRA_ARGS` environment variable can be used to override the default chart values.
To do so, set `HELM_UPGRADE_EXTRA_ARGS` to `--values my-values.yaml`.
### Custom Helm chart per environment
You can specify the use of a custom Helm chart per environment by scoping the environment variable
to the desired environment. See [Limiting environment scopes of variables](../../ci/variables/README.md#limiting-environment-scopes-of-environment-variables).
### Customizing `.gitlab-ci.yml`
Auto DevOps is completely customizable because the [Auto DevOps template](https://gitlab.com/gitlab-org/gitlab/blob/master/lib/gitlab/ci/templates/Auto-DevOps.gitlab-ci.yml):
- Is just an implementation of a [`.gitlab-ci.yml`](../../ci/yaml/README.md) file.
- Uses only features available to any implementation of `.gitlab-ci.yml`.
If you want to modify the CI/CD pipeline used by Auto DevOps, you can [`include`
the template](../../ci/yaml/README.md#includetemplate) and customize as
needed. To do this, add a `.gitlab-ci.yml` file to the root of your repository
containing the following:
```yml
include:
- template: Auto-DevOps.gitlab-ci.yml
```
Then add any extra changes you want. Your additions will be merged with the
[Auto DevOps template](https://gitlab.com/gitlab-org/gitlab/blob/master/lib/gitlab/ci/templates/Auto-DevOps.gitlab-ci.yml) using the behaviour described for
[`include`](../../ci/yaml/README.md#include).
It is also possible to copy and paste the contents of the [Auto DevOps template](https://gitlab.com/gitlab-org/gitlab/blob/master/lib/gitlab/ci/templates/Auto-DevOps.gitlab-ci.yml)
into your project and edit this as needed. You may prefer to do it
that way if you want to specifically remove any part of it.
### Customizing the Kubernetes namespace
> [Introduced](https://gitlab.com/gitlab-org/gitlab/issues/27630) in GitLab 12.6.
For **non**-GitLab-managed clusters, the namespace can be customized using
`.gitlab-ci.yml` by specifying
[`environment:kubernetes:namespace`](../../ci/environments.md#configuring-kubernetes-deployments).
For example, the following configuration overrides the namespace used for
`production` deployments:
```yaml
include:
- template: Auto-DevOps.gitlab-ci.yml
production:
environment:
kubernetes:
namespace: production
```
When deploying to a custom namespace with Auto DevOps, the service account
provided with the cluster needs at least the `edit` role within the namespace.
- If the service account can create namespaces, then the namespace can be created on-demand.
- Otherwise, the namespace must exist prior to deployment.
### Using components of Auto DevOps
If you only require a subset of the features offered by Auto DevOps, you can include
individual Auto DevOps jobs into your own `.gitlab-ci.yml`. Each component job relies
on a stage that should be defined in the `.gitlab-ci.yml` that includes the template.
For example, to make use of [Auto Build](stages.md#auto-build), you can add the following to
your `.gitlab-ci.yml`:
```yaml
stages:
- build
include:
- template: Jobs/Build.gitlab-ci.yml
```
Consult the [Auto DevOps template](https://gitlab.com/gitlab-org/gitlab/blob/master/lib/gitlab/ci/templates/Auto-DevOps.gitlab-ci.yml) for information on available jobs.
### PostgreSQL database support
In order to support applications that require a database,
[PostgreSQL](https://www.postgresql.org/) is provisioned by default. The credentials to access
the database are preconfigured, but can be customized by setting the associated
[variables](#environment-variables). These credentials can be used for defining a
`DATABASE_URL` of the format:
```yaml
postgres://user:password@postgres-host:postgres-port/postgres-database
```
#### Upgrading PostgresSQL
CAUTION: **Deprecation**
The variable `AUTO_DEVOPS_POSTGRES_CHANNEL` that controls default provisioned
PostgreSQL currently defaults to `1`. This is scheduled to change to `2` in
[GitLab 13.0](https://gitlab.com/gitlab-org/gitlab/-/issues/210499).
The version of the chart used to provision PostgreSQL:
- Is 0.7.1 in GitLab 12.8 and earlier.
- Can be set to from 0.7.1 to 8.2.1 in GitLab 12.9 and later.
GitLab encourages users to [migrate their database](upgrading_postgresql.md)
to the newer PostgreSQL.
To use the new PostgreSQL:
- New projects can set the `AUTO_DEVOPS_POSTGRES_CHANNEL` variable to `2`.
- Old projects can be upgraded by following the guide to
[upgrading PostgresSQL](upgrading_postgresql.md).
#### Using external PostgreSQL database providers
While Auto DevOps provides out-of-the-box support for a PostgreSQL container for
production environments, for some use-cases it may not be sufficiently secure or
resilient and you may wish to use an external managed provider for PostgreSQL.
For example, AWS Relational Database Service.
You will need to define environment-scoped variables for `POSTGRES_ENABLED` and `DATABASE_URL` in your project's CI/CD settings.
To achieve this:
1. Disable the built-in PostgreSQL installation for the required environments using
scoped [environment variables](../../ci/environments.md#scoping-environments-with-specs).
For this use case, it's likely that only `production` will need to be added to this
list as the builtin PostgreSQL setup for Review Apps and staging will be sufficient
as a high availability setup is not required.
![Auto Metrics](img/disable_postgres.png)
1. Define the `DATABASE_URL` CI variable as a scoped environment variable that will be
available to your application. This should be a URL in the following format:
```yaml
postgres://user:password@postgres-host:postgres-port/postgres-database
```
You will need to ensure that your Kubernetes cluster has network access to wherever
PostgreSQL is hosted.
### Environment variables
The following variables can be used for setting up the Auto DevOps domain,
providing a custom Helm chart, or scaling your application. PostgreSQL can
also be customized, and you can easily use a [custom buildpack](#custom-buildpacks).
#### Build and deployment
The following table lists variables related to building and deploying
applications.
| **Variable** | **Description** |
|-----------------------------------------|------------------------------------|
| `ADDITIONAL_HOSTS` | Fully qualified domain names specified as a comma-separated list that are added to the Ingress hosts. |
| `<ENVIRONMENT>_ADDITIONAL_HOSTS` | For a specific environment, the fully qualified domain names specified as a comma-separated list that are added to the Ingress hosts. This takes precedence over `ADDITIONAL_HOSTS`. |
| `AUTO_DEVOPS_BUILD_IMAGE_CNB_ENABLED` | When set to a non-empty value and no `Dockerfile` is present, Auto Build builds your application using Cloud Native Buildpacks instead of Herokuish. [More details](stages.md#auto-build-using-cloud-native-buildpacks-beta). |
| `AUTO_DEVOPS_BUILD_IMAGE_EXTRA_ARGS` | Extra arguments to be passed to the `docker build` command. Note that using quotes will not prevent word splitting. [More details](#passing-arguments-to-docker-build). |
| `AUTO_DEVOPS_BUILD_IMAGE_FORWARDED_CI_VARIABLES` | A [comma-separated list of CI variable names](#passing-secrets-to-docker-build) to be passed to the `docker build` command as secrets. |
| `AUTO_DEVOPS_CHART` | Helm Chart used to deploy your apps. Defaults to the one [provided by GitLab](https://gitlab.com/gitlab-org/charts/auto-deploy-app). |
| `AUTO_DEVOPS_CHART_REPOSITORY` | Helm Chart repository used to search for charts. Defaults to `https://charts.gitlab.io`. |
| `AUTO_DEVOPS_CHART_REPOSITORY_NAME` | From GitLab 11.11, used to set the name of the Helm repository. Defaults to `gitlab`. |
| `AUTO_DEVOPS_CHART_REPOSITORY_USERNAME` | From GitLab 11.11, used to set a username to connect to the Helm repository. Defaults to no credentials. Also set `AUTO_DEVOPS_CHART_REPOSITORY_PASSWORD`. |
| `AUTO_DEVOPS_CHART_REPOSITORY_PASSWORD` | From GitLab 11.11, used to set a password to connect to the Helm repository. Defaults to no credentials. Also set `AUTO_DEVOPS_CHART_REPOSITORY_USERNAME`. |
| `AUTO_DEVOPS_MODSECURITY_SEC_RULE_ENGINE` | From GitLab 12.5, used in combination with [Modsecurity feature flag](../../user/clusters/applications.md#web-application-firewall-modsecurity) to toggle [Modsecurity's `SecRuleEngine`](https://github.com/SpiderLabs/ModSecurity/wiki/Reference-Manual-(v2.x)#SecRuleEngine) behavior. Defaults to `DetectionOnly`. |
| `BUILDPACK_URL` | Buildpack's full URL. Can point to either Git repositories or a tarball URL. For Git repositories, it is possible to point to a specific `ref`. For example `https://github.com/heroku/heroku-buildpack-ruby.git#v142`. |
| `CANARY_ENABLED` | From GitLab 11.0, used to define a [deploy policy for canary environments](#deploy-policy-for-canary-environments-premium). |
| `CANARY_PRODUCTION_REPLICAS` | Number of canary replicas to deploy for [Canary Deployments](../../user/project/canary_deployments.md) in the production environment. Takes precedence over `CANARY_REPLICAS`. Defaults to 1. |
| `CANARY_REPLICAS` | Number of canary replicas to deploy for [Canary Deployments](../../user/project/canary_deployments.md). Defaults to 1. |
| `HELM_RELEASE_NAME` | From GitLab 12.1, allows the `helm` release name to be overridden. Can be used to assign unique release names when deploying multiple projects to a single namespace. |
| `HELM_UPGRADE_VALUES_FILE` | From GitLab 12.6, allows the `helm upgrade` values file to be overridden. Defaults to `.gitlab/auto-deploy-values.yaml`. |
| `HELM_UPGRADE_EXTRA_ARGS` | From GitLab 11.11, allows extra arguments in `helm` commands when deploying the application. Note that using quotes will not prevent word splitting. **Tip:** you can use this variable to [customize the Auto Deploy Helm chart](#custom-helm-chart) by applying custom override values with `--values my-values.yaml`. |
| `INCREMENTAL_ROLLOUT_MODE` | From GitLab 11.4, if present, can be used to enable an [incremental rollout](#incremental-rollout-to-production-premium) of your application for the production environment. Set to `manual` for manual deployment jobs or `timed` for automatic rollout deployments with a 5 minute delay each one. |
| `K8S_SECRET_*` | From GitLab 11.7, any variable prefixed with [`K8S_SECRET_`](#application-secret-variables) will be made available by Auto DevOps as environment variables to the deployed application. |
| `KUBE_INGRESS_BASE_DOMAIN` | From GitLab 11.8, can be used to set a domain per cluster. See [cluster domains](../../user/project/clusters/index.md#base-domain) for more information. |
| `PRODUCTION_REPLICAS` | Number of replicas to deploy in the production environment. Takes precedence over `REPLICAS` and defaults to 1. For zero downtime upgrades, set to 2 or greater. |
| `REPLICAS` | Number of replicas to deploy. Defaults to 1. |
| `ROLLOUT_RESOURCE_TYPE` | From GitLab 11.9, allows specification of the resource type being deployed when using a custom Helm chart. Default value is `deployment`. |
| `ROLLOUT_STATUS_DISABLED` | From GitLab 12.0, used to disable rollout status check because it doesn't support all resource types, for example, `cronjob`. |
| `STAGING_ENABLED` | From GitLab 10.8, used to define a [deploy policy for staging and production environments](#deploy-policy-for-staging-and-production-environments). |
TIP: **Tip:**
Set up the replica variables using a
[project variable](../../ci/variables/README.md#gitlab-cicd-environment-variables)
and scale your application by just redeploying it!
CAUTION: **Caution:**
You should *not* scale your application using Kubernetes directly. This can
cause confusion with Helm not detecting the change, and subsequent deploys with
Auto DevOps can undo your changes.
#### Database
The following table lists variables related to the database.
| **Variable** | **Description** |
|-----------------------------------------|------------------------------------|
| `DB_INITIALIZE` | From GitLab 11.4, used to specify the command to run to initialize the application's PostgreSQL database. Runs inside the application pod. |
| `DB_MIGRATE` | From GitLab 11.4, used to specify the command to run to migrate the application's PostgreSQL database. Runs inside the application pod. |
| `POSTGRES_ENABLED` | Whether PostgreSQL is enabled. Defaults to `"true"`. Set to `false` to disable the automatic deployment of PostgreSQL. |
| `POSTGRES_USER` | The PostgreSQL user. Defaults to `user`. Set it to use a custom username. |
| `POSTGRES_PASSWORD` | The PostgreSQL password. Defaults to `testing-password`. Set it to use a custom password. |
| `POSTGRES_DB` | The PostgreSQL database name. Defaults to the value of [`$CI_ENVIRONMENT_SLUG`](../../ci/variables/README.md#predefined-environment-variables). Set it to use a custom database name. |
| `POSTGRES_VERSION` | Tag for the [`postgres` Docker image](https://hub.docker.com/_/postgres) to use. Defaults to `9.6.2`. |
#### Security tools
The following table lists variables related to security tools.
| **Variable** | **Description** |
|-----------------------------------------|------------------------------------|
| `SAST_CONFIDENCE_LEVEL` | Minimum confidence level of security issues you want to be reported; `1` for Low, `2` for Medium, `3` for High. Defaults to `3`. |
#### Disable jobs
The following table lists variables used to disable jobs.
| **Variable** | **Description** |
|-----------------------------------------|------------------------------------|
| `CODE_QUALITY_DISABLED` | From GitLab 11.0, used to disable the `codequality` job. If the variable is present, the job will not be created. |
| `CONTAINER_SCANNING_DISABLED` | From GitLab 11.0, used to disable the `sast:container` job. If the variable is present, the job will not be created. |
| `DAST_DISABLED` | From GitLab 11.0, used to disable the `dast` job. If the variable is present, the job will not be created. |
| `DEPENDENCY_SCANNING_DISABLED` | From GitLab 11.0, used to disable the `dependency_scanning` job. If the variable is present, the job will not be created. |
| `LICENSE_MANAGEMENT_DISABLED` | From GitLab 11.0, used to disable the `license_management` job. If the variable is present, the job will not be created. |
| `PERFORMANCE_DISABLED` | From GitLab 11.0, used to disable the `performance` job. If the variable is present, the job will not be created. |
| `REVIEW_DISABLED` | From GitLab 11.0, used to disable the `review` and the manual `review:stop` job. If the variable is present, these jobs will not be created. |
| `SAST_DISABLED` | From GitLab 11.0, used to disable the `sast` job. If the variable is present, the job will not be created. |
| `TEST_DISABLED` | From GitLab 11.0, used to disable the `test` job. If the variable is present, the job will not be created. |
#### Application secret variables
> [Introduced](https://gitlab.com/gitlab-org/gitlab-foss/issues/49056) in GitLab 11.7.
Some applications need to define secret variables that are
accessible by the deployed application. Auto DevOps detects variables where the key starts with
`K8S_SECRET_` and make these prefixed variables available to the
deployed application, as environment variables.
To configure your application variables:
1. Go to your project's **Settings > CI/CD**, then expand the section
called **Variables**.
1. Create a CI Variable, ensuring the key is prefixed with
`K8S_SECRET_`. For example, you can create a variable with key
`K8S_SECRET_RAILS_MASTER_KEY`.
1. Run an Auto Devops pipeline either by manually creating a new
pipeline or by pushing a code change to GitLab.
Auto DevOps pipelines will take your application secret variables to
populate a Kubernetes secret. This secret is unique per environment.
When deploying your application, the secret is loaded as environment
variables in the container running the application. Following the
example above, you can see the secret below containing the
`RAILS_MASTER_KEY` variable.
```shell
$ kubectl get secret production-secret -n minimal-ruby-app-54 -o yaml
apiVersion: v1
data:
RAILS_MASTER_KEY: MTIzNC10ZXN0
kind: Secret
metadata:
creationTimestamp: 2018-12-20T01:48:26Z
name: production-secret
namespace: minimal-ruby-app-54
resourceVersion: "429422"
selfLink: /api/v1/namespaces/minimal-ruby-app-54/secrets/production-secret
uid: 57ac2bfd-03f9-11e9-b812-42010a9400e4
type: Opaque
```
Environment variables are generally considered immutable in a Kubernetes
pod. Therefore, if you update an application secret without changing any
code then manually create a new pipeline, you will find that any running
application pods will not have the updated secrets. In this case, you
can either push a code update to GitLab to force the Kubernetes
Deployment to recreate pods or manually delete running pods to
cause Kubernetes to create new pods with updated secrets.
NOTE: **Note:**
Variables with multiline values are not currently supported due to
limitations with the current Auto DevOps scripting environment.
#### Advanced replica variables setup
Apart from the two replica-related variables for production mentioned above,
you can also use others for different environments.
There's a very specific mapping between Kubernetes' label named `track`,
GitLab CI/CD environment names, and the replicas environment variable.
The general rule is: `TRACK_ENV_REPLICAS`. Where:
- `TRACK`: The capitalized value of the `track`
[Kubernetes label](https://kubernetes.io/docs/concepts/overview/working-with-objects/labels/)
in the Helm Chart app definition. If not set, it will not be taken into account
to the variable name.
- `ENV`: The capitalized environment name of the deploy job that is set in
`.gitlab-ci.yml`.
That way, you can define your own `TRACK_ENV_REPLICAS` variables with which
you will be able to scale the pod's replicas easily.
In the example below, the environment's name is `qa` and it deploys the track
`foo` which would result in looking for the `FOO_QA_REPLICAS` environment
variable:
```yaml
QA testing:
stage: deploy
environment:
name: qa
script:
- deploy foo
```
The track `foo` being referenced would also need to be defined in the
application's Helm chart, like:
```yaml
replicaCount: 1
image:
repository: gitlab.example.com/group/project
tag: stable
pullPolicy: Always
secrets:
- name: gitlab-registry
application:
track: foo
tier: web
service:
enabled: true
name: web
type: ClusterIP
url: http://my.host.com/
externalPort: 5000
internalPort: 5000
```
#### Deploy policy for staging and production environments
> [Introduced](https://gitlab.com/gitlab-org/gitlab-ci-yml/-/merge_requests/160) in GitLab 10.8.
TIP: **Tip:**
You can also set this inside your [project's settings](#deployment-strategy).
The normal behavior of Auto DevOps is to use Continuous Deployment, pushing
automatically to the `production` environment every time a new pipeline is run
on the default branch. However, there are cases where you might want to use a
staging environment and deploy to production manually. For this scenario, the
`STAGING_ENABLED` environment variable was introduced.
If `STAGING_ENABLED` is defined in your project (e.g., set `STAGING_ENABLED` to
`1` as a CI/CD variable), then the application will be automatically deployed
to a `staging` environment, and a `production_manual` job will be created for
you when you're ready to manually deploy to production.
#### Deploy policy for canary environments **(PREMIUM)**
> [Introduced](https://gitlab.com/gitlab-org/gitlab-ci-yml/-/merge_requests/171) in GitLab 11.0.
A [canary environment](../../user/project/canary_deployments.md) can be used
before any changes are deployed to production.
If `CANARY_ENABLED` is defined in your project (e.g., set `CANARY_ENABLED` to
`1` as a CI/CD variable) then two manual jobs will be created:
- `canary` which will deploy the application to the canary environment
- `production_manual` which is to be used by you when you're ready to manually
deploy to production.
#### Incremental rollout to production **(PREMIUM)**
> [Introduced](https://gitlab.com/gitlab-org/gitlab/issues/5415) in GitLab 10.8.
TIP: **Tip:**
You can also set this inside your [project's settings](#deployment-strategy).
When you have a new version of your app to deploy in production, you may want
to use an incremental rollout to replace just a few pods with the latest code.
This will allow you to first check how the app is behaving, and later manually
increasing the rollout up to 100%.
If `INCREMENTAL_ROLLOUT_MODE` is set to `manual` in your project, then instead
of the standard `production` job, 4 different
[manual jobs](../../ci/pipelines/index.md#manual-actions-from-pipeline-graphs)
will be created:
1. `rollout 10%`
1. `rollout 25%`
1. `rollout 50%`
1. `rollout 100%`
The percentage is based on the `REPLICAS` variable and defines the number of
pods you want to have for your deployment. If you say `10`, and then you run
the `10%` rollout job, there will be `1` new pod + `9` old ones.
To start a job, click on the play icon next to the job's name. You are not
required to go from `10%` to `100%`, you can jump to whatever job you want.
You can also scale down by running a lower percentage job, just before hitting
`100%`. Once you get to `100%`, you cannot scale down, and you'd have to roll
back by redeploying the old version using the
[rollback button](../../ci/environments.md#retrying-and-rolling-back) in the
environment page.
Below, you can see how the pipeline will look if the rollout or staging
variables are defined.
Without `INCREMENTAL_ROLLOUT_MODE` and without `STAGING_ENABLED`:
![Staging and rollout disabled](img/rollout_staging_disabled.png)
Without `INCREMENTAL_ROLLOUT_MODE` and with `STAGING_ENABLED`:
![Staging enabled](img/staging_enabled.png)
With `INCREMENTAL_ROLLOUT_MODE` set to `manual` and without `STAGING_ENABLED`:
![Rollout enabled](img/rollout_enabled.png)
With `INCREMENTAL_ROLLOUT_MODE` set to `manual` and with `STAGING_ENABLED`
![Rollout and staging enabled](img/rollout_staging_enabled.png)
CAUTION: **Caution:**
Before GitLab 11.4 this feature was enabled by the presence of the
`INCREMENTAL_ROLLOUT_ENABLED` environment variable.
This configuration is deprecated and will be removed in the future.
#### Timed incremental rollout to production **(PREMIUM)**
> [Introduced](https://gitlab.com/gitlab-org/gitlab/issues/7545) in GitLab 11.4.
TIP: **Tip:**
You can also set this inside your [project's settings](#deployment-strategy).
This configuration is based on
[incremental rollout to production](#incremental-rollout-to-production-premium).
Everything behaves the same way, except:
- It's enabled by setting the `INCREMENTAL_ROLLOUT_MODE` variable to `timed`.
- Instead of the standard `production` job, the following jobs are created with a 5 minute delay between each :
1. `timed rollout 10%`
1. `timed rollout 25%`
1. `timed rollout 50%`
1. `timed rollout 100%`
### Auto DevOps banner
The following Auto DevOps banner will show for maintainers+ on new projects when Auto DevOps is not
enabled:
![Auto DevOps banner](img/autodevops_banner_v12_6.png)
The banner can be disabled for:
- A user when they dismiss it themselves.
- A project by explicitly [disabling Auto DevOps](#enablingdisabling-auto-devops).
- An entire GitLab instance:
- By an administrator running the following in a Rails console:
```ruby
Feature.get(:auto_devops_banner_disabled).enable
```
- Through the REST API with an admin access token:
```shell
curl --data "value=true" --header "PRIVATE-TOKEN: <personal_access_token>" https://gitlab.example.com/api/v4/features/auto_devops_banner_disabled
```
## Currently supported languages ## Currently supported languages
Note that not all buildpacks support Auto Test yet, as it's a relatively new Note that not all buildpacks support Auto Test yet, as it's a relatively new
...@@ -1024,7 +423,7 @@ spec: ...@@ -1024,7 +423,7 @@ spec:
key files the buildpack is looking for. For example, for ruby apps, you must key files the buildpack is looking for. For example, for ruby apps, you must
have a `Gemfile` to be properly detected, even though it is possible to write a have a `Gemfile` to be properly detected, even though it is possible to write a
Ruby app without a `Gemfile`. Try specifying a [custom Ruby app without a `Gemfile`. Try specifying a [custom
buildpack](#custom-buildpacks). buildpack](customize.md#custom-buildpacks).
- Auto Test may fail because of a mismatch between testing frameworks. In this - Auto Test may fail because of a mismatch between testing frameworks. In this
case, you may need to customize your `.gitlab-ci.yml` with your test commands. case, you may need to customize your `.gitlab-ci.yml` with your test commands.
- Auto Deploy will fail if GitLab can not create a Kubernetes namespace and - Auto Deploy will fail if GitLab can not create a Kubernetes namespace and
......
...@@ -207,7 +207,7 @@ deployment and clicking a square will take you to the pod's logs page. ...@@ -207,7 +207,7 @@ deployment and clicking a square will take you to the pod's logs page.
TIP: **Tip:** TIP: **Tip:**
There is only one pod hosting the application at the moment, but you can add There is only one pod hosting the application at the moment, but you can add
more pods by defining the [`REPLICAS` variable](index.md#environment-variables) more pods by defining the [`REPLICAS` variable](customize.md#environment-variables)
under **Settings > CI/CD > Environment variables**. under **Settings > CI/CD > Environment variables**.
### Working with branches ### Working with branches
...@@ -278,8 +278,8 @@ and customized to fit your workflow. Here are some helpful resources for further ...@@ -278,8 +278,8 @@ and customized to fit your workflow. Here are some helpful resources for further
1. [Auto DevOps](index.md) 1. [Auto DevOps](index.md)
1. [Multiple Kubernetes clusters](index.md#using-multiple-kubernetes-clusters-premium) **(PREMIUM)** 1. [Multiple Kubernetes clusters](index.md#using-multiple-kubernetes-clusters-premium) **(PREMIUM)**
1. [Incremental rollout to production](index.md#incremental-rollout-to-production-premium) **(PREMIUM)** 1. [Incremental rollout to production](customize.md#incremental-rollout-to-production-premium) **(PREMIUM)**
1. [Disable jobs you don't need with environment variables](index.md#environment-variables) 1. [Disable jobs you don't need with environment variables](customize.md#environment-variables)
1. [Use a static IP for your cluster](../../user/clusters/applications.md#using-a-static-ip) 1. [Use a static IP for your cluster](../../user/clusters/applications.md#using-a-static-ip)
1. [Use your own buildpacks to build your application](index.md#custom-buildpacks) 1. [Use your own buildpacks to build your application](customize.md#custom-buildpacks)
1. [Prometheus monitoring](../../user/project/integrations/prometheus.md) 1. [Prometheus monitoring](../../user/project/integrations/prometheus.md)
...@@ -19,7 +19,7 @@ If a project's repository contains a `Dockerfile` at its root, Auto Build will u ...@@ -19,7 +19,7 @@ If a project's repository contains a `Dockerfile` at its root, Auto Build will u
If you are also using Auto Review Apps and Auto Deploy and choose to provide If you are also using Auto Review Apps and Auto Deploy and choose to provide
your own `Dockerfile`, make sure you expose your application to port your own `Dockerfile`, make sure you expose your application to port
`5000` as this is the port assumed by the `5000` as this is the port assumed by the
[default Helm chart](https://gitlab.com/gitlab-org/charts/auto-deploy-app). Alternatively you can override the default values by [customizing the Auto Deploy Helm chart](index.md#custom-helm-chart) [default Helm chart](https://gitlab.com/gitlab-org/charts/auto-deploy-app). Alternatively you can override the default values by [customizing the Auto Deploy Helm chart](customize.md#custom-helm-chart)
### Auto Build using Heroku buildpacks ### Auto Build using Heroku buildpacks
...@@ -78,7 +78,7 @@ Auto Test automatically runs the appropriate tests for your application using ...@@ -78,7 +78,7 @@ Auto Test automatically runs the appropriate tests for your application using
buildpacks](https://devcenter.heroku.com/articles/buildpacks) by analyzing buildpacks](https://devcenter.heroku.com/articles/buildpacks) by analyzing
your project to detect the language and framework. Several languages and your project to detect the language and framework. Several languages and
frameworks are detected automatically, but if your language is not detected, frameworks are detected automatically, but if your language is not detected,
you may succeed with a [custom buildpack](index.md#custom-buildpacks). Check the you may succeed with a [custom buildpack](customize.md#custom-buildpacks). Check the
[currently supported languages](index.md#currently-supported-languages). [currently supported languages](index.md#currently-supported-languages).
Auto Test uses tests you already have in your application. If there are no Auto Test uses tests you already have in your application. If there are no
...@@ -182,7 +182,7 @@ be deleted. ...@@ -182,7 +182,7 @@ be deleted.
Review apps are deployed using the Review apps are deployed using the
[auto-deploy-app](https://gitlab.com/gitlab-org/charts/auto-deploy-app) chart with [auto-deploy-app](https://gitlab.com/gitlab-org/charts/auto-deploy-app) chart with
Helm, which can be [customized](index.md#custom-helm-chart). The app will be deployed into the [Kubernetes Helm, which can be [customized](customize.md#custom-helm-chart). The app will be deployed into the [Kubernetes
namespace](../../user/project/clusters/index.md#deployment-variables) namespace](../../user/project/clusters/index.md#deployment-variables)
for the environment. for the environment.
...@@ -265,8 +265,8 @@ Auto Deploy doesn't include deployments to staging or canary by default, but the ...@@ -265,8 +265,8 @@ Auto Deploy doesn't include deployments to staging or canary by default, but the
[Auto DevOps template](https://gitlab.com/gitlab-org/gitlab/blob/master/lib/gitlab/ci/templates/Auto-DevOps.gitlab-ci.yml) contains job definitions for these tasks if you want to [Auto DevOps template](https://gitlab.com/gitlab-org/gitlab/blob/master/lib/gitlab/ci/templates/Auto-DevOps.gitlab-ci.yml) contains job definitions for these tasks if you want to
enable them. enable them.
You can make use of [environment variables](index.md#environment-variables) to automatically You can make use of [environment variables](customize.md#environment-variables) to automatically
scale your pod replicas and to apply custom arguments to the Auto DevOps `helm upgrade` commands. This is an easy way to [customize the Auto Deploy Helm chart](index.md#custom-helm-chart). scale your pod replicas and to apply custom arguments to the Auto DevOps `helm upgrade` commands. This is an easy way to [customize the Auto Deploy Helm chart](customize.md#custom-helm-chart).
Apps are deployed using the Apps are deployed using the
[auto-deploy-app](https://gitlab.com/gitlab-org/charts/auto-deploy-app) chart with [auto-deploy-app](https://gitlab.com/gitlab-org/charts/auto-deploy-app) chart with
...@@ -315,7 +315,7 @@ including support for `Deployment` in the `extensions/v1beta1` version. ...@@ -315,7 +315,7 @@ including support for `Deployment` in the `extensions/v1beta1` version.
To use Auto Deploy on a Kubernetes 1.16+ cluster, you must: To use Auto Deploy on a Kubernetes 1.16+ cluster, you must:
1. Set the following in the [`.gitlab/auto-deploy-values.yaml` file](index.md#customize-values-for-helm-chart): 1. Set the following in the [`.gitlab/auto-deploy-values.yaml` file](customize.md#customize-values-for-helm-chart):
```yml ```yml
deploymentApiVersion: apps/v1 deploymentApiVersion: apps/v1
...@@ -391,7 +391,7 @@ need to: ...@@ -391,7 +391,7 @@ need to:
Once you have configured your worker to respond to health checks, run a Sidekiq Once you have configured your worker to respond to health checks, run a Sidekiq
worker for your Rails application. You can enable workers by setting the worker for your Rails application. You can enable workers by setting the
following in the [`.gitlab/auto-deploy-values.yaml` file](index.md#customize-values-for-helm-chart): following in the [`.gitlab/auto-deploy-values.yaml` file](customize.md#customize-values-for-helm-chart):
```yml ```yml
workers: workers:
......
# Upgrading PostgreSQL for Auto DevOps # Upgrading PostgreSQL for Auto DevOps
Auto DevOps provides an [in-cluster PostgreSQL database](index.md#postgresql-database-support) Auto DevOps provides an [in-cluster PostgreSQL database](customize.md#postgresql-database-support)
for your application. for your application.
The version of the chart used to provision PostgreSQL: The version of the chart used to provision PostgreSQL:
......
...@@ -14,7 +14,7 @@ In this tutorial, we will show you different ways of undoing your work in Git, f ...@@ -14,7 +14,7 @@ In this tutorial, we will show you different ways of undoing your work in Git, f
we will assume you have a basic working knowledge of. Check GitLab's we will assume you have a basic working knowledge of. Check GitLab's
[Git documentation](../index.md) for reference. [Git documentation](../index.md) for reference.
Also, we will only provide some general info of the commands, which is enough Also, we will only provide some general information of the commands, which is enough
to get you started for the easy cases/examples, but for anything more advanced to get you started for the easy cases/examples, but for anything more advanced
please refer to the [Git book](https://git-scm.com/book/en/v2). please refer to the [Git book](https://git-scm.com/book/en/v2).
......
...@@ -116,7 +116,7 @@ Once saved, you can see the build quota in the group admin view. ...@@ -116,7 +116,7 @@ Once saved, you can see the build quota in the group admin view.
The quota can also be viewed in the project admin view if shared Runners The quota can also be viewed in the project admin view if shared Runners
are enabled. are enabled.
![Project admin info](img/admin_project_quota_view.png) ![Project admin information](img/admin_project_quota_view.png)
You can see an overview of the pipeline minutes quota of all projects of You can see an overview of the pipeline minutes quota of all projects of
a group in the **Usage Quotas** page available to the group page settings list. a group in the **Usage Quotas** page available to the group page settings list.
......
...@@ -367,8 +367,8 @@ the report JSON unless stated otherwise. Presence of optional fields depends on ...@@ -367,8 +367,8 @@ the report JSON unless stated otherwise. Presence of optional fields depends on
| `vulnerabilities[].message` | A short text that describes the vulnerability, it may include occurrence's specific information. Optional. | | `vulnerabilities[].message` | A short text that describes the vulnerability, it may include occurrence's specific information. Optional. |
| `vulnerabilities[].description` | A long text that describes the vulnerability. Optional. | | `vulnerabilities[].description` | A long text that describes the vulnerability. Optional. |
| `vulnerabilities[].cve` | A fingerprint string value that represents a concrete occurrence of the vulnerability. It's used to determine whether two vulnerability occurrences are same or different. May not be 100% accurate. **This is NOT a [CVE](https://cve.mitre.org/)**. | | `vulnerabilities[].cve` | A fingerprint string value that represents a concrete occurrence of the vulnerability. It's used to determine whether two vulnerability occurrences are same or different. May not be 100% accurate. **This is NOT a [CVE](https://cve.mitre.org/)**. |
| `vulnerabilities[].severity` | How much the vulnerability impacts the software. Possible values: `Undefined` (an analyzer has not provided this info), `Info`, `Unknown`, `Low`, `Medium`, `High`, `Critical`. **Note:** Our current container scanning tool based on [klar](https://github.com/optiopay/klar) only provides the following levels: `Unknown`, `Low`, `Medium`, `High`, `Critical`. | | `vulnerabilities[].severity` | How much the vulnerability impacts the software. Possible values: `Undefined` (an analyzer has not provided this information), `Info`, `Unknown`, `Low`, `Medium`, `High`, `Critical`. **Note:** Our current container scanning tool based on [klar](https://github.com/optiopay/klar) only provides the following levels: `Unknown`, `Low`, `Medium`, `High`, `Critical`. |
| `vulnerabilities[].confidence` | How reliable the vulnerability's assessment is. Possible values: `Undefined` (an analyzer has not provided this info), `Ignore`, `Unknown`, `Experimental`, `Low`, `Medium`, `High`, `Confirmed`. **Note:** Our current container scanning tool based on [klar](https://github.com/optiopay/klar) does not provide a confidence level, so this value is currently hardcoded to `Unknown`. | | `vulnerabilities[].confidence` | How reliable the vulnerability's assessment is. Possible values: `Undefined` (an analyzer has not provided this information), `Ignore`, `Unknown`, `Experimental`, `Low`, `Medium`, `High`, `Confirmed`. **Note:** Our current container scanning tool based on [klar](https://github.com/optiopay/klar) does not provide a confidence level, so this value is currently hardcoded to `Unknown`. |
| `vulnerabilities[].solution` | Explanation of how to fix the vulnerability. Optional. | | `vulnerabilities[].solution` | Explanation of how to fix the vulnerability. Optional. |
| `vulnerabilities[].scanner` | A node that describes the analyzer used to find this vulnerability. | | `vulnerabilities[].scanner` | A node that describes the analyzer used to find this vulnerability. |
| `vulnerabilities[].scanner.id` | Id of the scanner as a snake_case string. | | `vulnerabilities[].scanner.id` | Id of the scanner as a snake_case string. |
......
...@@ -365,8 +365,8 @@ the report JSON unless stated otherwise. Presence of optional fields depends on ...@@ -365,8 +365,8 @@ the report JSON unless stated otherwise. Presence of optional fields depends on
| `vulnerabilities[].message` | A short text that describes the vulnerability, it may include occurrence's specific information. Optional. | | `vulnerabilities[].message` | A short text that describes the vulnerability, it may include occurrence's specific information. Optional. |
| `vulnerabilities[].description` | A long text that describes the vulnerability. Optional. | | `vulnerabilities[].description` | A long text that describes the vulnerability. Optional. |
| `vulnerabilities[].cve` | A fingerprint string value that represents a concrete occurrence of the vulnerability. It's used to determine whether two vulnerability occurrences are same or different. May not be 100% accurate. **This is NOT a [CVE](https://cve.mitre.org/)**. | | `vulnerabilities[].cve` | A fingerprint string value that represents a concrete occurrence of the vulnerability. It's used to determine whether two vulnerability occurrences are same or different. May not be 100% accurate. **This is NOT a [CVE](https://cve.mitre.org/)**. |
| `vulnerabilities[].severity` | How much the vulnerability impacts the software. Possible values: `Undefined` (an analyzer has not provided this info), `Info`, `Unknown`, `Low`, `Medium`, `High`, `Critical`. | | `vulnerabilities[].severity` | How much the vulnerability impacts the software. Possible values: `Undefined` (an analyzer has not provided this information), `Info`, `Unknown`, `Low`, `Medium`, `High`, `Critical`. |
| `vulnerabilities[].confidence` | How reliable the vulnerability's assessment is. Possible values: `Undefined` (an analyzer has not provided this info), `Ignore`, `Unknown`, `Experimental`, `Low`, `Medium`, `High`, `Confirmed`. | | `vulnerabilities[].confidence` | How reliable the vulnerability's assessment is. Possible values: `Undefined` (an analyzer has not provided this information), `Ignore`, `Unknown`, `Experimental`, `Low`, `Medium`, `High`, `Confirmed`. |
| `vulnerabilities[].solution` | Explanation of how to fix the vulnerability. Optional. | | `vulnerabilities[].solution` | Explanation of how to fix the vulnerability. Optional. |
| `vulnerabilities[].scanner` | A node that describes the analyzer used to find this vulnerability. | | `vulnerabilities[].scanner` | A node that describes the analyzer used to find this vulnerability. |
| `vulnerabilities[].scanner.id` | Id of the scanner as a snake_case string. | | `vulnerabilities[].scanner.id` | Id of the scanner as a snake_case string. |
......
...@@ -73,7 +73,7 @@ hosted within your network. ...@@ -73,7 +73,7 @@ hosted within your network.
## Specific scanner instructions ## Specific scanner instructions
Each individual scanner may be slightly different than the steps described Each individual scanner may be slightly different than the steps described
above. You can find more info at each of the pages below: above. You can find more information at each of the pages below:
- [Container scanning offline directions](../container_scanning/index.md#running-container-scanning-in-an-offline-environment) - [Container scanning offline directions](../container_scanning/index.md#running-container-scanning-in-an-offline-environment)
- [SAST offline directions](../sast/index.md#gitlab-sast-in-an-offline-environment) - [SAST offline directions](../sast/index.md#gitlab-sast-in-an-offline-environment)
......
...@@ -443,8 +443,8 @@ the report JSON unless stated otherwise. Presence of optional fields depends on ...@@ -443,8 +443,8 @@ the report JSON unless stated otherwise. Presence of optional fields depends on
| `vulnerabilities[].message` | A short text that describes the vulnerability, it may include the occurrence's specific information. Optional. | | `vulnerabilities[].message` | A short text that describes the vulnerability, it may include the occurrence's specific information. Optional. |
| `vulnerabilities[].description` | A long text that describes the vulnerability. Optional. | | `vulnerabilities[].description` | A long text that describes the vulnerability. Optional. |
| `vulnerabilities[].cve` | A fingerprint string value that represents a concrete occurrence of the vulnerability. Is used to determine whether two vulnerability occurrences are same or different. May not be 100% accurate. **This is NOT a [CVE](https://cve.mitre.org/)**. | | `vulnerabilities[].cve` | A fingerprint string value that represents a concrete occurrence of the vulnerability. Is used to determine whether two vulnerability occurrences are same or different. May not be 100% accurate. **This is NOT a [CVE](https://cve.mitre.org/)**. |
| `vulnerabilities[].severity` | How much the vulnerability impacts the software. Possible values: `Undefined` (an analyzer has not provided this info), `Info`, `Unknown`, `Low`, `Medium`, `High`, `Critical`. | | `vulnerabilities[].severity` | How much the vulnerability impacts the software. Possible values: `Undefined` (an analyzer has not provided this information), `Info`, `Unknown`, `Low`, `Medium`, `High`, `Critical`. |
| `vulnerabilities[].confidence` | How reliable the vulnerability's assessment is. Possible values: `Undefined` (an analyzer has not provided this info), `Ignore`, `Unknown`, `Experimental`, `Low`, `Medium`, `High`, `Confirmed`. | | `vulnerabilities[].confidence` | How reliable the vulnerability's assessment is. Possible values: `Undefined` (an analyzer has not provided this information), `Ignore`, `Unknown`, `Experimental`, `Low`, `Medium`, `High`, `Confirmed`. |
| `vulnerabilities[].solution` | Explanation of how to fix the vulnerability. Optional. | | `vulnerabilities[].solution` | Explanation of how to fix the vulnerability. Optional. |
| `vulnerabilities[].scanner` | A node that describes the analyzer used to find this vulnerability. | | `vulnerabilities[].scanner` | A node that describes the analyzer used to find this vulnerability. |
| `vulnerabilities[].scanner.id` | Id of the scanner as a snake_case string. | | `vulnerabilities[].scanner.id` | Id of the scanner as a snake_case string. |
......
...@@ -156,7 +156,7 @@ That way, reports are created even if no code change happens. ...@@ -156,7 +156,7 @@ That way, reports are created even if no code change happens.
## Security scans using Auto DevOps ## Security scans using Auto DevOps
When using [Auto DevOps](../../../topics/autodevops/index.md), use When using [Auto DevOps](../../../topics/autodevops/index.md), use
[special environment variables](../../../topics/autodevops/index.md#environment-variables) [special environment variables](../../../topics/autodevops/customize.md#environment-variables)
to configure daily security scans. to configure daily security scans.
<!-- ## Troubleshooting <!-- ## Troubleshooting
......
...@@ -99,7 +99,7 @@ To create a new Group, either: ...@@ -99,7 +99,7 @@ To create a new Group, either:
Add the following information: Add the following information:
![new group info](img/create_new_group_info.png) ![new group information](img/create_new_group_info.png)
1. The **Group name** will automatically populate the URL. Optionally, you can change it. 1. The **Group name** will automatically populate the URL. Optionally, you can change it.
This is the name that displays in group views. This is the name that displays in group views.
......
...@@ -100,7 +100,7 @@ directly from GitLab. No third-party integrations needed. ...@@ -100,7 +100,7 @@ directly from GitLab. No third-party integrations needed.
There is a lot you can customize and configure There is a lot you can customize and configure
to enjoy the best of GitLab. to enjoy the best of GitLab.
- [Settings](profile/index.md): Manage your user settings to change your personal info, - [Settings](profile/index.md): Manage your user settings to change your personal information,
personal access tokens, authorized applications, etc. personal access tokens, authorized applications, etc.
- [Authentication](../topics/authentication/index.md): Read through the authentication - [Authentication](../topics/authentication/index.md): Read through the authentication
methods available in GitLab. methods available in GitLab.
......
...@@ -110,7 +110,7 @@ changing how standard Markdown is used: ...@@ -110,7 +110,7 @@ changing how standard Markdown is used:
| [emphasis](#emphasis) | [multiple underscores in words](#multiple-underscores-in-words-and-mid-word-emphasis) | [emphasis](#emphasis) | [multiple underscores in words](#multiple-underscores-in-words-and-mid-word-emphasis)
| [headers](#headers) | [linkable Header IDs](#header-ids-and-links) | | [headers](#headers) | [linkable Header IDs](#header-ids-and-links) |
| [images](#images) | [embedded videos](#videos) and [audio](#audio) | | [images](#images) | [embedded videos](#videos) and [audio](#audio) |
| [linebreaks](#line-breaks) | [more linebreak control](#newlines) | | [line breaks](#line-breaks) | [more line break control](#newlines) |
| [links](#links) | [automatically linking URLs](#url-auto-linking) | | [links](#links) | [automatically linking URLs](#url-auto-linking) |
## New GFM Markdown extensions ## New GFM Markdown extensions
...@@ -256,7 +256,7 @@ when rendered within GitLab, may appear different depending on the OS and browse ...@@ -256,7 +256,7 @@ when rendered within GitLab, may appear different depending on the OS and browse
Most emoji are natively supported on macOS, Windows, iOS, Android and will fallback to image-based emoji where there is lack of support. Most emoji are natively supported on macOS, Windows, iOS, Android and will fallback to image-based emoji where there is lack of support.
NOTE: **Note:** On Linux, you can download [Noto Color Emoji](https://www.google.com/get/noto/help/emoji/) NOTE: **Note:** On Linux, you can download [Noto Color Emoji](https://www.google.com/get/noto/help/emoji/)
to get full native emoji support. Ubuntu 18.04 (like many modern Linux distros) has to get full native emoji support. Ubuntu 18.04 (like many modern Linux distributions) has
this font installed by default. this font installed by default.
### Front matter ### Front matter
...@@ -1148,7 +1148,7 @@ GFM adheres to the Markdown specification in how [paragraphs and line breaks are ...@@ -1148,7 +1148,7 @@ GFM adheres to the Markdown specification in how [paragraphs and line breaks are
A paragraph is one or more consecutive lines of text, separated by one or A paragraph is one or more consecutive lines of text, separated by one or
more blank lines (two newlines at the end of the first paragraph), as [explained above](#line-breaks). more blank lines (two newlines at the end of the first paragraph), as [explained above](#line-breaks).
If you need more control over line-breaks or soft returns, you can add a single line-break If you need more control over line breaks or soft returns, you can add a single line break
by ending a line with a backslash, or two or more spaces. Two newlines in a row will create a new by ending a line with a backslash, or two or more spaces. Two newlines in a row will create a new
paragraph, with a blank line in between: paragraph, with a blank line in between:
...@@ -1163,21 +1163,6 @@ Another line, this time ending with a backslash.\ ...@@ -1163,21 +1163,6 @@ Another line, this time ending with a backslash.\
A new line due to the previous backslash. A new line due to the previous backslash.
``` ```
<!-- (Do *NOT* remove the two ending whitespaces in the third line) -->
<!-- (They are needed for the Markdown text to render correctly) -->
First paragraph.
Another line in the same paragraph.
A third line in the same paragraph, but this time ending with two spaces.
A new line directly under the first paragraph.
<!-- (Do *NOT* remove the two ending whitespaces in the second line) -->
<!-- (They are needed for the Markdown text to render correctly on docs.gitlab.com, the backslash works fine inside GitLab itself) -->
Second paragraph.
Another line, this time ending with a backslash.
A new line due to the previous backslash.
### Links ### Links
There are two ways to create links, inline-style and reference-style: There are two ways to create links, inline-style and reference-style:
...@@ -1406,7 +1391,7 @@ Example: ...@@ -1406,7 +1391,7 @@ Example:
| header 1 | header 2 | header 3 | | header 1 | header 2 | header 3 |
| --- | ------ |---------:| | --- | ------ |---------:|
| cell 1 | cell 2 | cell 3 | | cell 1 | cell 2 | cell 3 |
| cell 4 | cell 5 is longer | cell 6 is much longer than the others, but that's ok. It will eventually wrap the text when the cell is too large for the display size. | | cell 4 | cell 5 is longer | cell 6 is much longer than the others, but that's okay. It will eventually wrap the text when the cell is too large for the display size. |
| cell 7 | | cell <br> 9 | | cell 7 | | cell <br> 9 |
Additionally, you can choose the alignment of text within columns by adding colons (`:`) Additionally, you can choose the alignment of text within columns by adding colons (`:`)
......
...@@ -246,9 +246,9 @@ conan search Hello/0.1@my-group+my-project/beta --all --remote=gitlab ...@@ -246,9 +246,9 @@ conan search Hello/0.1@my-group+my-project/beta --all --remote=gitlab
The scope of your search will include all projects you have permission to access, this includes your private projects as well as all public projects. The scope of your search will include all projects you have permission to access, this includes your private projects as well as all public projects.
## Fetching Conan package info from the GitLab Package Registry ## Fetching Conan package information from the GitLab Package Registry
The `conan info` command will return info about a given package: The `conan info` command will return information about a given package:
```shell ```shell
conan info Hello/0.1@my-group+my-project/beta conan info Hello/0.1@my-group+my-project/beta
...@@ -261,7 +261,7 @@ The GitLab Conan repository supports the following Conan CLI commands: ...@@ -261,7 +261,7 @@ The GitLab Conan repository supports the following Conan CLI commands:
- `conan upload`: Upload your recipe and package files to the GitLab Package Registry. - `conan upload`: Upload your recipe and package files to the GitLab Package Registry.
- `conan install`: Install a conan package from the GitLab Package Registry, this includes using the `conanfile.txt` file. - `conan install`: Install a conan package from the GitLab Package Registry, this includes using the `conanfile.txt` file.
- `conan search`: Search the GitLab Package Registry for public packages, and private packages you have permission to view. - `conan search`: Search the GitLab Package Registry for public packages, and private packages you have permission to view.
- `conan info`: View the info on a given package from the GitLab Package Registry. - `conan info`: View the information on a given package from the GitLab Package Registry.
- `conan remove`: Delete the package from the GitLab Package Registry. - `conan remove`: Delete the package from the GitLab Package Registry.
## Using GitLab CI with Conan packages ## Using GitLab CI with Conan packages
......
...@@ -279,7 +279,7 @@ If a default Storage Class doesn't already exist and is desired, follow Amazon's ...@@ -279,7 +279,7 @@ If a default Storage Class doesn't already exist and is desired, follow Amazon's
to create one. to create one.
Alternatively, disable PostgreSQL by setting the project variable Alternatively, disable PostgreSQL by setting the project variable
[`POSTGRES_ENABLED`](../../../topics/autodevops/#environment-variables) to `false`. [`POSTGRES_ENABLED`](../../../topics/autodevops/customize.md#environment-variables) to `false`.
### Deploy the app to EKS ### Deploy the app to EKS
......
...@@ -4,7 +4,7 @@ GitLab provides a way to push update messages to an Irker server. When ...@@ -4,7 +4,7 @@ GitLab provides a way to push update messages to an Irker server. When
configured, pushes to a project will trigger the service to send data directly configured, pushes to a project will trigger the service to send data directly
to the Irker server. to the Irker server.
See the project homepage for further info: <https://gitlab.com/esr/irker> See the project homepage for further information: <https://gitlab.com/esr/irker>
## Needed setup ## Needed setup
......
...@@ -775,7 +775,7 @@ GitLab unfurls the link as an embedded metrics panel: ...@@ -775,7 +775,7 @@ GitLab unfurls the link as an embedded metrics panel:
![Embedded Metrics Rendered](img/embedded_metrics_rendered_v12_8.png) ![Embedded Metrics Rendered](img/embedded_metrics_rendered_v12_8.png)
You can also embed a single chart. To get a link to a chart, click the You can also embed a single chart. To get a link to a chart, click the
**{ellipsis_v}** **More info** menu in the upper right corner of the chart, **{ellipsis_v}** **More actions** menu in the upper right corner of the chart,
and select **Copy link to chart**, as shown in this example: and select **Copy link to chart**, as shown in this example:
![Copy Link To Chart](img/copy_link_to_chart_v12_10.png) ![Copy Link To Chart](img/copy_link_to_chart_v12_10.png)
......
...@@ -116,7 +116,7 @@ environment. ...@@ -116,7 +116,7 @@ environment.
Deployments that are ongoing will be shown, as well as the deploying/deployed state Deployments that are ongoing will be shown, as well as the deploying/deployed state
for environments. If it's the first time the branch is deployed, the link for environments. If it's the first time the branch is deployed, the link
will return a `404` error until done. During the deployment, the stop button will will return a `404` error until done. During the deployment, the stop button will
be disabled. If the pipeline fails to deploy, the deployment info will be hidden. be disabled. If the pipeline fails to deploy, the deployment information will be hidden.
![Merge request pipeline](img/merge_request_pipeline.png) ![Merge request pipeline](img/merge_request_pipeline.png)
......
...@@ -270,7 +270,7 @@ terminal: ...@@ -270,7 +270,7 @@ terminal:
- The `webide-file-sync` executable must start **after** the project - The `webide-file-sync` executable must start **after** the project
directory is available. This is why we need to add `sleep 5` to the `command`. directory is available. This is why we need to add `sleep 5` to the `command`.
See [this issue](https://gitlab.com/gitlab-org/webide-file-sync/issues/7) for See [this issue](https://gitlab.com/gitlab-org/webide-file-sync/issues/7) for
more info. more information.
- `$CI_PROJECT_DIR` is a - `$CI_PROJECT_DIR` is a
[predefined environment variable](../../../ci/variables/predefined_variables.md) [predefined environment variable](../../../ci/variables/predefined_variables.md)
for GitLab Runners. This is where your project's repository will be. for GitLab Runners. This is where your project's repository will be.
......
# frozen_string_literal: true
module Gitlab
module Graphql
module Connections
def self.use(_schema)
GraphQL::Relay::BaseConnection.register_connection_implementation(
ActiveRecord::Relation,
Gitlab::Graphql::Connections::Keyset::Connection
)
GraphQL::Relay::BaseConnection.register_connection_implementation(
Gitlab::Graphql::FilterableArray,
Gitlab::Graphql::Connections::FilterableArrayConnection
)
GraphQL::Relay::BaseConnection.register_connection_implementation(
Gitlab::Graphql::ExternallyPaginatedArray,
Gitlab::Graphql::Connections::ExternallyPaginatedArrayConnection
)
end
end
end
end
# frozen_string_literal: true
module Gitlab
module Graphql
module Pagination
module Connections
def self.use(schema)
schema.connections.add(
ActiveRecord::Relation,
Gitlab::Graphql::Pagination::Keyset::Connection)
schema.connections.add(
Gitlab::Graphql::FilterableArray,
Gitlab::Graphql::Pagination::FilterableArrayConnection)
schema.connections.add(
Gitlab::Graphql::ExternallyPaginatedArray,
Gitlab::Graphql::Pagination::ExternallyPaginatedArrayConnection)
end
end
end
end
end
...@@ -3,20 +3,14 @@ ...@@ -3,20 +3,14 @@
# Make a customized connection type # Make a customized connection type
module Gitlab module Gitlab
module Graphql module Graphql
module Connections module Pagination
class ExternallyPaginatedArrayConnection < GraphQL::Relay::ArrayConnection class ExternallyPaginatedArrayConnection < GraphQL::Pagination::ArrayConnection
# As the pagination happens externally
# we just return all the nodes here.
def sliced_nodes
@nodes
end
def start_cursor def start_cursor
nodes.previous_cursor items.previous_cursor
end end
def end_cursor def end_cursor
nodes.next_cursor items.next_cursor
end end
def next_page? def next_page?
......
...@@ -2,14 +2,14 @@ ...@@ -2,14 +2,14 @@
module Gitlab module Gitlab
module Graphql module Graphql
module Connections module Pagination
# FilterableArrayConnection is useful especially for lazy-loaded values. # FilterableArrayConnection is useful especially for lazy-loaded values.
# It allows us to call a callback only on the slice of array being # It allows us to call a callback only on the slice of array being
# rendered in the "after loaded" phase. For example we can check # rendered in the "after loaded" phase. For example we can check
# permissions only on a small subset of items. # permissions only on a small subset of items.
class FilterableArrayConnection < GraphQL::Relay::ArrayConnection class FilterableArrayConnection < GraphQL::Pagination::ArrayConnection
def paged_nodes def nodes
@filtered_nodes ||= nodes.filter_callback.call(super) @nodes ||= items.filter_callback.call(super)
end end
end end
end end
......
...@@ -2,7 +2,7 @@ ...@@ -2,7 +2,7 @@
module Gitlab module Gitlab
module Graphql module Graphql
module Connections module Pagination
module Keyset module Keyset
module Conditions module Conditions
class BaseCondition class BaseCondition
......
...@@ -2,7 +2,7 @@ ...@@ -2,7 +2,7 @@
module Gitlab module Gitlab
module Graphql module Graphql
module Connections module Pagination
module Keyset module Keyset
module Conditions module Conditions
class NotNullCondition < BaseCondition class NotNullCondition < BaseCondition
......
...@@ -2,7 +2,7 @@ ...@@ -2,7 +2,7 @@
module Gitlab module Gitlab
module Graphql module Graphql
module Connections module Pagination
module Keyset module Keyset
module Conditions module Conditions
class NullCondition < BaseCondition class NullCondition < BaseCondition
......
...@@ -27,21 +27,21 @@ ...@@ -27,21 +27,21 @@
# #
module Gitlab module Gitlab
module Graphql module Graphql
module Connections module Pagination
module Keyset module Keyset
class Connection < GraphQL::Relay::BaseConnection class Connection < GraphQL::Pagination::ActiveRecordRelationConnection
include Gitlab::Utils::StrongMemoize include Gitlab::Utils::StrongMemoize
def cursor_from_node(node) def cursor_for(node)
encoded_json_from_ordering(node) encoded_json_from_ordering(node)
end end
def sliced_nodes def sliced_nodes
@sliced_nodes ||= @sliced_nodes ||=
begin begin
OrderInfo.validate_ordering(ordered_nodes, order_list) OrderInfo.validate_ordering(ordered_items, order_list)
sliced = ordered_nodes sliced = ordered_items
sliced = slice_nodes(sliced, before, :before) if before.present? sliced = slice_nodes(sliced, before, :before) if before.present?
sliced = slice_nodes(sliced, after, :after) if after.present? sliced = slice_nodes(sliced, after, :after) if after.present?
...@@ -49,12 +49,12 @@ module Gitlab ...@@ -49,12 +49,12 @@ module Gitlab
end end
end end
def paged_nodes def nodes
# These are the nodes that will be loaded into memory for rendering # These are the nodes that will be loaded into memory for rendering
# So we're ok loading them into memory here as that's bound to happen # So we're ok loading them into memory here as that's bound to happen
# anyway. Having them ready means we can modify the result while # anyway. Having them ready means we can modify the result while
# rendering the fields. # rendering the fields.
@paged_nodes ||= load_paged_nodes.to_a @nodes ||= load_paged_nodes.to_a
end end
private private
...@@ -85,31 +85,31 @@ module Gitlab ...@@ -85,31 +85,31 @@ module Gitlab
@limit_value ||= [first, last, max_page_size].compact.min @limit_value ||= [first, last, max_page_size].compact.min
end end
def ordered_nodes def ordered_items
strong_memoize(:order_nodes) do strong_memoize(:ordered_items) do
unless nodes.primary_key.present? unless items.primary_key.present?
raise ArgumentError.new('Relation must have a primary key') raise ArgumentError.new('Relation must have a primary key')
end end
list = OrderInfo.build_order_list(nodes) list = OrderInfo.build_order_list(items)
# ensure there is a primary key ordering # ensure there is a primary key ordering
if list&.last&.attribute_name != nodes.primary_key if list&.last&.attribute_name != items.primary_key
nodes.order(arel_table[nodes.primary_key].desc) # rubocop: disable CodeReuse/ActiveRecord items.order(arel_table[items.primary_key].desc) # rubocop: disable CodeReuse/ActiveRecord
else else
nodes items
end end
end end
end end
def order_list def order_list
strong_memoize(:order_list) do strong_memoize(:order_list) do
OrderInfo.build_order_list(ordered_nodes) OrderInfo.build_order_list(ordered_items)
end end
end end
def arel_table def arel_table
nodes.arel_table items.arel_table
end end
# Storing the current order values in the cursor allows us to # Storing the current order values in the cursor allows us to
......
...@@ -2,7 +2,7 @@ ...@@ -2,7 +2,7 @@
module Gitlab module Gitlab
module Graphql module Graphql
module Connections module Pagination
module Keyset module Keyset
class OrderInfo class OrderInfo
attr_reader :attribute_name, :sort_direction, :named_function attr_reader :attribute_name, :sort_direction, :named_function
......
...@@ -2,7 +2,7 @@ ...@@ -2,7 +2,7 @@
module Gitlab module Gitlab
module Graphql module Graphql
module Connections module Pagination
module Keyset module Keyset
class QueryBuilder class QueryBuilder
def initialize(arel_table, order_list, decoded_cursor, before_or_after) def initialize(arel_table, order_list, decoded_cursor, before_or_after)
......
...@@ -20,6 +20,7 @@ module Gitlab ...@@ -20,6 +20,7 @@ module Gitlab
def restore def restore
@group_attributes = relation_reader.consume_attributes(nil) @group_attributes = relation_reader.consume_attributes(nil)
@group_members = relation_reader.consume_relation(nil, 'members') @group_members = relation_reader.consume_relation(nil, 'members')
.map(&:first)
# We need to remove `name` and `path` as we did consume it in previous pass # We need to remove `name` and `path` as we did consume it in previous pass
@group_attributes.delete('name') @group_attributes.delete('name')
......
...@@ -53,6 +53,7 @@ module Gitlab ...@@ -53,6 +53,7 @@ module Gitlab
def initialize(relation_names:, allowed_path:) def initialize(relation_names:, allowed_path:)
@relation_names = relation_names.map(&:to_s) @relation_names = relation_names.map(&:to_s)
@consumed_relations = Set.new
# This is legacy reader, to be used in transition # This is legacy reader, to be used in transition
# period before `.ndjson`, # period before `.ndjson`,
...@@ -81,17 +82,19 @@ module Gitlab ...@@ -81,17 +82,19 @@ module Gitlab
raise ArgumentError, "Invalid #{importable_name} passed to `consume_relation`. Use #{@allowed_path} instead." raise ArgumentError, "Invalid #{importable_name} passed to `consume_relation`. Use #{@allowed_path} instead."
end end
value = relations.delete(key) Enumerator.new do |documents|
next unless @consumed_relations.add?("#{importable_path}/#{key}")
return value unless block_given? value = relations.delete(key)
return if value.nil? next if value.nil?
if value.is_a?(Array) if value.is_a?(Array)
value.each.with_index do |item, idx| value.each.with_index do |item, idx|
yield(item, idx) documents << [item, idx]
end end
else else
yield(value, 0) documents << [value, 0]
end
end end
end end
......
# frozen_string_literal: true
module Gitlab
module ImportExport
module JSON
class NdjsonReader
MAX_JSON_DOCUMENT_SIZE = 50.megabytes
attr_reader :dir_path
def initialize(dir_path)
@dir_path = dir_path
@consumed_relations = Set.new
end
def exist?
Dir.exist?(@dir_path)
end
# This can be removed once legacy_reader is deprecated.
def legacy?
false
end
def consume_attributes(importable_path)
# This reads from `tree/project.json`
path = file_path("#{importable_path}.json")
data = File.read(path, MAX_JSON_DOCUMENT_SIZE)
json_decode(data)
end
def consume_relation(importable_path, key)
Enumerator.new do |documents|
next unless @consumed_relations.add?("#{importable_path}/#{key}")
# This reads from `tree/project/merge_requests.ndjson`
path = file_path(importable_path, "#{key}.ndjson")
next unless File.exist?(path)
File.foreach(path, MAX_JSON_DOCUMENT_SIZE).with_index do |line, line_num|
documents << [json_decode(line), line_num]
end
end
end
private
def json_decode(string)
ActiveSupport::JSON.decode(string)
rescue ActiveSupport::JSON.parse_error => e
Gitlab::ErrorTracking.log_exception(e)
raise Gitlab::ImportExport::Error, 'Incorrect JSON format'
end
def file_path(*path)
File.join(dir_path, *path)
end
end
end
end
end
...@@ -17,8 +17,13 @@ module Gitlab ...@@ -17,8 +17,13 @@ module Gitlab
end end
def restore def restore
unless relation_reader
raise Gitlab::ImportExport::Error, 'invalid import format'
end
@project_attributes = relation_reader.consume_attributes(importable_path) @project_attributes = relation_reader.consume_attributes(importable_path)
@project_members = relation_reader.consume_relation(importable_path, 'project_members') @project_members = relation_reader.consume_relation(importable_path, 'project_members')
.map(&:first)
if relation_tree_restorer.restore if relation_tree_restorer.restore
import_failure_service.with_retry(action: 'set_latest_merge_request_diff_ids!') do import_failure_service.with_retry(action: 'set_latest_merge_request_diff_ids!') do
...@@ -38,13 +43,26 @@ module Gitlab ...@@ -38,13 +43,26 @@ module Gitlab
def relation_reader def relation_reader
strong_memoize(:relation_reader) do strong_memoize(:relation_reader) do
[ndjson_relation_reader, legacy_relation_reader]
.compact.find(&:exist?)
end
end
def ndjson_relation_reader
return unless Feature.enabled?(:project_import_ndjson, project.namespace)
ImportExport::JSON::NdjsonReader.new(
File.join(shared.export_path, 'tree')
)
end
def legacy_relation_reader
ImportExport::JSON::LegacyReader::File.new( ImportExport::JSON::LegacyReader::File.new(
File.join(shared.export_path, 'project.json'), File.join(shared.export_path, 'project.json'),
relation_names: reader.project_relation_names, relation_names: reader.project_relation_names,
allowed_path: importable_path allowed_path: importable_path
) )
end end
end
def relation_tree_restorer def relation_tree_restorer
@relation_tree_restorer ||= RelationTreeRestorer.new( @relation_tree_restorer ||= RelationTreeRestorer.new(
......
...@@ -67,7 +67,7 @@ module Gitlab ...@@ -67,7 +67,7 @@ module Gitlab
end end
def process_relation!(relation_key, relation_definition) def process_relation!(relation_key, relation_definition)
@relation_reader.consume_relation(@importable_path, relation_key) do |data_hash, relation_index| @relation_reader.consume_relation(@importable_path, relation_key).each do |data_hash, relation_index|
process_relation_item!(relation_key, relation_definition, relation_index, data_hash) process_relation_item!(relation_key, relation_definition, relation_index, data_hash)
end end
end end
......
...@@ -186,5 +186,23 @@ ...@@ -186,5 +186,23 @@
} }
], ],
"snippets": [], "snippets": [],
"hooks": [] "hooks": [],
"custom_attributes": [
{
"id": 201,
"project_id": 5,
"created_at": "2016-06-14T15:01:51.315Z",
"updated_at": "2016-06-14T15:01:51.315Z",
"key": "color",
"value": "red"
},
{
"id": 202,
"project_id": 5,
"created_at": "2016-06-14T15:01:51.315Z",
"updated_at": "2016-06-14T15:01:51.315Z",
"key": "size",
"value": "small"
}
]
} }
...@@ -3,7 +3,7 @@ ...@@ -3,7 +3,7 @@
require 'spec_helper' require 'spec_helper'
describe GitlabSchema do describe GitlabSchema do
let_it_be(:implementations) { GraphQL::Relay::BaseConnection::CONNECTION_IMPLEMENTATIONS } let_it_be(:connections) { GitlabSchema.connections.all_wrappers }
let(:user) { build :user } let(:user) { build :user }
it 'uses batch loading' do it 'uses batch loading' do
...@@ -34,22 +34,22 @@ describe GitlabSchema do ...@@ -34,22 +34,22 @@ describe GitlabSchema do
expect(described_class.query).to eq(::Types::QueryType) expect(described_class.query).to eq(::Types::QueryType)
end end
it 'paginates active record relations using `Connections::Keyset::Connection`' do it 'paginates active record relations using `Pagination::Keyset::Connection`' do
connection = implementations[ActiveRecord::Relation.name] connection = connections[ActiveRecord::Relation]
expect(connection).to eq(Gitlab::Graphql::Connections::Keyset::Connection) expect(connection).to eq(Gitlab::Graphql::Pagination::Keyset::Connection)
end end
it 'paginates ExternallyPaginatedArray using `Connections::ExternallyPaginatedArrayConnection`' do it 'paginates ExternallyPaginatedArray using `Pagination::ExternallyPaginatedArrayConnection`' do
connection = implementations[Gitlab::Graphql::ExternallyPaginatedArray.name] connection = connections[Gitlab::Graphql::ExternallyPaginatedArray]
expect(connection).to eq(Gitlab::Graphql::Connections::ExternallyPaginatedArrayConnection) expect(connection).to eq(Gitlab::Graphql::Pagination::ExternallyPaginatedArrayConnection)
end end
it 'paginates FilterableArray using `Connections::FilterableArrayConnection`' do it 'paginates FilterableArray using `Pagination::FilterableArrayConnection`' do
connection = implementations[Gitlab::Graphql::FilterableArray.name] connection = connections[Gitlab::Graphql::FilterableArray]
expect(connection).to eq(Gitlab::Graphql::Connections::FilterableArrayConnection) expect(connection).to eq(Gitlab::Graphql::Pagination::FilterableArrayConnection)
end end
describe '.execute' do describe '.execute' do
......
...@@ -2,7 +2,7 @@ ...@@ -2,7 +2,7 @@
require 'spec_helper' require 'spec_helper'
describe Gitlab::Graphql::Connections::ExternallyPaginatedArrayConnection do describe Gitlab::Graphql::Pagination::ExternallyPaginatedArrayConnection do
let(:prev_cursor) { 1 } let(:prev_cursor) { 1 }
let(:next_cursor) { 6 } let(:next_cursor) { 6 }
let(:values) { [2, 3, 4, 5] } let(:values) { [2, 3, 4, 5] }
...@@ -10,21 +10,13 @@ describe Gitlab::Graphql::Connections::ExternallyPaginatedArrayConnection do ...@@ -10,21 +10,13 @@ describe Gitlab::Graphql::Connections::ExternallyPaginatedArrayConnection do
let(:arguments) { {} } let(:arguments) { {} }
subject(:connection) do subject(:connection) do
described_class.new(all_nodes, arguments) described_class.new(all_nodes, { max_page_size: values.size }.merge(arguments))
end end
describe '#sliced_nodes' do describe '#nodes' do
let(:sliced_nodes) { connection.sliced_nodes } let(:paged_nodes) { connection.nodes }
it 'returns all the nodes' do it_behaves_like 'connection with paged nodes' do
expect(connection.sliced_nodes).to eq(values)
end
end
describe '#paged_nodes' do
let(:paged_nodes) { connection.send(:paged_nodes) }
it_behaves_like "connection with paged nodes" do
let(:paged_nodes_size) { values.size } let(:paged_nodes_size) { values.size }
end end
end end
......
...@@ -2,19 +2,19 @@ ...@@ -2,19 +2,19 @@
require 'spec_helper' require 'spec_helper'
describe Gitlab::Graphql::Connections::FilterableArrayConnection do describe Gitlab::Graphql::Pagination::FilterableArrayConnection do
let(:callback) { proc { |nodes| nodes } } let(:callback) { proc { |nodes| nodes } }
let(:all_nodes) { Gitlab::Graphql::FilterableArray.new(callback, 1, 2, 3, 4, 5) } let(:all_nodes) { Gitlab::Graphql::FilterableArray.new(callback, 1, 2, 3, 4, 5) }
let(:arguments) { {} } let(:arguments) { {} }
subject(:connection) do subject(:connection) do
described_class.new(all_nodes, arguments, max_page_size: 3) described_class.new(all_nodes, { max_page_size: 3 }.merge(arguments))
end end
describe '#paged_nodes' do describe '#nodes' do
let(:paged_nodes) { subject.paged_nodes } let(:paged_nodes) { subject.nodes }
it_behaves_like "connection with paged nodes" do it_behaves_like 'connection with paged nodes' do
let(:paged_nodes_size) { 3 } let(:paged_nodes_size) { 3 }
end end
...@@ -22,7 +22,7 @@ describe Gitlab::Graphql::Connections::FilterableArrayConnection do ...@@ -22,7 +22,7 @@ describe Gitlab::Graphql::Connections::FilterableArrayConnection do
let(:callback) { proc { |nodes| nodes[1..-1] } } let(:callback) { proc { |nodes| nodes[1..-1] } }
it 'does not return filtered elements' do it 'does not return filtered elements' do
expect(subject.paged_nodes).to contain_exactly(all_nodes[1], all_nodes[2]) expect(subject.nodes).to contain_exactly(all_nodes[1], all_nodes[2])
end end
end end
end end
......
...@@ -2,7 +2,7 @@ ...@@ -2,7 +2,7 @@
require 'spec_helper' require 'spec_helper'
describe Gitlab::Graphql::Connections::Keyset::Conditions::NotNullCondition do describe Gitlab::Graphql::Pagination::Keyset::Conditions::NotNullCondition do
describe '#build' do describe '#build' do
let(:operators) { ['>', '>'] } let(:operators) { ['>', '>'] }
let(:before_or_after) { :after } let(:before_or_after) { :after }
...@@ -75,7 +75,7 @@ describe Gitlab::Graphql::Connections::Keyset::Conditions::NotNullCondition do ...@@ -75,7 +75,7 @@ describe Gitlab::Graphql::Connections::Keyset::Conditions::NotNullCondition do
context 'when ordering by LOWER' do context 'when ordering by LOWER' do
let(:arel_table) { Project.arel_table } let(:arel_table) { Project.arel_table }
let(:relation) { Project.order(arel_table['name'].lower.asc).order(:id) } let(:relation) { Project.order(arel_table['name'].lower.asc).order(:id) }
let(:order_list) { Gitlab::Graphql::Connections::Keyset::OrderInfo.build_order_list(relation) } let(:order_list) { Gitlab::Graphql::Pagination::Keyset::OrderInfo.build_order_list(relation) }
let(:values) { ['Test', 500] } let(:values) { ['Test', 500] }
context 'when :after' do context 'when :after' do
......
...@@ -2,7 +2,7 @@ ...@@ -2,7 +2,7 @@
require 'spec_helper' require 'spec_helper'
describe Gitlab::Graphql::Connections::Keyset::Conditions::NullCondition do describe Gitlab::Graphql::Pagination::Keyset::Conditions::NullCondition do
describe '#build' do describe '#build' do
let(:values) { [nil, 500] } let(:values) { [nil, 500] }
let(:operators) { [nil, '>'] } let(:operators) { [nil, '>'] }
...@@ -58,7 +58,7 @@ describe Gitlab::Graphql::Connections::Keyset::Conditions::NullCondition do ...@@ -58,7 +58,7 @@ describe Gitlab::Graphql::Connections::Keyset::Conditions::NullCondition do
context 'when ordering by LOWER' do context 'when ordering by LOWER' do
let(:arel_table) { Project.arel_table } let(:arel_table) { Project.arel_table }
let(:relation) { Project.order(arel_table['name'].lower.asc).order(:id) } let(:relation) { Project.order(arel_table['name'].lower.asc).order(:id) }
let(:order_list) { Gitlab::Graphql::Connections::Keyset::OrderInfo.build_order_list(relation) } let(:order_list) { Gitlab::Graphql::Pagination::Keyset::OrderInfo.build_order_list(relation) }
context 'when :after' do context 'when :after' do
it 'generates sql' do it 'generates sql' do
......
...@@ -2,25 +2,28 @@ ...@@ -2,25 +2,28 @@
require 'spec_helper' require 'spec_helper'
describe Gitlab::Graphql::Connections::Keyset::Connection do describe Gitlab::Graphql::Pagination::Keyset::Connection do
let(:nodes) { Project.all.order(id: :asc) } let(:nodes) { Project.all.order(id: :asc) }
let(:arguments) { {} } let(:arguments) { {} }
let(:query_type) { GraphQL::ObjectType.new }
let(:schema) { GraphQL::Schema.define(query: query_type, mutation: nil)}
let(:context) { GraphQL::Query::Context.new(query: OpenStruct.new(schema: schema), values: nil, object: nil) }
subject(:connection) do subject(:connection) do
described_class.new(nodes, arguments, max_page_size: 3) described_class.new(nodes, { context: context, max_page_size: 3 }.merge(arguments))
end end
def encoded_cursor(node) def encoded_cursor(node)
described_class.new(nodes, {}).cursor_from_node(node) described_class.new(nodes, { context: context }).cursor_for(node)
end end
def decoded_cursor(cursor) def decoded_cursor(cursor)
JSON.parse(Base64Bp.urlsafe_decode64(cursor)) JSON.parse(Base64Bp.urlsafe_decode64(cursor))
end end
describe '#cursor_from_nodes' do describe '#cursor_for' do
let(:project) { create(:project) } let(:project) { create(:project) }
let(:cursor) { connection.cursor_from_node(project) } let(:cursor) { connection.cursor_for(project) }
it 'returns an encoded ID' do it 'returns an encoded ID' do
expect(decoded_cursor(cursor)).to eq('id' => project.id.to_s) expect(decoded_cursor(cursor)).to eq('id' => project.id.to_s)
...@@ -264,11 +267,11 @@ describe Gitlab::Graphql::Connections::Keyset::Connection do ...@@ -264,11 +267,11 @@ describe Gitlab::Graphql::Connections::Keyset::Connection do
end end
end end
describe '#paged_nodes' do describe '#nodes' do
let_it_be(:all_nodes) { create_list(:project, 5) } let_it_be(:all_nodes) { create_list(:project, 5) }
let(:paged_nodes) { subject.paged_nodes } let(:paged_nodes) { subject.nodes }
it_behaves_like "connection with paged nodes" do it_behaves_like 'connection with paged nodes' do
let(:paged_nodes_size) { 3 } let(:paged_nodes_size) { 3 }
end end
......
...@@ -2,7 +2,7 @@ ...@@ -2,7 +2,7 @@
require 'spec_helper' require 'spec_helper'
describe Gitlab::Graphql::Connections::Keyset::OrderInfo do describe Gitlab::Graphql::Pagination::Keyset::OrderInfo do
describe '#build_order_list' do describe '#build_order_list' do
let(:order_list) { described_class.build_order_list(relation) } let(:order_list) { described_class.build_order_list(relation) }
......
...@@ -2,7 +2,7 @@ ...@@ -2,7 +2,7 @@
require 'spec_helper' require 'spec_helper'
describe Gitlab::Graphql::Connections::Keyset::QueryBuilder do describe Gitlab::Graphql::Pagination::Keyset::QueryBuilder do
context 'when number of ordering fields is 0' do context 'when number of ordering fields is 0' do
it 'raises an error' do it 'raises an error' do
expect { described_class.new(Issue.arel_table, [], {}, :after) } expect { described_class.new(Issue.arel_table, [], {}, :after) }
...@@ -12,7 +12,7 @@ describe Gitlab::Graphql::Connections::Keyset::QueryBuilder do ...@@ -12,7 +12,7 @@ describe Gitlab::Graphql::Connections::Keyset::QueryBuilder do
describe '#conditions' do describe '#conditions' do
let(:relation) { Issue.order(relative_position: :desc).order(:id) } let(:relation) { Issue.order(relative_position: :desc).order(:id) }
let(:order_list) { Gitlab::Graphql::Connections::Keyset::OrderInfo.build_order_list(relation) } let(:order_list) { Gitlab::Graphql::Pagination::Keyset::OrderInfo.build_order_list(relation) }
let(:arel_table) { Issue.arel_table } let(:arel_table) { Issue.arel_table }
let(:builder) { described_class.new(arel_table, order_list, decoded_cursor, before_or_after) } let(:builder) { described_class.new(arel_table, order_list, decoded_cursor, before_or_after) }
let(:before_or_after) { :after } let(:before_or_after) { :after }
......
...@@ -15,7 +15,6 @@ RSpec.shared_examples 'import/export json legacy reader' do ...@@ -15,7 +15,6 @@ RSpec.shared_examples 'import/export json legacy reader' do
subject { legacy_reader.consume_attributes("project") } subject { legacy_reader.consume_attributes("project") }
context 'no excluded attributes' do context 'no excluded attributes' do
let(:excluded_attributes) { [] }
let(:relation_names) { [] } let(:relation_names) { [] }
it 'returns the whole tree from parsed JSON' do it 'returns the whole tree from parsed JSON' do
...@@ -42,60 +41,53 @@ RSpec.shared_examples 'import/export json legacy reader' do ...@@ -42,60 +41,53 @@ RSpec.shared_examples 'import/export json legacy reader' do
describe '#consume_relation' do describe '#consume_relation' do
context 'when valid path is passed' do context 'when valid path is passed' do
let(:key) { 'description' } let(:key) { 'labels' }
context 'block not given' do subject { legacy_reader.consume_relation("project", key) }
it 'returns value of the key' do
expect(legacy_reader).to receive(:relations).and_return({ key => 'test value' }) context 'key has not been consumed' do
expect(legacy_reader.consume_relation("project", key)).to eq('test value') it 'returns an Enumerator' do
end expect(subject).to be_an_instance_of(Enumerator)
end end
context 'key has been consumed' do context 'value is nil' do
before do before do
legacy_reader.consume_relation("project", key) expect(legacy_reader).to receive(:relations).and_return({ key => nil })
end end
it 'does not yield' do it 'yields nothing to the Enumerator' do
expect do |blk| expect(subject.to_a).to eq([])
legacy_reader.consume_relation("project", key, &blk)
end.not_to yield_control
end end
end end
context 'value is nil' do context 'value is an array' do
before do before do
expect(legacy_reader).to receive(:relations).and_return({ key => nil }) expect(legacy_reader).to receive(:relations).and_return({ key => %w[label1 label2] })
end end
it 'does not yield' do it 'yields every relation value to the Enumerator' do
expect do |blk| expect(subject.to_a).to eq([['label1', 0], ['label2', 1]])
legacy_reader.consume_relation("project", key, &blk)
end.not_to yield_control
end end
end end
context 'value is not array' do context 'value is not array' do
before do before do
expect(legacy_reader).to receive(:relations).and_return({ key => 'value' }) expect(legacy_reader).to receive(:relations).and_return({ key => 'non-array value' })
end end
it 'yield the value with index 0' do it 'yields the value with index 0 to the Enumerator' do
expect do |blk| expect(subject.to_a).to eq([['non-array value', 0]])
legacy_reader.consume_relation("project", key, &blk) end
end.to yield_with_args('value', 0)
end end
end end
context 'value is an array' do context 'key has been consumed' do
before do before do
expect(legacy_reader).to receive(:relations).and_return({ key => %w[item1 item2 item3] }) legacy_reader.consume_relation("project", key).first
end end
it 'yield each array element with index' do it 'yields nothing to the Enumerator' do
expect do |blk| expect(subject.to_a).to eq([])
legacy_reader.consume_relation("project", key, &blk)
end.to yield_successive_args(['item1', 0], ['item2', 1], ['item3', 2])
end end
end end
end end
......
# frozen_string_literal: true
require 'spec_helper'
describe Gitlab::ImportExport::JSON::NdjsonReader do
include ImportExport::CommonUtil
let(:fixture) { 'spec/fixtures/lib/gitlab/import_export/light/tree' }
let(:root_tree) { JSON.parse(File.read(File.join(fixture, 'project.json'))) }
let(:ndjson_reader) { described_class.new(dir_path) }
let(:importable_path) { 'project' }
before :all do
extract_archive('spec/fixtures/lib/gitlab/import_export/light', 'tree.tar.gz')
end
after :all do
cleanup_artifacts_from_extract_archive('light')
end
describe '#exist?' do
subject { ndjson_reader.exist? }
context 'given valid dir_path' do
let(:dir_path) { fixture }
it { is_expected.to be true }
end
context 'given invalid dir_path' do
let(:dir_path) { 'invalid-dir-path' }
it { is_expected.to be false }
end
end
describe '#legacy?' do
let(:dir_path) { fixture }
subject { ndjson_reader.legacy? }
it { is_expected.to be false }
end
describe '#consume_attributes' do
let(:dir_path) { fixture }
subject { ndjson_reader.consume_attributes(importable_path) }
it 'returns the whole root tree from parsed JSON' do
expect(subject).to eq(root_tree)
end
end
describe '#consume_relation' do
let(:dir_path) { fixture }
subject { ndjson_reader.consume_relation(importable_path, key) }
context 'given any key' do
let(:key) { 'any-key' }
it 'returns an Enumerator' do
expect(subject).to be_an_instance_of(Enumerator)
end
end
context 'key has been consumed' do
let(:key) { 'issues' }
before do
ndjson_reader.consume_relation(importable_path, key).first
end
it 'yields nothing to the Enumerator' do
expect(subject.to_a).to eq([])
end
end
context 'key has not been consumed' do
context 'relation file does not exist' do
let(:key) { 'non-exist-relation-file-name' }
before do
relation_file_path = File.join(dir_path, importable_path, "#{key}.ndjson")
expect(File).to receive(:exist?).with(relation_file_path).and_return(false)
end
it 'yields nothing to the Enumerator' do
expect(subject.to_a).to eq([])
end
end
context 'relation file is empty' do
let(:key) { 'empty' }
it 'yields nothing to the Enumerator' do
expect(subject.to_a).to eq([])
end
end
context 'relation file contains multiple lines' do
let(:key) { 'custom_attributes' }
let(:attr_1) { JSON.parse('{"id":201,"project_id":5,"created_at":"2016-06-14T15:01:51.315Z","updated_at":"2016-06-14T15:01:51.315Z","key":"color","value":"red"}') }
let(:attr_2) { JSON.parse('{"id":202,"project_id":5,"created_at":"2016-06-14T15:01:51.315Z","updated_at":"2016-06-14T15:01:51.315Z","key":"size","value":"small"}') }
it 'yields every relation value to the Enumerator' do
expect(subject.to_a).to eq([[attr_1, 0], [attr_2, 1]])
end
end
end
end
end
...@@ -11,6 +11,7 @@ describe Gitlab::ImportExport::Project::TreeRestorer do ...@@ -11,6 +11,7 @@ describe Gitlab::ImportExport::Project::TreeRestorer do
let(:shared) { project.import_export_shared } let(:shared) { project.import_export_shared }
RSpec.shared_examples 'project tree restorer work properly' do |reader|
describe 'restore project tree' do describe 'restore project tree' do
before_all do before_all do
# Using an admin for import, so we can check assignment of existing members # Using an admin for import, so we can check assignment of existing members
...@@ -25,6 +26,7 @@ describe Gitlab::ImportExport::Project::TreeRestorer do ...@@ -25,6 +26,7 @@ describe Gitlab::ImportExport::Project::TreeRestorer do
@shared = @project.import_export_shared @shared = @project.import_export_shared
setup_import_export_config('complex') setup_import_export_config('complex')
setup_reader(reader)
allow_any_instance_of(Repository).to receive(:fetch_source_branch!).and_return(true) allow_any_instance_of(Repository).to receive(:fetch_source_branch!).and_return(true)
allow_any_instance_of(Gitlab::Git::Repository).to receive(:branch_exists?).and_return(false) allow_any_instance_of(Gitlab::Git::Repository).to receive(:branch_exists?).and_return(false)
...@@ -38,6 +40,10 @@ describe Gitlab::ImportExport::Project::TreeRestorer do ...@@ -38,6 +40,10 @@ describe Gitlab::ImportExport::Project::TreeRestorer do
end end
end end
after(:context) do
cleanup_artifacts_from_extract_archive('complex')
end
context 'JSON' do context 'JSON' do
it 'restores models based on JSON' do it 'restores models based on JSON' do
expect(@restored_project_json).to be_truthy expect(@restored_project_json).to be_truthy
...@@ -82,6 +88,7 @@ describe Gitlab::ImportExport::Project::TreeRestorer do ...@@ -82,6 +88,7 @@ describe Gitlab::ImportExport::Project::TreeRestorer do
expect(merge_request_note.note_html).to match(/#{note_content}/) expect(merge_request_note.note_html).to match(/#{note_content}/)
end end
end
context 'merge request system note metadata' do context 'merge request system note metadata' do
it 'restores title action for unmark wip' do it 'restores title action for unmark wip' do
...@@ -103,7 +110,6 @@ describe Gitlab::ImportExport::Project::TreeRestorer do ...@@ -103,7 +110,6 @@ describe Gitlab::ImportExport::Project::TreeRestorer do
end end
end end
end end
end
it 'creates a valid pipeline note' do it 'creates a valid pipeline note' do
expect(Ci::Pipeline.find_by_sha('sha-notes').notes).not_to be_empty expect(Ci::Pipeline.find_by_sha('sha-notes').notes).not_to be_empty
...@@ -507,7 +513,7 @@ describe Gitlab::ImportExport::Project::TreeRestorer do ...@@ -507,7 +513,7 @@ describe Gitlab::ImportExport::Project::TreeRestorer do
allow(shared).to receive(:export_path).and_call_original allow(shared).to receive(:export_path).and_call_original
expect(project_tree_restorer.restore).to eq(false) expect(project_tree_restorer.restore).to eq(false)
expect(shared.errors).to include('Incorrect JSON format') expect(shared.errors).to include('invalid import format')
end end
end end
end end
...@@ -521,16 +527,14 @@ describe Gitlab::ImportExport::Project::TreeRestorer do ...@@ -521,16 +527,14 @@ describe Gitlab::ImportExport::Project::TreeRestorer do
context 'with a simple project' do context 'with a simple project' do
before do before do
setup_import_export_config('light') setup_import_export_config('light')
setup_reader(reader)
expect(restored_project_json).to eq(true) expect(restored_project_json).to eq(true)
end end
it_behaves_like 'restores project successfully', after do
issues: 1, cleanup_artifacts_from_extract_archive('light')
labels: 2, end
label_with_priorities: 'A project label',
milestones: 1,
first_issue_labels: 1,
services: 1
it 'issue system note metadata restored successfully' do it 'issue system note metadata restored successfully' do
note_content = 'created merge request !1 to address this issue' note_content = 'created merge request !1 to address this issue'
...@@ -542,6 +546,20 @@ describe Gitlab::ImportExport::Project::TreeRestorer do ...@@ -542,6 +546,20 @@ describe Gitlab::ImportExport::Project::TreeRestorer do
expect(note.system_note_metadata.commit_count).to be_nil expect(note.system_note_metadata.commit_count).to be_nil
end end
context 'when there is an existing build with build token' do
before do
create(:ci_build, token: 'abcd')
end
it_behaves_like 'restores project successfully',
issues: 1,
labels: 2,
label_with_priorities: 'A project label',
milestones: 1,
first_issue_labels: 1,
services: 1
end
context 'when there is an existing build with build token' do context 'when there is an existing build with build token' do
before do before do
create(:ci_build, token: 'abcd') create(:ci_build, token: 'abcd')
...@@ -559,9 +577,15 @@ describe Gitlab::ImportExport::Project::TreeRestorer do ...@@ -559,9 +577,15 @@ describe Gitlab::ImportExport::Project::TreeRestorer do
context 'multiple pipelines reference the same external pull request' do context 'multiple pipelines reference the same external pull request' do
before do before do
setup_import_export_config('multi_pipeline_ref_one_external_pr') setup_import_export_config('multi_pipeline_ref_one_external_pr')
setup_reader(reader)
expect(restored_project_json).to eq(true) expect(restored_project_json).to eq(true)
end end
after do
cleanup_artifacts_from_extract_archive('multi_pipeline_ref_one_external_pr')
end
it_behaves_like 'restores project successfully', it_behaves_like 'restores project successfully',
issues: 0, issues: 0,
labels: 0, labels: 0,
...@@ -585,11 +609,17 @@ describe Gitlab::ImportExport::Project::TreeRestorer do ...@@ -585,11 +609,17 @@ describe Gitlab::ImportExport::Project::TreeRestorer do
before do before do
setup_import_export_config('light') setup_import_export_config('light')
setup_reader(reader)
expect(project) expect(project)
.to receive(:merge_requests) .to receive(:merge_requests)
.and_raise(exception) .and_raise(exception)
end end
after do
cleanup_artifacts_from_extract_archive('light')
end
it 'report post import error' do it 'report post import error' do
expect(restored_project_json).to eq(false) expect(restored_project_json).to eq(false)
expect(shared.errors).to include('post_import_error') expect(shared.errors).to include('post_import_error')
...@@ -601,6 +631,8 @@ describe Gitlab::ImportExport::Project::TreeRestorer do ...@@ -601,6 +631,8 @@ describe Gitlab::ImportExport::Project::TreeRestorer do
before do before do
setup_import_export_config('light') setup_import_export_config('light')
setup_reader(reader)
expect(project) expect(project)
.to receive(:merge_requests) .to receive(:merge_requests)
.and_raise(exception) .and_raise(exception)
...@@ -610,6 +642,10 @@ describe Gitlab::ImportExport::Project::TreeRestorer do ...@@ -610,6 +642,10 @@ describe Gitlab::ImportExport::Project::TreeRestorer do
expect(restored_project_json).to eq(true) expect(restored_project_json).to eq(true)
end end
after do
cleanup_artifacts_from_extract_archive('light')
end
it_behaves_like 'restores project successfully', it_behaves_like 'restores project successfully',
issues: 1, issues: 1,
labels: 2, labels: 2,
...@@ -635,6 +671,11 @@ describe Gitlab::ImportExport::Project::TreeRestorer do ...@@ -635,6 +671,11 @@ describe Gitlab::ImportExport::Project::TreeRestorer do
context 'when the project has overridden params in import data' do context 'when the project has overridden params in import data' do
before do before do
setup_import_export_config('light') setup_import_export_config('light')
setup_reader(reader)
end
after do
cleanup_artifacts_from_extract_archive('light')
end end
it 'handles string versions of visibility_level' do it 'handles string versions of visibility_level' do
...@@ -697,9 +738,15 @@ describe Gitlab::ImportExport::Project::TreeRestorer do ...@@ -697,9 +738,15 @@ describe Gitlab::ImportExport::Project::TreeRestorer do
before do before do
setup_import_export_config('group') setup_import_export_config('group')
setup_reader(reader)
expect(restored_project_json).to eq(true) expect(restored_project_json).to eq(true)
end end
after do
cleanup_artifacts_from_extract_archive('group')
end
it_behaves_like 'restores project successfully', it_behaves_like 'restores project successfully',
issues: 3, issues: 3,
labels: 2, labels: 2,
...@@ -730,6 +777,11 @@ describe Gitlab::ImportExport::Project::TreeRestorer do ...@@ -730,6 +777,11 @@ describe Gitlab::ImportExport::Project::TreeRestorer do
before do before do
setup_import_export_config('light') setup_import_export_config('light')
setup_reader(reader)
end
after do
cleanup_artifacts_from_extract_archive('light')
end end
it 'does not import any templated services' do it 'does not import any templated services' do
...@@ -776,6 +828,11 @@ describe Gitlab::ImportExport::Project::TreeRestorer do ...@@ -776,6 +828,11 @@ describe Gitlab::ImportExport::Project::TreeRestorer do
before do before do
setup_import_export_config('milestone-iid') setup_import_export_config('milestone-iid')
setup_reader(reader)
end
after do
cleanup_artifacts_from_extract_archive('milestone-iid')
end end
it 'preserves the project milestone IID' do it 'preserves the project milestone IID' do
...@@ -791,6 +848,11 @@ describe Gitlab::ImportExport::Project::TreeRestorer do ...@@ -791,6 +848,11 @@ describe Gitlab::ImportExport::Project::TreeRestorer do
context 'with external authorization classification labels' do context 'with external authorization classification labels' do
before do before do
setup_import_export_config('light') setup_import_export_config('light')
setup_reader(reader)
end
after do
cleanup_artifacts_from_extract_archive('light')
end end
it 'converts empty external classification authorization labels to nil' do it 'converts empty external classification authorization labels to nil' do
...@@ -818,7 +880,8 @@ describe Gitlab::ImportExport::Project::TreeRestorer do ...@@ -818,7 +880,8 @@ describe Gitlab::ImportExport::Project::TreeRestorer do
end end
before do before do
allow_any_instance_of(Gitlab::ImportExport::JSON::LegacyReader::File).to receive(:valid?).and_return(true) allow_any_instance_of(Gitlab::ImportExport::JSON::LegacyReader::File).to receive(:exist?).and_return(true)
allow_any_instance_of(Gitlab::ImportExport::JSON::NdjsonReader).to receive(:exist?).and_return(false)
allow_any_instance_of(Gitlab::ImportExport::JSON::LegacyReader::File).to receive(:tree_hash) { tree_hash } allow_any_instance_of(Gitlab::ImportExport::JSON::LegacyReader::File).to receive(:tree_hash) { tree_hash }
end end
...@@ -901,10 +964,15 @@ describe Gitlab::ImportExport::Project::TreeRestorer do ...@@ -901,10 +964,15 @@ describe Gitlab::ImportExport::Project::TreeRestorer do
before do before do
setup_import_export_config('with_invalid_records') setup_import_export_config('with_invalid_records')
setup_reader(reader)
subject subject
end end
after do
cleanup_artifacts_from_extract_archive('with_invalid_records')
end
context 'when failures occur because a relation fails to be processed' do context 'when failures occur because a relation fails to be processed' do
it_behaves_like 'restores project successfully', it_behaves_like 'restores project successfully',
issues: 0, issues: 0,
...@@ -928,4 +996,26 @@ describe Gitlab::ImportExport::Project::TreeRestorer do ...@@ -928,4 +996,26 @@ describe Gitlab::ImportExport::Project::TreeRestorer do
end end
end end
end end
end
context 'enable ndjson import' do
before_all do
# Test suite `restore project tree` run `project_tree_restorer.restore` in `before_all`.
# `Enable all features by default for testing` happens in `before(:each)`
# So it requires manually enable feature flag to allow ndjson_reader
Feature.enable(:project_import_ndjson)
end
it_behaves_like 'project tree restorer work properly', :legacy_reader
it_behaves_like 'project tree restorer work properly', :ndjson_reader
end
context 'disable ndjson import' do
before do
stub_feature_flags(project_import_ndjson: false)
end
it_behaves_like 'project tree restorer work properly', :legacy_reader
end
end end
...@@ -14,7 +14,7 @@ describe Gitlab::ImportExport::RelationTreeRestorer do ...@@ -14,7 +14,7 @@ describe Gitlab::ImportExport::RelationTreeRestorer do
let(:user) { create(:user) } let(:user) { create(:user) }
let(:shared) { Gitlab::ImportExport::Shared.new(importable) } let(:shared) { Gitlab::ImportExport::Shared.new(importable) }
let(:attributes) { {} } let(:attributes) { relation_reader.consume_attributes(importable_name) }
let(:members_mapper) do let(:members_mapper) do
Gitlab::ImportExport::MembersMapper.new(exported_members: {}, user: user, importable: importable) Gitlab::ImportExport::MembersMapper.new(exported_members: {}, user: user, importable: importable)
...@@ -30,7 +30,7 @@ describe Gitlab::ImportExport::RelationTreeRestorer do ...@@ -30,7 +30,7 @@ describe Gitlab::ImportExport::RelationTreeRestorer do
relation_factory: relation_factory, relation_factory: relation_factory,
reader: reader, reader: reader,
importable: importable, importable: importable,
importable_path: nil, importable_path: importable_path,
importable_attributes: attributes importable_attributes: attributes
) )
end end
...@@ -94,21 +94,24 @@ describe Gitlab::ImportExport::RelationTreeRestorer do ...@@ -94,21 +94,24 @@ describe Gitlab::ImportExport::RelationTreeRestorer do
end end
context 'when restoring a project' do context 'when restoring a project' do
let(:path) { 'spec/fixtures/lib/gitlab/import_export/complex/project.json' }
let(:importable) { create(:project, :builds_enabled, :issues_disabled, name: 'project', path: 'project') } let(:importable) { create(:project, :builds_enabled, :issues_disabled, name: 'project', path: 'project') }
let(:importable_name) { 'project' }
let(:importable_path) { 'project' }
let(:object_builder) { Gitlab::ImportExport::Project::ObjectBuilder } let(:object_builder) { Gitlab::ImportExport::Project::ObjectBuilder }
let(:relation_factory) { Gitlab::ImportExport::Project::RelationFactory } let(:relation_factory) { Gitlab::ImportExport::Project::RelationFactory }
let(:reader) { Gitlab::ImportExport::Reader.new(shared: shared) } let(:reader) { Gitlab::ImportExport::Reader.new(shared: shared) }
context 'using legacy reader' do context 'using legacy reader' do
let(:path) { 'spec/fixtures/lib/gitlab/import_export/complex/project.json' }
let(:relation_reader) do let(:relation_reader) do
Gitlab::ImportExport::JSON::LegacyReader::File.new( Gitlab::ImportExport::JSON::LegacyReader::File.new(
path, path,
relation_names: reader.project_relation_names relation_names: reader.project_relation_names,
allowed_path: 'project'
) )
end end
let(:attributes) { relation_reader.consume_attributes(nil) } let(:attributes) { relation_reader.consume_attributes('project') }
it_behaves_like 'import project successfully' it_behaves_like 'import project successfully'
...@@ -118,6 +121,21 @@ describe Gitlab::ImportExport::RelationTreeRestorer do ...@@ -118,6 +121,21 @@ describe Gitlab::ImportExport::RelationTreeRestorer do
include_examples 'logging of relations creation' include_examples 'logging of relations creation'
end end
context 'using ndjson reader' do
let(:path) { 'spec/fixtures/lib/gitlab/import_export/complex/tree' }
let(:relation_reader) { Gitlab::ImportExport::JSON::NdjsonReader.new(path) }
before :all do
extract_archive('spec/fixtures/lib/gitlab/import_export/complex', 'tree.tar.gz')
end
after :all do
cleanup_artifacts_from_extract_archive('complex')
end
it_behaves_like 'import project successfully'
end
end end
end end
...@@ -125,9 +143,16 @@ describe Gitlab::ImportExport::RelationTreeRestorer do ...@@ -125,9 +143,16 @@ describe Gitlab::ImportExport::RelationTreeRestorer do
let(:path) { 'spec/fixtures/lib/gitlab/import_export/group_exports/no_children/group.json' } let(:path) { 'spec/fixtures/lib/gitlab/import_export/group_exports/no_children/group.json' }
let(:group) { create(:group) } let(:group) { create(:group) }
let(:importable) { create(:group, parent: group) } let(:importable) { create(:group, parent: group) }
let(:importable_name) { nil }
let(:importable_path) { nil }
let(:object_builder) { Gitlab::ImportExport::Group::ObjectBuilder } let(:object_builder) { Gitlab::ImportExport::Group::ObjectBuilder }
let(:relation_factory) { Gitlab::ImportExport::Group::RelationFactory } let(:relation_factory) { Gitlab::ImportExport::Group::RelationFactory }
let(:relation_reader) { Gitlab::ImportExport::JSON::LegacyReader::File.new(path, relation_names: reader.group_relation_names) } let(:relation_reader) do
Gitlab::ImportExport::JSON::LegacyReader::File.new(
path,
relation_names: reader.group_relation_names)
end
let(:reader) do let(:reader) do
Gitlab::ImportExport::Reader.new( Gitlab::ImportExport::Reader.new(
shared: shared, shared: shared,
...@@ -135,6 +160,10 @@ describe Gitlab::ImportExport::RelationTreeRestorer do ...@@ -135,6 +160,10 @@ describe Gitlab::ImportExport::RelationTreeRestorer do
) )
end end
it 'restores group tree' do
expect(subject).to eq(true)
end
include_examples 'logging of relations creation' include_examples 'logging of relations creation'
end end
end end
...@@ -378,8 +378,9 @@ module GraphqlHelpers ...@@ -378,8 +378,9 @@ module GraphqlHelpers
def execute_query(query_type) def execute_query(query_type)
schema = Class.new(GraphQL::Schema) do schema = Class.new(GraphQL::Schema) do
use GraphQL::Pagination::Connections
use Gitlab::Graphql::Authorize use Gitlab::Graphql::Authorize
use Gitlab::Graphql::Connections use Gitlab::Graphql::Pagination::Connections
query(query_type) query(query_type)
end end
......
...@@ -15,9 +15,39 @@ module ImportExport ...@@ -15,9 +15,39 @@ module ImportExport
export_path = [prefix, 'spec', 'fixtures', 'lib', 'gitlab', 'import_export', name].compact export_path = [prefix, 'spec', 'fixtures', 'lib', 'gitlab', 'import_export', name].compact
export_path = File.join(*export_path) export_path = File.join(*export_path)
extract_archive(export_path, 'tree.tar.gz')
allow_any_instance_of(Gitlab::ImportExport).to receive(:export_path) { export_path } allow_any_instance_of(Gitlab::ImportExport).to receive(:export_path) { export_path }
end end
def extract_archive(path, archive)
if File.exist?(File.join(path, archive))
system("cd #{path}; tar xzvf #{archive} &> /dev/null")
end
end
def cleanup_artifacts_from_extract_archive(name, prefix = nil)
export_path = [prefix, 'spec', 'fixtures', 'lib', 'gitlab', 'import_export', name].compact
export_path = File.join(*export_path)
if File.exist?(File.join(export_path, 'tree.tar.gz'))
system("cd #{export_path}; rm -fr tree &> /dev/null")
end
end
def setup_reader(reader)
case reader
when :legacy_reader
allow_any_instance_of(Gitlab::ImportExport::JSON::LegacyReader::File).to receive(:exist?).and_return(true)
allow_any_instance_of(Gitlab::ImportExport::JSON::NdjsonReader).to receive(:exist?).and_return(false)
when :ndjson_reader
allow_any_instance_of(Gitlab::ImportExport::JSON::LegacyReader::File).to receive(:exist?).and_return(false)
allow_any_instance_of(Gitlab::ImportExport::JSON::NdjsonReader).to receive(:exist?).and_return(true)
else
raise "invalid reader #{reader}. Supported readers: :legacy_reader, :ndjson_reader"
end
end
def fixtures_path def fixtures_path
"spec/fixtures/lib/gitlab/import_export" "spec/fixtures/lib/gitlab/import_export"
end end
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment