Commit 1b6c2951 authored by GitLab Bot's avatar GitLab Bot

Automatic merge of gitlab-org/gitlab master

parents c34a9aa3 cca4398f
...@@ -10,29 +10,31 @@ class Projects::RawController < Projects::ApplicationController ...@@ -10,29 +10,31 @@ class Projects::RawController < Projects::ApplicationController
prepend_before_action(only: [:show]) { authenticate_sessionless_user!(:blob) } prepend_before_action(only: [:show]) { authenticate_sessionless_user!(:blob) }
before_action :set_ref_and_path
before_action :require_non_empty_project before_action :require_non_empty_project
before_action :authorize_download_code! before_action :authorize_download_code!
before_action :show_rate_limit, only: [:show], unless: :external_storage_request? before_action :show_rate_limit, only: [:show], unless: :external_storage_request?
before_action :assign_ref_vars
before_action :redirect_to_external_storage, only: :show, if: :static_objects_external_storage_enabled? before_action :redirect_to_external_storage, only: :show, if: :static_objects_external_storage_enabled?
feature_category :source_code_management feature_category :source_code_management
def show def show
@blob = @repository.blob_at(@commit.id, @path) @blob = @repository.blob_at(@ref, @path)
send_blob(@repository, @blob, inline: (params[:inline] != 'false'), allow_caching: @project.public?) send_blob(@repository, @blob, inline: (params[:inline] != 'false'), allow_caching: @project.public?)
end end
private private
def show_rate_limit def set_ref_and_path
# This bypasses assign_ref_vars to avoid a Gitaly FindCommit lookup. # This bypasses assign_ref_vars to avoid a Gitaly FindCommit lookup.
# When rate limiting, we really don't care if a different commit is # We don't need to find the commit to either rate limit or send the
# being requested. # blob.
_ref, path = extract_ref(get_id) @ref, @path = extract_ref(get_id)
end
if rate_limiter.throttled?(:show_raw_controller, scope: [@project, path], threshold: raw_blob_request_limit) def show_rate_limit
if rate_limiter.throttled?(:show_raw_controller, scope: [@project, @path], threshold: raw_blob_request_limit)
rate_limiter.log_request(request, :raw_blob_request_limit, current_user) rate_limiter.log_request(request, :raw_blob_request_limit, current_user)
render plain: _('You cannot access the raw file. Please wait a minute.'), status: :too_many_requests render plain: _('You cannot access the raw file. Please wait a minute.'), status: :too_many_requests
......
...@@ -62,6 +62,14 @@ module Clusters ...@@ -62,6 +62,14 @@ module Clusters
end end
end end
def uninstall_command
helm_command_module::DeleteCommand.new(
name: name,
rbac: cluster.platform_kubernetes_rbac?,
files: files
)
end
def prepare_uninstall def prepare_uninstall
# Override if your application needs any action before # Override if your application needs any action before
# being uninstalled by Helm # being uninstalled by Helm
......
...@@ -3,14 +3,6 @@ ...@@ -3,14 +3,6 @@
module Clusters module Clusters
module Concerns module Concerns
module ApplicationData module ApplicationData
def uninstall_command
helm_command_module::DeleteCommand.new(
name: name,
rbac: cluster.platform_kubernetes_rbac?,
files: files
)
end
def repository def repository
nil nil
end end
......
...@@ -513,6 +513,9 @@ class Repository ...@@ -513,6 +513,9 @@ class Repository
# Don't attempt to return a special result if there is no blob at all # Don't attempt to return a special result if there is no blob at all
return unless blob return unless blob
# Don't attempt to return a special result if this can't be a README
return blob unless Gitlab::FileDetector.type_of(blob.name) == :readme
# Don't attempt to return a special result unless we're looking at HEAD # Don't attempt to return a special result unless we're looking at HEAD
return blob unless head_commit&.sha == sha return blob unless head_commit&.sha == sha
......
...@@ -19,7 +19,7 @@ ...@@ -19,7 +19,7 @@
= link_to 'Remove header logo', header_logos_admin_appearances_path, data: { confirm: "Header logo will be removed. Are you sure?"}, method: :delete, class: "btn gl-button btn-danger btn-danger-secondary btn-sm" = link_to 'Remove header logo', header_logos_admin_appearances_path, data: { confirm: "Header logo will be removed. Are you sure?"}, method: :delete, class: "btn gl-button btn-danger btn-danger-secondary btn-sm"
%hr %hr
= f.hidden_field :header_logo_cache = f.hidden_field :header_logo_cache
= f.file_field :header_logo, class: "" = f.file_field :header_logo, class: "", accept: 'image/*'
.hint .hint
Maximum file size is 1MB. Pages are optimized for a 28px tall header logo Maximum file size is 1MB. Pages are optimized for a 28px tall header logo
%hr %hr
...@@ -38,7 +38,7 @@ ...@@ -38,7 +38,7 @@
= link_to 'Remove favicon', favicon_admin_appearances_path, data: { confirm: "Favicon will be removed. Are you sure?"}, method: :delete, class: "btn gl-button btn-danger btn-danger-secondary btn-sm" = link_to 'Remove favicon', favicon_admin_appearances_path, data: { confirm: "Favicon will be removed. Are you sure?"}, method: :delete, class: "btn gl-button btn-danger btn-danger-secondary btn-sm"
%hr %hr
= f.hidden_field :favicon_cache = f.hidden_field :favicon_cache
= f.file_field :favicon, class: '' = f.file_field :favicon, class: '', accept: 'image/*'
.hint .hint
Maximum file size is 1MB. Image size must be 32x32px. Allowed image formats are #{favicon_extension_whitelist}. Maximum file size is 1MB. Image size must be 32x32px. Allowed image formats are #{favicon_extension_whitelist}.
%br %br
...@@ -70,7 +70,7 @@ ...@@ -70,7 +70,7 @@
= link_to 'Remove logo', logo_admin_appearances_path, data: { confirm: "Logo will be removed. Are you sure?"}, method: :delete, class: "btn gl-button btn-danger btn-danger-secondary btn-sm remove-logo" = link_to 'Remove logo', logo_admin_appearances_path, data: { confirm: "Logo will be removed. Are you sure?"}, method: :delete, class: "btn gl-button btn-danger btn-danger-secondary btn-sm remove-logo"
%hr %hr
= f.hidden_field :logo_cache = f.hidden_field :logo_cache
= f.file_field :logo, class: "" = f.file_field :logo, class: "", accept: 'image/*'
.hint .hint
Maximum file size is 1MB. Pages are optimized for a 640x360 px logo. Maximum file size is 1MB. Pages are optimized for a 640x360 px logo.
......
---
title: Add type filtering in appearance page of the admin panel
merge_request: 48709
author: Paul Ungureanu @ungps
type: fixed
---
title: Remove unnecessary Gitaly calls from raw endpoint
merge_request: 48917
author:
type: performance
This diff is collapsed.
...@@ -53,7 +53,7 @@ helpful to others and, when properly explained, its benefits outweigh the risks. ...@@ -53,7 +53,7 @@ helpful to others and, when properly explained, its benefits outweigh the risks.
If you think you have found an exception to this rule, contact the If you think you have found an exception to this rule, contact the
Technical Writing team. Technical Writing team.
We will add all troubleshooting information to the documentation, no matter how GitLab adds all troubleshooting information to the documentation, no matter how
unlikely a user is to encounter a situation. For the [Troubleshooting sections](#troubleshooting), unlikely a user is to encounter a situation. For the [Troubleshooting sections](#troubleshooting),
people in GitLab Support can merge additions themselves. people in GitLab Support can merge additions themselves.
...@@ -83,7 +83,7 @@ different types. For example, [Divio recommends](https://www.divio.com/blog/docu ...@@ -83,7 +83,7 @@ different types. For example, [Divio recommends](https://www.divio.com/blog/docu
At GitLab, we have so many product changes in our monthly releases that we can't At GitLab, we have so many product changes in our monthly releases that we can't
afford to continuously update multiple types of information. If we have multiple afford to continuously update multiple types of information. If we have multiple
types, the information will become outdated. Therefore, we have a types, the information becomes outdated. Therefore, we have a
[single template](../structure.md) for documentation. [single template](../structure.md) for documentation.
We currently do not distinguish specific document types, although we are open to We currently do not distinguish specific document types, although we are open to
...@@ -93,8 +93,8 @@ improvement efforts, that point hasn't been reached. ...@@ -93,8 +93,8 @@ improvement efforts, that point hasn't been reached.
### Link instead of summarize ### Link instead of summarize
There is a temptation to summarize the information on another page. This will There is a temptation to summarize the information on another page, which
cause the information to live in two places. Instead, link to the single source causes the information to live in two places. Instead, link to the single source
of truth and explain why it is important to consume the information. of truth and explain why it is important to consume the information.
### Organize by topic, not by type ### Organize by topic, not by type
...@@ -137,7 +137,7 @@ that among any other documentation changes, you can either: ...@@ -137,7 +137,7 @@ that among any other documentation changes, you can either:
our [documentation template](../structure.md#template-for-new-docs), if present. our [documentation template](../structure.md#template-for-new-docs), if present.
The more we reflexively add useful information to the documentation, the more The more we reflexively add useful information to the documentation, the more
(and more successfully) the documentation will be used to efficiently accomplish the documentation helps others efficiently accomplish
tasks and solve problems. tasks and solve problems.
If you have questions when considering, authoring, or editing documentation, ask If you have questions when considering, authoring, or editing documentation, ask
...@@ -160,11 +160,11 @@ Markdown rendering engine. For a complete Kramdown reference, see the ...@@ -160,11 +160,11 @@ Markdown rendering engine. For a complete Kramdown reference, see the
[GitLab Markdown Kramdown Guide](https://about.gitlab.com/handbook/markdown-guide/). [GitLab Markdown Kramdown Guide](https://about.gitlab.com/handbook/markdown-guide/).
The [`gitlab-kramdown`](https://gitlab.com/gitlab-org/gitlab_kramdown) Ruby gem The [`gitlab-kramdown`](https://gitlab.com/gitlab-org/gitlab_kramdown) Ruby gem
will support all [GFM markup](../../../user/markdown.md) in the future, which is will support all [GitLab Flavored Markdown](../../../user/markdown.md) in the future, which is
all markup supported for display in the GitLab application itself. For now, use all Markdown supported for display in the GitLab application itself. For now, use
regular Markdown markup, following the rules in the linked style guide. regular Markdown, following the rules in the linked style guide.
Note that Kramdown-specific markup (for example, `{:.class}`) doesn't render Kramdown-specific markup (for example, `{:.class}`) doesn't render
properly on GitLab instances under [`/help`](../index.md#gitlab-help). properly on GitLab instances under [`/help`](../index.md#gitlab-help).
### HTML in Markdown ### HTML in Markdown
...@@ -183,7 +183,7 @@ GitLab ensures that the Markdown used across all documentation is consistent, as ...@@ -183,7 +183,7 @@ GitLab ensures that the Markdown used across all documentation is consistent, as
well as easy to review and maintain, by [testing documentation changes](../testing.md) well as easy to review and maintain, by [testing documentation changes](../testing.md)
with [markdownlint](../testing.md#markdownlint). This lint test fails when any with [markdownlint](../testing.md#markdownlint). This lint test fails when any
document has an issue with Markdown formatting that may cause the page to render document has an issue with Markdown formatting that may cause the page to render
incorrectly within GitLab. It will also fail when a document is using incorrectly within GitLab. It also fails when a document has
non-standard Markdown (which may render correctly, but is not the current non-standard Markdown (which may render correctly, but is not the current
standard for GitLab documentation). standard for GitLab documentation).
...@@ -193,7 +193,7 @@ A rule that could cause confusion is `MD044/proper-names`, as it might not be ...@@ -193,7 +193,7 @@ A rule that could cause confusion is `MD044/proper-names`, as it might not be
immediately clear what caused markdownlint to fail, or how to correct the immediately clear what caused markdownlint to fail, or how to correct the
failure. This rule checks a list of known words, listed in the `.markdownlint.json` failure. This rule checks a list of known words, listed in the `.markdownlint.json`
file in each project, to verify proper use of capitalization and backticks. file in each project, to verify proper use of capitalization and backticks.
Words in backticks will be ignored by markdownlint. Words in backticks are ignored by markdownlint.
In general, product names should follow the exact capitalization of the official In general, product names should follow the exact capitalization of the official
names of the products, protocols, and so on. See [`.markdownlint.json`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/.markdownlint.json) names of the products, protocols, and so on. See [`.markdownlint.json`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/.markdownlint.json)
...@@ -210,7 +210,7 @@ included in backticks. For example: ...@@ -210,7 +210,7 @@ included in backticks. For example:
- "Change the `needs` keyword in your `.gitlab.yml`..." - "Change the `needs` keyword in your `.gitlab.yml`..."
- `needs` is a parameter, and `.gitlab.yml` is a file, so both need backticks. - `needs` is a parameter, and `.gitlab.yml` is a file, so both need backticks.
Additionally, `.gitlab.yml` will fail markdownlint without backticks as it Additionally, `.gitlab.yml` without backticks fails markdownlint because it
does not have capital G or L. does not have capital G or L.
- "Run `git clone` to clone a Git repository..." - "Run `git clone` to clone a Git repository..."
- `git clone` is a command, so it must be lowercase, while Git is the product, - `git clone` is a command, so it must be lowercase, while Git is the product,
...@@ -286,14 +286,14 @@ Refer to the following items when working with directories and files: ...@@ -286,14 +286,14 @@ Refer to the following items when working with directories and files:
located at `doc/user/admin_area/settings/visibility_and_access_controls.md`. located at `doc/user/admin_area/settings/visibility_and_access_controls.md`.
1. The `doc/topics/` directory holds topic-related technical content. Create 1. The `doc/topics/` directory holds topic-related technical content. Create
`doc/topics/topic_name/subtopic_name/index.md` when subtopics become necessary. `doc/topics/topic_name/subtopic_name/index.md` when subtopics become necessary.
General user- and admin- related documentation, should be placed accordingly. General user and administrator documentation should be placed accordingly.
1. The `/university/` directory is *deprecated* and the majority of its documentation 1. The `/university/` directory is *deprecated* and the majority of its documentation
has been moved. has been moved.
If you're unsure where to place a document or a content addition, this shouldn't If you're unsure where to place a document or a content addition, this shouldn't
stop you from authoring and contributing. Use your best judgment, and then ask stop you from authoring and contributing. Use your best judgment, and then ask
the reviewer of your MR to confirm your decision, or ask a technical writer at the reviewer of your MR to confirm your decision, or ask a technical writer at
any stage in the process. The technical writing team will review all any stage in the process. The technical writing team reviews all
documentation changes, regardless, and can move content if there is a better documentation changes, regardless, and can move content if there is a better
place for it. place for it.
...@@ -542,8 +542,8 @@ tenses, words, and phrases: ...@@ -542,8 +542,8 @@ tenses, words, and phrases:
- Avoid use of the future tense: - Avoid use of the future tense:
- Instead of "after you execute this command, GitLab will display the - Instead of "after you execute this command, GitLab will display the
result", use "after you execute this command, GitLab displays the result". result", use "after you execute this command, GitLab displays the result".
- Only use the future tense to convey when the action or result will actually - Only use the future tense to convey when the action or result actually
occur at a future time. occurs at a future time.
- Don't use slashes to clump different words together or as a replacement for - Don't use slashes to clump different words together or as a replacement for
the word "or": the word "or":
- Instead of "and/or," consider using "or," or use another sensible - Instead of "and/or," consider using "or," or use another sensible
...@@ -566,7 +566,7 @@ tenses, words, and phrases: ...@@ -566,7 +566,7 @@ tenses, words, and phrases:
- Avoid using the word *currently* when talking about the product or its - Avoid using the word *currently* when talking about the product or its
features. The documentation describes the product as it is, and not as it features. The documentation describes the product as it is, and not as it
will be at some indeterminate point in the future. will be at some indeterminate point in the future.
- Avoid using the word scalability when talking about increasing GitLab - Avoid using the word *scalability* when talking about increasing GitLab
performance for additional users. The words scale or scaling are sometimes performance for additional users. The words scale or scaling are sometimes
acceptable, but references to increasing GitLab performance for additional acceptable, but references to increasing GitLab performance for additional
users should direct readers to the GitLab users should direct readers to the GitLab
...@@ -673,8 +673,8 @@ Follow these guidelines for punctuation: ...@@ -673,8 +673,8 @@ Follow these guidelines for punctuation:
### Placeholder text ### Placeholder text
Often in examples, a writer will provide a command or configuration that You might want to provide a command or configuration that
uses values specific to the reader. uses specific values.
In these cases, use [`<` and `>`](https://en.wikipedia.org/wiki/Usage_message#Pattern) In these cases, use [`<` and `>`](https://en.wikipedia.org/wiki/Usage_message#Pattern)
to call out where a reader must replace text with their own value. to call out where a reader must replace text with their own value.
......
...@@ -11,7 +11,7 @@ GitLab creates a new Project with an associated Git repository that is a ...@@ -11,7 +11,7 @@ GitLab creates a new Project with an associated Git repository that is a
copy of the original project at the time of the fork. If a large project copy of the original project at the time of the fork. If a large project
gets forked often, this can lead to a quick increase in Git repository gets forked often, this can lead to a quick increase in Git repository
storage disk use. To counteract this problem, we are adding Git object storage disk use. To counteract this problem, we are adding Git object
deduplication for forks to GitLab. In this document, we will describe how deduplication for forks to GitLab. In this document, we describe how
GitLab implements Git object deduplication. GitLab implements Git object deduplication.
## Pool repositories ## Pool repositories
...@@ -27,9 +27,9 @@ If we want repository A to borrow from repository B, we first write a ...@@ -27,9 +27,9 @@ If we want repository A to borrow from repository B, we first write a
path that resolves to `B.git/objects` in the special file path that resolves to `B.git/objects` in the special file
`A.git/objects/info/alternates`. This establishes the alternates link. `A.git/objects/info/alternates`. This establishes the alternates link.
Next, we must perform a Git repack in A. After the repack, any objects Next, we must perform a Git repack in A. After the repack, any objects
that are duplicated between A and B will get deleted from A. Repository that are duplicated between A and B are deleted from A. Repository
A is now no longer self-contained, but it still has its own refs and A is now no longer self-contained, but it still has its own refs and
configuration. Objects in A that are not in B will remain in A. For this configuration. Objects in A that are not in B remain in A. For this
to work, it is of course critical that **no objects ever get deleted from to work, it is of course critical that **no objects ever get deleted from
B** because A might need them. B** because A might need them.
...@@ -49,7 +49,7 @@ repositories** which are hidden from the user. We then use Git ...@@ -49,7 +49,7 @@ repositories** which are hidden from the user. We then use Git
alternates to let a collection of project repositories borrow from a alternates to let a collection of project repositories borrow from a
single pool repository. We call such a collection of project single pool repository. We call such a collection of project
repositories a pool. Pools form star-shaped networks of repositories repositories a pool. Pools form star-shaped networks of repositories
that borrow from a single pool, which will resemble (but not be that borrow from a single pool, which resemble (but not be
identical to) the fork networks that get formed when users fork identical to) the fork networks that get formed when users fork
projects. projects.
...@@ -72,9 +72,9 @@ across a collection of GitLab project repositories at the Git level: ...@@ -72,9 +72,9 @@ across a collection of GitLab project repositories at the Git level:
The effectiveness of Git object deduplication in GitLab depends on the The effectiveness of Git object deduplication in GitLab depends on the
amount of overlap between the pool repository and each of its amount of overlap between the pool repository and each of its
participants. Each time garbage collection runs on the source project, participants. Each time garbage collection runs on the source project,
Git objects from the source project will get migrated to the pool Git objects from the source project are migrated to the pool
repository. One by one, as garbage collection runs, other member repository. One by one, as garbage collection runs, other member
projects will benefit from the new objects that got added to the pool. projects benefit from the new objects that got added to the pool.
## SQL model ## SQL model
...@@ -123,19 +123,19 @@ are as follows: ...@@ -123,19 +123,19 @@ are as follows:
the fork parent and the fork child project become members of the new the fork parent and the fork child project become members of the new
pool. pool.
- Once project A has become the source project of a pool, all future - Once project A has become the source project of a pool, all future
eligible forks of A will become pool members. eligible forks of A become pool members.
- If the fork source is itself a fork, the resulting repository will - If the fork source is itself a fork, the resulting repository will
neither join the repository nor will a new pool repository be neither join the repository nor is a new pool repository
seeded. seeded.
eg: Such as:
Suppose fork A is part of a pool repository, any forks created off Suppose fork A is part of a pool repository, any forks created off
of fork A *will not* be a part of the pool repository that fork A is of fork A *are not* a part of the pool repository that fork A is
a part of. a part of.
Suppose B is a fork of A, and A does not belong to an object pool. Suppose B is a fork of A, and A does not belong to an object pool.
Now C gets created as a fork of B. C will not be part of a pool Now C gets created as a fork of B. C is not part of a pool
repository. repository.
> TODO should forks of forks be deduplicated? > TODO should forks of forks be deduplicated?
...@@ -146,11 +146,11 @@ are as follows: ...@@ -146,11 +146,11 @@ are as follows:
- If a normal Project participating in a pool gets moved to another - If a normal Project participating in a pool gets moved to another
Gitaly storage shard, its "belongs to PoolRepository" relation will Gitaly storage shard, its "belongs to PoolRepository" relation will
be broken. Because of the way moving repositories between shard is be broken. Because of the way moving repositories between shard is
implemented, we will automatically get a fresh self-contained copy implemented, we get a fresh self-contained copy
of the project's repository on the new storage shard. of the project's repository on the new storage shard.
- If the source project of a pool gets moved to another Gitaly storage - If the source project of a pool gets moved to another Gitaly storage
shard or is deleted the "source project" relation is not broken. shard or is deleted the "source project" relation is not broken.
However, as of GitLab 12.0 a pool will not fetch from a source However, as of GitLab 12.0 a pool does not fetch from a source
unless the source is on the same Gitaly shard. unless the source is on the same Gitaly shard.
## Consistency between the SQL pool relation and Gitaly ## Consistency between the SQL pool relation and Gitaly
...@@ -163,7 +163,7 @@ repository and a pool. ...@@ -163,7 +163,7 @@ repository and a pool.
### Pool existence ### Pool existence
If GitLab thinks a pool repository exists (i.e. it exists according to If GitLab thinks a pool repository exists (i.e. it exists according to
SQL), but it does not on the Gitaly server, then it will be created on SQL), but it does not on the Gitaly server, then it is created on
the fly by Gitaly. the fly by Gitaly.
### Pool relation existence ### Pool relation existence
...@@ -173,27 +173,27 @@ There are three different things that can go wrong here. ...@@ -173,27 +173,27 @@ There are three different things that can go wrong here.
#### 1. SQL says repo A belongs to pool P but Gitaly says A has no alternate objects #### 1. SQL says repo A belongs to pool P but Gitaly says A has no alternate objects
In this case, we miss out on disk space savings but all RPC's on A In this case, we miss out on disk space savings but all RPC's on A
itself will function fine. The next time garbage collection runs on A, itself function fine. The next time garbage collection runs on A,
the alternates connection gets established in Gitaly. This is done by the alternates connection gets established in Gitaly. This is done by
`Projects::GitDeduplicationService` in GitLab Rails. `Projects::GitDeduplicationService` in GitLab Rails.
#### 2. SQL says repo A belongs to pool P1 but Gitaly says A has alternate objects in pool P2 #### 2. SQL says repo A belongs to pool P1 but Gitaly says A has alternate objects in pool P2
In this case `Projects::GitDeduplicationService` will throw an exception. In this case `Projects::GitDeduplicationService` throws an exception.
#### 3. SQL says repo A does not belong to any pool but Gitaly says A belongs to P #### 3. SQL says repo A does not belong to any pool but Gitaly says A belongs to P
In this case `Projects::GitDeduplicationService` will try to In this case `Projects::GitDeduplicationService` tries to
"re-duplicate" the repository A using the DisconnectGitAlternates RPC. "re-duplicate" the repository A using the DisconnectGitAlternates RPC.
## Git object deduplication and GitLab Geo ## Git object deduplication and GitLab Geo
When a pool repository record is created in SQL on a Geo primary, this When a pool repository record is created in SQL on a Geo primary, this
will eventually trigger an event on the Geo secondary. The Geo secondary eventually triggers an event on the Geo secondary. The Geo secondary
will then create the pool repository in Gitaly. This leads to an then creates the pool repository in Gitaly. This leads to an
"eventually consistent" situation because as each pool participant gets "eventually consistent" situation because as each pool participant gets
synchronized, Geo will eventually trigger garbage collection in Gitaly on synchronized, Geo eventually triggers garbage collection in Gitaly on
the secondary, at which stage Git objects will get deduplicated. the secondary, at which stage Git objects are deduplicated.
> TODO How do we handle the edge case where at the time the Geo > TODO How do we handle the edge case where at the time the Geo
> secondary tries to create the pool repository, the source project does > secondary tries to create the pool repository, the source project does
......
...@@ -46,8 +46,8 @@ and running. ...@@ -46,8 +46,8 @@ and running.
Can the queries used potentially take down any critical services and result in Can the queries used potentially take down any critical services and result in
engineers being woken up in the night? Can a malicious user abuse the code to engineers being woken up in the night? Can a malicious user abuse the code to
take down a GitLab instance? Will my changes simply make loading a certain page take down a GitLab instance? Do my changes simply make loading a certain page
slower? Will execution time grow exponentially given enough load or data in the slower? Does execution time grow exponentially given enough load or data in the
database? database?
These are all questions one should ask themselves before submitting a merge These are all questions one should ask themselves before submitting a merge
...@@ -67,14 +67,14 @@ in turn can request a performance specialist to review the changes. ...@@ -67,14 +67,14 @@ in turn can request a performance specialist to review the changes.
## Think outside of the box ## Think outside of the box
Everyone has their own perception how the new feature is going to be used. Everyone has their own perception of how to use the new feature.
Always consider how users might be using the feature instead. Usually, Always consider how users might be using the feature instead. Usually,
users test our features in a very unconventional way, users test our features in a very unconventional way,
like by brute forcing or abusing edge conditions that we have. like by brute forcing or abusing edge conditions that we have.
## Data set ## Data set
The data set that will be processed by the merge request should be known The data set the merge request processes should be known
and documented. The feature should clearly document what the expected and documented. The feature should clearly document what the expected
data set is for this feature to process, and what problems it might cause. data set is for this feature to process, and what problems it might cause.
...@@ -86,8 +86,8 @@ from the repository and perform search for the set of files. ...@@ -86,8 +86,8 @@ from the repository and perform search for the set of files.
As an author you should in context of that problem consider As an author you should in context of that problem consider
the following: the following:
1. What repositories are going to be supported? 1. What repositories are planned to be supported?
1. How long it will take for big repositories like Linux kernel? 1. How long it do big repositories like Linux kernel take?
1. Is there something that we can do differently to not process such a 1. Is there something that we can do differently to not process such a
big data set? big data set?
1. Should we build some fail-safe mechanism to contain 1. Should we build some fail-safe mechanism to contain
...@@ -96,28 +96,28 @@ the following: ...@@ -96,28 +96,28 @@ the following:
## Query plans and database structure ## Query plans and database structure
The query plan can tell us if we will need additional The query plan can tell us if we need additional
indexes, or expensive filtering (such as using sequential scans). indexes, or expensive filtering (such as using sequential scans).
Each query plan should be run against substantial size of data set. Each query plan should be run against substantial size of data set.
For example, if you look for issues with specific conditions, For example, if you look for issues with specific conditions,
you should consider validating a query against you should consider validating a query against
a small number (a few hundred) and a big number (100_000) of issues. a small number (a few hundred) and a big number (100_000) of issues.
See how the query will behave if the result will be a few See how the query behaves if the result is a few
and a few thousand. and a few thousand.
This is needed as we have users using GitLab for very big projects and This is needed as we have users using GitLab for very big projects and
in a very unconventional way. Even if it seems that it's unlikely in a very unconventional way. Even if it seems that it's unlikely
that such a big data set will be used, it's still plausible that one that such a big data set is used, it's still plausible that one
of our customers will encounter a problem with the feature. of our customers could encounter a problem with the feature.
Understanding ahead of time how it's going to behave at scale, even if we accept it, Understanding ahead of time how it behaves at scale, even if we accept it,
is the desired outcome. We should always have a plan or understanding of what it will take is the desired outcome. We should always have a plan or understanding of what is needed
to optimize the feature for higher usage patterns. to optimize the feature for higher usage patterns.
Every database structure should be optimized and sometimes even over-described Every database structure should be optimized and sometimes even over-described
in preparation for easy extension. The hardest part after some point is in preparation for easy extension. The hardest part after some point is
data migration. Migrating millions of rows will always be troublesome and data migration. Migrating millions of rows is always troublesome and
can have a negative impact on the application. can have a negative impact on the application.
To better understand how to get help with the query plan reviews To better understand how to get help with the query plan reviews
...@@ -130,7 +130,7 @@ queries unless absolutely necessary. ...@@ -130,7 +130,7 @@ queries unless absolutely necessary.
The total number of queries executed by the code modified or added by a merge request The total number of queries executed by the code modified or added by a merge request
must not increase unless absolutely necessary. When building features it's must not increase unless absolutely necessary. When building features it's
entirely possible you will need some extra queries, but you should try to keep entirely possible you need some extra queries, but you should try to keep
this at a minimum. this at a minimum.
As an example, say you introduce a feature that updates a number of database As an example, say you introduce a feature that updates a number of database
...@@ -144,7 +144,7 @@ objects_to_update.each do |object| ...@@ -144,7 +144,7 @@ objects_to_update.each do |object|
end end
``` ```
This will end up running one query for every object to update. This code can This means running one query for every object to update. This code can
easily overload a database given enough rows to update or many instances of this easily overload a database given enough rows to update or many instances of this
code running in parallel. This particular problem is known as the code running in parallel. This particular problem is known as the
["N+1 query problem"](https://guides.rubyonrails.org/active_record_querying.html#eager-loading-associations). You can write a test with [QueryRecorder](query_recorder.md) to detect this and prevent regressions. ["N+1 query problem"](https://guides.rubyonrails.org/active_record_querying.html#eager-loading-associations). You can write a test with [QueryRecorder](query_recorder.md) to detect this and prevent regressions.
...@@ -162,10 +162,10 @@ query. This in turn makes it much harder for this code to overload a database. ...@@ -162,10 +162,10 @@ query. This in turn makes it much harder for this code to overload a database.
**Summary:** a merge request **should not** execute duplicated cached queries. **Summary:** a merge request **should not** execute duplicated cached queries.
Rails provides an [SQL Query Cache](cached_queries.md#cached-queries-guidelines), Rails provides an [SQL Query Cache](cached_queries.md#cached-queries-guidelines),
used to cache the results of database queries for the duration of the request. used to cache the results of database queries for the duration of the request.
See [why cached queries are considered bad](cached_queries.md#why-cached-queries-are-considered-bad) and See [why cached queries are considered bad](cached_queries.md#why-cached-queries-are-considered-bad) and
[how to detect them](cached_queries.md#how-to-detect-cached-queries). [how to detect them](cached_queries.md#how-to-detect-cached-queries).
The code introduced by a merge request, should not execute multiple duplicated cached queries. The code introduced by a merge request, should not execute multiple duplicated cached queries.
...@@ -189,8 +189,8 @@ build.project == pipeline_project ...@@ -189,8 +189,8 @@ build.project == pipeline_project
# => true # => true
``` ```
When we call `build.project`, it will not hit the database, it will use the cached result, but it will re-instantiate When we call `build.project`, it doesn't hit the database, it uses the cached result, but it re-instantiates
same pipeline project object. It turns out that associated objects do not point to the same in-memory object. the same pipeline project object. It turns out that associated objects do not point to the same in-memory object.
If we try to serialize each build: If we try to serialize each build:
...@@ -200,7 +200,7 @@ pipeline.builds.each do |build| ...@@ -200,7 +200,7 @@ pipeline.builds.each do |build|
end end
``` ```
It will re-instantiate project object for each build, instead of using the same in-memory object. It re-instantiates project object for each build, instead of using the same in-memory object.
In this particular case the workaround is fairly easy: In this particular case the workaround is fairly easy:
...@@ -212,7 +212,7 @@ end ...@@ -212,7 +212,7 @@ end
``` ```
We can assign `pipeline.project` to each `build.project`, since we know it should point to the same project. We can assign `pipeline.project` to each `build.project`, since we know it should point to the same project.
This will allow us that each build point to the same in-memory project, This allows us that each build point to the same in-memory project,
avoiding the cached SQL query and re-instantiation of the project object for each build. avoiding the cached SQL query and re-instantiation of the project object for each build.
## Executing Queries in Loops ## Executing Queries in Loops
...@@ -323,7 +323,7 @@ Certain UI elements may not always be needed. For example, when hovering over a ...@@ -323,7 +323,7 @@ Certain UI elements may not always be needed. For example, when hovering over a
diff line there's a small icon displayed that can be used to create a new diff line there's a small icon displayed that can be used to create a new
comment. Instead of always rendering these kind of elements they should only be comment. Instead of always rendering these kind of elements they should only be
rendered when actually needed. This ensures we don't spend time generating rendered when actually needed. This ensures we don't spend time generating
Haml/HTML when it's not going to be used. Haml/HTML when it's not used.
## Instrumenting New Code ## Instrumenting New Code
...@@ -411,8 +411,8 @@ The quota should be optimised to a level that we consider the feature to ...@@ -411,8 +411,8 @@ The quota should be optimised to a level that we consider the feature to
be performant and usable for the user, but **not limiting**. be performant and usable for the user, but **not limiting**.
**We want the features to be fully usable for the users.** **We want the features to be fully usable for the users.**
**However, we want to ensure that the feature will continue to perform well if used at its limit** **However, we want to ensure that the feature continues to perform well if used at its limit**
**and it won't cause availability issues.** **and it doesn't cause availability issues.**
Consider that it's always better to start with some kind of limitation, Consider that it's always better to start with some kind of limitation,
instead of later introducing a breaking change that would result in some instead of later introducing a breaking change that would result in some
...@@ -433,11 +433,11 @@ The intent of quotas could be different: ...@@ -433,11 +433,11 @@ The intent of quotas could be different:
Examples: Examples:
1. Pipeline Schedules: It's very unlikely that user will want to create 1. Pipeline Schedules: It's very unlikely that user wants to create
more than 50 schedules. more than 50 schedules.
In such cases it's rather expected that this is either misuse In such cases it's rather expected that this is either misuse
or abuse of the feature. Lack of the upper limit can result or abuse of the feature. Lack of the upper limit can result
in service degradation as the system will try to process all schedules in service degradation as the system tries to process all schedules
assigned the project. assigned the project.
1. GitLab CI/CD includes: We started with the limit of maximum of 50 nested includes. 1. GitLab CI/CD includes: We started with the limit of maximum of 50 nested includes.
...@@ -477,7 +477,7 @@ We can consider the following types of storages: ...@@ -477,7 +477,7 @@ We can consider the following types of storages:
for most of our implementations. Even though this allows the above limit to be significantly larger, for most of our implementations. Even though this allows the above limit to be significantly larger,
it does not really mean that you can use more. The shared temporary storage is shared by it does not really mean that you can use more. The shared temporary storage is shared by
all nodes. Thus, the job that uses significant amount of that space or performs a lot all nodes. Thus, the job that uses significant amount of that space or performs a lot
of operations will create a contention on execution of all other jobs and request of operations creates a contention on execution of all other jobs and request
across the whole application, this can easily impact stability of the whole GitLab. across the whole application, this can easily impact stability of the whole GitLab.
Be respectful of that. Be respectful of that.
......
...@@ -467,7 +467,7 @@ of the `gitlab-org/gitlab-foss` project. These jobs are only created in the foll ...@@ -467,7 +467,7 @@ of the `gitlab-org/gitlab-foss` project. These jobs are only created in the foll
The `* as-if-foss` jobs are run in addition to the regular EE-context jobs. They have the `FOSS_ONLY='1'` variable The `* as-if-foss` jobs are run in addition to the regular EE-context jobs. They have the `FOSS_ONLY='1'` variable
set and get their EE-specific folders removed before the tests start running. set and get their EE-specific folders removed before the tests start running.
The intent is to ensure that a change won't introduce a failure once the `gitlab-org/gitlab` project will be synced to The intent is to ensure that a change doesn't introduce a failure after the `gitlab-org/gitlab` project is synced to
the `gitlab-org/gitlab-foss` project. the `gitlab-org/gitlab-foss` project.
## Performance ## Performance
...@@ -502,7 +502,7 @@ request, be sure to start the `dont-interrupt-me` job before pushing. ...@@ -502,7 +502,7 @@ request, be sure to start the `dont-interrupt-me` job before pushing.
- `update-assets-compile-production-cache`, defined in [`.gitlab/ci/frontend.gitlab-ci.yml`](https://gitlab.com/gitlab-org/gitlab/blob/master/.gitlab/ci/frontend.gitlab-ci.yml). - `update-assets-compile-production-cache`, defined in [`.gitlab/ci/frontend.gitlab-ci.yml`](https://gitlab.com/gitlab-org/gitlab/blob/master/.gitlab/ci/frontend.gitlab-ci.yml).
- `update-assets-compile-test-cache`, defined in [`.gitlab/ci/frontend.gitlab-ci.yml`](https://gitlab.com/gitlab-org/gitlab/blob/master/.gitlab/ci/frontend.gitlab-ci.yml). - `update-assets-compile-test-cache`, defined in [`.gitlab/ci/frontend.gitlab-ci.yml`](https://gitlab.com/gitlab-org/gitlab/blob/master/.gitlab/ci/frontend.gitlab-ci.yml).
- `update-yarn-cache`, defined in [`.gitlab/ci/frontend.gitlab-ci.yml`](https://gitlab.com/gitlab-org/gitlab/blob/master/.gitlab/ci/frontend.gitlab-ci.yml). - `update-yarn-cache`, defined in [`.gitlab/ci/frontend.gitlab-ci.yml`](https://gitlab.com/gitlab-org/gitlab/blob/master/.gitlab/ci/frontend.gitlab-ci.yml).
1. These jobs will run in merge requests whose title include `UPDATE CACHE`. 1. These jobs run in merge requests whose title include `UPDATE CACHE`.
### Pre-clone step ### Pre-clone step
...@@ -546,8 +546,7 @@ on a scheduled pipeline, it does the following: ...@@ -546,8 +546,7 @@ on a scheduled pipeline, it does the following:
1. Saves the data as a `.tar.gz`. 1. Saves the data as a `.tar.gz`.
1. Uploads it into the Google Cloud Storage bucket. 1. Uploads it into the Google Cloud Storage bucket.
When a CI job runs with this configuration, you'll see something like When a CI job runs with this configuration, the output looks something like this:
this:
```shell ```shell
$ eval "$CI_PRE_CLONE_SCRIPT" $ eval "$CI_PRE_CLONE_SCRIPT"
...@@ -568,7 +567,7 @@ GitLab Team Member, find credentials in the ...@@ -568,7 +567,7 @@ GitLab Team Member, find credentials in the
[GitLab shared 1Password account](https://about.gitlab.com/handbook/security/#1password-for-teams). [GitLab shared 1Password account](https://about.gitlab.com/handbook/security/#1password-for-teams).
Note that this bucket should be located in the same continent as the Note that this bucket should be located in the same continent as the
runner, or [network egress charges will apply](https://cloud.google.com/storage/pricing). runner, or [you can incur network egress charges](https://cloud.google.com/storage/pricing).
## CI configuration internals ## CI configuration internals
...@@ -662,7 +661,7 @@ and included in `rules` definitions via [YAML anchors](../ci/yaml/README.md#anch ...@@ -662,7 +661,7 @@ and included in `rules` definitions via [YAML anchors](../ci/yaml/README.md#anch
| `if-not-canonical-namespace` | Matches if the project isn't in the canonical (`gitlab-org/`) or security (`gitlab-org/security`) namespace. | Use to create a job for forks (by using `when: on_success\|manual`), or **not** create a job for forks (by using `when: never`). | | `if-not-canonical-namespace` | Matches if the project isn't in the canonical (`gitlab-org/`) or security (`gitlab-org/security`) namespace. | Use to create a job for forks (by using `when: on_success\|manual`), or **not** create a job for forks (by using `when: never`). |
| `if-not-ee` | Matches if the project isn't EE (i.e. project name isn't `gitlab` or `gitlab-ee`). | Use to create a job only in the FOSS project (by using `when: on_success|manual`), or **not** create a job if the project is EE (by using `when: never`). | | `if-not-ee` | Matches if the project isn't EE (i.e. project name isn't `gitlab` or `gitlab-ee`). | Use to create a job only in the FOSS project (by using `when: on_success|manual`), or **not** create a job if the project is EE (by using `when: never`). |
| `if-not-foss` | Matches if the project isn't FOSS (i.e. project name isn't `gitlab-foss`, `gitlab-ce`, or `gitlabhq`). | Use to create a job only in the EE project (by using `when: on_success|manual`), or **not** create a job if the project is FOSS (by using `when: never`). | | `if-not-foss` | Matches if the project isn't FOSS (i.e. project name isn't `gitlab-foss`, `gitlab-ce`, or `gitlabhq`). | Use to create a job only in the EE project (by using `when: on_success|manual`), or **not** create a job if the project is FOSS (by using `when: never`). |
| `if-default-refs` | Matches if the pipeline is for `master`, `/^[\d-]+-stable(-ee)?$/` (stable branches), `/^\d+-\d+-auto-deploy-\d+$/` (auto-deploy branches), `/^security\//` (security branches), merge requests, and tags. | Note that jobs won't be created for branches with this default configuration. | | `if-default-refs` | Matches if the pipeline is for `master`, `/^[\d-]+-stable(-ee)?$/` (stable branches), `/^\d+-\d+-auto-deploy-\d+$/` (auto-deploy branches), `/^security\//` (security branches), merge requests, and tags. | Note that jobs aren't created for branches with this default configuration. |
| `if-master-refs` | Matches if the current branch is `master`. | | | `if-master-refs` | Matches if the current branch is `master`. | |
| `if-master-or-tag` | Matches if the pipeline is for the `master` branch or for a tag. | | | `if-master-or-tag` | Matches if the pipeline is for the `master` branch or for a tag. | |
| `if-merge-request` | Matches if the pipeline is for a merge request. | | | `if-merge-request` | Matches if the pipeline is for a merge request. | |
......
...@@ -52,11 +52,11 @@ version. The range of supported versions is based on the evaluation of: ...@@ -52,11 +52,11 @@ version. The range of supported versions is based on the evaluation of:
GitLab supports the following Kubernetes versions, and you can upgrade your GitLab supports the following Kubernetes versions, and you can upgrade your
Kubernetes version to any supported version at any time: Kubernetes version to any supported version at any time:
- 1.18
- 1.17 - 1.17
- 1.16 - 1.16
- 1.15 - 1.15
- 1.14 (deprecated, support ends on December 22, 2020) - 1.14 (deprecated, support ends on December 22, 2020)
- 1.13 (deprecated, support ends on November 22, 2020)
Some GitLab features may support versions outside the range provided here. Some GitLab features may support versions outside the range provided here.
......
...@@ -190,6 +190,22 @@ Expect the output: ...@@ -190,6 +190,22 @@ Expect the output:
_gitlab-pages-verification-code.<YOUR-PAGES-DOMAIN>. 300 IN TXT "gitlab-pages-verification-code=<YOUR-VERIFICATION-CODE>" _gitlab-pages-verification-code.<YOUR-PAGES-DOMAIN>. 300 IN TXT "gitlab-pages-verification-code=<YOUR-VERIFICATION-CODE>"
``` ```
In some cases it can help to add the verification code with the same domain name as you are trying to register.
For a root domain:
| From | DNS Record | To |
| ------------------------------------------------- | ---------- | ---------------------- |
| `example.com` | TXT | `gitlab-pages-verification-code=00112233445566778899aabbccddeeff` |
| `_gitlab-pages-verification-code.example.com` | TXT | `gitlab-pages-verification-code=00112233445566778899aabbccddeeff` |
For a subdomain:
| From | DNS Record | To |
| ------------------------------------------------- | ---------- | ---------------------- |
| `www.example.com` | TXT | `gitlab-pages-verification-code=00112233445566778899aabbccddeeff` |
| `_gitlab-pages-verification-code.www.example.com` | TXT | `gitlab-pages-verification-code=00112233445566778899aabbccddeeff` |
### Adding more domain aliases ### Adding more domain aliases
You can add more than one alias (custom domains and subdomains) to the same project. You can add more than one alias (custom domains and subdomains) to the same project.
......
...@@ -5,11 +5,11 @@ require 'spec_helper' ...@@ -5,11 +5,11 @@ require 'spec_helper'
RSpec.describe Projects::RawController do RSpec.describe Projects::RawController do
include RepoHelpers include RepoHelpers
let(:project) { create(:project, :public, :repository) } let_it_be(:project) { create(:project, :public, :repository) }
let(:inline) { nil } let(:inline) { nil }
describe 'GET #show' do describe 'GET #show' do
subject do def get_show
get(:show, get(:show,
params: { params: {
namespace_id: project.namespace, namespace_id: project.namespace,
...@@ -19,6 +19,18 @@ RSpec.describe Projects::RawController do ...@@ -19,6 +19,18 @@ RSpec.describe Projects::RawController do
}) })
end end
subject { get_show }
shared_examples 'single Gitaly request' do
it 'makes a single Gitaly request', :request_store, :clean_gitlab_redis_cache do
# Warm up to populate repository cache
get_show
RequestStore.clear!
expect { get_show }.to change { Gitlab::GitalyClient.get_request_count }.by(1)
end
end
context 'regular filename' do context 'regular filename' do
let(:filepath) { 'master/README.md' } let(:filepath) { 'master/README.md' }
...@@ -33,6 +45,7 @@ RSpec.describe Projects::RawController do ...@@ -33,6 +45,7 @@ RSpec.describe Projects::RawController do
it_behaves_like 'project cache control headers' it_behaves_like 'project cache control headers'
it_behaves_like 'content disposition headers' it_behaves_like 'content disposition headers'
include_examples 'single Gitaly request'
end end
context 'image header' do context 'image header' do
...@@ -48,6 +61,7 @@ RSpec.describe Projects::RawController do ...@@ -48,6 +61,7 @@ RSpec.describe Projects::RawController do
it_behaves_like 'project cache control headers' it_behaves_like 'project cache control headers'
it_behaves_like 'content disposition headers' it_behaves_like 'content disposition headers'
include_examples 'single Gitaly request'
end end
context 'with LFS files' do context 'with LFS files' do
...@@ -56,6 +70,7 @@ RSpec.describe Projects::RawController do ...@@ -56,6 +70,7 @@ RSpec.describe Projects::RawController do
it_behaves_like 'a controller that can serve LFS files' it_behaves_like 'a controller that can serve LFS files'
it_behaves_like 'project cache control headers' it_behaves_like 'project cache control headers'
include_examples 'single Gitaly request'
end end
context 'when the endpoint receives requests above the limit', :clean_gitlab_redis_cache do context 'when the endpoint receives requests above the limit', :clean_gitlab_redis_cache do
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment