Commit 8df072db authored by Adam Hegyi's avatar Adam Hegyi

Merge branch 'rackattack-git_lfs_rate_limit' into 'master'

Add specific rate limit for Git LFS requests

See merge request gitlab-org/gitlab!68642
parents b2a851ad f55a2d21
......@@ -311,6 +311,9 @@ module ApplicationSettingsHelper
:throttle_authenticated_api_enabled,
:throttle_authenticated_api_period_in_seconds,
:throttle_authenticated_api_requests_per_period,
:throttle_authenticated_git_lfs_enabled,
:throttle_authenticated_git_lfs_period_in_seconds,
:throttle_authenticated_git_lfs_requests_per_period,
:throttle_authenticated_web_enabled,
:throttle_authenticated_web_period_in_seconds,
:throttle_authenticated_web_requests_per_period,
......
......@@ -497,6 +497,14 @@ class ApplicationSetting < ApplicationRecord
presence: true,
numericality: { only_integer: true, greater_than: 0 }
validates :throttle_authenticated_git_lfs_requests_per_period,
presence: true,
numericality: { only_integer: true, greater_than: 0 }
validates :throttle_authenticated_git_lfs_period_in_seconds,
presence: true,
numericality: { only_integer: true, greater_than: 0 }
validates :throttle_authenticated_web_requests_per_period,
presence: true,
numericality: { only_integer: true, greater_than: 0 }
......
......@@ -163,6 +163,9 @@ module ApplicationSettingImplementation
throttle_authenticated_api_enabled: false,
throttle_authenticated_api_period_in_seconds: 3600,
throttle_authenticated_api_requests_per_period: 7200,
throttle_authenticated_git_lfs_enabled: false,
throttle_authenticated_git_lfs_period_in_seconds: 60,
throttle_authenticated_git_lfs_requests_per_period: 1000,
throttle_authenticated_web_enabled: false,
throttle_authenticated_web_period_in_seconds: 3600,
throttle_authenticated_web_requests_per_period: 7200,
......
= form_for @application_setting, url: network_admin_application_settings_path(anchor: 'js-git-lfs-limits-settings'), html: { class: 'fieldset-form' } do |f|
= form_errors(@application_setting)
%fieldset
%h5
= _('Authenticated Git LFS request rate limit')
.form-group
.form-check
= f.check_box :throttle_authenticated_git_lfs_enabled, class: 'form-check-input', data: { qa_selector: 'throttle_authenticated_git_lfs_checkbox' }
= f.label :throttle_authenticated_git_lfs_enabled, class: 'form-check-label gl-font-weight-bold' do
= _('Enable authenticated Git LFS request rate limit')
%span.form-text.gl-text-gray-600
= _('Helps reduce request volume (for example, from crawlers or abusive bots)')
.form-group
= f.label :throttle_authenticated_git_lfs_requests_per_period, _('Max authenticated Git LFS requests per period per user'), class: 'gl-font-weight-bold'
= f.number_field :throttle_authenticated_git_lfs_requests_per_period, class: 'form-control gl-form-input'
.form-group
= f.label :throttle_authenticated_git_lfs_period_in_seconds, _('Authenticated Git LFS rate limit period in seconds'), class: 'gl-font-weight-bold'
= f.number_field :throttle_authenticated_git_lfs_period_in_seconds, class: 'form-control gl-form-input'
= f.submit _('Save changes'), class: "gl-button btn btn-confirm", data: { qa_selector: 'save_changes_button' }
......@@ -35,6 +35,18 @@
.settings-content
= render 'package_registry_limits'
%section.settings.as-git-lfs-limits.no-animate#js-git-lfs-limits-settings{ class: ('expanded' if expanded_by_default?), data: { qa_selector: 'git_lfs_limits_content' } }
.settings-header
%h4
= _('Git LFS Rate Limits')
%button.btn.gl-button.btn-default.js-settings-toggle{ type: 'button' }
= expanded_by_default? ? _('Collapse') : _('Expand')
%p
= _('Configure specific limits for Git LFS requests that supersede the general user and IP rate limits.')
= link_to _('Learn more.'), help_page_path('user/admin_area/settings/git_lfs_rate_limits.md'), target: '_blank', rel: 'noopener noreferrer'
.settings-content
= render 'git_lfs_limits'
%section.settings.as-outbound.no-animate#js-outbound-settings{ class: ('expanded' if expanded_by_default?), data: { qa_selector: 'outbound_requests_content' } }
.settings-header
%h4
......
# frozen_string_literal: true
class AddThrottleAuthenticatedGitLfsColumns < ActiveRecord::Migration[6.1]
include Gitlab::Database::MigrationHelpers
disable_ddl_transaction!
def up
with_lock_retries do
add_column :application_settings, :throttle_authenticated_git_lfs_requests_per_period, :integer, default: 1000, null: false
add_column :application_settings, :throttle_authenticated_git_lfs_period_in_seconds, :integer, default: 60, null: false
add_column :application_settings, :throttle_authenticated_git_lfs_enabled, :boolean, default: false, null: false
end
end
def down
with_lock_retries do
remove_column :application_settings, :throttle_authenticated_git_lfs_requests_per_period
remove_column :application_settings, :throttle_authenticated_git_lfs_period_in_seconds
remove_column :application_settings, :throttle_authenticated_git_lfs_enabled, :boolean
end
end
end
cf1a51194961500cb63d848afb1d5ebbf2ef77f54d60957e92c9dd6a667913ea
\ No newline at end of file
......@@ -9614,6 +9614,9 @@ CREATE TABLE application_settings (
throttle_authenticated_files_api_period_in_seconds integer DEFAULT 15 NOT NULL,
throttle_unauthenticated_files_api_enabled boolean DEFAULT false NOT NULL,
throttle_authenticated_files_api_enabled boolean DEFAULT false NOT NULL,
throttle_authenticated_git_lfs_requests_per_period integer DEFAULT 1000 NOT NULL,
throttle_authenticated_git_lfs_period_in_seconds integer DEFAULT 60 NOT NULL,
throttle_authenticated_git_lfs_enabled boolean DEFAULT false NOT NULL,
CONSTRAINT app_settings_container_reg_cleanup_tags_max_list_size_positive CHECK ((container_registry_cleanup_tags_service_max_list_size >= 0)),
CONSTRAINT app_settings_ext_pipeline_validation_service_url_text_limit CHECK ((char_length(external_pipeline_validation_service_url) <= 255)),
CONSTRAINT app_settings_registry_exp_policies_worker_capacity_positive CHECK ((container_registry_expiration_policies_worker_capacity >= 0)),
......@@ -78,6 +78,16 @@ This setting limits the request rate on the Packages API per user or IP. For mor
- **Default rate limit**: Disabled by default.
### Git LFS
> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/68642) in GitLab 14.3.
This setting limits the request rate on the [Git LFS](../topics/git/lfs/index.md)
requests per user. For more information, read
[GitLab Git Large File Storage (LFS) Administration](../administration/lfs/index.md).
- **Default rate limit**: Disabled by default.
### Import/Export
> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/35728) in GitLab 13.2.
......
......@@ -34,6 +34,7 @@ These are rate limits you can set in the Admin Area of your instance:
- [Raw endpoints rate limits](../user/admin_area/settings/rate_limits_on_raw_endpoints.md)
- [User and IP rate limits](../user/admin_area/settings/user_and_ip_rate_limits.md)
- [Package registry rate limits](../user/admin_area/settings/package_registry_rate_limits.md)
- [Git LFS rate limits](../user/admin_area/settings/git_lfs_rate_limits.md)
## Non-configurable limits
......
---
stage: Create
group: Source Code
info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments
type: reference
---
# Git LFS Rate Limits **(FREE SELF)**
[Git LFS (Large File Storage)](../../../topics/git/lfs/index.md) is a Git extension
for handling large files. If you use Git LFS in your repository, common Git operations
can generate many Git LFS requests. You can enforce
[general user and IP rate limits](user_and_ip_rate_limits.md), but you can also
override the general setting to enforce additional limits on Git LFS requests. This
override can improve the security and durability of your web application. Aside from
precedence, this configuration provides the same features as the general user and IP
rate limits.
## Configure Git LFS rate limits
Git LFS rate limits are disabled by default. If enabled and configured, these limits
supersede the [general user and IP rate limits](user_and_ip_rate_limits.md):
1. On the top bar, select **Menu >** **{admin}** **Admin**.
1. On the left sidebar, select **Settings > Network**.
1. Expand **Git LFS Rate Limits**.
1. Select **Enable authenticated Git LFS request rate limit**.
1. Enter a value for **Max authenticated Git LFS requests per period per user**.
1. Enter a value for **Authenticated Git LFS rate limit period in seconds**.
1. Select **Save changes**.
## Resources
- [Rate limiting](../../../security/rate_limits.md)
- [User and IP rate limits](user_and_ip_rate_limits.md)
......@@ -96,6 +96,7 @@ To access the default page for Admin Area settings:
| Performance optimization | [Write to "authorized_keys" file](../../../administration/operations/fast_ssh_key_lookup.md#setting-up-fast-lookup-via-gitlab-shell) and [Push event activities limit and bulk push events](push_event_activities_limit.md). Various settings that affect GitLab performance. |
| [User and IP rate limits](user_and_ip_rate_limits.md) | Configure limits for web and API requests. |
| [Package Registry Rate Limits](package_registry_rate_limits.md) | Configure specific limits for Packages API requests that supersede the user and IP rate limits. |
| [Git LFS Rate Limits](git_lfs_rate_limits.md) | Configure specific limits for Git LFS requests that supersede the user and IP rate limits. |
| [Outbound requests](../../../security/webhooks.md) | Allow requests to the local network from hooks and services. |
| [Protected Paths](protected_paths.md) | Configure paths to be protected by Rack Attack. |
| [Incident Management](../../../operations/incident_management/index.md) Limits | Limit the number of inbound alerts that can be sent to a project. |
......
......@@ -136,6 +136,7 @@ The possible names are:
- `throttle_authenticated_protected_paths_web`
- `throttle_unauthenticated_packages_api`
- `throttle_authenticated_packages_api`
- `throttle_authenticated_git_lfs`
For example, to try out throttles for all authenticated requests to
non-protected paths can be done by setting
......
......@@ -338,6 +338,10 @@ module Gitlab
Gitlab::PathRegex.repository_git_route_regex.match?(current_request.path)
end
def git_lfs_request?
Gitlab::PathRegex.repository_git_lfs_route_regex.match?(current_request.path)
end
def archive_request?
current_request.path.include?('/-/archive/')
end
......
......@@ -20,7 +20,8 @@ module Gitlab
:throttle_authenticated_web,
:throttle_authenticated_protected_paths_api,
:throttle_authenticated_protected_paths_web,
:throttle_authenticated_packages_api
:throttle_authenticated_packages_api,
:throttle_authenticated_git_lfs
].freeze
PAYLOAD_KEYS = [
......
......@@ -184,6 +184,10 @@ module Gitlab
@repository_git_route_regex ||= /#{repository_route_regex}\.git/.freeze
end
def repository_git_lfs_route_regex
@repository_git_lfs_route_regex ||= %r{#{repository_git_route_regex}\/(info\/lfs|gitlab-lfs)\/}.freeze
end
def repository_wiki_git_route_regex
@repository_wiki_git_route_regex ||= /#{full_namespace_route_regex}\.*\.wiki\.git/.freeze
end
......
......@@ -139,6 +139,12 @@ module Gitlab
end
end
throttle_or_track(rack_attack, 'throttle_authenticated_git_lfs', Gitlab::Throttle.throttle_authenticated_git_lfs_options) do |req|
if req.throttle_authenticated_git_lfs?
req.throttled_user_id([:api])
end
end
rack_attack.safelist('throttle_bypass_header') do |req|
Gitlab::Throttle.bypass_header.present? &&
req.get_header(Gitlab::Throttle.bypass_header) == '1'
......
......@@ -73,6 +73,7 @@ module Gitlab
def throttle_authenticated_web?
web_request? &&
!throttle_authenticated_git_lfs? &&
Gitlab::Throttle.settings.throttle_authenticated_web_enabled
end
......@@ -109,6 +110,11 @@ module Gitlab
Gitlab::Throttle.settings.throttle_authenticated_packages_api_enabled
end
def throttle_authenticated_git_lfs?
git_lfs_path? &&
Gitlab::Throttle.settings.throttle_authenticated_git_lfs_enabled
end
private
def authenticated_user_id(request_formats)
......@@ -130,6 +136,10 @@ module Gitlab
def packages_api_path?
path =~ ::Gitlab::Regex::Packages::API_PATH_REGEX
end
def git_lfs_path?
path =~ Gitlab::PathRegex.repository_git_lfs_route_regex
end
end
end
end
......
......@@ -63,6 +63,13 @@ module Gitlab
{ limit: limit_proc, period: period_proc }
end
def self.throttle_authenticated_git_lfs_options
limit_proc = proc { |req| settings.throttle_authenticated_git_lfs_requests_per_period }
period_proc = proc { |req| settings.throttle_authenticated_git_lfs_period_in_seconds.seconds }
{ limit: limit_proc, period: period_proc }
end
def self.rate_limiting_response_text
(settings.rate_limiting_response_text.presence || DEFAULT_RATE_LIMITING_RESPONSE_TEXT) + "\n"
end
......
......@@ -4769,6 +4769,12 @@ msgstr ""
msgid "Authenticated API requests"
msgstr ""
msgid "Authenticated Git LFS rate limit period in seconds"
msgstr ""
msgid "Authenticated Git LFS request rate limit"
msgstr ""
msgid "Authenticated web rate limit period in seconds"
msgstr ""
......@@ -8506,6 +8512,9 @@ msgstr ""
msgid "Configure settings for Advanced Search with Elasticsearch."
msgstr ""
msgid "Configure specific limits for Git LFS requests that supersede the general user and IP rate limits."
msgstr ""
msgid "Configure specific limits for Packages API requests that supersede the general user and IP rate limits."
msgstr ""
......@@ -12413,6 +12422,9 @@ msgstr ""
msgid "Enable authenticated API request rate limit"
msgstr ""
msgid "Enable authenticated Git LFS request rate limit"
msgstr ""
msgid "Enable authentication"
msgstr ""
......@@ -15203,6 +15215,9 @@ msgstr ""
msgid "Git GC period"
msgstr ""
msgid "Git LFS Rate Limits"
msgstr ""
msgid "Git LFS is not enabled on this GitLab server, contact your admin."
msgstr ""
......@@ -16526,6 +16541,9 @@ msgstr ""
msgid "Helps reduce request volume (e.g. from crawlers or abusive bots)"
msgstr ""
msgid "Helps reduce request volume (for example, from crawlers or abusive bots)"
msgstr ""
msgid "Helps reduce request volume for protected paths"
msgstr ""
......@@ -20604,6 +20622,9 @@ msgstr ""
msgid "Max authenticated API requests per period per user"
msgstr ""
msgid "Max authenticated Git LFS requests per period per user"
msgstr ""
msgid "Max authenticated web requests per period per user"
msgstr ""
......
......@@ -468,6 +468,7 @@ RSpec.describe Gitlab::PathRegex do
end
let_it_be(:git_paths) { container_paths.map { |path| path + '.git' } }
let_it_be(:git_lfs_paths) { git_paths.flat_map { |path| [path + '/info/lfs/', path + '/gitlab-lfs/'] } }
let_it_be(:snippet_paths) { container_paths.grep(%r{snippets/\d}) }
let_it_be(:wiki_git_paths) { (container_paths - snippet_paths).map { |path| path + '.wiki.git' } }
let_it_be(:invalid_git_paths) { invalid_paths.map { |path| path + '.git' } }
......@@ -498,6 +499,15 @@ RSpec.describe Gitlab::PathRegex do
end
end
describe '.repository_git_lfs_route_regex' do
subject { %r{\A#{described_class.repository_git_lfs_route_regex}\z} }
it 'matches the expected paths' do
expect_route_match(git_lfs_paths)
expect_no_route_match(container_paths + invalid_paths + git_paths + invalid_git_paths)
end
end
describe '.repository_wiki_git_route_regex' do
subject { %r{\A#{described_class.repository_wiki_git_route_regex}\z} }
......
......@@ -939,6 +939,8 @@ RSpec.describe ApplicationSetting do
throttle_unauthenticated_files_api_period_in_seconds
throttle_authenticated_files_api_requests_per_period
throttle_authenticated_files_api_period_in_seconds
throttle_authenticated_git_lfs_requests_per_period
throttle_authenticated_git_lfs_period_in_seconds
]
end
......
......@@ -22,7 +22,9 @@ RSpec.describe 'Rack Attack global throttles', :use_clean_rails_memory_store_cac
throttle_unauthenticated_packages_api_requests_per_period: 100,
throttle_unauthenticated_packages_api_period_in_seconds: 1,
throttle_authenticated_packages_api_requests_per_period: 100,
throttle_authenticated_packages_api_period_in_seconds: 1
throttle_authenticated_packages_api_period_in_seconds: 1,
throttle_authenticated_git_lfs_requests_per_period: 100,
throttle_authenticated_git_lfs_period_in_seconds: 1
}
end
......@@ -620,6 +622,95 @@ RSpec.describe 'Rack Attack global throttles', :use_clean_rails_memory_store_cac
end
end
describe 'authenticated git lfs requests', :api do
let_it_be(:project) { create(:project, :internal) }
let_it_be(:user) { create(:user) }
let_it_be(:token) { create(:personal_access_token, user: user) }
let_it_be(:other_user) { create(:user) }
let_it_be(:other_user_token) { create(:personal_access_token, user: other_user) }
let(:request_method) { 'GET' }
let(:throttle_setting_prefix) { 'throttle_authenticated_git_lfs' }
let(:git_lfs_url) { "/#{project.full_path}.git/info/lfs/locks" }
before do
allow(Gitlab.config.lfs).to receive(:enabled).and_return(true)
stub_application_setting(settings_to_set)
end
context 'with regular login' do
let(:url_that_requires_authentication) { git_lfs_url }
it_behaves_like 'rate-limited web authenticated requests'
end
context 'with the token in the headers' do
let(:request_args) { [git_lfs_url, { headers: basic_auth_headers(user, token) }] }
let(:other_user_request_args) { [git_lfs_url, { headers: basic_auth_headers(other_user, other_user_token) }] }
it_behaves_like 'rate-limited token-authenticated requests'
end
context 'precedence over authenticated web throttle' do
before do
settings_to_set[:throttle_authenticated_git_lfs_requests_per_period] = requests_per_period
settings_to_set[:throttle_authenticated_git_lfs_period_in_seconds] = period_in_seconds
end
def do_request
get git_lfs_url, headers: basic_auth_headers(user, token)
end
context 'when authenticated git lfs throttle is enabled' do
before do
settings_to_set[:throttle_authenticated_git_lfs_enabled] = true
end
context 'when authenticated web throttle is lower' do
before do
settings_to_set[:throttle_authenticated_web_requests_per_period] = 0
settings_to_set[:throttle_authenticated_web_period_in_seconds] = period_in_seconds
settings_to_set[:throttle_authenticated_web_enabled] = true
stub_application_setting(settings_to_set)
end
it 'ignores authenticated web throttle' do
requests_per_period.times do
do_request
expect(response).to have_gitlab_http_status(:ok)
end
expect_rejection { do_request }
end
end
end
context 'when authenticated git lfs throttle is disabled' do
before do
settings_to_set[:throttle_authenticated_git_lfs_enabled] = false
end
context 'when authenticated web throttle is enabled' do
before do
settings_to_set[:throttle_authenticated_web_requests_per_period] = requests_per_period
settings_to_set[:throttle_authenticated_web_period_in_seconds] = period_in_seconds
settings_to_set[:throttle_authenticated_web_enabled] = true
stub_application_setting(settings_to_set)
end
it 'rejects requests over the authenticated web rate limit' do
requests_per_period.times do
do_request
expect(response).to have_gitlab_http_status(:ok)
end
expect_rejection { do_request }
end
end
end
end
end
describe 'throttle bypass header' do
let(:headers) { {} }
let(:bypass_header) { 'gitlab-bypass-rate-limiting' }
......
......@@ -388,6 +388,26 @@ RSpec.describe ApplicationSettings::UpdateService do
end
end
context 'when git lfs rate limits are passed' do
let(:params) do
{
throttle_authenticated_git_lfs_enabled: 1,
throttle_authenticated_git_lfs_period_in_seconds: 600,
throttle_authenticated_git_lfs_requests_per_period: 10
}
end
it 'updates git lfs throttle settings' do
subject.execute
application_settings.reload
expect(application_settings.throttle_authenticated_git_lfs_enabled).to be_truthy
expect(application_settings.throttle_authenticated_git_lfs_period_in_seconds).to eq(600)
expect(application_settings.throttle_authenticated_git_lfs_requests_per_period).to eq(10)
end
end
context 'when issues_create_limit is passed' do
let(:params) do
{
......
# frozen_string_literal: true
#
# Requires let variables:
# * throttle_setting_prefix: "throttle_authenticated_api", "throttle_authenticated_web", "throttle_protected_paths", "throttle_authenticated_packages_api"
# * throttle_setting_prefix: "throttle_authenticated_api", "throttle_authenticated_web", "throttle_protected_paths", "throttle_authenticated_packages_api", "throttle_authenticated_git_lfs"
# * request_method
# * request_args
# * other_user_request_args
......@@ -14,7 +14,8 @@ RSpec.shared_examples 'rate-limited token-authenticated requests' do
"throttle_protected_paths" => "throttle_authenticated_protected_paths_api",
"throttle_authenticated_api" => "throttle_authenticated_api",
"throttle_authenticated_web" => "throttle_authenticated_web",
"throttle_authenticated_packages_api" => "throttle_authenticated_packages_api"
"throttle_authenticated_packages_api" => "throttle_authenticated_packages_api",
"throttle_authenticated_git_lfs" => "throttle_authenticated_git_lfs"
}
end
......@@ -165,7 +166,7 @@ RSpec.shared_examples 'rate-limited token-authenticated requests' do
end
# Requires let variables:
# * throttle_setting_prefix: "throttle_authenticated_web" or "throttle_protected_paths"
# * throttle_setting_prefix: "throttle_authenticated_web", "throttle_protected_paths", "throttle_authenticated_git_lfs"
# * user
# * url_that_requires_authentication
# * request_method
......@@ -176,7 +177,8 @@ RSpec.shared_examples 'rate-limited web authenticated requests' do
let(:throttle_types) do
{
"throttle_protected_paths" => "throttle_authenticated_protected_paths_web",
"throttle_authenticated_web" => "throttle_authenticated_web"
"throttle_authenticated_web" => "throttle_authenticated_web",
"throttle_authenticated_git_lfs" => "throttle_authenticated_git_lfs"
}
end
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment