Merge branch 'qmnguyen0711/731-add-ratelimit-headers-to-rackattack-responses' into 'master'

Add RateLimit-* headers to RackAttack responses

See merge request gitlab-org/gitlab!50833

app/models/application_setting.rb

@@ -404,6 +404,38 @@ class ApplicationSetting < ApplicationRecord
             length: { maximum: 255, message: _('is too long (maximum is %{count} characters)') },
             allow_blank: true
 
+  validates :throttle_unauthenticated_requests_per_period,
+            presence: true,
+            numericality: { only_integer: true, greater_than: 0 }
+
+  validates :throttle_unauthenticated_period_in_seconds,
+            presence: true,
+            numericality: { only_integer: true, greater_than: 0 }
+
+  validates :throttle_authenticated_api_requests_per_period,
+            presence: true,
+            numericality: { only_integer: true, greater_than: 0 }
+
+  validates :throttle_authenticated_api_period_in_seconds,
+            presence: true,
+            numericality: { only_integer: true, greater_than: 0 }
+
+  validates :throttle_authenticated_web_requests_per_period,
+            presence: true,
+            numericality: { only_integer: true, greater_than: 0 }
+
+  validates :throttle_authenticated_web_period_in_seconds,
+            presence: true,
+            numericality: { only_integer: true, greater_than: 0 }
+
+  validates :throttle_protected_paths_requests_per_period,
+            presence: true,
+            numericality: { only_integer: true, greater_than: 0 }
+
+  validates :throttle_protected_paths_period_in_seconds,
+            presence: true,
+            numericality: { only_integer: true, greater_than: 0 }
+
   attr_encrypted :asset_proxy_secret_key,
                  mode: :per_attribute_iv,
                  key: Settings.attr_encrypted_db_key_base_truncated,

changelogs/unreleased/qmnguyen0711-731-add-ratelimit-headers-to-rackattack-responses.yml

+---
+title: Add RateLimit-* headers to RackAttack responses
+merge_request: 50833
+author: qmnguyen0711
+type: fixed

doc/security/rack_attack.md

@@ -54,11 +54,7 @@ By default, protected paths are:
 - `/import/github/personal_access_token`
 - `/admin/session`
 
-This header is included in responses to blocked requests:
-
-```plaintext
-Retry-After: 60
-```
+See [User and IP rate limits](../user/admin_area/settings/user_and_ip_rate_limits.md#response-headers) for the headers responded to blocked requests.
 
 For example, the following are limited to a maximum 10 requests per minute:
 

doc/user/admin_area/settings/protected_paths.md

@@ -28,11 +28,7 @@ GitLab rate limits the following paths with Rack Attack by default:
 GitLab responds with HTTP status code `429` to POST requests at protected paths
 that exceed 10 requests per minute per IP address.
 
-This header is included in responses to blocked requests:
-
-```plaintext
-Retry-After: 60
-```
+See [User and IP rate limits](../../admin_area/settings/user_and_ip_rate_limits.md#response-headers) for the headers responded to blocked requests.
 
 For example, the following are limited to a maximum 10 requests per minute:
 

doc/user/admin_area/settings/user_and_ip_rate_limits.md

@@ -36,6 +36,25 @@ Retry later
 
 It is possible to customize this response text in the admin area.
 
+## Response headers
+
+> [Introduced](https://gitlab.com/gitlab-com/gl-infra/scalability/-/issues/731) in GitLab 13.8, the `Rate-Limit` headers. `Retry-After` was introduced in an earlier version.
+
+When a client exceeds the associated rate limit, the following requests are
+blocked. The server may respond with rate-limiting information allowing the
+requester to retry after a specific period of time. These information are
+attached into the response headers.
+
+| Header                | Example                         | Description                                                                                                                                                                                                             |
+|:----------------------|:--------------------------------|:-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
+| `RateLimit-Limit`     | `60`                            | The request quota for the client **each minute**. If the rate limit period set in the admin area is different from 1 minute, the value of this header is adjusted to approximately the nearest 60-minute period. |
+| `RateLimit-Name`      | `throttle_authenticated_web`    | Name of the throttle blocking the requests.                                                                                                                                                                          |
+| `RateLimit-Observed`  | `67`                            | Number of requests associated to the client in the time window.                                                                                                                                       |
+| `RateLimit-Remaining` | `0`                             | Remaining quota in the time window. The result of `RateLimit-Limit` - `RateLimit-Remaining`.                                                                                                             |
+| `RateLimit-Reset`     | `30`                            | An alias of `Retry-After` header.                                                                                                                                                                                        |
+| `RateLimit-ResetTime` | `Tue, 05 Jan 2021 11:00:00 GMT` | [RFC2616](https://tools.ietf.org/html/rfc2616#section-3.3.1)-formatted date and time when the request quota is reset.                                                     |
+| `Retry-After`         | `30`                            | Remaining duration **in seconds** until the quota is reset. This is a [standard HTTP header](https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Retry-After).                                    |
+
 ## Use an HTTP header to bypass rate limiting
 
 > [Introduced](https://gitlab.com/gitlab-com/gl-infra/scalability/-/issues/622) in GitLab 13.6.

doc/user/gitlab_com/index.md

@@ -532,13 +532,10 @@ endpoints](../../user/admin_area/settings/rate_limits_on_raw_endpoints.md).
 
 ### Rate limiting responses
 
-The [`Retry-After`
-header](https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Retry-After)
-indicates when the client should retry.
+For information on rate limiting responses, see:
 
-Rate limits applied by HAProxy (instead of Cloudflare or the
-GitLab application) have `RateLimit-Reset` and `RateLimit-ResetTime`
-headers.
+- [List of headers on responses to blocked requests](../admin_area/settings/user_and_ip_rate_limits.md#response-headers).
+- [Customizable response text](../admin_area/settings/user_and_ip_rate_limits.md#response-text).
 
 ### Protected paths throttle
 
@@ -548,11 +545,7 @@ paths that exceed 10 requests per **minute** per IP address.
 See the source below for which paths are protected. This includes user creation,
 user confirmation, user sign in, and password reset.
 
-This header is included in responses to blocked requests:
-
-```plaintext
-Retry-After: 60
-```
+[User and IP rate limits](../admin_area/settings/user_and_ip_rate_limits.md#response-headers) includes a list of the headers responded to blocked requests.
 
 See [Protected Paths](../admin_area/settings/protected_paths.md) for more details.
 

lib/gitlab/rack_attack.rb

@@ -13,12 +13,10 @@ module Gitlab
 
       # This is Rack::Attack::DEFAULT_THROTTLED_RESPONSE, modified to allow a custom response
       Rack::Attack.throttled_response = lambda do |env|
-        # Send the Retry-After header so clients (e.g. python-gitlab) can make good choices about delays
-        match_data = env['rack.attack.match_data']
-        now = match_data[:epoch_time]
-        retry_after = match_data[:period] - (now % match_data[:period])
-
-        [429, { 'Content-Type' => 'text/plain', 'Retry-After' => retry_after.to_s }, [Gitlab::Throttle.rate_limiting_response_text]]
+        throttled_headers = Gitlab::RackAttack.throttled_response_headers(
+          env['rack.attack.matched'], env['rack.attack.match_data']
+        )
+        [429, { 'Content-Type' => 'text/plain' }.merge(throttled_headers), [Gitlab::Throttle.rate_limiting_response_text]]
       end
 
       # Configure the throttles
@@ -27,6 +25,55 @@ module Gitlab
       configure_user_allowlist
     end
 
+    # Rate Limit HTTP headers are not standardized anywhere. This is the latest
+    # draft submitted to IETF:
+    # https://github.com/ietf-wg-httpapi/ratelimit-headers/blob/main/draft-ietf-httpapi-ratelimit-headers.md
+    #
+    # This method implement the most viable parts of the headers. Those headers
+    # will be sent back to the client when it gets throttled.
+    #
+    # - RateLimit-Limit: indicates the request quota associated to the client
+    # in 60 seconds. The time window for the quota here is supposed to be
+    # mirrored to throttle_*_period_in_seconds application settings.  However,
+    # our HAProxy as well as some ecosystem libraries are using a fixed
+    # 60-second window. Therefore, the returned limit is approximately rounded
+    # up to fit into that window.
+    #
+    # - RateLimit-Observed: indicates the current request amount associated to
+    # the client within the time window.
+    #
+    # - RateLimit-Remaining: indicates the remaining quota within the time
+    # window. It is the result of RateLimit-Limit - RateLimit-Remaining
+    #
+    # - Retry-After: the remaining duration in seconds until the quota is
+    # reset. This is a standardized HTTP header:
+    # https://tools.ietf.org/html/rfc7231#page-69
+    #
+    # - RateLimit-Reset: Similar to Retry-After.
+    #
+    # - RateLimit-ResetTime: the point of time that the quest quota is reset.
+    def self.throttled_response_headers(matched, match_data)
+      # Match data example:
+      # {:discriminator=>"127.0.0.1", :count=>12, :period=>60 seconds, :limit=>1, :epoch_time=>1609833930}
+      # Source: https://github.com/rack/rack-attack/blob/v6.3.0/lib/rack/attack/throttle.rb#L33
+      period = match_data[:period]
+      limit = match_data[:limit]
+      rounded_limit = (limit.to_f * 1.minute / match_data[:period]).ceil
+      observed = match_data[:count]
+      now = match_data[:epoch_time]
+      retry_after = period - (now % period)
+      reset_time = now + (period - now % period)
+      {
+        'RateLimit-Name' => matched.to_s,
+        'RateLimit-Limit' => rounded_limit.to_s,
+        'RateLimit-Observed' => observed.to_s,
+        'RateLimit-Remaining' => (limit > observed ? limit - observed : 0).to_s,
+        'RateLimit-Reset' => retry_after.to_s,
+        'RateLimit-ResetTime' => Time.at(reset_time).httpdate,
+        'Retry-After' => retry_after.to_s
+      }
+    end
+
     def self.configure_user_allowlist
       @user_allowlist = nil
       user_allowlist

spec/lib/gitlab/rack_attack_spec.rb

@@ -98,4 +98,207 @@ RSpec.describe Gitlab::RackAttack, :aggregate_failures do
       end
     end
   end
+
+  describe '.throttled_response_headers' do
+    where(:matched, :match_data, :headers) do
+      [
+        [
+          'throttle_unauthenticated',
+          {
+            discriminator: '127.0.0.1',
+            count: 3700,
+            period: 1.hour,
+            limit: 3600,
+            epoch_time: Time.utc(2021, 1, 5, 10, 29, 30).to_i
+          },
+          {
+            'RateLimit-Name' => 'throttle_unauthenticated',
+            'RateLimit-Limit' => '60',
+            'RateLimit-Observed' => '3700',
+            'RateLimit-Remaining' => '0',
+            'RateLimit-Reset' => '1830',
+            'RateLimit-ResetTime' => 'Tue, 05 Jan 2021 11:00:00 GMT',
+            'Retry-After' => '1830'
+          }
+        ],
+        [
+          'throttle_unauthenticated',
+          {
+            discriminator: '127.0.0.1',
+            count: 3700,
+            period: 1.hour,
+            limit: 3600,
+            epoch_time: Time.utc(2021, 1, 5, 10, 59, 59).to_i
+          },
+          {
+            'RateLimit-Name' => 'throttle_unauthenticated',
+            'RateLimit-Limit' => '60',
+            'RateLimit-Observed' => '3700',
+            'RateLimit-Remaining' => '0',
+            'RateLimit-Reset' => '1',
+            'RateLimit-ResetTime' => 'Tue, 05 Jan 2021 11:00:00 GMT',
+            'Retry-After' => '1'
+          }
+        ],
+        [
+          'throttle_unauthenticated',
+          {
+            discriminator: '127.0.0.1',
+            count: 3700,
+            period: 1.hour,
+            limit: 3600,
+            epoch_time: Time.utc(2021, 1, 5, 10, 0, 0).to_i
+          },
+          {
+            'RateLimit-Name' => 'throttle_unauthenticated',
+            'RateLimit-Limit' => '60',
+            'RateLimit-Observed' => '3700',
+            'RateLimit-Remaining' => '0',
+            'RateLimit-Reset' => '3600',
+            'RateLimit-ResetTime' => 'Tue, 05 Jan 2021 11:00:00 GMT',
+            'Retry-After' => '3600'
+          }
+        ],
+        [
+          'throttle_unauthenticated',
+          {
+            discriminator: '127.0.0.1',
+            count: 3700,
+            period: 1.hour,
+            limit: 3600,
+            epoch_time: Time.utc(2021, 1, 5, 23, 30, 0).to_i
+          },
+          {
+            'RateLimit-Name' => 'throttle_unauthenticated',
+            'RateLimit-Limit' => '60',
+            'RateLimit-Observed' => '3700',
+            'RateLimit-Remaining' => '0',
+            'RateLimit-Reset' => '1800',
+            'RateLimit-ResetTime' => 'Wed, 06 Jan 2021 00:00:00 GMT', # Next day
+            'Retry-After' => '1800'
+          }
+        ],
+        [
+          'throttle_unauthenticated',
+          {
+            discriminator: '127.0.0.1',
+            count: 3700,
+            period: 1.hour,
+            limit: 3400,
+            epoch_time: Time.utc(2021, 1, 5, 10, 30, 0).to_i
+          },
+          {
+            'RateLimit-Name' => 'throttle_unauthenticated',
+            'RateLimit-Limit' => '57', # 56.66 requests per minute
+            'RateLimit-Observed' => '3700',
+            'RateLimit-Remaining' => '0',
+            'RateLimit-Reset' => '1800',
+            'RateLimit-ResetTime' => 'Tue, 05 Jan 2021 11:00:00 GMT',
+            'Retry-After' => '1800'
+          }
+        ],
+        [
+          'throttle_unauthenticated',
+          {
+            discriminator: '127.0.0.1',
+            count: 3700,
+            period: 1.hour,
+            limit: 3700,
+            epoch_time: Time.utc(2021, 1, 5, 10, 30, 0).to_i
+          },
+          {
+            'RateLimit-Name' => 'throttle_unauthenticated',
+            'RateLimit-Limit' => '62', # 61.66 requests per minute
+            'RateLimit-Observed' => '3700',
+            'RateLimit-Remaining' => '0',
+            'RateLimit-Reset' => '1800',
+            'RateLimit-ResetTime' => 'Tue, 05 Jan 2021 11:00:00 GMT',
+            'Retry-After' => '1800'
+          }
+        ],
+        [
+          'throttle_unauthenticated',
+          {
+            discriminator: '127.0.0.1',
+            count: 3700,
+            period: 1.hour,
+            limit: 59,
+            epoch_time: Time.utc(2021, 1, 5, 10, 30, 0).to_i
+          },
+          {
+            'RateLimit-Name' => 'throttle_unauthenticated',
+            'RateLimit-Limit' => '1', # 0.9833 requests per minute
+            'RateLimit-Observed' => '3700',
+            'RateLimit-Remaining' => '0',
+            'RateLimit-Reset' => '1800',
+            'RateLimit-ResetTime' => 'Tue, 05 Jan 2021 11:00:00 GMT',
+            'Retry-After' => '1800'
+          }
+        ],
+        [
+          'throttle_unauthenticated',
+          {
+            discriminator: '127.0.0.1',
+            count: 3700,
+            period: 1.hour,
+            limit: 61,
+            epoch_time: Time.utc(2021, 1, 5, 10, 30, 0).to_i
+          },
+          {
+            'RateLimit-Name' => 'throttle_unauthenticated',
+            'RateLimit-Limit' => '2', # 1.016 requests per minute
+            'RateLimit-Observed' => '3700',
+            'RateLimit-Remaining' => '0',
+            'RateLimit-Reset' => '1800',
+            'RateLimit-ResetTime' => 'Tue, 05 Jan 2021 11:00:00 GMT',
+            'Retry-After' => '1800'
+          }
+        ],
+        [
+          'throttle_unauthenticated',
+          {
+            discriminator: '127.0.0.1',
+            count: 3700,
+            period: 15.seconds,
+            limit: 10,
+            epoch_time: Time.utc(2021, 1, 5, 10, 30, 0).to_i
+          },
+          {
+            'RateLimit-Name' => 'throttle_unauthenticated',
+            'RateLimit-Limit' => '40',
+            'RateLimit-Observed' => '3700',
+            'RateLimit-Remaining' => '0',
+            'RateLimit-Reset' => '15',
+            'RateLimit-ResetTime' => 'Tue, 05 Jan 2021 10:30:15 GMT',
+            'Retry-After' => '15'
+          }
+        ],
+        [
+          'throttle_unauthenticated',
+          {
+            discriminator: '127.0.0.1',
+            count: 3700,
+            period: 27.seconds,
+            limit: 10,
+            epoch_time: Time.utc(2021, 1, 5, 10, 30, 0).to_i
+          },
+          {
+            'RateLimit-Name' => 'throttle_unauthenticated',
+            'RateLimit-Limit' => '23',
+            'RateLimit-Observed' => '3700',
+            'RateLimit-Remaining' => '0',
+            'RateLimit-Reset' => '27',
+            'RateLimit-ResetTime' => 'Tue, 05 Jan 2021 10:30:27 GMT',
+            'Retry-After' => '27'
+          }
+        ]
+      ]
+    end
+
+    with_them do
+      it 'generates accurate throttled headers' do
+        expect(described_class.throttled_response_headers(matched, match_data)).to eql(headers)
+      end
+    end
+  end
 end

spec/models/application_setting_spec.rb

@@ -733,6 +733,27 @@ RSpec.describe ApplicationSetting do
         is_expected.to be_invalid
       end
     end
+
+    context 'throttle_* settings' do
+      where(:throttle_setting) do
+        %i[
+          throttle_unauthenticated_requests_per_period
+          throttle_unauthenticated_period_in_seconds
+          throttle_authenticated_api_requests_per_period
+          throttle_authenticated_api_period_in_seconds
+          throttle_authenticated_web_requests_per_period
+          throttle_authenticated_web_period_in_seconds
+        ]
+      end
+
+      with_them do
+        it { is_expected.to allow_value(3).for(throttle_setting) }
+        it { is_expected.not_to allow_value(-3).for(throttle_setting) }
+        it { is_expected.not_to allow_value(0).for(throttle_setting) }
+        it { is_expected.not_to allow_value('three').for(throttle_setting) }
+        it { is_expected.not_to allow_value(nil).for(throttle_setting) }
+      end
+    end
   end
 
   context 'restrict creating duplicates' do

spec/support/helpers/rack_attack_spec_helpers.rb

@@ -30,7 +30,19 @@ module RackAttackSpecHelpers
     yield
 
     expect(response).to have_gitlab_http_status(:too_many_requests)
-    expect(response).to have_header('Retry-After')
+
+    expect(response.headers.to_h).to include(
+      'RateLimit-Limit' => a_string_matching(/^\d+$/),
+      'RateLimit-Name' => a_string_matching(/^throttle_.*$/),
+      'RateLimit-Observed' => a_string_matching(/^\d+$/),
+      'RateLimit-Remaining' => a_string_matching(/^\d+$/),
+      'RateLimit-Reset' => a_string_matching(/^\d+$/),
+      'Retry-After' => a_string_matching(/^\d+$/)
+    )
+    expect(response).to have_header('RateLimit-ResetTime')
+    expect do
+      Time.httpdate(response.headers['RateLimit-ResetTime'])
+    end.not_to raise_error
   end
 
   def expect_ok(&block)