Merge branch 'add-rate-limiting-redis-instance' into 'master'

Add Gitlab::Redis::RateLimiting Redis instance

See merge request gitlab-org/gitlab!70414

.gitignore

@@ -40,10 +40,7 @@ eslint-report.html
 /config/initializers/smtp_settings.rb
 /config/initializers/relative_url.rb
 /config/resque.yml
-/config/redis.cache.yml
-/config/redis.queues.yml
-/config/redis.shared_state.yml
-/config/redis.trace_chunks.yml
+/config/redis.*.yml
 /config/unicorn.rb
 /config/puma.rb
 /config/secrets.yml

app/controllers/admin/dashboard_controller.rb

@@ -15,7 +15,13 @@ class Admin::DashboardController < Admin::ApplicationController
     @groups = Group.order_id_desc.with_route.limit(10)
     @notices = Gitlab::ConfigChecker::PumaRuggedChecker.check
     @notices += Gitlab::ConfigChecker::ExternalDatabaseChecker.check
-    @redis_versions = [Gitlab::Redis::Queues, Gitlab::Redis::SharedState, Gitlab::Redis::Cache, Gitlab::Redis::TraceChunks].map(&:version).uniq
+    @redis_versions = [
+      Gitlab::Redis::Queues,
+      Gitlab::Redis::SharedState,
+      Gitlab::Redis::Cache,
+      Gitlab::Redis::TraceChunks,
+      Gitlab::Redis::RateLimiting
+    ].map(&:version).uniq
   end
   # rubocop: enable CodeReuse/ActiveRecord
 

app/controllers/health_controller.rb

@@ -16,6 +16,7 @@ class HealthController < ActionController::Base
     Gitlab::HealthChecks::Redis::QueuesCheck,
     Gitlab::HealthChecks::Redis::SharedStateCheck,
     Gitlab::HealthChecks::Redis::TraceChunksCheck,
+    Gitlab::HealthChecks::Redis::RateLimitingCheck,
     Gitlab::HealthChecks::GitalyCheck
   ].freeze
 

config/README.md

 # Configuration files Documentation
 
 Note that most configuration files (`config/*.*`) committed into
-[gitlab-ce](https://gitlab.com/gitlab-org/gitlab-foss) **will not be used** for
+[gitlab-foss](https://gitlab.com/gitlab-org/gitlab-foss) **will not be used** for
 [omnibus-gitlab](https://gitlab.com/gitlab-org/omnibus-gitlab). Configuration
-files committed into gitlab-ce are only used for development.
+files committed into gitlab-foss are only used for development.
 
 ## gitlab.yml
 
-You can find most of GitLab configuration settings here.
+You can find most of the GitLab configuration settings here.
 
 ## mail_room.yml
 
@@ -21,7 +21,7 @@ This file is called `resque.yml` for historical reasons. We are **NOT**
 using Resque at the moment. It is used to specify Redis configuration
 values when a single database instance of Redis is desired.
 
-# Advanced Redis configuration files
+## Advanced Redis configuration files
 
 In more advanced configurations of Redis key-value storage, it is desirable
 to separate the keys by lifecycle and intended use to ease provisioning and
@@ -40,7 +40,7 @@ If desired, the routing URL provided by these settings can be used with:
     2. TCP port number for each Redis instance desired
     3. `database number` for each Redis instance desired
 
-## Example URL attribute formats for GitLab Redis `.yml` configuration files
+### Example URL attribute formats for GitLab Redis `.yml` configuration files
 * Unix Socket, default Redis database (0)
     * `url: unix:/path/to/redis.sock`
     * `url: unix:/path/to/redis.sock?db=`
@@ -52,129 +52,37 @@ If desired, the routing URL provided by these settings can be used with:
 * TCP Socket for Redis on remote host `myserver`, port 6379, database 33
     * `url: redis://:mynewpassword@myserver:6379/33`
 
-## redis.cache.yml
-
-If configured, `redis.cache.yml` overrides the
-`resque.yml` settings to configure the Redis database instance
-used for `Rails.cache` and other volatile non-persistent data which enhances
-the performance of GitLab.
-Settings here can be overridden by the environment variable
-`GITLAB_REDIS_CACHE_CONFIG_FILE` which provides
-an alternate location for configuration settings.
-
-The order of precedence for the URL used to connect to the Redis instance
-used for `cache` is:
-1. URL from a configuration file pointed to by the
-`GITLAB_REDIS_CACHE_CONFIG_FILE` environment variable
-2. URL from `redis.cache.yml`
-3. URL from a configuration file pointed to by the
-`GITLAB_REDIS_CONFIG_FILE` environment variable
-4. URL from `resque.yml`
-5. `redis://localhost:6380`
-
-The order of precedence for all other configuration settings for `cache`
-are selected from only the first of the following files found (if a setting
-is not provided in an earlier file, the remainder of the files are not
-searched):
-1. the configuration file pointed to by the
-`GITLAB_REDIS_CACHE_CONFIG_FILE` environment variable
-2. the configuration file `redis.cache.yml`
-3. the configuration file pointed to by the
-`GITLAB_REDIS_CONFIG_FILE` environment variable
-4. the configuration file `resque.yml`
-
-## redis.queues.yml
-
-If configured, `redis.queues.yml` overrides the
-`resque.yml` settings to configure the Redis database instance
-used for clients of `::Gitlab::Redis::Queues`.
-These queues are intended to be the foundation
-of reliable inter-process communication between modules, whether on the same
-host node, or within a cluster.   The primary clients of the queues are
-SideKiq, Mailroom, CI Runner, Workhorse, and push services.  Settings here can
-be overridden by the environment variable
-`GITLAB_REDIS_QUEUES_CONFIG_FILE` which provides an alternate location for
-configuration settings.
-
-The order of precedence for the URL used to connect to the Redis instance
-used for `queues` is:
-1. URL from a configuration file pointed to by the
-`GITLAB_REDIS_QUEUES_CONFIG_FILE` environment variable
-2. URL from `redis.queues.yml`
-3. URL from a configuration file pointed to by the
-`GITLAB_REDIS_CONFIG_FILE` environment variable
-4. URL from `resque.yml`
-5. `redis://localhost:6381`
-
-The order of precedence for all other configuration settings for `queues`
-are selected from only the first of the following files found (if a setting
-is not provided in an earlier file, the remainder of the files are not
-searched):
-1. the configuration file pointed to by the
-`GITLAB_REDIS_QUEUES_CONFIG_FILE` environment variable
-2. the configuration file `redis.queues.yml`
-3. the configuration file pointed to by the
-`GITLAB_REDIS_CONFIG_FILE` environment variable
-4. the configuration file `resque.yml`
-
-## redis.shared_state.yml
-
-If configured, `redis.shared_state.yml` overrides the
-`resque.yml` settings to configure the Redis database instance
-used for clients of `::Gitlab::Redis::SharedState` such as session state,
-and rate limiting.
-Settings here can be overridden by the environment variable
-`GITLAB_REDIS_SHARED_STATE_CONFIG_FILE` which provides
-an alternate location for configuration settings.
-
-The order of precedence for the URL used to connect to the Redis instance
-used for `shared_state` is:
-1. URL from a configuration file pointed to by the
-`GITLAB_REDIS_SHARED_STATE_CONFIG_FILE` environment variable
-2. URL from `redis.shared_state.yml`
-3. URL from a configuration file pointed to by the
-`GITLAB_REDIS_CONFIG_FILE` environment variable
-4. URL from `resque.yml`
-5. `redis://localhost:6382`
-
-The order of precedence for all other configuration settings for `shared_state`
-are selected from only the first of the following files found (if a setting
-is not provided in an earlier file, the remainder of the files are not
-searched):
-1. the configuration file pointed to by the
-`GITLAB_REDIS_SHARED_STATE_CONFIG_FILE` environment variable
-2. the configuration file `redis.shared_state.yml`
-3. the configuration file pointed to by the
-`GITLAB_REDIS_CONFIG_FILE` environment variable
-4. the configuration file `resque.yml`
-
-## redis.trace_chunks.yml
-
-If configured, `redis.trace_chunks.yml` overrides the
-`resque.yml` settings to configure the Redis database instance
-used for clients of `::Gitlab::Redis::TraceChunks` which stores CI trace chunks.
-
-Settings here can be overridden by the environment variable
-`GITLAB_REDIS_TRACE_CHUNKS_CONFIG_FILE` which provides
-an alternate location for configuration settings.
-
-The order of precedence for the URL used to connect to the Redis instance
-used for `trace_chunks` is:
-1. URL from a configuration file pointed to by the
-`GITLAB_REDIS_TRACE_CHUNKS_CONFIG_FILE` environment variable
-2. URL from `redis.trace_chunks.yml`
-3. URL from a configuration file pointed to by the
-`GITLAB_REDIS_CONFIG_FILE` environment variable
-4. URL from `resque.yml`
-5. `redis://localhost:6383`
-
-The order of precedence for all other configuration settings for `trace_chunks`
-are selected from only the first of the following files found (if a setting
-is not provided in an earlier file, the remainder of the files are not
-searched):
-1. the configuration file pointed to by the
-`GITLAB_REDIS_TRACE_CHUNKS_CONFIG_FILE` environment variable
-2. the configuration file `redis.trace_chunks.yml`
-3. the configuration file pointed to by the
-`GITLAB_REDIS_CONFIG_FILE` environment variable
-4. the configuration file `resque.yml`
+## Available configuration files
+
+The Redis instances that can be configured are described in the table below. The
+order of precedence for configuration is described below, where `$NAME` and
+`$FALLBACK_NAME` are the upper-cased instance names from the table, and `$name`
+and `$fallback_name` are the lower-cased versions:
+
+1. The configuration file pointed to by the `GITLAB_REDIS_$NAME_CONFIG_FILE`
+   environment variable.
+1. The configuration file `redis.$name.yml`.
+1. **If a fallback instance is available**, the configuration file
+   `redis.$fallback_name.yml`.
+1. The configuration file pointed to by the `GITLAB_REDIS_CONFIG_FILE`
+environment variable.
+1. The configuration file `resque.yml`.
+
+An example configuration file for Redis is in this directory under the name
+`resque.yml.example`.
+
+| Name | Fallback instance | Purpose |
+| --- | --- | --- |
+| `cache` | | Volatile non-persistent data |
+| `queues` | | Background job processing queues |
+| `shared_state` | | Persistent application state |
+| `trace_chunks` | `shared_state` | [CI trace chunks](https://docs.gitlab.com/ee/administration/job_logs.html#incremental-logging-architecture) |
+| `rate_limiting` | `cache` | [Rate limiting](https://docs.gitlab.com/ee/user/admin_area/settings/user_and_ip_rate_limits.html) state |
+
+If no configuration is found, or no URL is found in the configuration
+file, the default URL used is:
+
+1. `redis://localhost:6380` for `cache`.
+1. `redis://localhost:6381` for `queues`.
+1. `redis://localhost:6382` for `shared_state`.
+1. The URL from the fallback instance for all other instances.

config/application.rb

@@ -23,6 +23,8 @@ module Gitlab
     require_dependency Rails.root.join('lib/gitlab/redis/cache')
     require_dependency Rails.root.join('lib/gitlab/redis/queues')
     require_dependency Rails.root.join('lib/gitlab/redis/shared_state')
+    require_dependency Rails.root.join('lib/gitlab/redis/trace_chunks')
+    require_dependency Rails.root.join('lib/gitlab/redis/rate_limiting')
     require_dependency Rails.root.join('lib/gitlab/current_settings')
     require_dependency Rails.root.join('lib/gitlab/middleware/read_only')
     require_dependency Rails.root.join('lib/gitlab/middleware/basic_health_check')

config/initializers/7_redis.rb

@@ -11,3 +11,4 @@ Gitlab::Redis::Cache.with { nil }
 Gitlab::Redis::Queues.with { nil }
 Gitlab::Redis::SharedState.with { nil }
 Gitlab::Redis::TraceChunks.with { nil }
+Gitlab::Redis::RateLimiting.with { nil }

config/redis.cache.yml.example

-# If you change this file in a merge request, please also create
-# a merge request on https://gitlab.com/gitlab-org/omnibus-gitlab/merge_requests
-#
-development:
-  url: redis://localhost:6379/10
-  #
-  # url: redis://localhost:6380
-  # sentinels:
-  #   -
-  #     host: localhost
-  #     port: 26380 # point to sentinel, not to redis port
-  #   -
-  #     host: replica2
-  #     port: 26380 # point to sentinel, not to redis port
-test:
-  url: redis://localhost:6379/10
-  #
-  # url: redis://localhost:6380
-production:
-  # Redis (single instance)
-  url: unix:/var/run/redis/redis.cache.sock
-  ##
-  # Redis + Sentinel (for HA)
-  #
-  # Please read instructions carefully before using it as you may lose data:
-  # http://redis.io/topics/sentinel
-  #
-  # You must specify a list of a few sentinels that will handle client connection
-  # please read here for more information: https://docs.gitlab.com/ee/administration/redis/index.html
-  ##
-  # url: redis://master:6380
-  # sentinels:
-  #   -
-  #     host: replica1
-  #     port: 26380 # point to sentinel, not to redis port
-  #   -
-  #     host: replica2
-  #     port: 26380 # point to sentinel, not to redis port

config/redis.queues.yml.example

-# If you change this file in a merge request, please also create
-# a merge request on https://gitlab.com/gitlab-org/omnibus-gitlab/merge_requests
-#
-development:
-  url: redis://localhost:6379/11
-  #
-  # url: redis://localhost:6381
-  # sentinels:
-  #   -
-  #     host: localhost
-  #     port: 26381 # point to sentinel, not to redis port
-  #   -
-  #     host: replica2
-  #     port: 26381 # point to sentinel, not to redis port
-test:
-  url: redis://localhost:6379/11
-  #
-  # url: redis://localhost:6381
-production:
-  # Redis (single instance)
-  url: unix:/var/run/redis/redis.queues.sock
-  ##
-  # Redis + Sentinel (for HA)
-  #
-  # Please read instructions carefully before using it as you may lose data:
-  # http://redis.io/topics/sentinel
-  #
-  # You must specify a list of a few sentinels that will handle client connection
-  # please read here for more information: https://docs.gitlab.com/ee/administration/redis/index.html
-  ##
-  # url: redis://master:6381
-  # sentinels:
-  #   -
-  #     host: replica1
-  #     port: 26381 # point to sentinel, not to redis port
-  #   -
-  #     host: replica2
-  #     port: 26381 # point to sentinel, not to redis port

config/redis.shared_state.yml.example

-# If you change this file in a merge request, please also create
-# a merge request on https://gitlab.com/gitlab-org/omnibus-gitlab/merge_requests
-#
-development:
-  url: redis://localhost:6379/12
-  #
-  # url: redis://localhost:6382
-  # sentinels:
-  #   -
-  #     host: localhost
-  #     port: 26382 # point to sentinel, not to redis port
-  #   -
-  #     host: replica2
-  #     port: 26382 # point to sentinel, not to redis port
-test:
-  url: redis://localhost:6379/12
-  #
-  # url: redis://localhost:6382
-production:
-  # Redis (single instance)
-  url: unix:/var/run/redis/redis.shared_state.sock
-  ##
-  # Redis + Sentinel (for HA)
-  #
-  # Please read instructions carefully before using it as you may lose data:
-  # http://redis.io/topics/sentinel
-  #
-  # You must specify a list of a few sentinels that will handle client connection
-  # please read here for more information: https://docs.gitlab.com/ee/administration/redis/index.html
-  ##
-  # url: redis://master:6382
-  # sentinels:
-  #   -
-  #     host: replica1
-  #     port: 26382 # point to sentinel, not to redis port
-  #   -
-  #     host: replica2
-  #     port: 26382 # point to sentinel, not to redis port

config/redis.trace_chunks.yml.example

-# If you change this file in a merge request, please also create
-# a merge request on https://gitlab.com/gitlab-org/omnibus-gitlab/merge_requests
-#
-development:
-  url: redis://localhost:6379/13
-  #
-  # url: redis://localhost:6382
-  # sentinels:
-  #   -
-  #     host: localhost
-  #     port: 26382 # point to sentinel, not to redis port
-  #   -
-  #     host: replica2
-  #     port: 26382 # point to sentinel, not to redis port
-test:
-  url: redis://localhost:6379/13
-  #
-  # url: redis://localhost:6382
-production:
-  # Redis (single instance)
-  url: unix:/var/run/redis/redis.trace_chunks.sock
-  ##
-  # Redis + Sentinel (for HA)
-  #
-  # Please read instructions carefully before using it as you may lose data:
-  # http://redis.io/topics/sentinel
-  #
-  # You must specify a list of a few sentinels that will handle client connection
-  # please read here for more information: https://docs.gitlab.com/ee/administration/redis/index.html
-  ##
-  # url: redis://master:6382
-  # sentinels:
-  #   -
-  #     host: replica1
-  #     port: 26382 # point to sentinel, not to redis port
-  #   -
-  #     host: replica2
-  #     port: 26382 # point to sentinel, not to redis port

doc/administration/redis/replication_and_failover.md

@@ -647,6 +647,7 @@ persistence classes.
 | `shared_state` | Store session-related and other persistent data. |
 | `actioncable`  | Pub/Sub queue backend for ActionCable.           |
 | `trace_chunks` | Store [CI trace chunks](../job_logs.md#enable-or-disable-incremental-logging) data. |
+| `rate_limiting` | Store [rate limiting](../../user/admin_area/settings/user_and_ip_rate_limits.md) state. |
 
 To make this work with Sentinel:
 
@@ -659,6 +660,7 @@ To make this work with Sentinel:
    gitlab_rails['redis_shared_state_instance'] = REDIS_SHARED_STATE_URL
    gitlab_rails['redis_actioncable_instance'] = REDIS_ACTIONCABLE_URL
    gitlab_rails['redis_trace_chunks_instance'] = REDIS_TRACE_CHUNKS_URL
+   gitlab_rails['redis_rate_limiting_instance'] = REDIS_RATE_LIMITING_URL
 
    # Configure the Sentinels
    gitlab_rails['redis_cache_sentinels'] = [
@@ -681,6 +683,10 @@ To make this work with Sentinel:
      { host: TRACE_CHUNKS_SENTINEL_HOST, port: 26379 },
      { host: TRACE_CHUNKS_SENTINEL_HOST2, port: 26379 }
    ]
+   gitlab_rails['redis_rate_limiting_sentinels'] = [
+     { host: RATE_LIMITING_SENTINEL_HOST, port: 26379 },
+     { host: RATE_LIMITING_SENTINEL_HOST2, port: 26379 }
+   ]
    ```
 
    - Redis URLs should be in the format: `redis://:PASSWORD@SENTINEL_PRIMARY_NAME`, where:

doc/development/redis.md

@@ -15,6 +15,8 @@ GitLab uses [Redis](https://redis.io) for the following distinct purposes:
 - To manage the shared application state.
 - To store CI trace chunks.
 - As a Pub/Sub queue backend for ActionCable.
+- CI trace chunk storage.
+- Rate limiting state storage.
 
 In most environments (including the GDK), all of these point to the same
 Redis instance.

lib/gitlab/health_checks/redis/rate_limiting_check.rb

+# frozen_string_literal: true
+
+module Gitlab
+  module HealthChecks
+    module Redis
+      class RateLimitingCheck
+        extend SimpleAbstractCheck
+
+        class << self
+          def check_up
+            check
+          end
+
+          private
+
+          def metric_prefix
+            'redis_rate_limiting_ping'
+          end
+
+          def successful?(result)
+            result == 'PONG'
+          end
+
+          # rubocop: disable CodeReuse/ActiveRecord
+          def check
+            catch_timeout 10.seconds do
+              Gitlab::Redis::RateLimiting.with(&:ping)
+            end
+          end
+          # rubocop: enable CodeReuse/ActiveRecord
+        end
+      end
+    end
+  end
+end

lib/gitlab/health_checks/redis/redis_check.rb

@@ -21,7 +21,8 @@ module Gitlab
             ::Gitlab::HealthChecks::Redis::CacheCheck.check_up &&
               ::Gitlab::HealthChecks::Redis::QueuesCheck.check_up &&
               ::Gitlab::HealthChecks::Redis::SharedStateCheck.check_up &&
-              ::Gitlab::HealthChecks::Redis::TraceChunksCheck.check_up
+              ::Gitlab::HealthChecks::Redis::TraceChunksCheck.check_up &&
+              ::Gitlab::HealthChecks::Redis::RateLimitingCheck.check_up
           end
         end
       end

lib/gitlab/instrumentation/redis.rb

@@ -9,8 +9,9 @@ module Gitlab
       Queues = Class.new(RedisBase)
       SharedState = Class.new(RedisBase).enable_redis_cluster_validation
       TraceChunks = Class.new(RedisBase).enable_redis_cluster_validation
+      RateLimiting = Class.new(RedisBase).enable_redis_cluster_validation
 
-      STORAGES = [ActionCable, Cache, Queues, SharedState, TraceChunks].freeze
+      STORAGES = [ActionCable, Cache, Queues, SharedState, TraceChunks, RateLimiting].freeze
 
       # Milliseconds represented in seconds (from 1 millisecond to 2 seconds).
       QUERY_TIME_BUCKETS = [0.001, 0.0025, 0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1, 2].freeze

lib/gitlab/redis/cache.rb

@@ -4,14 +4,6 @@ module Gitlab
   module Redis
     class Cache < ::Gitlab::Redis::Wrapper
       CACHE_NAMESPACE = 'cache:gitlab'
-
-      private
-
-      def raw_config_hash
-        config = super
-        config[:url] = 'redis://localhost:6380' if config[:url].blank?
-        config
-      end
     end
   end
 end

lib/gitlab/redis/queues.rb

@@ -2,21 +2,12 @@
 
 # We need this require for MailRoom
 require_relative 'wrapper' unless defined?(::Gitlab::Redis::Wrapper)
-require 'active_support/core_ext/object/blank'
 
 module Gitlab
   module Redis
     class Queues < ::Gitlab::Redis::Wrapper
       SIDEKIQ_NAMESPACE = 'resque:gitlab'
       MAILROOM_NAMESPACE = 'mail_room:gitlab'
-
-      private
-
-      def raw_config_hash
-        config = super
-        config[:url] = 'redis://localhost:6381' if config[:url].blank?
-        config
-      end
     end
   end
 end

lib/gitlab/redis/rate_limiting.rb

+# frozen_string_literal: true
+
+module Gitlab
+  module Redis
+    class RateLimiting < ::Gitlab::Redis::Wrapper
+      # The data we store on RateLimiting used to be stored on Cache.
+      def self.config_fallback
+        Cache
+      end
+    end
+  end
+end

lib/gitlab/redis/shared_state.rb

@@ -7,14 +7,6 @@ module Gitlab
       USER_SESSIONS_NAMESPACE = 'session:user:gitlab'
       USER_SESSIONS_LOOKUP_NAMESPACE = 'session:lookup:user:gitlab'
       IP_SESSIONS_LOOKUP_NAMESPACE = 'session:lookup:ip:gitlab2'
-
-      private
-
-      def raw_config_hash
-        config = super
-        config[:url] = 'redis://localhost:6382' if config[:url].blank?
-        config
-      end
     end
   end
 end

lib/gitlab/redis/wrapper.rb

@@ -6,6 +6,7 @@
 # Rails.
 require 'active_support/core_ext/hash/keys'
 require 'active_support/core_ext/module/delegation'
+require 'active_support/core_ext/object/blank'
 require 'active_support/core_ext/string/inflections'
 
 # Explicitly load Redis::Store::Factory so we can read Redis configuration in
@@ -150,11 +151,35 @@ module Gitlab
       def raw_config_hash
         config_data = fetch_config
 
-        if config_data
-          config_data.is_a?(String) ? { url: config_data } : config_data.deep_symbolize_keys
-        else
-          { url: '' }
+        config_hash =
+          if config_data
+            config_data.is_a?(String) ? { url: config_data } : config_data.deep_symbolize_keys
+          else
+            { url: '' }
+          end
+
+        if config_hash[:url].blank?
+          config_hash[:url] = legacy_fallback_urls[self.class.store_name] || legacy_fallback_urls[self.class.config_fallback.store_name]
         end
+
+        config_hash
+      end
+
+      # These URLs were defined for cache, queues, and shared_state in
+      # code. They are used only when no config file exists at all for a
+      # given instance. The configuration does not seem particularly
+      # useful - it uses different ports on localhost - but we cannot
+      # confidently delete it as we don't know if any instances rely on
+      # this.
+      #
+      # DO NOT ADD new instances here. All new instances should define a
+      # `.config_fallback`, which will then be used to look up this URL.
+      def legacy_fallback_urls
+        {
+          'Cache' => 'redis://localhost:6380',
+          'Queues' => 'redis://localhost:6381',
+          'SharedState' => 'redis://localhost:6382'
+        }
       end
 
       def fetch_config

scripts/prepare_build.sh

@@ -38,18 +38,6 @@ sed -i 's|url:.*$|url: redis://redis:6379|g' config/cable.yml
 cp config/resque.yml.example config/resque.yml
 sed -i 's|url:.*$|url: redis://redis:6379|g' config/resque.yml
 
-cp config/redis.cache.yml.example config/redis.cache.yml
-sed -i 's|url:.*$|url: redis://redis:6379/10|g' config/redis.cache.yml
-
-cp config/redis.queues.yml.example config/redis.queues.yml
-sed -i 's|url:.*$|url: redis://redis:6379/11|g' config/redis.queues.yml
-
-cp config/redis.shared_state.yml.example config/redis.shared_state.yml
-sed -i 's|url:.*$|url: redis://redis:6379/12|g' config/redis.shared_state.yml
-
-cp config/redis.trace_chunks.yml.example config/redis.trace_chunks.yml
-sed -i 's|url:.*$|url: redis://redis:6379/13|g' config/redis.trace_chunks.yml
-
 if [ "$SETUP_DB" != "false" ]; then
   setup_db
 elif getent hosts postgres; then

spec/lib/gitlab/health_checks/probes/collection_spec.rb

@@ -16,6 +16,8 @@ RSpec.describe Gitlab::HealthChecks::Probes::Collection do
           Gitlab::HealthChecks::Redis::CacheCheck,
           Gitlab::HealthChecks::Redis::QueuesCheck,
           Gitlab::HealthChecks::Redis::SharedStateCheck,
+          Gitlab::HealthChecks::Redis::TraceChunksCheck,
+          Gitlab::HealthChecks::Redis::RateLimitingCheck,
           Gitlab::HealthChecks::GitalyCheck
         ]
       end

spec/lib/gitlab/health_checks/redis/rate_limiting_check_spec.rb

+# frozen_string_literal: true
+
+require 'spec_helper'
+require_relative '../simple_check_shared'
+
+RSpec.describe Gitlab::HealthChecks::Redis::RateLimitingCheck do
+  include_examples 'simple_check', 'redis_rate_limiting_ping', 'RedisRateLimiting', 'PONG'
+end

spec/lib/gitlab/instrumentation/redis_spec.rb

@@ -76,7 +76,8 @@ RSpec.describe Gitlab::Instrumentation::Redis do
                             details_row.merge(storage: 'Cache'),
                             details_row.merge(storage: 'Queues'),
                             details_row.merge(storage: 'SharedState'),
-                            details_row.merge(storage: 'TraceChunks'))
+                            details_row.merge(storage: 'TraceChunks'),
+                            details_row.merge(storage: 'RateLimiting'))
     end
   end
 end

spec/lib/gitlab/redis/rate_limiting_spec.rb

+# frozen_string_literal: true
+
+require 'spec_helper'
+
+RSpec.describe Gitlab::Redis::RateLimiting do
+  let(:instance_specific_config_file) { "config/redis.rate_limiting.yml" }
+  let(:environment_config_file_name) { "GITLAB_REDIS_RATE_LIMITING_CONFIG_FILE" }
+  let(:cache_config_file) { nil }
+
+  before do
+    allow(Gitlab::Redis::Cache).to receive(:config_file_name).and_return(cache_config_file)
+  end
+
+  include_examples "redis_shared_examples"
+
+  describe '.config_file_name' do
+    subject { described_class.config_file_name }
+
+    let(:rails_root) { Dir.mktmpdir('redis_shared_examples') }
+
+    before do
+      # Undo top-level stub of config_file_name because we are testing that method now.
+      allow(described_class).to receive(:config_file_name).and_call_original
+
+      allow(described_class).to receive(:rails_root).and_return(rails_root)
+      FileUtils.mkdir_p(File.join(rails_root, 'config'))
+    end
+
+    after do
+      FileUtils.rm_rf(rails_root)
+    end
+
+    context 'when there is only a resque.yml' do
+      before do
+        FileUtils.touch(File.join(rails_root, 'config/resque.yml'))
+      end
+
+      it { expect(subject).to eq("#{rails_root}/config/resque.yml") }
+
+      context 'and there is a global env override' do
+        before do
+          stub_env('GITLAB_REDIS_CONFIG_FILE', 'global override')
+        end
+
+        it { expect(subject).to eq('global override') }
+
+        context 'and Cache has a different config file' do
+          let(:cache_config_file) { 'cache config file' }
+
+          it { expect(subject).to eq('cache config file') }
+        end
+      end
+    end
+  end
+end

spec/support/redis.rb

@@ -38,4 +38,12 @@ RSpec.configure do |config|
 
     redis_trace_chunks_cleanup!
   end
+
+  config.around(:each, :clean_gitlab_redis_rate_limiting) do |example|
+    redis_rate_limiting_cleanup!
+
+    example.run
+
+    redis_rate_limiting_cleanup!
+  end
 end

spec/support/redis/redis_helpers.rb

@@ -22,4 +22,9 @@ module RedisHelpers
   def redis_trace_chunks_cleanup!
     Gitlab::Redis::TraceChunks.with(&:flushdb)
   end
+
+  # Usage: rate limiting state (for Rack::Attack)
+  def redis_rate_limiting_cleanup!
+    Gitlab::Redis::RateLimiting.with(&:flushdb)
+  end
 end

spec/support/redis/redis_shared_examples.rb

@@ -327,6 +327,12 @@ RSpec.shared_examples "redis_shared_examples" do
 
       expect(subject.send(:fetch_config)).to eq false
     end
+
+    it 'has a value for the legacy default URL' do
+      allow(subject).to receive(:fetch_config) { false }
+
+      expect(subject.send(:raw_config_hash)).to include(url: a_string_matching(%r{\Aredis://localhost:638[012]\Z}))
+    end
   end
 
   def clear_raw_config