Commit 93571c43 authored by Robert Speicher's avatar Robert Speicher

Merge branch 'redis-connection-pool' into 'master'

Redis connection pool

Split from https://gitlab.com/gitlab-org/gitlab-ce/merge_requests/3232

Having an easily accessible Redis connection pool allows us to do more
cool stuff with Redis in GitLab (instead of having to go through e.g.
the Rails cache).

See merge request !3521
parents 833808d7 cf669551
No related merge requests found
......@@ -149,6 +149,10 @@ gem 'version_sorter', '~> 2.0.0'
# Cache
gem "redis-rails", '~> 4.0.0'
# Redis
gem 'redis', '~> 3.2'
gem 'connection_pool', '~> 2.0'
# Campfire integration
gem 'tinder', '~> 1.10.0'
......@@ -229,7 +233,6 @@ group :metrics do
gem 'allocations', '~> 1.0', require: false, platform: :mri
gem 'method_source', '~> 0.8', require: false
gem 'influxdb', '~> 0.2', require: false
gem 'connection_pool', '~> 2.0', require: false
end
group :development do
......
......@@ -999,6 +999,7 @@ DEPENDENCIES
rdoc (~> 3.6)
recaptcha
redcarpet (~> 3.3.3)
redis (~> 3.2)
redis-namespace
redis-rails (~> 4.0.0)
request_store (~> 1.3.0)
......
......@@ -4,11 +4,9 @@ require 'rails/all'
require 'devise'
I18n.config.enforce_available_locales = false
Bundler.require(:default, Rails.env)
require_relative '../lib/gitlab/redis_config'
require_relative '../lib/gitlab/redis'
module Gitlab
REDIS_CACHE_NAMESPACE = 'cache:gitlab'
class Application < Rails::Application
# Settings in config/environments/* take precedence over those specified here.
# Application configuration should go into files in config/initializers
......@@ -69,8 +67,8 @@ module Gitlab
end
end
redis_config_hash = Gitlab::RedisConfig.redis_store_options
redis_config_hash[:namespace] = REDIS_CACHE_NAMESPACE
redis_config_hash = Gitlab::Redis.redis_store_options
redis_config_hash[:namespace] = Gitlab::Redis::CACHE_NAMESPACE
redis_config_hash[:expires_in] = 2.weeks # Cache should not grow forever
config.cache_store = :redis_store, redis_config_hash
......
......@@ -13,7 +13,7 @@ end
if Rails.env.test?
Gitlab::Application.config.session_store :cookie_store, key: "_gitlab_session"
else
redis_config = Gitlab::RedisConfig.redis_store_options
redis_config = Gitlab::Redis.redis_store_options
redis_config[:namespace] = 'session:gitlab'
Gitlab::Application.config.session_store(
......
......@@ -2,7 +2,7 @@ SIDEKIQ_REDIS_NAMESPACE = 'resque:gitlab'
Sidekiq.configure_server do |config|
config.redis = {
url: Gitlab::RedisConfig.url,
url: Gitlab::Redis.url,
namespace: SIDEKIQ_REDIS_NAMESPACE
}
......@@ -29,7 +29,7 @@ end
Sidekiq.configure_client do |config|
config.redis = {
url: Gitlab::RedisConfig.url,
url: Gitlab::Redis.url,
namespace: SIDEKIQ_REDIS_NAMESPACE
}
end
......@@ -2,7 +2,7 @@
<%
require "yaml"
require "json"
require_relative "lib/gitlab/redis_config"
require_relative "lib/gitlab/redis"
rails_env = ENV["RAILS_ENV"] || ENV["RACK_ENV"] || "development"
......@@ -18,7 +18,7 @@ if File.exists?(config_file)
config['mailbox'] = "inbox" if config['mailbox'].nil?
if config['enabled'] && config['address']
redis_url = Gitlab::RedisConfig.new(rails_env).url
redis_url = Gitlab::Redis.new(rails_env).url
%>
-
:host: <%= config['host'].to_json %>
......
......@@ -43,8 +43,10 @@ module Gitlab
# false if the lease is already taken.
def try_obtain
# Performing a single SET is atomic
Gitlab::Redis.with do |redis|
!!redis.set(redis_key, '1', nx: true, ex: @timeout)
end
end
# No #cancel method. See comments above!
......
module Gitlab
class RedisConfig
class Redis
CACHE_NAMESPACE = 'cache:gitlab'
attr_reader :url
# To be thread-safe we must be careful when writing the class instance
# variables @url and @pool. Because @pool depends on @url we need two
# mutexes to prevent deadlock.
URL_MUTEX = Mutex.new
POOL_MUTEX = Mutex.new
private_constant :URL_MUTEX, :POOL_MUTEX
def self.url
new.url
@url || URL_MUTEX.synchronize { @url = new.url }
end
def self.with
if @pool.nil?
POOL_MUTEX.synchronize do
@pool = ConnectionPool.new { ::Redis.new(url: url) }
end
end
@pool.with { |redis| yield redis }
end
def self.redis_store_options
url = new.url
redis_config_hash = Redis::Store::Factory.extract_host_options_from_uri(url)
redis_config_hash = ::Redis::Store::Factory.extract_host_options_from_uri(url)
# Redis::Store does not handle Unix sockets well, so let's do it for them
redis_uri = URI.parse(url)
if redis_uri.scheme == 'unix'
......
......@@ -4,12 +4,12 @@ namespace :cache do
desc "GitLab | Clear redis cache"
task :clear => :environment do
redis = Redis.new(url: Gitlab::RedisConfig.url)
Gitlab::Redis.with do |redis|
cursor = REDIS_SCAN_START_STOP
loop do
cursor, keys = redis.scan(
cursor,
match: "#{Gitlab::REDIS_CACHE_NAMESPACE}*",
match: "#{Gitlab::Redis::CACHE_NAMESPACE}*",
count: CLEAR_BATCH_SIZE
)
......@@ -18,4 +18,5 @@ namespace :cache do
break if cursor == REDIS_SCAN_START_STOP
end
end
end
end
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment