Commit c7bb3a1f authored by Dmitriy Zaporozhets's avatar Dmitriy Zaporozhets

sidekiq

parent 7bfd38ea
...@@ -81,8 +81,10 @@ gem "acts-as-taggable-on", "2.3.3" ...@@ -81,8 +81,10 @@ gem "acts-as-taggable-on", "2.3.3"
gem "draper", "~> 0.18.0" gem "draper", "~> 0.18.0"
# Background jobs # Background jobs
gem "resque", git: "https://github.com/gitlabhq/resque.git", ref: "9ef4700306dd946a3ac000612428967ce0c32213" gem 'slim'
gem 'resque_mailer' gem 'sinatra', :require => nil
gem 'sidekiq', '2.6.4'
gem 'sidekiq_mailer'
# HTTP requests # HTTP requests
gem "httparty" gem "httparty"
...@@ -157,7 +159,6 @@ group :test do ...@@ -157,7 +159,6 @@ group :test do
gem "simplecov", require: false gem "simplecov", require: false
gem "shoulda-matchers", "1.3.0" gem "shoulda-matchers", "1.3.0"
gem 'email_spec' gem 'email_spec'
gem 'resque_spec'
gem "webmock" gem "webmock"
gem 'test_after_commit' gem 'test_after_commit'
end end
......
...@@ -54,17 +54,6 @@ GIT ...@@ -54,17 +54,6 @@ GIT
specs: specs:
raphael-rails (2.1.0) raphael-rails (2.1.0)
GIT
remote: https://github.com/gitlabhq/resque.git
revision: 9ef4700306dd946a3ac000612428967ce0c32213
ref: 9ef4700306dd946a3ac000612428967ce0c32213
specs:
resque (2.0.0.pre.1)
json
redis-namespace (~> 1.0)
sinatra (>= 0.9.2)
vegas (~> 0.1.2)
GIT GIT
remote: https://github.com/jonleighton/poltergeist.git remote: https://github.com/jonleighton/poltergeist.git
revision: 5c2e092001074a8cf09f332d3714e9ba150bc8ca revision: 5c2e092001074a8cf09f332d3714e9ba150bc8ca
...@@ -132,6 +121,9 @@ GEM ...@@ -132,6 +121,9 @@ GEM
carrierwave (0.7.1) carrierwave (0.7.1)
activemodel (>= 3.2.0) activemodel (>= 3.2.0)
activesupport (>= 3.2.0) activesupport (>= 3.2.0)
celluloid (0.12.4)
facter (>= 1.6.12)
timers (>= 1.0.0)
charlock_holmes (0.6.9) charlock_holmes (0.6.9)
childprocess (0.3.6) childprocess (0.3.6)
ffi (~> 1.0, >= 1.0.6) ffi (~> 1.0, >= 1.0.6)
...@@ -150,6 +142,7 @@ GEM ...@@ -150,6 +142,7 @@ GEM
coffee-script-source (1.4.0) coffee-script-source (1.4.0)
colored (1.2) colored (1.2)
colorize (0.5.8) colorize (0.5.8)
connection_pool (1.0.0)
crack (0.3.1) crack (0.3.1)
daemons (1.1.9) daemons (1.1.9)
devise (2.1.2) devise (2.1.2)
...@@ -169,6 +162,7 @@ GEM ...@@ -169,6 +162,7 @@ GEM
eventmachine (1.0.0) eventmachine (1.0.0)
execjs (1.4.0) execjs (1.4.0)
multi_json (~> 1.0) multi_json (~> 1.0)
facter (1.6.17)
factory_girl (4.1.0) factory_girl (4.1.0)
activesupport (>= 3.0.0) activesupport (>= 3.0.0)
factory_girl_rails (4.1.0) factory_girl_rails (4.1.0)
...@@ -370,11 +364,6 @@ GEM ...@@ -370,11 +364,6 @@ GEM
redis (3.0.2) redis (3.0.2)
redis-namespace (1.2.1) redis-namespace (1.2.1)
redis (~> 3.0.0) redis (~> 3.0.0)
resque_mailer (2.1.0)
actionmailer (~> 3.0)
resque_spec (0.12.5)
resque (>= 1.19.0)
rspec (>= 2.5.0)
rspec (2.12.0) rspec (2.12.0)
rspec-core (~> 2.12.0) rspec-core (~> 2.12.0)
rspec-expectations (~> 2.12.0) rspec-expectations (~> 2.12.0)
...@@ -412,6 +401,16 @@ GEM ...@@ -412,6 +401,16 @@ GEM
sexp_processor (4.1.3) sexp_processor (4.1.3)
shoulda-matchers (1.3.0) shoulda-matchers (1.3.0)
activesupport (>= 3.0.0) activesupport (>= 3.0.0)
sidekiq (2.6.4)
celluloid (~> 0.12.0)
connection_pool (~> 1.0)
multi_json (~> 1)
redis (~> 3)
redis-namespace
sidekiq_mailer (0.0.4)
actionmailer (~> 3.0)
activesupport (~> 3.0)
sidekiq (~> 2.3)
simplecov (0.7.1) simplecov (0.7.1)
multi_json (~> 1.0) multi_json (~> 1.0)
simplecov-html (~> 0.7.1) simplecov-html (~> 0.7.1)
...@@ -421,6 +420,9 @@ GEM ...@@ -421,6 +420,9 @@ GEM
rack-protection (~> 1.2) rack-protection (~> 1.2)
tilt (~> 1.3, >= 1.3.3) tilt (~> 1.3, >= 1.3.3)
six (0.2.0) six (0.2.0)
slim (1.3.6)
temple (~> 0.5.5)
tilt (~> 1.3.3)
slop (3.3.3) slop (3.3.3)
spinach (0.5.2) spinach (0.5.2)
colorize colorize
...@@ -435,6 +437,7 @@ GEM ...@@ -435,6 +437,7 @@ GEM
rack (~> 1.0) rack (~> 1.0)
tilt (~> 1.1, != 1.3.0) tilt (~> 1.1, != 1.3.0)
stamp (0.3.0) stamp (0.3.0)
temple (0.5.5)
test_after_commit (0.0.1) test_after_commit (0.0.1)
therubyracer (0.10.2) therubyracer (0.10.2)
libv8 (~> 3.3.10) libv8 (~> 3.3.10)
...@@ -444,6 +447,7 @@ GEM ...@@ -444,6 +447,7 @@ GEM
rack (>= 1.0.0) rack (>= 1.0.0)
thor (0.16.0) thor (0.16.0)
tilt (1.3.3) tilt (1.3.3)
timers (1.0.2)
treetop (1.4.12) treetop (1.4.12)
polyglot polyglot
polyglot (>= 0.3.1) polyglot (>= 0.3.1)
...@@ -455,8 +459,6 @@ GEM ...@@ -455,8 +459,6 @@ GEM
kgio (~> 2.6) kgio (~> 2.6)
rack rack
raindrops (~> 0.7) raindrops (~> 0.7)
vegas (0.1.11)
rack (>= 1.0.0)
virtus (0.5.2) virtus (0.5.2)
backports (~> 2.6.1) backports (~> 2.6.1)
warden (1.2.1) warden (1.2.1)
...@@ -534,17 +536,18 @@ DEPENDENCIES ...@@ -534,17 +536,18 @@ DEPENDENCIES
rb-fsevent rb-fsevent
rb-inotify rb-inotify
redcarpet (~> 2.2.2) redcarpet (~> 2.2.2)
resque!
resque_mailer
resque_spec
rspec-rails rspec-rails
sass-rails (~> 3.2.5) sass-rails (~> 3.2.5)
sdoc sdoc
seed-fu seed-fu
settingslogic settingslogic
shoulda-matchers (= 1.3.0) shoulda-matchers (= 1.3.0)
sidekiq (= 2.6.4)
sidekiq_mailer
simplecov simplecov
sinatra
six six
slim
spinach-rails spinach-rails
stamp stamp
test_after_commit test_after_commit
......
web: bundle exec rails s -p $PORT web: bundle exec rails s -p $PORT
worker: bundle exec rake environment resque:work QUEUE=* VVERBOSE=1 worker: bundle exec sidekiq -q post_receive,mailer,system_hook,common
...@@ -3,10 +3,6 @@ class Admin::DashboardController < AdminController ...@@ -3,10 +3,6 @@ class Admin::DashboardController < AdminController
@projects = Project.order("created_at DESC").limit(10) @projects = Project.order("created_at DESC").limit(10)
@users = User.order("created_at DESC").limit(10) @users = User.order("created_at DESC").limit(10)
@resque_accessible = true
@workers = Resque.workers
@pending_jobs = Resque.size(:post_receive)
rescue Redis::InheritedError rescue Redis::InheritedError
@resque_accessible = false @resque_accessible = false
end end
......
class Notify < ActionMailer::Base class Notify < ActionMailer::Base
include Resque::Mailer include Sidekiq::Mailer
add_template_helper ApplicationHelper add_template_helper ApplicationHelper
add_template_helper GitlabMarkdownHelper add_template_helper GitlabMarkdownHelper
......
...@@ -19,6 +19,6 @@ class SystemHook < WebHook ...@@ -19,6 +19,6 @@ class SystemHook < WebHook
end end
def async_execute(data) def async_execute(data)
Resque.enqueue(SystemHookWorker, id, data) Sidekiq::Client.enqueue(SystemHookWorker, id, data)
end end
end end
.admin_dash.row .admin_dash.row
.span3 .span4
.ui-box .ui-box
%h5.title Projects %h5.title Projects
.data.padded .data.padded
...@@ -7,7 +7,7 @@ ...@@ -7,7 +7,7 @@
%h1= Project.count %h1= Project.count
%hr %hr
= link_to 'New Project', new_project_path, class: "btn small" = link_to 'New Project', new_project_path, class: "btn small"
.span3 .span4
.ui-box .ui-box
%h5.title Groups %h5.title Groups
.data.padded .data.padded
...@@ -15,7 +15,7 @@ ...@@ -15,7 +15,7 @@
%h1= Group.count %h1= Group.count
%hr %hr
= link_to 'New Group', new_admin_group_path, class: "btn small" = link_to 'New Group', new_admin_group_path, class: "btn small"
.span3 .span4
.ui-box .ui-box
%h5.title Users %h5.title Users
.data.padded .data.padded
...@@ -23,25 +23,6 @@ ...@@ -23,25 +23,6 @@
%h1= User.count %h1= User.count
%hr %hr
= link_to 'New User', new_admin_user_path, class: "btn small" = link_to 'New User', new_admin_user_path, class: "btn small"
.span3
.ui-box
%h5.title
Resque Workers
.data.padded
- if @resque_accessible
= link_to admin_resque_path do
%h1{class: @workers.present? ? "cgreen" : "cred"}
= @workers.count
%hr
%p
%strong{class: @pending_jobs > 0 ? "cred" : "cgreen"}
#{@pending_jobs} post receive jobs waiting
- else
= link_to admin_resque_path do
%h1.cdark ?
%hr
%p
%strong Resque status unknown
.row .row
.span6 .span6
......
%h3.page_title Resque %h3.page_title Background Jobs
%br %br
.ui-box .ui-box
%iframe{src: resque_path, width: '100%', height: 600, style: "border: none"} %iframe{src: sidekiq_path, width: '100%', height: 900, style: "border: none"}
...@@ -19,6 +19,6 @@ ...@@ -19,6 +19,6 @@
= nav_link(controller: :hooks) do = nav_link(controller: :hooks) do
= link_to "Hooks", admin_hooks_path = link_to "Hooks", admin_hooks_path
= nav_link(controller: :resque) do = nav_link(controller: :resque) do
= link_to "Resque", admin_resque_path = link_to "Background Jobs", admin_resque_path
.content= yield .content= yield
class PostReceive class PostReceive
@queue = :post_receive include Sidekiq::Worker
def self.perform(repo_path, oldrev, newrev, ref, identifier) sidekiq_options queue: :post_receive
def perform(repo_path, oldrev, newrev, ref, identifier)
repo_path.gsub!(Gitlab.config.gitolite.repos_path.to_s, "") repo_path.gsub!(Gitlab.config.gitolite.repos_path.to_s, "")
repo_path.gsub!(/.git$/, "") repo_path.gsub!(/.git$/, "")
repo_path.gsub!(/^\//, "") repo_path.gsub!(/^\//, "")
......
class SystemHookWorker class SystemHookWorker
@queue = :system_hook include Sidekiq::Worker
def self.perform(hook_id, data) sidekiq_options queue: :system_hook
def perform(hook_id, data)
SystemHook.find(hook_id).execute data SystemHook.find(hook_id).execute data
end end
end end
# Custom Redis configuration # Custom Redis configuration
config_file = Rails.root.join('config', 'resque.yml') config_file = Rails.root.join('config', 'resque.yml')
if File.exists?(config_file) resque_url = if File.exists?(config_file)
resque_config = YAML.load_file(config_file) YAML.load_file(config_file)[Rails.env]
Resque.redis = resque_config[Rails.env] else
end "localhost:6379"
Resque.redis.namespace = 'resque:gitlab'
Resque.before_fork = Proc.new { ActiveRecord::Base.establish_connection }
# Authentication
require 'resque/server'
class ResqueAuthentication
def initialize(app)
@app = app
end end
def call(env) Sidekiq.configure_server do |config|
account = env['warden'].authenticate!(:database_authenticatable, :rememberable, scope: :user) config.redis = {
raise "Access denied" if !account.admin? url: "redis://#{resque_url}",
@app.call(env) namespace: 'resque:gitlab'
end }
end end
Resque::Server.use ResqueAuthentication Sidekiq.configure_client do |config|
config.redis = {
# Mailer url: "redis://#{resque_url}",
Resque::Mailer.excluded_environments = [] namespace: 'resque:gitlab'
}
end
require 'sidekiq/web'
Gitlab::Application.routes.draw do Gitlab::Application.routes.draw do
# #
# Search # Search
...@@ -8,9 +10,10 @@ Gitlab::Application.routes.draw do ...@@ -8,9 +10,10 @@ Gitlab::Application.routes.draw do
require 'api' require 'api'
mount Gitlab::API => '/api' mount Gitlab::API => '/api'
# Optionally, enable Resque here constraint = lambda { |request| request.env["warden"].authenticate? and request.env['warden'].user.admin? }
require 'resque/server' constraints constraint do
mount Resque::Server => '/info/resque', as: 'resque' mount Sidekiq::Web, at: "/admin/workers", as: :sidekiq
end
# Enable Grack support # Enable Grack support
mount Grack::Bundle.new({ mount Grack::Bundle.new({
......
...@@ -2,20 +2,20 @@ require 'resque/tasks' ...@@ -2,20 +2,20 @@ require 'resque/tasks'
namespace :resque do namespace :resque do
task setup: :environment do task setup: :environment do
Resque.before_fork = Proc.new { ActiveRecord::Base.establish_connection } #Resque.before_fork = Proc.new { ActiveRecord::Base.establish_connection }
end end
desc "Resque | kill all workers (using -QUIT), god will take care of them" desc "Resque | kill all workers (using -QUIT), god will take care of them"
task :stop_workers => :environment do task :stop_workers => :environment do
pids = Array.new #pids = Array.new
Resque.workers.each do |worker| #Resque.workers.each do |worker|
pids << worker.to_s.split(/:/).second #pids << worker.to_s.split(/:/).second
end #end
if pids.size > 0 #if pids.size > 0
system("kill -QUIT #{pids.join(' ')}") #system("kill -QUIT #{pids.join(' ')}")
end #end
end end
end end
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment