Merge branch 'stanhu/gitlab-ce-add-eager-load-lib' into 'master'

Add eager load paths to help prevent dependency load issues with Sidekiq workers

_Originally opened at !3545 by @stanhu._

- - -

Relevant resources:

- https://github.com/mperham/sidekiq/wiki/FAQ#why-doesnt-sidekiq-autoload-my-rails-application-code
- https://github.com/mperham/sidekiq/issues/1281#issuecomment-27129904
- http://blog.arkency.com/2014/11/dont-forget-about-eager-load-when-extending-autoload
- https://github.com/rails/rails/blob/52ce6ece8c8f74064bb64e0a0b1ddd83092718e1/railties/lib/rails/engine.rb#L472-L479
- https://github.com/rails/rails/blob/v4.2.6/railties/lib/rails/paths.rb

Attempts to address #3661, #11896, #12769, #13521, #14131, #14589, #14759, #14825.

See merge request !3724

CHANGELOG

@@ -32,6 +32,7 @@ v 8.8.0 (unreleased)
   - API: Expose Issue#user_notes_count. !3126 (Anton Popov)
   - Files over 5MB can only be viewed in their raw form, files over 1MB without highlighting !3718
   - Add support for supressing text diffs using .gitattributes on the default branch (Matt Oakes)
+  - Add eager load paths to help prevent dependency load issues in Sidekiq workers. !3724
   - Added multiple colors for labels in dropdowns when dups happen.
   - Improve description for the Two-factor Authentication sign-in screen. (Connor Shea)
   - API support for the 'since' and 'until' operators on commit requests (Paco Guzman)

app/models/repository.rb

@@ -81,7 +81,7 @@ class Repository
   def commit(id = 'HEAD')
     return nil unless exists?
     commit = Gitlab::Git::Commit.find(raw_repository, id)
-    commit = Commit.new(commit, @project) if commit
+    commit = ::Commit.new(commit, @project) if commit
     commit
   rescue Rugged::OdbError
     nil

config/application.rb

 require File.expand_path('../boot', __FILE__)
 
 require 'rails/all'
-require 'devise'
-I18n.config.enforce_available_locales = false
+
 Bundler.require(:default, Rails.env)
-require_relative '../lib/gitlab/redis'
 
 module Gitlab
   class Application < Rails::Application
+    require_dependency Rails.root.join('lib/gitlab/redis')
+
     # Settings in config/environments/* take precedence over those specified here.
     # Application configuration should go into files in config/initializers
     # -- all .rb files in that directory are automatically loaded.
 
-    # Custom directories with classes and modules you want to be autoloadable.
-    config.autoload_paths.push(*%W(#{config.root}/lib
+    # Sidekiq uses eager loading, but directories not in the standard Rails
+    # directories must be added to the eager load paths:
+    # https://github.com/mperham/sidekiq/wiki/FAQ#why-doesnt-sidekiq-autoload-my-rails-application-code
+    # Also, there is no need to add `lib` to autoload_paths since autoloading is
+    # configured to check for eager loaded paths:
+    # https://github.com/rails/rails/blob/v4.2.6/railties/lib/rails/engine.rb#L687
+    # This is a nice reference article on autoloading/eager loading:
+    # http://blog.arkency.com/2014/11/dont-forget-about-eager-load-when-extending-autoload
+    config.eager_load_paths.push(*%W(#{config.root}/lib
+                                     #{config.root}/app/models/ci
                                      #{config.root}/app/models/hooks
-                                   #{config.root}/app/models/concerns
-                                   #{config.root}/app/models/project_services
-                                   #{config.root}/app/models/members))
+                                     #{config.root}/app/models/members
+                                     #{config.root}/app/models/project_services))
 
     # Only load the plugins named here, in the order given (default is alphabetical).
     # :all can be used as a placeholder for all plugins not explicitly named.

config/initializers/1_settings.rb

-require 'gitlab' # Load lib/gitlab.rb as soon as possible
+require_dependency Rails.root.join('lib/gitlab') # Load Gitlab as soon as possible
 
 class Settings < Settingslogic
   source ENV.fetch('GITLAB_CONFIG') { "#{Rails.root}/config/gitlab.yml" }

config/initializers/5_backend.rb

 # GIT over HTTP
-require Rails.root.join("lib", "gitlab", "backend", "grack_auth")
+require_dependency Rails.root.join('lib/gitlab/backend/grack_auth')
 
 # GIT over SSH
-require Rails.root.join("lib", "gitlab", "backend", "shell")
+require_dependency Rails.root.join('lib/gitlab/backend/shell')
 
 # GitLab shell adapter
-require Rails.root.join("lib", "gitlab", "backend", "shell_adapter")
+require_dependency Rails.root.join('lib/gitlab/backend/shell_adapter')
 
 required_version = Gitlab::VersionInfo.parse(Gitlab::Shell.version_required)
 current_version = Gitlab::VersionInfo.parse(Gitlab::Shell.new.version)

lib/api/api.rb

-Dir["#{Rails.root}/lib/api/*.rb"].each {|file| require file}
-
 module API
   class API < Grape::API
     include APIGuard
@@ -25,38 +23,39 @@ module API
     format :json
     content_type :txt, "text/plain"
 
-    helpers Helpers
-
-    mount Groups
-    mount GroupMembers
-    mount Users
-    mount Projects
-    mount Repositories
-    mount Issues
-    mount Milestones
-    mount Session
-    mount MergeRequests
-    mount Notes
-    mount Internal
-    mount SystemHooks
-    mount ProjectSnippets
-    mount ProjectMembers
-    mount DeployKeys
-    mount ProjectHooks
-    mount Services
-    mount Files
-    mount Commits
-    mount CommitStatus
-    mount Namespaces
-    mount Branches
-    mount Labels
-    mount Settings
-    mount Keys
-    mount Tags
-    mount Triggers
-    mount Builds
-    mount Variables
-    mount Runners
-    mount Licenses
+    # Ensure the namespace is right, otherwise we might load Grape::API::Helpers
+    helpers ::API::Helpers
+
+    mount ::API::Groups
+    mount ::API::GroupMembers
+    mount ::API::Users
+    mount ::API::Projects
+    mount ::API::Repositories
+    mount ::API::Issues
+    mount ::API::Milestones
+    mount ::API::Session
+    mount ::API::MergeRequests
+    mount ::API::Notes
+    mount ::API::Internal
+    mount ::API::SystemHooks
+    mount ::API::ProjectSnippets
+    mount ::API::ProjectMembers
+    mount ::API::DeployKeys
+    mount ::API::ProjectHooks
+    mount ::API::Services
+    mount ::API::Files
+    mount ::API::Commits
+    mount ::API::CommitStatuses
+    mount ::API::Namespaces
+    mount ::API::Branches
+    mount ::API::Labels
+    mount ::API::Settings
+    mount ::API::Keys
+    mount ::API::Tags
+    mount ::API::Triggers
+    mount ::API::Builds
+    mount ::API::Variables
+    mount ::API::Runners
+    mount ::API::Licenses
   end
 end

lib/api/api_guard.rb

@@ -2,7 +2,8 @@
 
 require 'rack/oauth2'
 
-module APIGuard
+module API
+  module APIGuard
     extend ActiveSupport::Concern
 
     included do |base|
@@ -80,6 +81,7 @@ module APIGuard
       end
 
       private
+
       def find_access_token
         @access_token ||= Doorkeeper.authenticate(doorkeeper_request, Doorkeeper.configuration.access_token_methods)
       end
@@ -108,6 +110,7 @@ module APIGuard
       end
 
       private
+
       def install_error_responders(base)
         error_classes = [ MissingTokenError, TokenNotFoundError,
                           ExpiredError, RevokedError, InsufficientScopeError]
@@ -169,4 +172,5 @@ module APIGuard
         @scopes = scopes
       end
     end
+  end
 end

lib/api/commit_statuses.rb

@@ -2,7 +2,7 @@ require 'mime/types'
 
 module API
   # Project commit statuses API
-  class CommitStatus < Grape::API
+  class CommitStatuses < Grape::API
     resource :projects do
       before { authenticate! }
 

lib/ci/api/api.rb

-Dir["#{Rails.root}/lib/ci/api/*.rb"].each {|file| require file}
-
 module Ci
   module API
     class API < Grape::API
-      include APIGuard
+      include ::API::APIGuard
       version 'v1', using: :path
 
       rescue_from ActiveRecord::RecordNotFound do
@@ -31,9 +29,9 @@ module Ci
       helpers ::API::Helpers
       helpers Gitlab::CurrentSettings
 
-      mount Builds
-      mount Runners
-      mount Triggers
+      mount ::Ci::API::Builds
+      mount ::Ci::API::Runners
+      mount ::Ci::API::Triggers
     end
   end
 end

lib/gitlab.rb

-require 'gitlab/git'
+require_dependency 'gitlab/git'
 
 module Gitlab
   def self.com?

spec/requests/api/commit_status_spec.rb → spec/requests/api/commit_statuses_spec.rb

 require 'spec_helper'
 
-describe API::CommitStatus, api: true do
+describe API::CommitStatuses, api: true do
   include ApiHelpers
 
   let!(:project) { create(:project) }