Merge branch '344794-sidekiq-cluster-extraction' into 'master'

Extract sidekiq-cluster to top-level directory

See merge request gitlab-org/gitlab!73630

bin/sidekiq-cluster

 #!/usr/bin/env ruby
 # frozen_string_literal: true
 
-require 'optparse'
-require_relative '../lib/gitlab'
-require_relative '../lib/gitlab/utils'
-require_relative '../lib/gitlab/sidekiq_config/cli_methods'
-require_relative '../lib/gitlab/sidekiq_config/worker_matcher'
-require_relative '../lib/gitlab/sidekiq_cluster'
-require_relative '../lib/gitlab/sidekiq_cluster/cli'
+require_relative '../sidekiq_cluster/cli'
 
 Thread.abort_on_exception = true
 

lib/gitlab/sidekiq_cluster/cli.rb → sidekiq_cluster/cli.rb

@@ -4,27 +4,21 @@ require 'optparse'
 require 'logger'
 require 'time'
 
+# In environments where code is preloaded and cached such as `spring`,
+# we may run into "already initialized" warnings, hence the check.
+require_relative '../lib/gitlab' unless Object.const_defined?('Gitlab')
+require_relative '../lib/gitlab/utils'
+require_relative '../lib/gitlab/sidekiq_config/cli_methods'
+require_relative '../lib/gitlab/sidekiq_config/worker_matcher'
+require_relative '../lib/gitlab/sidekiq_logging/json_formatter'
+require_relative 'sidekiq_cluster'
+
 module Gitlab
   module SidekiqCluster
     class CLI
-      CHECK_TERMINATE_INTERVAL_SECONDS = 1
-
-      # How long to wait when asking for a clean termination.
-      # It maps the Sidekiq default timeout:
-      # https://github.com/mperham/sidekiq/wiki/Signals#term
-      #
-      # This value is passed to Sidekiq's `-t` if none
-      # is given through arguments.
-      DEFAULT_SOFT_TIMEOUT_SECONDS = 25
-
-      # After surpassing the soft timeout.
-      DEFAULT_HARD_TIMEOUT_SECONDS = 5
-
       CommandError = Class.new(StandardError)
 
       def initialize(log_output = $stderr)
-        require_relative '../../../lib/gitlab/sidekiq_logging/json_formatter'
-
         # As recommended by https://github.com/mperham/sidekiq/wiki/Advanced-Options#concurrency
         @max_concurrency = 50
         @min_concurrency = 0

sidekiq_cluster/dependencies.rb

+# rubocop:disable Naming/FileName
+# frozen_string_literal: true
+
+require 'shellwords'
+
+# rubocop:enable Naming/FileName

lib/gitlab/sidekiq_cluster.rb → sidekiq_cluster/sidekiq_cluster.rb

 # frozen_string_literal: true
 
-require 'shellwords'
+require_relative 'dependencies'
 
 module Gitlab
   module SidekiqCluster
+    CHECK_TERMINATE_INTERVAL_SECONDS = 1
+
+    # How long to wait when asking for a clean termination.
+    # It maps the Sidekiq default timeout:
+    # https://github.com/mperham/sidekiq/wiki/Signals#term
+    #
+    # This value is passed to Sidekiq's `-t` if none
+    # is given through arguments.
+    DEFAULT_SOFT_TIMEOUT_SECONDS = 25
+
+    # After surpassing the soft timeout.
+    DEFAULT_HARD_TIMEOUT_SECONDS = 5
+
     # The signals that should terminate both the master and workers.
     TERMINATE_SIGNALS = %i(INT TERM).freeze
 
@@ -62,7 +75,7 @@ module Gitlab
     # directory - The directory of the Rails application.
     #
     # Returns an Array containing the PIDs of the started processes.
-    def self.start(queues, env: :development, directory: Dir.pwd, max_concurrency: 50, min_concurrency: 0, timeout: CLI::DEFAULT_SOFT_TIMEOUT_SECONDS, dryrun: false)
+    def self.start(queues, env: :development, directory: Dir.pwd, max_concurrency: 50, min_concurrency: 0, timeout: DEFAULT_SOFT_TIMEOUT_SECONDS, dryrun: false)
       queues.map.with_index do |pair, index|
         start_sidekiq(pair, env: env,
                             directory: directory,

spec/lib/gitlab/sidekiq_cluster/cli_spec.rb → spec/commands/sidekiq_cluster/cli_spec.rb

@@ -3,9 +3,11 @@
 require 'fast_spec_helper'
 require 'rspec-parameterized'
 
-RSpec.describe Gitlab::SidekiqCluster::CLI do
+require_relative '../../../sidekiq_cluster/cli'
+
+RSpec.describe Gitlab::SidekiqCluster::CLI do # rubocop:disable RSpec/FilePath
   let(:cli) { described_class.new('/dev/null') }
-  let(:timeout) { described_class::DEFAULT_SOFT_TIMEOUT_SECONDS }
+  let(:timeout) { Gitlab::SidekiqCluster::DEFAULT_SOFT_TIMEOUT_SECONDS }
   let(:default_options) do
     { env: 'test', directory: Dir.pwd, max_concurrency: 50, min_concurrency: 0, dryrun: false, timeout: timeout }
   end
@@ -103,7 +105,7 @@ RSpec.describe Gitlab::SidekiqCluster::CLI do
 
         it 'when not given', 'starts Sidekiq workers with default timeout' do
           expect(Gitlab::SidekiqCluster).to receive(:start)
-            .with([['foo']], default_options.merge(timeout: described_class::DEFAULT_SOFT_TIMEOUT_SECONDS))
+            .with([['foo']], default_options.merge(timeout: Gitlab::SidekiqCluster::DEFAULT_SOFT_TIMEOUT_SECONDS))
 
           cli.run(%w(foo))
         end
@@ -271,7 +273,7 @@ RSpec.describe Gitlab::SidekiqCluster::CLI do
       expect(Gitlab::SidekiqCluster).to receive(:signal_processes)
         .with([], "-KILL")
 
-      stub_const("Gitlab::SidekiqCluster::CLI::CHECK_TERMINATE_INTERVAL_SECONDS", 0.1)
+      stub_const("Gitlab::SidekiqCluster::CHECK_TERMINATE_INTERVAL_SECONDS", 0.1)
       allow(cli).to receive(:terminate_timeout_seconds) { 1 }
 
       cli.wait_for_termination
@@ -301,7 +303,7 @@ RSpec.describe Gitlab::SidekiqCluster::CLI do
 
         cli.run(%w(foo))
 
-        stub_const("Gitlab::SidekiqCluster::CLI::CHECK_TERMINATE_INTERVAL_SECONDS", 0.1)
+        stub_const("Gitlab::SidekiqCluster::CHECK_TERMINATE_INTERVAL_SECONDS", 0.1)
         allow(cli).to receive(:terminate_timeout_seconds) { 1 }
 
         cli.wait_for_termination

spec/lib/gitlab/sidekiq_cluster_spec.rb → spec/sidekiq_cluster/sidekiq_cluster_spec.rb

 # frozen_string_literal: true
 
-require 'fast_spec_helper'
 require 'rspec-parameterized'
 
-RSpec.describe Gitlab::SidekiqCluster do
+require_relative '../../sidekiq_cluster/sidekiq_cluster'
+
+RSpec.describe Gitlab::SidekiqCluster do # rubocop:disable RSpec/FilePath
   describe '.trap_signals' do
     it 'traps the given signals' do
       expect(described_class).to receive(:trap).ordered.with(:INT)

spec/tooling/quality/test_level_spec.rb

@@ -49,7 +49,7 @@ RSpec.describe Quality::TestLevel do
     context 'when level is integration' do
       it 'returns a pattern' do
         expect(subject.pattern(:integration))
-          .to eq("spec/{controllers,mailers,requests}{,/**/}*_spec.rb")
+          .to eq("spec/{commands,controllers,mailers,requests}{,/**/}*_spec.rb")
       end
     end
 
@@ -131,7 +131,7 @@ RSpec.describe Quality::TestLevel do
     context 'when level is integration' do
       it 'returns a regexp' do
         expect(subject.regexp(:integration))
-          .to eq(%r{spec/(controllers|mailers|requests)})
+          .to eq(%r{spec/(commands|controllers|mailers|requests)})
       end
     end
 
@@ -204,6 +204,10 @@ RSpec.describe Quality::TestLevel do
       expect(subject.level_for('spec/mailers/abuse_report_mailer_spec.rb')).to eq(:integration)
     end
 
+    it 'returns the correct level for an integration test in a subfolder' do
+      expect(subject.level_for('spec/commands/sidekiq_cluster/cli.rb')).to eq(:integration)
+    end
+
     it 'returns the correct level for a system test' do
       expect(subject.level_for('spec/features/abuse_report_spec.rb')).to eq(:system)
     end

tooling/quality/test_level.rb

@@ -54,6 +54,7 @@ module Quality
         tooling
       ],
       integration: %w[
+        commands
         controllers
         mailers
         requests