Merge branch 'bvl-deduplication-ff' into 'master'

Add a feature flag to disable deduplication

See merge request gitlab-org/gitlab!32469

doc/development/sidekiq_style_guide.md

@@ -150,6 +150,24 @@ execute.
 More [deduplication strategies have been suggested](https://gitlab.com/gitlab-com/gl-infra/scalability/issues/195). If you are implementing a worker that
 could benefit from a different strategy, please comment in the issue.
 
+If the automatic deduplication were to cause issues in certain
+queues. This can be temporarily disabled by enabling a feature flag
+named `disable_<queue name>_deduplication`. For example to disable
+deduplication for the `AuthorizedProjectsWorker`, we would enable the
+feature flag `disable_authorized_projects_deduplication`.
+
+From chatops:
+
+```shell
+/chatops run feature set disable_authorized_projects_deduplication true
+```
+
+From the rails console:
+
+```ruby
+Feature.enable!(:disable_authorized_projects_deduplication)
+```
+
 ## Job urgency
 
 Jobs can have an `urgency` attribute set, which can be `:high`,

lib/gitlab/sidekiq_middleware/duplicate_jobs/duplicate_job.rb

@@ -67,7 +67,7 @@ module Gitlab
         end
 
         def droppable?
-          idempotent? && duplicate?
+          idempotent? && duplicate? && ::Feature.disabled?("disable_#{queue_name}_deduplication")
         end
 
         private

spec/lib/gitlab/sidekiq_middleware/duplicate_jobs/duplicate_job_spec.rb

@@ -113,22 +113,27 @@ describe Gitlab::SidekiqMiddleware::DuplicateJobs::DuplicateJob, :clean_gitlab_r
   end
 
   describe 'droppable?' do
-    where(:idempotent, :duplicate) do
-      # [true, false].repeated_permutation(2)
-      [[true, true],
-       [true, false],
-       [false, true],
-       [false, false]]
+    where(:idempotent, :duplicate, :prevent_deduplication) do
+      # [true, false].repeated_permutation(3)
+      [[true, true, true],
+       [true, true, false],
+       [true, false, true],
+       [true, false, false],
+       [false, true, true],
+       [false, true, false],
+       [false, false, true],
+       [false, false, false]]
     end
 
     with_them do
       before do
         allow(AuthorizedProjectsWorker).to receive(:idempotent?).and_return(idempotent)
         allow(duplicate_job).to receive(:duplicate?).and_return(duplicate)
+        stub_feature_flags("disable_#{queue}_deduplication" => prevent_deduplication)
       end
 
       it 'is droppable when all conditions are met' do
-        if idempotent && duplicate
+        if idempotent && duplicate && !prevent_deduplication
           expect(duplicate_job).to be_droppable
         else
           expect(duplicate_job).not_to be_droppable