Merge branch 'add-tags-to-queue-attributes' into 'master'

Add tags to queue selector attributes

See merge request gitlab-org/gitlab!32651

app/workers/concerns/worker_attributes.rb

@@ -111,6 +111,14 @@ module WorkerAttributes
         1
     end
 
+    def tags(*values)
+      worker_attributes[:tags] = values
+    end
+
+    def get_tags
+      Array(worker_attributes[:tags])
+    end
+
     protected
 
     # Returns a worker attribute declared on this class or its parent class.

changelogs/unreleased/add-tags-to-queue-attributes.yml

+---
+title: Add tags to experimental queue selector attributes
+merge_request: 32651
+author:
+type: added

doc/administration/operations/extra_sidekiq_processes.md

@@ -126,6 +126,8 @@ in a more general way using the following components:
 
 ### Available attributes
 
+- [Introduced](https://gitlab.com/gitlab-com/gl-infra/scalability/-/issues/261) in GitLab 13.1, `tags`.
+
 From the [list of all available
 attributes](https://gitlab.com/gitlab-org/gitlab/-/blob/master/app/workers/all_queues.yml),
 `experimental_queue_selector` allows selecting of queues by the
@@ -144,14 +146,21 @@ following attributes:
 - `name` - the queue name. The other attributes are typically more useful as
   they are more general, but this is available in case a particular queue needs
   to be selected.
-- `resource_boundary` - if the worker is bound by `cpu`, `memory`, or
+- `resource_boundary` - if the queue is bound by `cpu`, `memory`, or
   `unknown`. For example, the `project_export` queue is memory bound as it has
   to load data in memory before saving it for export.
+- `tags` - short-lived annotations for queues. These are expected to frequently
+  change from release to release, and may be removed entirely.
 
 `has_external_dependencies` is a boolean attribute: only the exact
 string `true` is considered true, and everything else is considered
 false.
 
+`tags` is a set, which means that `=` checks for intersecting sets, and
+`!=` checks for disjoint sets. For example, `tags=a,b` selects queues
+that have tags `a`, `b`, or both. `tags!=a,b` selects queues that have
+neither of those tags.
+
 ### Available operators
 
 `experimental_queue_selector` supports the following operators, listed

lib/gitlab/sidekiq_config.rb

@@ -14,8 +14,8 @@ module Gitlab
     ].compact.freeze
 
     DEFAULT_WORKERS = [
-      DummyWorker.new('default', weight: 1),
-      DummyWorker.new('mailers', weight: 2)
+      DummyWorker.new('default', weight: 1, tags: []),
+      DummyWorker.new('mailers', weight: 2, tags: [])
     ].map { |worker| Gitlab::SidekiqConfig::Worker.new(worker, ee: false) }.freeze
 
     class << self

lib/gitlab/sidekiq_config/cli_methods.rb

@@ -28,6 +28,7 @@ module Gitlab
         has_external_dependencies: lambda { |value| value == 'true' },
         name: :to_s,
         resource_boundary: :to_sym,
+        tags: :to_sym,
         urgency: :to_sym
       }.freeze
 
@@ -117,7 +118,11 @@ module Gitlab
 
         raise UnknownPredicate.new("Unknown predicate: #{lhs}") unless values_block
 
-        lambda { |queue| values.map(&values_block).include?(queue[lhs.to_sym]) }
+        lambda do |queue|
+          comparator = Array(queue[lhs.to_sym]).to_set
+
+          values.map(&values_block).to_set.intersect?(comparator)
+        end
       end
     end
   end

lib/gitlab/sidekiq_config/dummy_worker.rb

@@ -12,7 +12,8 @@ module Gitlab
         urgency: :get_urgency,
         resource_boundary: :get_worker_resource_boundary,
         idempotent: :idempotent?,
-        weight: :get_weight
+        weight: :get_weight,
+        tags: :get_tags
       }.freeze
 
       def initialize(queue, attributes = {})

lib/gitlab/sidekiq_config/worker.rb

@@ -6,7 +6,7 @@ module Gitlab
       include Comparable
 
       attr_reader :klass
-      delegate :feature_category_not_owned?, :get_feature_category,
+      delegate :feature_category_not_owned?, :get_feature_category, :get_tags,
                :get_urgency, :get_weight, :get_worker_resource_boundary,
                :idempotent?, :queue, :queue_namespace,
                :worker_has_external_dependencies?,
@@ -52,7 +52,8 @@ module Gitlab
           urgency: get_urgency,
           resource_boundary: get_worker_resource_boundary,
           weight: get_weight,
-          idempotent: idempotent?
+          idempotent: idempotent?,
+          tags: get_tags
         }
       end
 

spec/lib/gitlab/sidekiq_config/cli_methods_spec.rb

@@ -117,28 +117,32 @@ describe Gitlab::SidekiqConfig::CliMethods do
           feature_category: :category_a,
           has_external_dependencies: false,
           urgency: :low,
-          resource_boundary: :cpu
+          resource_boundary: :cpu,
+          tags: [:no_disk_io, :git_access]
         },
         {
           name: 'a:2',
           feature_category: :category_a,
           has_external_dependencies: false,
           urgency: :high,
-          resource_boundary: :none
+          resource_boundary: :none,
+          tags: [:git_access]
         },
         {
           name: 'b',
           feature_category: :category_b,
           has_external_dependencies: true,
           urgency: :high,
-          resource_boundary: :memory
+          resource_boundary: :memory,
+          tags: [:no_disk_io]
         },
         {
           name: 'c',
           feature_category: :category_c,
           has_external_dependencies: false,
           urgency: :throttled,
-          resource_boundary: :memory
+          resource_boundary: :memory,
+          tags: []
         }
       ]
     end
@@ -177,6 +181,18 @@ describe Gitlab::SidekiqConfig::CliMethods do
         'resource_boundary=memory|resource_boundary=cpu' | %w(a b c)
         'resource_boundary!=memory,cpu' | %w(a:2)
 
+        # tags
+        'tags=no_disk_io' | %w(a b)
+        'tags=no_disk_io,git_access' | %w(a a:2 b)
+        'tags=no_disk_io|tags=git_access' | %w(a a:2 b)
+        'tags=no_disk_io&tags=git_access' | %w(a)
+        'tags!=no_disk_io' | %w(a:2 c)
+        'tags!=no_disk_io,git_access' | %w(c)
+        'tags=unknown_tag' | []
+        'tags!=no_disk_io' | %w(a:2 c)
+        'tags!=no_disk_io,git_access' | %w(c)
+        'tags!=unknown_tag' | %w(a a:2 b c)
+
         # combinations
         'feature_category=category_a&urgency=high' | %w(a:2)
         'feature_category=category_a&urgency=high|feature_category=category_c' | %w(a:2 c)

spec/lib/gitlab/sidekiq_config/worker_spec.rb

@@ -13,7 +13,8 @@ describe Gitlab::SidekiqConfig::Worker do
       get_worker_resource_boundary: attributes[:resource_boundary],
       get_urgency: attributes[:urgency],
       worker_has_external_dependencies?: attributes[:has_external_dependencies],
-      idempotent?: attributes[:idempotent]
+      idempotent?: attributes[:idempotent],
+      get_tags: attributes[:tags]
     )
 
     described_class.new(inner_worker, ee: false)
@@ -91,7 +92,8 @@ describe Gitlab::SidekiqConfig::Worker do
         urgency: :low,
         resource_boundary: :memory,
         weight: 2,
-        idempotent: true
+        idempotent: true,
+        tags: []
       }
 
       attributes_b = {
@@ -100,7 +102,8 @@ describe Gitlab::SidekiqConfig::Worker do
         urgency: :high,
         resource_boundary: :unknown,
         weight: 3,
-        idempotent: false
+        idempotent: false,
+        tags: [:no_disk_io]
       }
 
       worker_a = create_worker(queue: 'a', **attributes_a)