Merge branch '205578-generate-pkg-events-file' into 'master'

Generate package events file

See merge request gitlab-org/gitlab!47118

app/models/packages/event.rb

@@ -3,6 +3,7 @@
 class Packages::Event < ApplicationRecord
   belongs_to :package, optional: true
 
+  UNIQUE_EVENTS_ALLOWED = %i[push_package delete_package pull_package].freeze
   EVENT_SCOPES = ::Packages::Package.package_types.merge(container: 1000, tag: 1001).freeze
 
   enum event_scope: EVENT_SCOPES
@@ -22,4 +23,18 @@ class Packages::Event < ApplicationRecord
   }
 
   enum originator_type: { user: 0, deploy_token: 1, guest: 2 }
+
+  def self.event_name(event_scope, originator, event_type)
+    # remove `package` from the event name to avoid issues with HLLRedisCounter class parsing
+    "i_package_#{event_scope}_#{originator}_#{event_type.gsub(/_packages?/, "")}"
+  end
+
+  # Remove some of the events, for now, so we don't hammer Redis too hard.
+  # See: https://gitlab.com/gitlab-org/gitlab/-/issues/280770
+  def self.event_allowed?(event_scope, event_type, originator)
+    return false if originator.to_sym == :guest
+    return true if UNIQUE_EVENTS_ALLOWED.include?(event_type.to_sym)
+
+    false
+  end
 end

changelogs/unreleased/205578-generate-pkg-events-file.yml

+---
+title: Adds rake task to generate package events file
+merge_request: 47118
+author:
+type: added

lib/gitlab/usage_data_counters/known_events/package_events.yml

+---
+- name: i_package_maven_user_push
+  category: maven_packages
+  aggregation: weekly
+  redis_slot: package
+- name: i_package_maven_deploy_token_push
+  category: maven_packages
+  aggregation: weekly
+  redis_slot: package
+- name: i_package_maven_user_delete
+  category: maven_packages
+  aggregation: weekly
+  redis_slot: package
+- name: i_package_maven_deploy_token_delete
+  category: maven_packages
+  aggregation: weekly
+  redis_slot: package
+- name: i_package_maven_user_pull
+  category: maven_packages
+  aggregation: weekly
+  redis_slot: package
+- name: i_package_maven_deploy_token_pull
+  category: maven_packages
+  aggregation: weekly
+  redis_slot: package
+- name: i_package_npm_user_push
+  category: npm_packages
+  aggregation: weekly
+  redis_slot: package
+- name: i_package_npm_deploy_token_push
+  category: npm_packages
+  aggregation: weekly
+  redis_slot: package
+- name: i_package_npm_user_delete
+  category: npm_packages
+  aggregation: weekly
+  redis_slot: package
+- name: i_package_npm_deploy_token_delete
+  category: npm_packages
+  aggregation: weekly
+  redis_slot: package
+- name: i_package_npm_user_pull
+  category: npm_packages
+  aggregation: weekly
+  redis_slot: package
+- name: i_package_npm_deploy_token_pull
+  category: npm_packages
+  aggregation: weekly
+  redis_slot: package
+- name: i_package_conan_user_push
+  category: conan_packages
+  aggregation: weekly
+  redis_slot: package
+- name: i_package_conan_deploy_token_push
+  category: conan_packages
+  aggregation: weekly
+  redis_slot: package
+- name: i_package_conan_user_delete
+  category: conan_packages
+  aggregation: weekly
+  redis_slot: package
+- name: i_package_conan_deploy_token_delete
+  category: conan_packages
+  aggregation: weekly
+  redis_slot: package
+- name: i_package_conan_user_pull
+  category: conan_packages
+  aggregation: weekly
+  redis_slot: package
+- name: i_package_conan_deploy_token_pull
+  category: conan_packages
+  aggregation: weekly
+  redis_slot: package
+- name: i_package_nuget_user_push
+  category: nuget_packages
+  aggregation: weekly
+  redis_slot: package
+- name: i_package_nuget_deploy_token_push
+  category: nuget_packages
+  aggregation: weekly
+  redis_slot: package
+- name: i_package_nuget_user_delete
+  category: nuget_packages
+  aggregation: weekly
+  redis_slot: package
+- name: i_package_nuget_deploy_token_delete
+  category: nuget_packages
+  aggregation: weekly
+  redis_slot: package
+- name: i_package_nuget_user_pull
+  category: nuget_packages
+  aggregation: weekly
+  redis_slot: package
+- name: i_package_nuget_deploy_token_pull
+  category: nuget_packages
+  aggregation: weekly
+  redis_slot: package
+- name: i_package_pypi_user_push
+  category: pypi_packages
+  aggregation: weekly
+  redis_slot: package
+- name: i_package_pypi_deploy_token_push
+  category: pypi_packages
+  aggregation: weekly
+  redis_slot: package
+- name: i_package_pypi_user_delete
+  category: pypi_packages
+  aggregation: weekly
+  redis_slot: package
+- name: i_package_pypi_deploy_token_delete
+  category: pypi_packages
+  aggregation: weekly
+  redis_slot: package
+- name: i_package_pypi_user_pull
+  category: pypi_packages
+  aggregation: weekly
+  redis_slot: package
+- name: i_package_pypi_deploy_token_pull
+  category: pypi_packages
+  aggregation: weekly
+  redis_slot: package
+- name: i_package_composer_user_push
+  category: composer_packages
+  aggregation: weekly
+  redis_slot: package
+- name: i_package_composer_deploy_token_push
+  category: composer_packages
+  aggregation: weekly
+  redis_slot: package
+- name: i_package_composer_user_delete
+  category: composer_packages
+  aggregation: weekly
+  redis_slot: package
+- name: i_package_composer_deploy_token_delete
+  category: composer_packages
+  aggregation: weekly
+  redis_slot: package
+- name: i_package_composer_user_pull
+  category: composer_packages
+  aggregation: weekly
+  redis_slot: package
+- name: i_package_composer_deploy_token_pull
+  category: composer_packages
+  aggregation: weekly
+  redis_slot: package
+- name: i_package_generic_user_push
+  category: generic_packages
+  aggregation: weekly
+  redis_slot: package
+- name: i_package_generic_deploy_token_push
+  category: generic_packages
+  aggregation: weekly
+  redis_slot: package
+- name: i_package_generic_user_delete
+  category: generic_packages
+  aggregation: weekly
+  redis_slot: package
+- name: i_package_generic_deploy_token_delete
+  category: generic_packages
+  aggregation: weekly
+  redis_slot: package
+- name: i_package_generic_user_pull
+  category: generic_packages
+  aggregation: weekly
+  redis_slot: package
+- name: i_package_generic_deploy_token_pull
+  category: generic_packages
+  aggregation: weekly
+  redis_slot: package
+- name: i_package_golang_user_push
+  category: golang_packages
+  aggregation: weekly
+  redis_slot: package
+- name: i_package_golang_deploy_token_push
+  category: golang_packages
+  aggregation: weekly
+  redis_slot: package
+- name: i_package_golang_user_delete
+  category: golang_packages
+  aggregation: weekly
+  redis_slot: package
+- name: i_package_golang_deploy_token_delete
+  category: golang_packages
+  aggregation: weekly
+  redis_slot: package
+- name: i_package_golang_user_pull
+  category: golang_packages
+  aggregation: weekly
+  redis_slot: package
+- name: i_package_golang_deploy_token_pull
+  category: golang_packages
+  aggregation: weekly
+  redis_slot: package
+- name: i_package_debian_user_push
+  category: debian_packages
+  aggregation: weekly
+  redis_slot: package
+- name: i_package_debian_deploy_token_push
+  category: debian_packages
+  aggregation: weekly
+  redis_slot: package
+- name: i_package_debian_user_delete
+  category: debian_packages
+  aggregation: weekly
+  redis_slot: package
+- name: i_package_debian_deploy_token_delete
+  category: debian_packages
+  aggregation: weekly
+  redis_slot: package
+- name: i_package_debian_user_pull
+  category: debian_packages
+  aggregation: weekly
+  redis_slot: package
+- name: i_package_debian_deploy_token_pull
+  category: debian_packages
+  aggregation: weekly
+  redis_slot: package
+- name: i_package_container_user_push
+  category: container_packages
+  aggregation: weekly
+  redis_slot: package
+- name: i_package_container_deploy_token_push
+  category: container_packages
+  aggregation: weekly
+  redis_slot: package
+- name: i_package_container_user_delete
+  category: container_packages
+  aggregation: weekly
+  redis_slot: package
+- name: i_package_container_deploy_token_delete
+  category: container_packages
+  aggregation: weekly
+  redis_slot: package
+- name: i_package_container_user_pull
+  category: container_packages
+  aggregation: weekly
+  redis_slot: package
+- name: i_package_container_deploy_token_pull
+  category: container_packages
+  aggregation: weekly
+  redis_slot: package
+- name: i_package_tag_user_push
+  category: tag_packages
+  aggregation: weekly
+  redis_slot: package
+- name: i_package_tag_deploy_token_push
+  category: tag_packages
+  aggregation: weekly
+  redis_slot: package
+- name: i_package_tag_user_delete
+  category: tag_packages
+  aggregation: weekly
+  redis_slot: package
+- name: i_package_tag_deploy_token_delete
+  category: tag_packages
+  aggregation: weekly
+  redis_slot: package
+- name: i_package_tag_user_pull
+  category: tag_packages
+  aggregation: weekly
+  redis_slot: package
+- name: i_package_tag_deploy_token_pull
+  category: tag_packages
+  aggregation: weekly
+  redis_slot: package

lib/tasks/gitlab/packages/events.rake

+require 'logger'
+
+desc "GitLab | Packages | Events | Generate hll counter events file for packages"
+namespace :gitlab do
+  namespace :packages do
+    namespace :events do
+      task generate: :environment do
+        logger = Logger.new(STDOUT)
+        logger.info('Building list of package events...')
+
+        path = File.join(File.dirname(::Gitlab::UsageDataCounters::HLLRedisCounter::KNOWN_EVENTS_PATH), 'package_events.yml')
+
+        File.open(path, "w") { |file| file << generate_unique_events_list.to_yaml }
+
+        logger.info("Events file `#{path}` generated successfully")
+      rescue => e
+        logger.error("Error building events list: #{e}")
+      end
+
+      def event_pairs
+        ::Packages::Event.event_types.keys.product(::Packages::Event.originator_types.keys)
+      end
+
+      def generate_unique_events_list
+        ::Packages::Event::EVENT_SCOPES.keys.each_with_object([]) do |event_scope, events|
+          event_pairs.each do |event_type, originator|
+            if ::Packages::Event.event_allowed?(event_scope, event_type, originator)
+              events << {
+                "name" => ::Packages::Event.event_name(event_scope, originator, event_type),
+                "category" => "#{event_scope}_packages",
+                "aggregation" => "weekly",
+                "redis_slot" => "package"
+              }
+            end
+          end
+        end
+      end
+    end
+  end
+end

spec/lib/gitlab/usage_data_counters/hll_redis_counter_spec.rb

@@ -21,8 +21,26 @@ RSpec.describe Gitlab::UsageDataCounters::HLLRedisCounter, :clean_gitlab_redis_s
   describe '.categories' do
     it 'gets all unique category names' do
       expect(described_class.categories).to contain_exactly(
-        'analytics', 'compliance', 'ide_edit', 'search', 'source_code',
-        'incident_management', 'issues_edit', 'testing', 'ci_secrets_management'
+        'compliance',
+        'analytics',
+        'ide_edit',
+        'search',
+        'source_code',
+        'incident_management',
+        'testing',
+        'issues_edit',
+        'ci_secrets_management',
+        'maven_packages',
+        'npm_packages',
+        'conan_packages',
+        'nuget_packages',
+        'pypi_packages',
+        'composer_packages',
+        'generic_packages',
+        'golang_packages',
+        'debian_packages',
+        'container_packages',
+        'tag_packages'
       )
     end
   end

spec/tasks/gitlab/packages/events_rake_spec.rb

+# frozen_string_literal: true
+
+require 'rake_helper'
+
+RSpec.describe 'gitlab:packages:events namespace rake task' do
+  before :all do
+    Rake.application.rake_require 'tasks/gitlab/packages/events'
+  end
+
+  describe 'generate' do
+    subject do
+      file = double('file')
+      yml_file = nil
+
+      allow(file).to receive(:<<) { |contents| yml_file = contents }
+      allow(File).to receive(:open).and_yield(file)
+
+      run_rake_task('gitlab:packages:events:generate')
+
+      YAML.safe_load(yml_file)
+    end
+
+    it 'excludes guest events' do
+      expect(subject.find { |event| event['name'].include?("guest") }).to be_nil
+    end
+
+    ::Packages::Event::EVENT_SCOPES.keys.each do |event_scope|
+      it "includes includes `#{event_scope}` scope" do
+        expect(subject.find { |event| event['name'].include?(event_scope) }).not_to be_nil
+      end
+    end
+
+    it 'excludes some event types' do
+      expect(subject.find { |event| event['name'].include?("search_package") }).to be_nil
+      expect(subject.find { |event| event['name'].include?("list_package") }).to be_nil
+    end
+  end
+end