Commit 13a30718 authored by Douglas Barbosa Alexandre's avatar Douglas Barbosa Alexandre

Merge branch '205578-generate-pkg-events-file' into 'master'

Generate package events file

See merge request gitlab-org/gitlab!47118
parents df48e12c 114bf455
......@@ -3,6 +3,7 @@
class Packages::Event < ApplicationRecord
belongs_to :package, optional: true
UNIQUE_EVENTS_ALLOWED = %i[push_package delete_package pull_package].freeze
EVENT_SCOPES = ::Packages::Package.package_types.merge(container: 1000, tag: 1001).freeze
enum event_scope: EVENT_SCOPES
......@@ -22,4 +23,18 @@ class Packages::Event < ApplicationRecord
}
enum originator_type: { user: 0, deploy_token: 1, guest: 2 }
def self.event_name(event_scope, originator, event_type)
# remove `package` from the event name to avoid issues with HLLRedisCounter class parsing
"i_package_#{event_scope}_#{originator}_#{event_type.gsub(/_packages?/, "")}"
end
# Remove some of the events, for now, so we don't hammer Redis too hard.
# See: https://gitlab.com/gitlab-org/gitlab/-/issues/280770
def self.event_allowed?(event_scope, event_type, originator)
return false if originator.to_sym == :guest
return true if UNIQUE_EVENTS_ALLOWED.include?(event_type.to_sym)
false
end
end
---
title: Adds rake task to generate package events file
merge_request: 47118
author:
type: added
---
- name: i_package_maven_user_push
category: maven_packages
aggregation: weekly
redis_slot: package
- name: i_package_maven_deploy_token_push
category: maven_packages
aggregation: weekly
redis_slot: package
- name: i_package_maven_user_delete
category: maven_packages
aggregation: weekly
redis_slot: package
- name: i_package_maven_deploy_token_delete
category: maven_packages
aggregation: weekly
redis_slot: package
- name: i_package_maven_user_pull
category: maven_packages
aggregation: weekly
redis_slot: package
- name: i_package_maven_deploy_token_pull
category: maven_packages
aggregation: weekly
redis_slot: package
- name: i_package_npm_user_push
category: npm_packages
aggregation: weekly
redis_slot: package
- name: i_package_npm_deploy_token_push
category: npm_packages
aggregation: weekly
redis_slot: package
- name: i_package_npm_user_delete
category: npm_packages
aggregation: weekly
redis_slot: package
- name: i_package_npm_deploy_token_delete
category: npm_packages
aggregation: weekly
redis_slot: package
- name: i_package_npm_user_pull
category: npm_packages
aggregation: weekly
redis_slot: package
- name: i_package_npm_deploy_token_pull
category: npm_packages
aggregation: weekly
redis_slot: package
- name: i_package_conan_user_push
category: conan_packages
aggregation: weekly
redis_slot: package
- name: i_package_conan_deploy_token_push
category: conan_packages
aggregation: weekly
redis_slot: package
- name: i_package_conan_user_delete
category: conan_packages
aggregation: weekly
redis_slot: package
- name: i_package_conan_deploy_token_delete
category: conan_packages
aggregation: weekly
redis_slot: package
- name: i_package_conan_user_pull
category: conan_packages
aggregation: weekly
redis_slot: package
- name: i_package_conan_deploy_token_pull
category: conan_packages
aggregation: weekly
redis_slot: package
- name: i_package_nuget_user_push
category: nuget_packages
aggregation: weekly
redis_slot: package
- name: i_package_nuget_deploy_token_push
category: nuget_packages
aggregation: weekly
redis_slot: package
- name: i_package_nuget_user_delete
category: nuget_packages
aggregation: weekly
redis_slot: package
- name: i_package_nuget_deploy_token_delete
category: nuget_packages
aggregation: weekly
redis_slot: package
- name: i_package_nuget_user_pull
category: nuget_packages
aggregation: weekly
redis_slot: package
- name: i_package_nuget_deploy_token_pull
category: nuget_packages
aggregation: weekly
redis_slot: package
- name: i_package_pypi_user_push
category: pypi_packages
aggregation: weekly
redis_slot: package
- name: i_package_pypi_deploy_token_push
category: pypi_packages
aggregation: weekly
redis_slot: package
- name: i_package_pypi_user_delete
category: pypi_packages
aggregation: weekly
redis_slot: package
- name: i_package_pypi_deploy_token_delete
category: pypi_packages
aggregation: weekly
redis_slot: package
- name: i_package_pypi_user_pull
category: pypi_packages
aggregation: weekly
redis_slot: package
- name: i_package_pypi_deploy_token_pull
category: pypi_packages
aggregation: weekly
redis_slot: package
- name: i_package_composer_user_push
category: composer_packages
aggregation: weekly
redis_slot: package
- name: i_package_composer_deploy_token_push
category: composer_packages
aggregation: weekly
redis_slot: package
- name: i_package_composer_user_delete
category: composer_packages
aggregation: weekly
redis_slot: package
- name: i_package_composer_deploy_token_delete
category: composer_packages
aggregation: weekly
redis_slot: package
- name: i_package_composer_user_pull
category: composer_packages
aggregation: weekly
redis_slot: package
- name: i_package_composer_deploy_token_pull
category: composer_packages
aggregation: weekly
redis_slot: package
- name: i_package_generic_user_push
category: generic_packages
aggregation: weekly
redis_slot: package
- name: i_package_generic_deploy_token_push
category: generic_packages
aggregation: weekly
redis_slot: package
- name: i_package_generic_user_delete
category: generic_packages
aggregation: weekly
redis_slot: package
- name: i_package_generic_deploy_token_delete
category: generic_packages
aggregation: weekly
redis_slot: package
- name: i_package_generic_user_pull
category: generic_packages
aggregation: weekly
redis_slot: package
- name: i_package_generic_deploy_token_pull
category: generic_packages
aggregation: weekly
redis_slot: package
- name: i_package_golang_user_push
category: golang_packages
aggregation: weekly
redis_slot: package
- name: i_package_golang_deploy_token_push
category: golang_packages
aggregation: weekly
redis_slot: package
- name: i_package_golang_user_delete
category: golang_packages
aggregation: weekly
redis_slot: package
- name: i_package_golang_deploy_token_delete
category: golang_packages
aggregation: weekly
redis_slot: package
- name: i_package_golang_user_pull
category: golang_packages
aggregation: weekly
redis_slot: package
- name: i_package_golang_deploy_token_pull
category: golang_packages
aggregation: weekly
redis_slot: package
- name: i_package_debian_user_push
category: debian_packages
aggregation: weekly
redis_slot: package
- name: i_package_debian_deploy_token_push
category: debian_packages
aggregation: weekly
redis_slot: package
- name: i_package_debian_user_delete
category: debian_packages
aggregation: weekly
redis_slot: package
- name: i_package_debian_deploy_token_delete
category: debian_packages
aggregation: weekly
redis_slot: package
- name: i_package_debian_user_pull
category: debian_packages
aggregation: weekly
redis_slot: package
- name: i_package_debian_deploy_token_pull
category: debian_packages
aggregation: weekly
redis_slot: package
- name: i_package_container_user_push
category: container_packages
aggregation: weekly
redis_slot: package
- name: i_package_container_deploy_token_push
category: container_packages
aggregation: weekly
redis_slot: package
- name: i_package_container_user_delete
category: container_packages
aggregation: weekly
redis_slot: package
- name: i_package_container_deploy_token_delete
category: container_packages
aggregation: weekly
redis_slot: package
- name: i_package_container_user_pull
category: container_packages
aggregation: weekly
redis_slot: package
- name: i_package_container_deploy_token_pull
category: container_packages
aggregation: weekly
redis_slot: package
- name: i_package_tag_user_push
category: tag_packages
aggregation: weekly
redis_slot: package
- name: i_package_tag_deploy_token_push
category: tag_packages
aggregation: weekly
redis_slot: package
- name: i_package_tag_user_delete
category: tag_packages
aggregation: weekly
redis_slot: package
- name: i_package_tag_deploy_token_delete
category: tag_packages
aggregation: weekly
redis_slot: package
- name: i_package_tag_user_pull
category: tag_packages
aggregation: weekly
redis_slot: package
- name: i_package_tag_deploy_token_pull
category: tag_packages
aggregation: weekly
redis_slot: package
require 'logger'
desc "GitLab | Packages | Events | Generate hll counter events file for packages"
namespace :gitlab do
namespace :packages do
namespace :events do
task generate: :environment do
logger = Logger.new(STDOUT)
logger.info('Building list of package events...')
path = File.join(File.dirname(::Gitlab::UsageDataCounters::HLLRedisCounter::KNOWN_EVENTS_PATH), 'package_events.yml')
File.open(path, "w") { |file| file << generate_unique_events_list.to_yaml }
logger.info("Events file `#{path}` generated successfully")
rescue => e
logger.error("Error building events list: #{e}")
end
def event_pairs
::Packages::Event.event_types.keys.product(::Packages::Event.originator_types.keys)
end
def generate_unique_events_list
::Packages::Event::EVENT_SCOPES.keys.each_with_object([]) do |event_scope, events|
event_pairs.each do |event_type, originator|
if ::Packages::Event.event_allowed?(event_scope, event_type, originator)
events << {
"name" => ::Packages::Event.event_name(event_scope, originator, event_type),
"category" => "#{event_scope}_packages",
"aggregation" => "weekly",
"redis_slot" => "package"
}
end
end
end
end
end
end
end
......@@ -21,8 +21,26 @@ RSpec.describe Gitlab::UsageDataCounters::HLLRedisCounter, :clean_gitlab_redis_s
describe '.categories' do
it 'gets all unique category names' do
expect(described_class.categories).to contain_exactly(
'analytics', 'compliance', 'ide_edit', 'search', 'source_code',
'incident_management', 'issues_edit', 'testing', 'ci_secrets_management'
'compliance',
'analytics',
'ide_edit',
'search',
'source_code',
'incident_management',
'testing',
'issues_edit',
'ci_secrets_management',
'maven_packages',
'npm_packages',
'conan_packages',
'nuget_packages',
'pypi_packages',
'composer_packages',
'generic_packages',
'golang_packages',
'debian_packages',
'container_packages',
'tag_packages'
)
end
end
......
# frozen_string_literal: true
require 'rake_helper'
RSpec.describe 'gitlab:packages:events namespace rake task' do
before :all do
Rake.application.rake_require 'tasks/gitlab/packages/events'
end
describe 'generate' do
subject do
file = double('file')
yml_file = nil
allow(file).to receive(:<<) { |contents| yml_file = contents }
allow(File).to receive(:open).and_yield(file)
run_rake_task('gitlab:packages:events:generate')
YAML.safe_load(yml_file)
end
it 'excludes guest events' do
expect(subject.find { |event| event['name'].include?("guest") }).to be_nil
end
::Packages::Event::EVENT_SCOPES.keys.each do |event_scope|
it "includes includes `#{event_scope}` scope" do
expect(subject.find { |event| event['name'].include?(event_scope) }).not_to be_nil
end
end
it 'excludes some event types' do
expect(subject.find { |event| event['name'].include?("search_package") }).to be_nil
expect(subject.find { |event| event['name'].include?("list_package") }).to be_nil
end
end
end
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment