Commit 1ea62abd authored by Alina Mihaila's avatar Alina Mihaila Committed by Dylan Griffith

Add fallback_totals to redis counters

Add fallback_totals in BaseCounter
Add fallback_totals in specific redis counters
Add tests

Fix rubocop offence
parent 972c347a
...@@ -140,7 +140,7 @@ module EE ...@@ -140,7 +140,7 @@ module EE
super.tap do |usage_data| super.tap do |usage_data|
usage_data[:counts].merge!( usage_data[:counts].merge!(
{ {
dependency_list_usages_total: ::Gitlab::UsageCounters::DependencyList.usage_totals[:total], dependency_list_usages_total: redis_usage_data { ::Gitlab::UsageCounters::DependencyList.usage_totals[:total] },
epics: count(::Epic), epics: count(::Epic),
feature_flags: count(Operations::FeatureFlag), feature_flags: count(Operations::FeatureFlag),
geo_nodes: count(::GeoNode), geo_nodes: count(::GeoNode),
...@@ -148,7 +148,7 @@ module EE ...@@ -148,7 +148,7 @@ module EE
issues_with_health_status: count(::Issue.with_health_status), issues_with_health_status: count(::Issue.with_health_status),
ldap_keys: count(::LDAPKey), ldap_keys: count(::LDAPKey),
ldap_users: count(::User.ldap, 'users.id'), ldap_users: count(::User.ldap, 'users.id'),
pod_logs_usages_total: ::Gitlab::UsageCounters::PodLogs.usage_totals[:total], pod_logs_usages_total: redis_usage_data { ::Gitlab::UsageCounters::PodLogs.usage_totals[:total] },
projects_enforcing_code_owner_approval: count(::Project.without_deleted.non_archived.requiring_code_owner_approval), projects_enforcing_code_owner_approval: count(::Project.without_deleted.non_archived.requiring_code_owner_approval),
merge_requests_with_optional_codeowners: distinct_count(::ApprovalMergeRequestRule.code_owner_approval_optional, :merge_request_id), merge_requests_with_optional_codeowners: distinct_count(::ApprovalMergeRequestRule.code_owner_approval_optional, :merge_request_id),
merge_requests_with_required_codeowners: distinct_count(::ApprovalMergeRequestRule.code_owner_approval_required, :merge_request_id), merge_requests_with_required_codeowners: distinct_count(::ApprovalMergeRequestRule.code_owner_approval_required, :merge_request_id),
......
...@@ -25,7 +25,17 @@ module Gitlab::UsageCounters ...@@ -25,7 +25,17 @@ module Gitlab::UsageCounters
end end
def totals def totals
KNOWN_EVENTS.map { |e| ["design_management_designs_#{e}".to_sym, read(e)] }.to_h KNOWN_EVENTS.map { |event| [counter_key(event), read(event)] }.to_h
end
def fallback_totals
KNOWN_EVENTS.map { |event| [counter_key(event), -1] }.to_h
end
private
def counter_key(event)
"design_management_designs_#{event}".to_sym
end end
end end
end end
......
# frozen_string_literal: true # frozen_string_literal: true
# For hardening usage ping and make it easier to add measures there is in place alt_usage_data method # For hardening usage ping and make it easier to add measures there is in place
# which handles StandardError and fallbacks into -1 # * alt_usage_data method
# this way not all measures fail if we encounter one exception # handles StandardError and fallbacks into -1 this way not all measures fail if we encounter one exception
# #
# Examples: # Examples:
# alt_usage_data { Gitlab::VERSION } # alt_usage_data { Gitlab::VERSION }
# alt_usage_data { Gitlab::CurrentSettings.uuid } # alt_usage_data { Gitlab::CurrentSettings.uuid }
#
# * redis_usage_data method
# handles ::Redis::CommandError, Gitlab::UsageDataCounters::BaseCounter::UnknownEvent
# returns -1 when a block is sent or hash with all values -1 when a counter is sent
# different behaviour due to 2 different implementations of redis counter
#
# Examples:
# redis_usage_data(Gitlab::UsageDataCounters::WikiPageCounter)
# redis_usage_data { ::Gitlab::UsageCounters::PodLogs.usage_totals[:total] }
module Gitlab module Gitlab
class UsageData class UsageData
BATCH_SIZE = 100 BATCH_SIZE = 100
...@@ -192,7 +201,7 @@ module Gitlab ...@@ -192,7 +201,7 @@ module Gitlab
# @return [Hash<Symbol, Integer>] # @return [Hash<Symbol, Integer>]
def usage_counters def usage_counters
usage_data_counters.map(&:totals).reduce({}) { |a, b| a.merge(b) } usage_data_counters.map { |counter| redis_usage_data(counter) }.reduce({}, :merge)
end end
# @return [Array<#totals>] An array of objects that respond to `#totals` # @return [Array<#totals>] An array of objects that respond to `#totals`
...@@ -388,8 +397,28 @@ module Gitlab ...@@ -388,8 +397,28 @@ module Gitlab
fallback fallback
end end
def redis_usage_data(counter = nil, &block)
if block_given?
redis_usage_counter(&block)
elsif counter.present?
redis_usage_data_totals(counter)
end
end
private private
def redis_usage_counter
yield
rescue ::Redis::CommandError, Gitlab::UsageDataCounters::BaseCounter::UnknownEvent
-1
end
def redis_usage_data_totals(counter)
counter.totals
rescue ::Redis::CommandError, Gitlab::UsageDataCounters::BaseCounter::UnknownEvent
counter.fallback_totals
end
def installation_type def installation_type
if Rails.env.production? if Rails.env.production?
Gitlab::INSTALLATION_TYPE Gitlab::INSTALLATION_TYPE
......
...@@ -22,11 +22,19 @@ module Gitlab::UsageDataCounters ...@@ -22,11 +22,19 @@ module Gitlab::UsageDataCounters
end end
def totals def totals
known_events.map { |e| ["#{prefix}_#{e}".to_sym, read(e)] }.to_h known_events.map { |event| [counter_key(event), read(event)] }.to_h
end
def fallback_totals
known_events.map { |event| [counter_key(event), -1] }.to_h
end end
private private
def counter_key(event)
"#{prefix}_#{event}".to_sym
end
def known_events def known_events
self::KNOWN_EVENTS self::KNOWN_EVENTS
end end
......
...@@ -25,12 +25,20 @@ module Gitlab::UsageDataCounters ...@@ -25,12 +25,20 @@ module Gitlab::UsageDataCounters
def totals def totals
COUNTABLE_TYPES.map do |countable_type| COUNTABLE_TYPES.map do |countable_type|
[:"#{countable_type.underscore}_comment", read(:create, countable_type)] [counter_key(countable_type), read(:create, countable_type)]
end.to_h end.to_h
end end
def fallback_totals
COUNTABLE_TYPES.map { |counter_key| [counter_key(counter_key), -1] }.to_h
end
private private
def counter_key(countable_type)
"#{countable_type.underscore}_comment".to_sym
end
def countable?(noteable_type) def countable?(noteable_type)
COUNTABLE_TYPES.include?(noteable_type.to_s) COUNTABLE_TYPES.include?(noteable_type.to_s)
end end
......
...@@ -21,6 +21,10 @@ module Gitlab ...@@ -21,6 +21,10 @@ module Gitlab
navbar_searches: total_navbar_searches_count navbar_searches: total_navbar_searches_count
} }
end end
def fallback_totals
{ navbar_searches: -1 }
end
end end
end end
end end
......
...@@ -4,54 +4,44 @@ module Gitlab ...@@ -4,54 +4,44 @@ module Gitlab
module UsageDataCounters module UsageDataCounters
class WebIdeCounter class WebIdeCounter
extend RedisCounter extend RedisCounter
KNOWN_EVENTS = %i[commits views merge_requests previews].freeze
COMMITS_COUNT_KEY = 'WEB_IDE_COMMITS_COUNT' PREFIX = 'web_ide'
MERGE_REQUEST_COUNT_KEY = 'WEB_IDE_MERGE_REQUESTS_COUNT'
VIEWS_COUNT_KEY = 'WEB_IDE_VIEWS_COUNT'
PREVIEW_COUNT_KEY = 'WEB_IDE_PREVIEWS_COUNT'
class << self class << self
def increment_commits_count def increment_commits_count
increment(COMMITS_COUNT_KEY) increment(redis_key('commits'))
end
def total_commits_count
total_count(COMMITS_COUNT_KEY)
end end
def increment_merge_requests_count def increment_merge_requests_count
increment(MERGE_REQUEST_COUNT_KEY) increment(redis_key('merge_requests'))
end
def total_merge_requests_count
total_count(MERGE_REQUEST_COUNT_KEY)
end end
def increment_views_count def increment_views_count
increment(VIEWS_COUNT_KEY) increment(redis_key('views'))
end
def total_views_count
total_count(VIEWS_COUNT_KEY)
end end
def increment_previews_count def increment_previews_count
return unless Gitlab::CurrentSettings.web_ide_clientside_preview_enabled? return unless Gitlab::CurrentSettings.web_ide_clientside_preview_enabled?
increment(PREVIEW_COUNT_KEY) increment(redis_key('previews'))
end end
def total_previews_count def totals
total_count(PREVIEW_COUNT_KEY) KNOWN_EVENTS.map { |event| [counter_key(event), total_count(redis_key(event))] }.to_h
end end
def totals def fallback_totals
{ KNOWN_EVENTS.map { |event| [counter_key(event), -1] }.to_h
web_ide_commits: total_commits_count, end
web_ide_views: total_views_count,
web_ide_merge_requests: total_merge_requests_count, private
web_ide_previews: total_previews_count
} def redis_key(event)
"#{PREFIX}_#{event}_count".upcase
end
def counter_key(event)
"#{PREFIX}_#{event}".to_sym
end end
end end
end end
......
...@@ -42,7 +42,7 @@ describe Projects::UsagePingController do ...@@ -42,7 +42,7 @@ describe Projects::UsagePingController do
it 'increments the counter' do it 'increments the counter' do
expect do expect do
subject subject
end.to change { Gitlab::UsageDataCounters::WebIdeCounter.total_previews_count }.by(1) end.to change { Gitlab::UsageDataCounters::WebIdeCounter.total_count('WEB_IDE_PREVIEWS_COUNT') }.by(1)
end end
end end
end end
......
...@@ -3,35 +3,27 @@ ...@@ -3,35 +3,27 @@
require 'spec_helper' require 'spec_helper'
describe Gitlab::UsageDataCounters::WebIdeCounter, :clean_gitlab_redis_shared_state do describe Gitlab::UsageDataCounters::WebIdeCounter, :clean_gitlab_redis_shared_state do
shared_examples 'counter examples' do shared_examples 'counter examples' do |event|
it 'increments counter and return the total count' do it 'increments counter and return the total count' do
expect(described_class.public_send(total_counter_method)).to eq(0) expect(described_class.public_send(:total_count, event)).to eq(0)
2.times { described_class.public_send(increment_counter_method) } 2.times { described_class.public_send(:"increment_#{event}_count") }
expect(described_class.public_send(total_counter_method)).to eq(2) redis_key = "web_ide_#{event}_count".upcase
expect(described_class.public_send(:total_count, redis_key)).to eq(2)
end end
end end
describe 'commits counter' do describe 'commits counter' do
let(:increment_counter_method) { :increment_commits_count } it_behaves_like 'counter examples', 'commits'
let(:total_counter_method) { :total_commits_count }
it_behaves_like 'counter examples'
end end
describe 'merge requests counter' do describe 'merge requests counter' do
let(:increment_counter_method) { :increment_merge_requests_count } it_behaves_like 'counter examples', 'merge_requests'
let(:total_counter_method) { :total_merge_requests_count }
it_behaves_like 'counter examples'
end end
describe 'views counter' do describe 'views counter' do
let(:increment_counter_method) { :increment_views_count } it_behaves_like 'counter examples', 'views'
let(:total_counter_method) { :total_views_count }
it_behaves_like 'counter examples'
end end
describe 'previews counter' do describe 'previews counter' do
...@@ -42,21 +34,19 @@ describe Gitlab::UsageDataCounters::WebIdeCounter, :clean_gitlab_redis_shared_st ...@@ -42,21 +34,19 @@ describe Gitlab::UsageDataCounters::WebIdeCounter, :clean_gitlab_redis_shared_st
end end
context 'when web ide clientside preview is enabled' do context 'when web ide clientside preview is enabled' do
let(:increment_counter_method) { :increment_previews_count } it_behaves_like 'counter examples', 'previews'
let(:total_counter_method) { :total_previews_count }
it_behaves_like 'counter examples'
end end
context 'when web ide clientside preview is not enabled' do context 'when web ide clientside preview is not enabled' do
let(:setting_enabled) { false } let(:setting_enabled) { false }
it 'does not increment the counter' do it 'does not increment the counter' do
expect(described_class.total_previews_count).to eq(0) redis_key = 'WEB_IDE_PREVIEWS_COUNT'
expect(described_class.total_count(redis_key)).to eq(0)
2.times { described_class.increment_previews_count } 2.times { described_class.increment_previews_count }
expect(described_class.total_previews_count).to eq(0) expect(described_class.total_count(redis_key)).to eq(0)
end end
end end
end end
......
...@@ -116,6 +116,7 @@ describe Gitlab::UsageData, :aggregate_failures do ...@@ -116,6 +116,7 @@ describe Gitlab::UsageData, :aggregate_failures do
subject { described_class.usage_data_counters } subject { described_class.usage_data_counters }
it { is_expected.to all(respond_to :totals) } it { is_expected.to all(respond_to :totals) }
it { is_expected.to all(respond_to :fallback_totals) }
describe 'the results of calling #totals on all objects in the array' do describe 'the results of calling #totals on all objects in the array' do
subject { described_class.usage_data_counters.map(&:totals) } subject { described_class.usage_data_counters.map(&:totals) }
...@@ -124,6 +125,13 @@ describe Gitlab::UsageData, :aggregate_failures do ...@@ -124,6 +125,13 @@ describe Gitlab::UsageData, :aggregate_failures do
it { is_expected.to all(have_attributes(keys: all(be_a Symbol), values: all(be_a Integer))) } it { is_expected.to all(have_attributes(keys: all(be_a Symbol), values: all(be_a Integer))) }
end end
describe 'the results of calling #fallback_totals on all objects in the array' do
subject { described_class.usage_data_counters.map(&:fallback_totals) }
it { is_expected.to all(be_a Hash) }
it { is_expected.to all(have_attributes(keys: all(be_a Symbol), values: all(eq(-1)))) }
end
it 'does not have any conflicts' do it 'does not have any conflicts' do
all_keys = subject.flat_map { |counter| counter.totals.keys } all_keys = subject.flat_map { |counter| counter.totals.keys }
...@@ -576,4 +584,28 @@ describe Gitlab::UsageData, :aggregate_failures do ...@@ -576,4 +584,28 @@ describe Gitlab::UsageData, :aggregate_failures do
expect(described_class.alt_usage_data(1)).to eq 1 expect(described_class.alt_usage_data(1)).to eq 1
end end
end end
describe '#redis_usage_data' do
context 'with block given' do
it 'returns the fallback when it gets an error' do
expect(described_class.redis_usage_data { raise ::Redis::CommandError } ).to eq(-1)
end
it 'returns the evaluated block when given' do
expect(described_class.redis_usage_data { 1 }).to eq(1)
end
end
context 'with counter given' do
it 'returns the falback values for all counter keys when it gets an error' do
allow(::Gitlab::UsageDataCounters::WikiPageCounter).to receive(:totals).and_raise(::Redis::CommandError)
expect(described_class.redis_usage_data(::Gitlab::UsageDataCounters::WikiPageCounter)).to eql(::Gitlab::UsageDataCounters::WikiPageCounter.fallback_totals)
end
it 'returns the totals when couter is given' do
allow(::Gitlab::UsageDataCounters::WikiPageCounter).to receive(:totals).and_return({ wiki_pages_create: 2 })
expect(described_class.redis_usage_data(::Gitlab::UsageDataCounters::WikiPageCounter)).to eql({ wiki_pages_create: 2 })
end
end
end
end end
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment