Commit 11b78958 authored by Alper Akgun's avatar Alper Akgun

Reorganize usage data util methods

- Moves the util methods to lib/gitlab/utils
- Moves specs
parent 4a064587
# frozen_string_literal: true # frozen_string_literal: true
# For hardening usage ping and make it easier to add measures there is in place # When developing usage data metrics use the below usage data interface methods
# * alt_usage_data method # unless you have good reasons to implement custom usage data
# handles StandardError and fallbacks into -1 this way not all measures fail if we encounter one exception # See `lib/gitlab/utils/usage_data.rb`
# #
# Examples: # Examples
# issues_using_zoom_quick_actions: distinct_count(ZoomMeeting, :issue_id),
# active_user_count: count(User.active)
# alt_usage_data { Gitlab::VERSION } # alt_usage_data { Gitlab::VERSION }
# alt_usage_data { Gitlab::CurrentSettings.uuid }
#
# * redis_usage_data method
# handles ::Redis::CommandError, Gitlab::UsageDataCounters::BaseCounter::UnknownEvent
# returns -1 when a block is sent or hash with all values -1 when a counter is sent
# different behaviour due to 2 different implementations of redis counter
#
# Examples:
# redis_usage_data(Gitlab::UsageDataCounters::WikiPageCounter) # redis_usage_data(Gitlab::UsageDataCounters::WikiPageCounter)
# redis_usage_data { ::Gitlab::UsageCounters::PodLogs.usage_totals[:total] } # redis_usage_data { ::Gitlab::UsageCounters::PodLogs.usage_totals[:total] }
module Gitlab module Gitlab
class UsageData class UsageData
BATCH_SIZE = 100 BATCH_SIZE = 100
FALLBACK = -1
class << self class << self
include Gitlab::Utils::UsageData
def data(force_refresh: false) def data(force_refresh: false)
Rails.cache.fetch('usage_data', force: force_refresh, expires_in: 2.weeks) do Rails.cache.fetch('usage_data', force: force_refresh, expires_in: 2.weeks) do
uncached_data uncached_data
...@@ -382,58 +378,6 @@ module Gitlab ...@@ -382,58 +378,6 @@ module Gitlab
{} # augmented in EE {} # augmented in EE
end end
def count(relation, column = nil, batch: true, start: nil, finish: nil)
if batch && Feature.enabled?(:usage_ping_batch_counter, default_enabled: true)
Gitlab::Database::BatchCount.batch_count(relation, column, start: start, finish: finish)
else
relation.count
end
rescue ActiveRecord::StatementInvalid
FALLBACK
end
def distinct_count(relation, column = nil, batch: true, start: nil, finish: nil)
if batch && Feature.enabled?(:usage_ping_batch_counter, default_enabled: true)
Gitlab::Database::BatchCount.batch_distinct_count(relation, column, start: start, finish: finish)
else
relation.distinct_count_by(column)
end
rescue ActiveRecord::StatementInvalid
FALLBACK
end
def alt_usage_data(value = nil, fallback: FALLBACK, &block)
if block_given?
yield
else
value
end
rescue
fallback
end
def redis_usage_data(counter = nil, &block)
if block_given?
redis_usage_counter(&block)
elsif counter.present?
redis_usage_data_totals(counter)
end
end
private
def redis_usage_counter
yield
rescue ::Redis::CommandError, Gitlab::UsageDataCounters::BaseCounter::UnknownEvent
FALLBACK
end
def redis_usage_data_totals(counter)
counter.totals
rescue ::Redis::CommandError, Gitlab::UsageDataCounters::BaseCounter::UnknownEvent
counter.fallback_totals
end
def installation_type def installation_type
if Rails.env.production? if Rails.env.production?
Gitlab::INSTALLATION_TYPE Gitlab::INSTALLATION_TYPE
......
# frozen_string_literal: true
# Usage data utilities
#
# * distinct_count(relation, column = nil, batch: true, start: nil, finish: nil)
# Does a distinct batch count, smartly reduces batch_size and handles errors
#
# Examples:
# issues_using_zoom_quick_actions: distinct_count(ZoomMeeting, :issue_id),
#
# * count(relation, column = nil, batch: true, start: nil, finish: nil)
# Does a non-distinct batch count, smartly reduces batch_size and handles errors
#
# Examples:
# active_user_count: count(User.active)
#
# * alt_usage_data method
# handles StandardError and fallbacks into -1 this way not all measures fail if we encounter one exception
#
# Examples:
# alt_usage_data { Gitlab::VERSION }
# alt_usage_data { Gitlab::CurrentSettings.uuid }
#
# * redis_usage_data method
# handles ::Redis::CommandError, Gitlab::UsageDataCounters::BaseCounter::UnknownEvent
# returns -1 when a block is sent or hash with all values -1 when a counter is sent
# different behaviour due to 2 different implementations of redis counter
#
# Examples:
# redis_usage_data(Gitlab::UsageDataCounters::WikiPageCounter)
# redis_usage_data { ::Gitlab::UsageCounters::PodLogs.usage_totals[:total] }
module Gitlab
module Utils
module UsageData
extend self
FALLBACK = -1
def count(relation, column = nil, batch: true, start: nil, finish: nil)
if batch && Feature.enabled?(:usage_ping_batch_counter, default_enabled: true)
Gitlab::Database::BatchCount.batch_count(relation, column, start: start, finish: finish)
else
relation.count
end
rescue ActiveRecord::StatementInvalid
FALLBACK
end
def distinct_count(relation, column = nil, batch: true, start: nil, finish: nil)
if batch && Feature.enabled?(:usage_ping_batch_counter, default_enabled: true)
Gitlab::Database::BatchCount.batch_distinct_count(relation, column, start: start, finish: finish)
else
relation.distinct_count_by(column)
end
rescue ActiveRecord::StatementInvalid
FALLBACK
end
def alt_usage_data(value = nil, fallback: FALLBACK, &block)
if block_given?
yield
else
value
end
rescue
fallback
end
def redis_usage_data(counter = nil, &block)
if block_given?
redis_usage_counter(&block)
elsif counter.present?
redis_usage_data_totals(counter)
end
end
private
def redis_usage_counter
yield
rescue ::Redis::CommandError, Gitlab::UsageDataCounters::BaseCounter::UnknownEvent
FALLBACK
end
def redis_usage_data_totals(counter)
counter.totals
rescue ::Redis::CommandError, Gitlab::UsageDataCounters::BaseCounter::UnknownEvent
counter.fallback_totals
end
end
end
end
...@@ -549,40 +549,6 @@ describe Gitlab::UsageData, :aggregate_failures do ...@@ -549,40 +549,6 @@ describe Gitlab::UsageData, :aggregate_failures do
end end
end end
end end
describe '#count' do
let(:relation) { double(:relation) }
it 'returns the count when counting succeeds' do
allow(relation).to receive(:count).and_return(1)
expect(described_class.count(relation, batch: false)).to eq(1)
end
it 'returns the fallback value when counting fails' do
stub_const("Gitlab::UsageData::FALLBACK", 15)
allow(relation).to receive(:count).and_raise(ActiveRecord::StatementInvalid.new(''))
expect(described_class.count(relation, batch: false)).to eq(15)
end
end
describe '#distinct_count' do
let(:relation) { double(:relation) }
it 'returns the count when counting succeeds' do
allow(relation).to receive(:distinct_count_by).and_return(1)
expect(described_class.distinct_count(relation, batch: false)).to eq(1)
end
it 'returns the fallback value when counting fails' do
stub_const("Gitlab::UsageData::FALLBACK", 15)
allow(relation).to receive(:distinct_count_by).and_raise(ActiveRecord::StatementInvalid.new(''))
expect(described_class.distinct_count(relation, batch: false)).to eq(15)
end
end
end end
end end
...@@ -601,42 +567,4 @@ describe Gitlab::UsageData, :aggregate_failures do ...@@ -601,42 +567,4 @@ describe Gitlab::UsageData, :aggregate_failures do
it_behaves_like 'usage data execution' it_behaves_like 'usage data execution'
end end
describe '#alt_usage_data' do
it 'returns the fallback when it gets an error' do
expect(described_class.alt_usage_data { raise StandardError } ).to eq(-1)
end
it 'returns the evaluated block when give' do
expect(described_class.alt_usage_data { Gitlab::CurrentSettings.uuid } ).to eq(Gitlab::CurrentSettings.uuid)
end
it 'returns the value when given' do
expect(described_class.alt_usage_data(1)).to eq 1
end
end
describe '#redis_usage_data' do
context 'with block given' do
it 'returns the fallback when it gets an error' do
expect(described_class.redis_usage_data { raise ::Redis::CommandError } ).to eq(-1)
end
it 'returns the evaluated block when given' do
expect(described_class.redis_usage_data { 1 }).to eq(1)
end
end
context 'with counter given' do
it 'returns the falback values for all counter keys when it gets an error' do
allow(::Gitlab::UsageDataCounters::WikiPageCounter).to receive(:totals).and_raise(::Redis::CommandError)
expect(described_class.redis_usage_data(::Gitlab::UsageDataCounters::WikiPageCounter)).to eql(::Gitlab::UsageDataCounters::WikiPageCounter.fallback_totals)
end
it 'returns the totals when couter is given' do
allow(::Gitlab::UsageDataCounters::WikiPageCounter).to receive(:totals).and_return({ wiki_pages_create: 2 })
expect(described_class.redis_usage_data(::Gitlab::UsageDataCounters::WikiPageCounter)).to eql({ wiki_pages_create: 2 })
end
end
end
end end
# frozen_string_literal: true
require 'spec_helper'
describe Gitlab::Utils::UsageData do
describe '#count' do
let(:relation) { double(:relation) }
it 'returns the count when counting succeeds' do
allow(relation).to receive(:count).and_return(1)
expect(described_class.count(relation, batch: false)).to eq(1)
end
it 'returns the fallback value when counting fails' do
stub_const("Gitlab::Utils::UsageData::FALLBACK", 15)
allow(relation).to receive(:count).and_raise(ActiveRecord::StatementInvalid.new(''))
expect(described_class.count(relation, batch: false)).to eq(15)
end
end
describe '#distinct_count' do
let(:relation) { double(:relation) }
it 'returns the count when counting succeeds' do
allow(relation).to receive(:distinct_count_by).and_return(1)
expect(described_class.distinct_count(relation, batch: false)).to eq(1)
end
it 'returns the fallback value when counting fails' do
stub_const("Gitlab::Utils::UsageData::FALLBACK", 15)
allow(relation).to receive(:distinct_count_by).and_raise(ActiveRecord::StatementInvalid.new(''))
expect(described_class.distinct_count(relation, batch: false)).to eq(15)
end
end
describe '#alt_usage_data' do
it 'returns the fallback when it gets an error' do
expect(described_class.alt_usage_data { raise StandardError } ).to eq(-1)
end
it 'returns the evaluated block when give' do
expect(described_class.alt_usage_data { Gitlab::CurrentSettings.uuid } ).to eq(Gitlab::CurrentSettings.uuid)
end
it 'returns the value when given' do
expect(described_class.alt_usage_data(1)).to eq 1
end
end
describe '#redis_usage_data' do
context 'with block given' do
it 'returns the fallback when it gets an error' do
expect(described_class.redis_usage_data { raise ::Redis::CommandError } ).to eq(-1)
end
it 'returns the evaluated block when given' do
expect(described_class.redis_usage_data { 1 }).to eq(1)
end
end
context 'with counter given' do
it 'returns the falback values for all counter keys when it gets an error' do
allow(::Gitlab::UsageDataCounters::WikiPageCounter).to receive(:totals).and_raise(::Redis::CommandError)
expect(described_class.redis_usage_data(::Gitlab::UsageDataCounters::WikiPageCounter)).to eql(::Gitlab::UsageDataCounters::WikiPageCounter.fallback_totals)
end
it 'returns the totals when couter is given' do
allow(::Gitlab::UsageDataCounters::WikiPageCounter).to receive(:totals).and_return({ wiki_pages_create: 2 })
expect(described_class.redis_usage_data(::Gitlab::UsageDataCounters::WikiPageCounter)).to eql({ wiki_pages_create: 2 })
end
end
end
end
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment