Skip to content

G
gitlab-ce
Commit f048036e authored by Bob Van Landuyt's avatar Bob Van Landuyt
Browse files
Options

Merge branch 'mwaw/use_postgres_hll_batch_counting_in_usage_ping' into 'master' 

Apply Postgres HLL distributed batch counter to Usage Ping secure pipelines metrics

See merge request gitlab-org/gitlab!48233
parents 91fd7b72 7ead5a08
Hide whitespace changes
Inline Side-by-side
Showing with 344 additions and 50 deletions
config/feature_flags/development/postgres_hll_batch_counting.yml 0 → 100644
View file @ f048036e
---
name: postgres_hll_batch_counting
introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/48233
rollout_issue_url: https://gitlab.com/gitlab-org/gitlab/-/issues/285485
milestone: '13.7'
type: development
group: group::product analytics
default_enabled: false
ee/lib/ee/gitlab/usage_data.rb
View file @ f048036e
...@@ -396,16 +396,44 @@ module EE ...@@ -396,16 +396,44 @@ module EE
def count_secure_pipelines(time_period) def count_secure_pipelines(time_period)
return {} if time_period.blank? return {} if time_period.blank?
start = ::Ci::Pipeline.minimum(:id)
finish = ::Ci::Pipeline.maximum(:id)
pipelines_with_secure_jobs = {} pipelines_with_secure_jobs = {}
::Security::Scan.scan_types.each do |name, scan_type| # HLL batch counting always iterate over pkey of
relation = ::Ci::Build.joins(:security_scans) # given relation, while ordinary batch count
.where(status: 'success', retried: [nil, false]) # iterated over counted attribute, one-to-many joins
.where('security_scans.scan_type = ?', scan_type) # can break batch size limitation, and lead to
.where(time_period) # time outing batch queries, to avoid that
pipelines_with_secure_jobs["#{name}_pipeline".to_sym] = distinct_count(relation, :commit_id, start: start, finish: finish, batch: false) # different join strategy is used for HLL counter
if ::Feature.enabled?(:postgres_hll_batch_counting)
relation = ::Security::Scan.where(time_period).group(:created_at)
start = relation.select('MIN(id) as min_id').order('min_id ASC').first&.min_id
finish = relation.select('MAX(id) as max_id').order('max_id DESC').first&.max_id
::Security::Scan.scan_types.each do |name, scan_type|
relation = ::Security::Scan.joins(:build)
.where(ci_builds: { status: 'success', retried: [nil, false] })
.where('security_scans.scan_type = ?', scan_type)
.where(security_scans: time_period)
pipelines_with_secure_jobs["#{name}_pipeline".to_sym] =
if start && finish
estimate_batch_distinct_count(relation, :commit_id, batch_size: 1000, start: start, finish: finish)
else
0
end
end
else
start = ::Ci::Pipeline.minimum(:id)
finish = ::Ci::Pipeline.maximum(:id)
::Security::Scan.scan_types.each do |name, scan_type|
relation = ::Ci::Build.joins(:security_scans)
.where(status: 'success', retried: [nil, false])
.where('security_scans.scan_type = ?', scan_type)
.where(time_period)
pipelines_with_secure_jobs["#{name}_pipeline".to_sym] = distinct_count(relation, :commit_id, start: start, finish: finish, batch: false)
end
end end
pipelines_with_secure_jobs pipelines_with_secure_jobs
......
ee/spec/lib/ee/gitlab/usage_data_spec.rb
View file @ f048036e
...@@ -504,6 +504,7 @@ RSpec.describe Gitlab::UsageData do ...@@ -504,6 +504,7 @@ RSpec.describe Gitlab::UsageData do
end end
describe 'usage_activity_by_stage_secure' do describe 'usage_activity_by_stage_secure' do
let_it_be(:error_rate) { Gitlab::Database::PostgresHllBatchDistinctCounter::ERROR_RATE }
let_it_be(:days_ago_within_monthly_time_period) { 3.days.ago } let_it_be(:days_ago_within_monthly_time_period) { 3.days.ago }
let_it_be(:user) { create(:user, group_view: :security_dashboard, created_at: days_ago_within_monthly_time_period) } let_it_be(:user) { create(:user, group_view: :security_dashboard, created_at: days_ago_within_monthly_time_period) }
let_it_be(:user2) { create(:user, group_view: :security_dashboard, created_at: days_ago_within_monthly_time_period) } let_it_be(:user2) { create(:user, group_view: :security_dashboard, created_at: days_ago_within_monthly_time_period) }
...@@ -524,7 +525,7 @@ RSpec.describe Gitlab::UsageData do ...@@ -524,7 +525,7 @@ RSpec.describe Gitlab::UsageData do
end end
it 'includes accurate usage_activity_by_stage data' do it 'includes accurate usage_activity_by_stage data' do
expect(described_class.usage_activity_by_stage_secure(described_class.last_28_days_time_period)).to eq( expect(described_class.usage_activity_by_stage_secure(described_class.last_28_days_time_period)).to include(
user_preferences_group_overview_security_dashboard: 3, user_preferences_group_overview_security_dashboard: 3,
user_container_scanning_jobs: 1, user_container_scanning_jobs: 1,
user_api_fuzzing_jobs: 1, user_api_fuzzing_jobs: 1,
...@@ -535,19 +536,19 @@ RSpec.describe Gitlab::UsageData do ...@@ -535,19 +536,19 @@ RSpec.describe Gitlab::UsageData do
user_license_management_jobs: 1, user_license_management_jobs: 1,
user_sast_jobs: 1, user_sast_jobs: 1,
user_secret_detection_jobs: 1, user_secret_detection_jobs: 1,
sast_pipeline: 0, sast_pipeline: be_within(error_rate).percent_of(0),
sast_scans: 0, sast_scans: 0,
dependency_scanning_pipeline: 0, dependency_scanning_pipeline: be_within(error_rate).percent_of(0),
dependency_scanning_scans: 0, dependency_scanning_scans: 0,
container_scanning_pipeline: 0, container_scanning_pipeline: be_within(error_rate).percent_of(0),
container_scanning_scans: 0, container_scanning_scans: 0,
dast_pipeline: 0, dast_pipeline: be_within(error_rate).percent_of(0),
dast_scans: 0, dast_scans: 0,
secret_detection_pipeline: 0, secret_detection_pipeline: be_within(error_rate).percent_of(0),
secret_detection_scans: 0, secret_detection_scans: 0,
coverage_fuzzing_pipeline: 0, coverage_fuzzing_pipeline: be_within(error_rate).percent_of(0),
coverage_fuzzing_scans: 0, coverage_fuzzing_scans: 0,
api_fuzzing_pipeline: 0, api_fuzzing_pipeline: be_within(error_rate).percent_of(0),
api_fuzzing_scans: 0, api_fuzzing_scans: 0,
user_unique_users_all_secure_scanners: 1 user_unique_users_all_secure_scanners: 1
) )
...@@ -597,13 +598,13 @@ RSpec.describe Gitlab::UsageData do ...@@ -597,13 +598,13 @@ RSpec.describe Gitlab::UsageData do
user_license_management_jobs: 1, user_license_management_jobs: 1,
user_sast_jobs: 1, user_sast_jobs: 1,
user_secret_detection_jobs: 1, user_secret_detection_jobs: 1,
sast_pipeline: 0, sast_pipeline: be_within(error_rate).percent_of(0),
dependency_scanning_pipeline: 1, dependency_scanning_pipeline: be_within(error_rate).percent_of(1),
container_scanning_pipeline: 1, container_scanning_pipeline: be_within(error_rate).percent_of(1),
dast_pipeline: 0, dast_pipeline: be_within(error_rate).percent_of(0),
secret_detection_pipeline: 1, secret_detection_pipeline: be_within(error_rate).percent_of(1),
coverage_fuzzing_pipeline: 0, coverage_fuzzing_pipeline: be_within(error_rate).percent_of(0),
api_fuzzing_pipeline: 0, api_fuzzing_pipeline: be_within(error_rate).percent_of(0),
user_unique_users_all_secure_scanners: 1, user_unique_users_all_secure_scanners: 1,
sast_scans: 0, sast_scans: 0,
dependency_scanning_scans: 2, dependency_scanning_scans: 2,
...@@ -622,7 +623,7 @@ RSpec.describe Gitlab::UsageData do ...@@ -622,7 +623,7 @@ RSpec.describe Gitlab::UsageData do
create(:ci_build, name: 'dast', user: user3) create(:ci_build, name: 'dast', user: user3)
end end
expect(described_class.usage_activity_by_stage_secure(described_class.last_28_days_time_period)).to eq( expect(described_class.usage_activity_by_stage_secure(described_class.last_28_days_time_period)).to include(
user_preferences_group_overview_security_dashboard: 3, user_preferences_group_overview_security_dashboard: 3,
user_api_fuzzing_jobs: 1, user_api_fuzzing_jobs: 1,
user_api_fuzzing_dnd_jobs: 1, user_api_fuzzing_dnd_jobs: 1,
...@@ -633,19 +634,19 @@ RSpec.describe Gitlab::UsageData do ...@@ -633,19 +634,19 @@ RSpec.describe Gitlab::UsageData do
user_license_management_jobs: 1, user_license_management_jobs: 1,
user_sast_jobs: 2, user_sast_jobs: 2,
user_secret_detection_jobs: 1, user_secret_detection_jobs: 1,
sast_pipeline: 0, sast_pipeline: be_within(error_rate).percent_of(0),
sast_scans: 0, sast_scans: 0,
dependency_scanning_pipeline: 0, dependency_scanning_pipeline: be_within(error_rate).percent_of(0),
dependency_scanning_scans: 0, dependency_scanning_scans: 0,
container_scanning_pipeline: 0, container_scanning_pipeline: be_within(error_rate).percent_of(0),
container_scanning_scans: 0, container_scanning_scans: 0,
dast_pipeline: 0, dast_pipeline: be_within(error_rate).percent_of(0),
dast_scans: 0, dast_scans: 0,
secret_detection_pipeline: 0, secret_detection_pipeline: be_within(error_rate).percent_of(0),
secret_detection_scans: 0, secret_detection_scans: 0,
coverage_fuzzing_pipeline: 0, coverage_fuzzing_pipeline: be_within(error_rate).percent_of(0),
coverage_fuzzing_scans: 0, coverage_fuzzing_scans: 0,
api_fuzzing_pipeline: 0, api_fuzzing_pipeline: be_within(error_rate).percent_of(0),
api_fuzzing_scans: 0, api_fuzzing_scans: 0,
user_unique_users_all_secure_scanners: 3 user_unique_users_all_secure_scanners: 3
) )
...@@ -656,7 +657,7 @@ RSpec.describe Gitlab::UsageData do ...@@ -656,7 +657,7 @@ RSpec.describe Gitlab::UsageData do
create(:ci_build, name: 'license_scanning', user: user) create(:ci_build, name: 'license_scanning', user: user)
end end
expect(described_class.usage_activity_by_stage_secure(described_class.last_28_days_time_period)).to eq( expect(described_class.usage_activity_by_stage_secure(described_class.last_28_days_time_period)).to include(
user_preferences_group_overview_security_dashboard: 3, user_preferences_group_overview_security_dashboard: 3,
user_api_fuzzing_jobs: 1, user_api_fuzzing_jobs: 1,
user_api_fuzzing_dnd_jobs: 1, user_api_fuzzing_dnd_jobs: 1,
...@@ -667,30 +668,35 @@ RSpec.describe Gitlab::UsageData do ...@@ -667,30 +668,35 @@ RSpec.describe Gitlab::UsageData do
user_license_management_jobs: 2, user_license_management_jobs: 2,
user_sast_jobs: 1, user_sast_jobs: 1,
user_secret_detection_jobs: 1, user_secret_detection_jobs: 1,
sast_pipeline: 0, sast_pipeline: be_within(error_rate).percent_of(0),
sast_scans: 0, sast_scans: 0,
dependency_scanning_pipeline: 0, dependency_scanning_pipeline: be_within(error_rate).percent_of(0),
dependency_scanning_scans: 0, dependency_scanning_scans: 0,
container_scanning_pipeline: 0, container_scanning_pipeline: be_within(error_rate).percent_of(0),
container_scanning_scans: 0, container_scanning_scans: 0,
dast_pipeline: 0, dast_pipeline: be_within(error_rate).percent_of(0),
dast_scans: 0, dast_scans: 0,
secret_detection_pipeline: 0, secret_detection_pipeline: be_within(error_rate).percent_of(0),
secret_detection_scans: 0, secret_detection_scans: 0,
coverage_fuzzing_pipeline: 0, coverage_fuzzing_pipeline: be_within(error_rate).percent_of(0),
coverage_fuzzing_scans: 0, coverage_fuzzing_scans: 0,
api_fuzzing_pipeline: 0, api_fuzzing_pipeline: be_within(error_rate).percent_of(0),
api_fuzzing_scans: 0, api_fuzzing_scans: 0,
user_unique_users_all_secure_scanners: 1 user_unique_users_all_secure_scanners: 1
) )
end end
it 'has to resort to 0 for counting license scan' do it 'has to resort to 0 for counting license scan' do
for_defined_days_back do
create(:security_scan)
end
allow(Gitlab::Database::BatchCount).to receive(:batch_distinct_count).and_raise(ActiveRecord::StatementInvalid) allow(Gitlab::Database::BatchCount).to receive(:batch_distinct_count).and_raise(ActiveRecord::StatementInvalid)
allow(Gitlab::Database::BatchCount).to receive(:batch_count).and_raise(ActiveRecord::StatementInvalid) allow(Gitlab::Database::BatchCount).to receive(:batch_count).and_raise(ActiveRecord::StatementInvalid)
allow(Gitlab::Database::PostgresHllBatchDistinctCounter).to receive(:new).and_raise(ActiveRecord::StatementInvalid)
allow(::Ci::Build).to receive(:distinct_count_by).and_raise(ActiveRecord::StatementInvalid) allow(::Ci::Build).to receive(:distinct_count_by).and_raise(ActiveRecord::StatementInvalid)
expect(described_class.usage_activity_by_stage_secure(described_class.last_28_days_time_period)).to eq( expect(described_class.usage_activity_by_stage_secure(described_class.last_28_days_time_period)).to include(
user_preferences_group_overview_security_dashboard: -1, user_preferences_group_overview_security_dashboard: -1,
user_api_fuzzing_jobs: -1, user_api_fuzzing_jobs: -1,
user_api_fuzzing_dnd_jobs: -1, user_api_fuzzing_dnd_jobs: -1,
...@@ -718,6 +724,208 @@ RSpec.describe Gitlab::UsageData do ...@@ -718,6 +724,208 @@ RSpec.describe Gitlab::UsageData do
user_unique_users_all_secure_scanners: -1 user_unique_users_all_secure_scanners: -1
) )
end end
context 'with feature flag: postgres_hll_batch_counting is disabled' do
before do
stub_feature_flags(postgres_hll_batch_counting: false)
end
it 'includes accurate usage_activity_by_stage data' do
expect(described_class.usage_activity_by_stage_secure(described_class.last_28_days_time_period)).to eq(
user_preferences_group_overview_security_dashboard: 3,
user_container_scanning_jobs: 1,
user_api_fuzzing_jobs: 1,
user_api_fuzzing_dnd_jobs: 1,
user_coverage_fuzzing_jobs: 1,
user_dast_jobs: 1,
user_dependency_scanning_jobs: 1,
user_license_management_jobs: 1,
user_sast_jobs: 1,
user_secret_detection_jobs: 1,
sast_pipeline: 0,
sast_scans: 0,
dependency_scanning_pipeline: 0,
dependency_scanning_scans: 0,
container_scanning_pipeline: 0,
container_scanning_scans: 0,
dast_pipeline: 0,
dast_scans: 0,
secret_detection_pipeline: 0,
secret_detection_scans: 0,
coverage_fuzzing_pipeline: 0,
coverage_fuzzing_scans: 0,
api_fuzzing_pipeline: 0,
api_fuzzing_scans: 0,
user_unique_users_all_secure_scanners: 1
)
end
it 'counts pipelines that have security jobs' do
for_defined_days_back do
ds_build = create(:ci_build, name: 'retirejs', user: user, status: 'success')
ds_bundler_audit_build = create(:ci_build, :failed, user: user, name: 'retirejs')
ds_bundler_build = create(:ci_build, name: 'bundler-audit', user: user, commit_id: ds_build.pipeline.id, status: 'success')
secret_detection_build = create(:ci_build, name: 'secret', user: user, commit_id: ds_build.pipeline.id, status: 'success')
cs_build = create(:ci_build, name: 'klar', user: user, status: 'success')
sast_build = create(:ci_build, name: 'sast', user: user, status: 'success', retried: true)
create(:security_scan, build: ds_build, scan_type: 'dependency_scanning' )
create(:security_scan, build: ds_bundler_build, scan_type: 'dependency_scanning')
create(:security_scan, build: secret_detection_build, scan_type: 'secret_detection')
create(:security_scan, build: cs_build, scan_type: 'container_scanning')
create(:security_scan, build: sast_build, scan_type: 'sast')
create(:security_scan, build: ds_bundler_audit_build, scan_type: 'dependency_scanning')
end
expect(described_class.usage_activity_by_stage_secure({})).to include(
user_preferences_group_overview_security_dashboard: 3,
user_container_scanning_jobs: 1,
user_dast_jobs: 1,
user_dependency_scanning_jobs: 1,
user_license_management_jobs: 1,
user_sast_jobs: 1,
user_secret_detection_jobs: 1,
user_unique_users_all_secure_scanners: 1,
sast_scans: 0,
dependency_scanning_scans: 4,
container_scanning_scans: 2,
dast_scans: 0,
secret_detection_scans: 2,
coverage_fuzzing_scans: 0
)
expect(described_class.usage_activity_by_stage_secure(described_class.last_28_days_time_period)).to include(
user_preferences_group_overview_security_dashboard: 3,
user_api_fuzzing_jobs: 1,
user_api_fuzzing_dnd_jobs: 1,
user_container_scanning_jobs: 1,
user_coverage_fuzzing_jobs: 1,
user_dast_jobs: 1,
user_dependency_scanning_jobs: 1,
user_license_management_jobs: 1,
user_sast_jobs: 1,
user_secret_detection_jobs: 1,
sast_pipeline: 0,
dependency_scanning_pipeline: 1,
container_scanning_pipeline: 1,
dast_pipeline: 0,
secret_detection_pipeline: 1,
coverage_fuzzing_pipeline: 0,
api_fuzzing_pipeline: 0,
user_unique_users_all_secure_scanners: 1,
sast_scans: 0,
dependency_scanning_scans: 2,
container_scanning_scans: 1,
dast_scans: 0,
secret_detection_scans: 1,
coverage_fuzzing_scans: 0,
api_fuzzing_scans: 0
)
end
it 'counts unique users correctly across multiple scanners' do
for_defined_days_back do
create(:ci_build, name: 'sast', user: user2)
create(:ci_build, name: 'dast', user: user2)
create(:ci_build, name: 'dast', user: user3)
end
expect(described_class.usage_activity_by_stage_secure(described_class.last_28_days_time_period)).to eq(
user_preferences_group_overview_security_dashboard: 3,
user_api_fuzzing_jobs: 1,
user_api_fuzzing_dnd_jobs: 1,
user_container_scanning_jobs: 1,
user_coverage_fuzzing_jobs: 1,
user_dast_jobs: 3,
user_dependency_scanning_jobs: 1,
user_license_management_jobs: 1,
user_sast_jobs: 2,
user_secret_detection_jobs: 1,
sast_pipeline: 0,
sast_scans: 0,
dependency_scanning_pipeline: 0,
dependency_scanning_scans: 0,
container_scanning_pipeline: 0,
container_scanning_scans: 0,
dast_pipeline: 0,
dast_scans: 0,
secret_detection_pipeline: 0,
secret_detection_scans: 0,
coverage_fuzzing_pipeline: 0,
coverage_fuzzing_scans: 0,
api_fuzzing_pipeline: 0,
api_fuzzing_scans: 0,
user_unique_users_all_secure_scanners: 3
)
end
it 'combines license_scanning into license_management' do
for_defined_days_back do
create(:ci_build, name: 'license_scanning', user: user)
end
expect(described_class.usage_activity_by_stage_secure(described_class.last_28_days_time_period)).to eq(
user_preferences_group_overview_security_dashboard: 3,
user_api_fuzzing_jobs: 1,
user_api_fuzzing_dnd_jobs: 1,
user_container_scanning_jobs: 1,
user_coverage_fuzzing_jobs: 1,
user_dast_jobs: 1,
user_dependency_scanning_jobs: 1,
user_license_management_jobs: 2,
user_sast_jobs: 1,
user_secret_detection_jobs: 1,
sast_pipeline: 0,
sast_scans: 0,
dependency_scanning_pipeline: 0,
dependency_scanning_scans: 0,
container_scanning_pipeline: 0,
container_scanning_scans: 0,
dast_pipeline: 0,
dast_scans: 0,
secret_detection_pipeline: 0,
secret_detection_scans: 0,
coverage_fuzzing_pipeline: 0,
coverage_fuzzing_scans: 0,
api_fuzzing_pipeline: 0,
api_fuzzing_scans: 0,
user_unique_users_all_secure_scanners: 1
)
end
it 'has to resort to 0 for counting license scan' do
allow(Gitlab::Database::BatchCount).to receive(:batch_distinct_count).and_raise(ActiveRecord::StatementInvalid)
allow(Gitlab::Database::BatchCount).to receive(:batch_count).and_raise(ActiveRecord::StatementInvalid)
allow(::Ci::Build).to receive(:distinct_count_by).and_raise(ActiveRecord::StatementInvalid)
expect(described_class.usage_activity_by_stage_secure(described_class.last_28_days_time_period)).to eq(
user_preferences_group_overview_security_dashboard: -1,
user_api_fuzzing_jobs: -1,
user_api_fuzzing_dnd_jobs: -1,
user_container_scanning_jobs: -1,
user_coverage_fuzzing_jobs: -1,
user_dast_jobs: -1,
user_dependency_scanning_jobs: -1,
user_license_management_jobs: -1,
user_sast_jobs: -1,
user_secret_detection_jobs: -1,
sast_pipeline: -1,
sast_scans: -1,
dependency_scanning_pipeline: -1,
dependency_scanning_scans: -1,
container_scanning_pipeline: -1,
container_scanning_scans: -1,
dast_pipeline: -1,
dast_scans: -1,
secret_detection_pipeline: -1,
secret_detection_scans: -1,
coverage_fuzzing_pipeline: -1,
coverage_fuzzing_scans: -1,
api_fuzzing_pipeline: -1,
api_fuzzing_scans: -1,
user_unique_users_all_secure_scanners: -1
)
end
end
end end
describe 'usage_activity_by_stage_verify' do describe 'usage_activity_by_stage_verify' do
......
lib/gitlab/database/postgres_hll_batch_distinct_counter.rb
View file @ f048036e
...@@ -28,13 +28,14 @@ module Gitlab ...@@ -28,13 +28,14 @@ module Gitlab
# for given implementation no higher value was reported (https://gitlab.com/gitlab-org/gitlab/-/merge_requests/45673#accuracy-estimation) than 5.3% # for given implementation no higher value was reported (https://gitlab.com/gitlab-org/gitlab/-/merge_requests/45673#accuracy-estimation) than 5.3%
# for the most of a cases this value is lower. However, if the exact value is necessary other tools has to be used. # for the most of a cases this value is lower. However, if the exact value is necessary other tools has to be used.
class PostgresHllBatchDistinctCounter class PostgresHllBatchDistinctCounter
ERROR_RATE = 4.9 # max encountered empirical error rate, used in tests
FALLBACK = -1 FALLBACK = -1
MIN_REQUIRED_BATCH_SIZE = 1_250 MIN_REQUIRED_BATCH_SIZE = 750
MAX_ALLOWED_LOOPS = 10_000
SLEEP_TIME_IN_SECONDS = 0.01 # 10 msec sleep SLEEP_TIME_IN_SECONDS = 0.01 # 10 msec sleep
MAX_DATA_VOLUME = 4_000_000_000
# Each query should take < 500ms https://gitlab.com/gitlab-org/gitlab/-/merge_requests/22705 # Each query should take < 500ms https://gitlab.com/gitlab-org/gitlab/-/merge_requests/22705
DEFAULT_BATCH_SIZE = 100_000 DEFAULT_BATCH_SIZE = 10_000
BIT_31_MASK = "B'0#{'1' * 31}'" BIT_31_MASK = "B'0#{'1' * 31}'"
BIT_9_MASK = "B'#{'0' * 23}#{'1' * 9}'" BIT_9_MASK = "B'#{'0' * 23}#{'1' * 9}'"
...@@ -49,7 +50,7 @@ module Gitlab ...@@ -49,7 +50,7 @@ module Gitlab
SELECT (attr_hash_32_bits & #{BIT_9_MASK})::int AS bucket_num, SELECT (attr_hash_32_bits & #{BIT_9_MASK})::int AS bucket_num,
(31 - floor(log(2, min((attr_hash_32_bits & #{BIT_31_MASK})::int))))::int as bucket_hash (31 - floor(log(2, min((attr_hash_32_bits & #{BIT_31_MASK})::int))))::int as bucket_hash
FROM hashed_attributes FROM hashed_attributes
GROUP BY 1 ORDER BY 1 GROUP BY 1
SQL SQL
TOTAL_BUCKETS_NUMBER = 512 TOTAL_BUCKETS_NUMBER = 512
...@@ -61,7 +62,7 @@ module Gitlab ...@@ -61,7 +62,7 @@ module Gitlab
def unwanted_configuration?(finish, batch_size, start) def unwanted_configuration?(finish, batch_size, start)
batch_size <= MIN_REQUIRED_BATCH_SIZE || batch_size <= MIN_REQUIRED_BATCH_SIZE ||
(finish - start) / batch_size >= MAX_ALLOWED_LOOPS || (finish - start) >= MAX_DATA_VOLUME ||
start > finish start > finish
end end
...@@ -101,7 +102,7 @@ module Gitlab ...@@ -101,7 +102,7 @@ module Gitlab
num_uniques = ( num_uniques = (
((TOTAL_BUCKETS_NUMBER**2) * (0.7213 / (1 + 1.079 / TOTAL_BUCKETS_NUMBER))) / ((TOTAL_BUCKETS_NUMBER**2) * (0.7213 / (1 + 1.079 / TOTAL_BUCKETS_NUMBER))) /
(num_zero_buckets + hll_blob.values.sum { |bucket_hash, _| 2**(-1 * bucket_hash)} ) (num_zero_buckets + hll_blob.values.sum { |bucket_hash| 2**(-1 * bucket_hash)} )
).to_i ).to_i
if num_zero_buckets > 0 && num_uniques < 2.5 * TOTAL_BUCKETS_NUMBER if num_zero_buckets > 0 && num_uniques < 2.5 * TOTAL_BUCKETS_NUMBER
......
lib/gitlab/usage_data_queries.rb
View file @ f048036e
...@@ -25,6 +25,13 @@ module Gitlab ...@@ -25,6 +25,13 @@ module Gitlab
relation.select(relation.all.table[column].sum).to_sql relation.select(relation.all.table[column].sum).to_sql
end end
# For estimated distinct count use exact query instead of hll
# buckets query, because it can't be used to obtain estimations without
# supplementary ruby code present in Gitlab::Database::PostgresHllBatchDistinctCounter
def estimate_batch_distinct_count(relation, column = nil, *rest)
raw_sql(relation, column, :distinct)
end
private private
def raw_sql(relation, column, distinct = nil) def raw_sql(relation, column, distinct = nil)
......
lib/gitlab/utils/usage_data.rb
View file @ f048036e
...@@ -38,6 +38,7 @@ module Gitlab ...@@ -38,6 +38,7 @@ module Gitlab
extend self extend self
FALLBACK = -1 FALLBACK = -1
DISTRIBUTED_HLL_FALLBACK = -2
def count(relation, column = nil, batch: true, batch_size: nil, start: nil, finish: nil) def count(relation, column = nil, batch: true, batch_size: nil, start: nil, finish: nil)
if batch if batch
...@@ -59,6 +60,17 @@ module Gitlab ...@@ -59,6 +60,17 @@ module Gitlab
FALLBACK FALLBACK
end end
def estimate_batch_distinct_count(relation, column = nil, batch_size: nil, start: nil, finish: nil)
Gitlab::Database::PostgresHllBatchDistinctCounter.new(relation, column).estimate_distinct_count(batch_size: batch_size, start: start, finish: finish)
rescue ActiveRecord::StatementInvalid
FALLBACK
# catch all rescue should be removed as a part of feature flag rollout issue
# https://gitlab.com/gitlab-org/gitlab/-/issues/285485
rescue StandardError => error
Gitlab::ErrorTracking.track_and_raise_for_dev_exception(error)
DISTRIBUTED_HLL_FALLBACK
end
def sum(relation, column, batch_size: nil, start: nil, finish: nil) def sum(relation, column, batch_size: nil, start: nil, finish: nil)
Gitlab::Database::BatchCount.batch_sum(relation, column, batch_size: batch_size, start: start, finish: finish) Gitlab::Database::BatchCount.batch_sum(relation, column, batch_size: batch_size, start: start, finish: finish)
rescue ActiveRecord::StatementInvalid rescue ActiveRecord::StatementInvalid
......
spec/lib/gitlab/database/postgres_hll_batch_distinct_counter_spec.rb
View file @ f048036e
...@@ -3,9 +3,9 @@ ...@@ -3,9 +3,9 @@
require 'spec_helper' require 'spec_helper'
RSpec.describe Gitlab::Database::PostgresHllBatchDistinctCounter do RSpec.describe Gitlab::Database::PostgresHllBatchDistinctCounter do
let_it_be(:error_rate) { 4.9 } # HyperLogLog is a probabilistic algorithm, which provides estimated data, with given error margin let_it_be(:error_rate) { described_class::ERROR_RATE } # HyperLogLog is a probabilistic algorithm, which provides estimated data, with given error margin
let_it_be(:fallback) { ::Gitlab::Database::BatchCounter::FALLBACK } let_it_be(:fallback) { ::Gitlab::Database::BatchCounter::FALLBACK }
let_it_be(:small_batch_size) { calculate_batch_size(::Gitlab::Database::BatchCounter::MIN_REQUIRED_BATCH_SIZE) } let_it_be(:small_batch_size) { calculate_batch_size(described_class::MIN_REQUIRED_BATCH_SIZE) }
let(:model) { Issue } let(:model) { Issue }
let(:column) { :author_id } let(:column) { :author_id }
...@@ -118,8 +118,8 @@ RSpec.describe Gitlab::Database::PostgresHllBatchDistinctCounter do ...@@ -118,8 +118,8 @@ RSpec.describe Gitlab::Database::PostgresHllBatchDistinctCounter do
expect(described_class.new(model, column).estimate_distinct_count(start: 1, finish: 0)).to eq(fallback) expect(described_class.new(model, column).estimate_distinct_count(start: 1, finish: 0)).to eq(fallback)
end end
it 'returns fallback if loops more than allowed' do it 'returns fallback if data volume exceeds upper limit' do
large_finish = Gitlab::Database::PostgresHllBatchDistinctCounter::MAX_ALLOWED_LOOPS * default_batch_size + 1 large_finish = Gitlab::Database::PostgresHllBatchDistinctCounter::MAX_DATA_VOLUME + 1
expect(described_class.new(model, column).estimate_distinct_count(start: 1, finish: large_finish)).to eq(fallback) expect(described_class.new(model, column).estimate_distinct_count(start: 1, finish: large_finish)).to eq(fallback)
end end
......
spec/lib/gitlab/utils/usage_data_spec.rb
View file @ f048036e
...@@ -37,6 +37,36 @@ RSpec.describe Gitlab::Utils::UsageData do ...@@ -37,6 +37,36 @@ RSpec.describe Gitlab::Utils::UsageData do
end end
end end
describe '#estimate_batch_distinct_count' do
let(:relation) { double(:relation) }
it 'delegates counting to counter class instance' do
expect_next_instance_of(Gitlab::Database::PostgresHllBatchDistinctCounter, relation, 'column') do |instance|
expect(instance).to receive(:estimate_distinct_count)
.with(batch_size: nil, start: nil, finish: nil)
.and_return(5)
end
expect(described_class.estimate_batch_distinct_count(relation, 'column')).to eq(5)
end
it 'returns default fallback value when counting fails due to database error' do
stub_const("Gitlab::Utils::UsageData::FALLBACK", 15)
allow(Gitlab::Database::PostgresHllBatchDistinctCounter).to receive(:new).and_raise(ActiveRecord::StatementInvalid.new(''))
expect(described_class.estimate_batch_distinct_count(relation)).to eq(15)
end
it 'logs error and returns DISTRIBUTED_HLL_FALLBACK value when counting raises any error', :aggregate_failures do
error = StandardError.new('')
stub_const("Gitlab::Utils::UsageData::DISTRIBUTED_HLL_FALLBACK", 15)
allow(Gitlab::Database::PostgresHllBatchDistinctCounter).to receive(:new).and_raise(error)
expect(Gitlab::ErrorTracking).to receive(:track_and_raise_for_dev_exception).with(error)
expect(described_class.estimate_batch_distinct_count(relation)).to eq(15)
end
end
describe '#sum' do describe '#sum' do
let(:relation) { double(:relation) } let(:relation) { double(:relation) }
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment
GitLab Nexedi Edition | About GitLab | About Nexedi | 沪ICP备2021021310号-2 | 沪ICP备2021021310号-7