Skip to content
Projects
Groups
Snippets
Help
Loading...
Help
Support
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in / Register
Toggle navigation
G
gitlab-ce
Project overview
Project overview
Details
Activity
Releases
Repository
Repository
Files
Commits
Branches
Tags
Contributors
Graph
Compare
Issues
0
Issues
0
List
Boards
Labels
Milestones
Merge Requests
1
Merge Requests
1
Analytics
Analytics
Repository
Value Stream
Wiki
Wiki
Snippets
Snippets
Members
Members
Collapse sidebar
Close sidebar
Activity
Graph
Create a new issue
Commits
Issue Boards
Open sidebar
nexedi
gitlab-ce
Commits
e51d2609
Commit
e51d2609
authored
May 13, 2021
by
Piotr Skorupa
Browse files
Options
Browse Files
Download
Email Patches
Plain Diff
Add TimeFrame module for metric time periods SSOT
parent
1a6bb212
Changes
12
Show whitespace changes
Inline
Side-by-side
Showing
12 changed files
with
83 additions
and
78 deletions
+83
-78
ee/spec/lib/ee/gitlab/usage_data_spec.rb
ee/spec/lib/ee/gitlab/usage_data_spec.rb
+20
-20
lib/gitlab/usage/metrics/aggregates/aggregate.rb
lib/gitlab/usage/metrics/aggregates/aggregate.rb
+5
-7
lib/gitlab/usage/metrics/aggregates/sources/postgres_hll.rb
lib/gitlab/usage/metrics/aggregates/sources/postgres_hll.rb
+3
-3
lib/gitlab/usage/metrics/instrumentations/base_metric.rb
lib/gitlab/usage/metrics/instrumentations/base_metric.rb
+1
-0
lib/gitlab/usage/metrics/instrumentations/database_metric.rb
lib/gitlab/usage/metrics/instrumentations/database_metric.rb
+1
-1
lib/gitlab/usage/metrics/instrumentations/redis_hll_metric.rb
...gitlab/usage/metrics/instrumentations/redis_hll_metric.rb
+2
-2
lib/gitlab/usage/time_frame.rb
lib/gitlab/usage/time_frame.rb
+25
-0
lib/gitlab/usage_data.rb
lib/gitlab/usage_data.rb
+11
-14
lib/gitlab/usage_data_counters/hll_redis_counter.rb
lib/gitlab/usage_data_counters/hll_redis_counter.rb
+1
-0
lib/gitlab/utils/usage_data.rb
lib/gitlab/utils/usage_data.rb
+0
-17
spec/config/metrics/aggregates/aggregated_metrics_spec.rb
spec/config/metrics/aggregates/aggregated_metrics_spec.rb
+4
-4
spec/lib/gitlab/usage_data_spec.rb
spec/lib/gitlab/usage_data_spec.rb
+10
-10
No files found.
ee/spec/lib/ee/gitlab/usage_data_spec.rb
View file @
e51d2609
...
...
@@ -270,7 +270,7 @@ RSpec.describe Gitlab::UsageData do
projects_slack_slash_active:
2
,
projects_with_prometheus_alerts:
2
)
expect
(
described_class
.
usage_activity_by_stage_configure
(
described_class
.
last_28_days_time_period
)).
to
include
(
expect
(
described_class
.
usage_activity_by_stage_configure
(
described_class
.
monthly_time_range_db_params
)).
to
include
(
projects_slack_notifications_active:
1
,
projects_slack_slash_active:
1
,
projects_with_prometheus_alerts:
1
...
...
@@ -341,7 +341,7 @@ RSpec.describe Gitlab::UsageData do
total_number_of_path_locks:
20
,
total_number_of_locked_files:
14
)
expect
(
described_class
.
usage_activity_by_stage_create
(
described_class
.
last_28_days_time_period
)).
to
include
(
expect
(
described_class
.
usage_activity_by_stage_create
(
described_class
.
monthly_time_range_db_params
)).
to
include
(
approval_project_rules:
10
,
approval_project_rules_with_target_branch:
2
,
approval_project_rules_with_more_approvers_than_required:
2
,
...
...
@@ -385,7 +385,7 @@ RSpec.describe Gitlab::UsageData do
end
subject
do
described_class
.
usage_activity_by_stage_enablement
(
described_class
.
last_28_days_time_period
)
described_class
.
usage_activity_by_stage_enablement
(
described_class
.
monthly_time_range_db_params
)
end
it
'excludes data outside of the date range'
do
...
...
@@ -431,7 +431,7 @@ RSpec.describe Gitlab::UsageData do
ldap_admin_sync_enabled:
true
,
group_saml_enabled:
true
)
expect
(
described_class
.
usage_activity_by_stage_manage
(
described_class
.
last_28_days_time_period
)).
to
include
(
expect
(
described_class
.
usage_activity_by_stage_manage
(
described_class
.
monthly_time_range_db_params
)).
to
include
(
ldap_keys:
1
,
ldap_users:
1
,
value_stream_management_customized_group_stages:
2
,
...
...
@@ -476,7 +476,7 @@ RSpec.describe Gitlab::UsageData do
operations_dashboard_users_with_projects_added:
2
,
projects_incident_sla_enabled:
2
)
expect
(
described_class
.
usage_activity_by_stage_monitor
(
described_class
.
last_28_days_time_period
)).
to
include
(
expect
(
described_class
.
usage_activity_by_stage_monitor
(
described_class
.
monthly_time_range_db_params
)).
to
include
(
operations_dashboard_users_with_projects_added:
1
,
projects_incident_sla_enabled:
2
)
...
...
@@ -503,7 +503,7 @@ RSpec.describe Gitlab::UsageData do
label_lists:
2
,
milestone_lists:
2
)
expect
(
described_class
.
usage_activity_by_stage_plan
(
described_class
.
last_28_days_time_period
)).
to
include
(
expect
(
described_class
.
usage_activity_by_stage_plan
(
described_class
.
monthly_time_range_db_params
)).
to
include
(
assignee_lists:
1
,
epics:
1
,
label_lists:
1
,
...
...
@@ -521,7 +521,7 @@ RSpec.describe Gitlab::UsageData do
expect
(
described_class
.
usage_activity_by_stage_release
({})).
to
include
(
projects_mirrored_with_pipelines_enabled:
2
)
expect
(
described_class
.
usage_activity_by_stage_release
(
described_class
.
last_28_days_time_period
)).
to
include
(
expect
(
described_class
.
usage_activity_by_stage_release
(
described_class
.
monthly_time_range_db_params
)).
to
include
(
projects_mirrored_with_pipelines_enabled:
1
)
end
...
...
@@ -549,7 +549,7 @@ RSpec.describe Gitlab::UsageData do
end
it
'includes accurate usage_activity_by_stage data'
do
expect
(
described_class
.
usage_activity_by_stage_secure
(
described_class
.
last_28_days_time_period
)).
to
include
(
expect
(
described_class
.
usage_activity_by_stage_secure
(
described_class
.
monthly_time_range_db_params
)).
to
include
(
user_preferences_group_overview_security_dashboard:
3
,
user_container_scanning_jobs:
1
,
user_api_fuzzing_jobs:
1
,
...
...
@@ -611,7 +611,7 @@ RSpec.describe Gitlab::UsageData do
coverage_fuzzing_scans:
0
)
expect
(
described_class
.
usage_activity_by_stage_secure
(
described_class
.
last_28_days_time_period
)).
to
include
(
expect
(
described_class
.
usage_activity_by_stage_secure
(
described_class
.
monthly_time_range_db_params
)).
to
include
(
user_preferences_group_overview_security_dashboard:
3
,
user_api_fuzzing_jobs:
1
,
user_api_fuzzing_dnd_jobs:
1
,
...
...
@@ -647,7 +647,7 @@ RSpec.describe Gitlab::UsageData do
create
(
:ci_build
,
name:
'dast'
,
user:
user3
)
end
expect
(
described_class
.
usage_activity_by_stage_secure
(
described_class
.
last_28_days_time_period
)).
to
include
(
expect
(
described_class
.
usage_activity_by_stage_secure
(
described_class
.
monthly_time_range_db_params
)).
to
include
(
user_preferences_group_overview_security_dashboard:
3
,
user_api_fuzzing_jobs:
1
,
user_api_fuzzing_dnd_jobs:
1
,
...
...
@@ -681,7 +681,7 @@ RSpec.describe Gitlab::UsageData do
create
(
:ci_build
,
name:
'license_scanning'
,
user:
user
)
end
expect
(
described_class
.
usage_activity_by_stage_secure
(
described_class
.
last_28_days_time_period
)).
to
include
(
expect
(
described_class
.
usage_activity_by_stage_secure
(
described_class
.
monthly_time_range_db_params
)).
to
include
(
user_preferences_group_overview_security_dashboard:
3
,
user_api_fuzzing_jobs:
1
,
user_api_fuzzing_dnd_jobs:
1
,
...
...
@@ -720,7 +720,7 @@ RSpec.describe Gitlab::UsageData do
allow
(
Gitlab
::
Database
::
PostgresHll
::
BatchDistinctCounter
).
to
receive
(
:new
).
and_raise
(
ActiveRecord
::
StatementInvalid
)
allow
(
::
Ci
::
Build
).
to
receive
(
:distinct_count_by
).
and_raise
(
ActiveRecord
::
StatementInvalid
)
expect
(
described_class
.
usage_activity_by_stage_secure
(
described_class
.
last_28_days_time_period
)).
to
include
(
expect
(
described_class
.
usage_activity_by_stage_secure
(
described_class
.
monthly_time_range_db_params
)).
to
include
(
user_preferences_group_overview_security_dashboard:
-
1
,
user_api_fuzzing_jobs:
-
1
,
user_api_fuzzing_dnd_jobs:
-
1
,
...
...
@@ -765,7 +765,7 @@ RSpec.describe Gitlab::UsageData do
create
(
:ee_ci_build
,
:secret_detection
,
:failed
,
user:
user2
)
end
expect
(
described_class
.
usage_activity_by_stage_secure
(
described_class
.
last_28_days_time_period
)).
to
include
(
expect
(
described_class
.
usage_activity_by_stage_secure
(
described_class
.
monthly_time_range_db_params
)).
to
include
(
user_api_fuzzing_scans:
be_within
(
error_rate
).
percent_of
(
1
),
user_container_scanning_scans:
be_within
(
error_rate
).
percent_of
(
1
),
user_coverage_fuzzing_scans:
be_within
(
error_rate
).
percent_of
(
1
),
...
...
@@ -794,7 +794,7 @@ RSpec.describe Gitlab::UsageData do
create
(
:ee_ci_build
,
:secret_detection
,
:failed
,
user:
user3
)
end
expect
(
described_class
.
usage_activity_by_stage_secure
(
described_class
.
last_28_days_time_period
)).
not_to
include
(
expect
(
described_class
.
usage_activity_by_stage_secure
(
described_class
.
monthly_time_range_db_params
)).
not_to
include
(
:user_api_fuzzing_scans
,
:user_container_scanning_scans
,
:user_coverage_fuzzing_scans
,
...
...
@@ -806,7 +806,7 @@ RSpec.describe Gitlab::UsageData do
end
it
'includes accurate usage_activity_by_stage data'
do
expect
(
described_class
.
usage_activity_by_stage_secure
(
described_class
.
last_28_days_time_period
)).
to
include
(
expect
(
described_class
.
usage_activity_by_stage_secure
(
described_class
.
monthly_time_range_db_params
)).
to
include
(
user_preferences_group_overview_security_dashboard:
3
,
user_container_scanning_jobs:
1
,
user_api_fuzzing_jobs:
1
,
...
...
@@ -868,7 +868,7 @@ RSpec.describe Gitlab::UsageData do
coverage_fuzzing_scans:
0
)
expect
(
described_class
.
usage_activity_by_stage_secure
(
described_class
.
last_28_days_time_period
)).
to
include
(
expect
(
described_class
.
usage_activity_by_stage_secure
(
described_class
.
monthly_time_range_db_params
)).
to
include
(
user_preferences_group_overview_security_dashboard:
3
,
user_api_fuzzing_jobs:
1
,
user_api_fuzzing_dnd_jobs:
1
,
...
...
@@ -904,7 +904,7 @@ RSpec.describe Gitlab::UsageData do
create
(
:ci_build
,
name:
'dast'
,
user:
user3
)
end
expect
(
described_class
.
usage_activity_by_stage_secure
(
described_class
.
last_28_days_time_period
)).
to
include
(
expect
(
described_class
.
usage_activity_by_stage_secure
(
described_class
.
monthly_time_range_db_params
)).
to
include
(
user_preferences_group_overview_security_dashboard:
3
,
user_api_fuzzing_jobs:
1
,
user_api_fuzzing_dnd_jobs:
1
,
...
...
@@ -938,7 +938,7 @@ RSpec.describe Gitlab::UsageData do
create
(
:ci_build
,
name:
'license_scanning'
,
user:
user
)
end
expect
(
described_class
.
usage_activity_by_stage_secure
(
described_class
.
last_28_days_time_period
)).
to
include
(
expect
(
described_class
.
usage_activity_by_stage_secure
(
described_class
.
monthly_time_range_db_params
)).
to
include
(
user_preferences_group_overview_security_dashboard:
3
,
user_api_fuzzing_jobs:
1
,
user_api_fuzzing_dnd_jobs:
1
,
...
...
@@ -972,7 +972,7 @@ RSpec.describe Gitlab::UsageData do
allow
(
Gitlab
::
Database
::
BatchCount
).
to
receive
(
:batch_count
).
and_raise
(
ActiveRecord
::
StatementInvalid
)
allow
(
::
Ci
::
Build
).
to
receive
(
:distinct_count_by
).
and_raise
(
ActiveRecord
::
StatementInvalid
)
expect
(
described_class
.
usage_activity_by_stage_secure
(
described_class
.
last_28_days_time_period
)).
to
eq
(
expect
(
described_class
.
usage_activity_by_stage_secure
(
described_class
.
monthly_time_range_db_params
)).
to
eq
(
user_preferences_group_overview_security_dashboard:
-
1
,
user_api_fuzzing_jobs:
-
1
,
user_api_fuzzing_dnd_jobs:
-
1
,
...
...
@@ -1012,7 +1012,7 @@ RSpec.describe Gitlab::UsageData do
expect
(
described_class
.
usage_activity_by_stage_verify
({})).
to
include
(
projects_reporting_ci_cd_back_to_github:
2
)
expect
(
described_class
.
usage_activity_by_stage_verify
(
described_class
.
last_28_days_time_period
)).
to
include
(
expect
(
described_class
.
usage_activity_by_stage_verify
(
described_class
.
monthly_time_range_db_params
)).
to
include
(
projects_reporting_ci_cd_back_to_github:
1
)
end
...
...
lib/gitlab/usage/metrics/aggregates/aggregate.rb
View file @
e51d2609
...
...
@@ -22,9 +22,7 @@ module Gitlab
}.
freeze
class
Aggregate
delegate
:weekly_time_range
,
:monthly_time_range
,
to:
Gitlab
::
Utils
::
UsageData
include
Gitlab
::
Usage
::
TimeFrame
def
initialize
(
recorded_at
)
@aggregated_metrics
=
load_metrics
(
AGGREGATED_METRICS_PATH
)
...
...
@@ -32,15 +30,15 @@ module Gitlab
end
def
all_time_data
aggregated_metrics_data
(
start_date:
nil
,
end_date:
nil
,
time_frame:
Gitlab
::
U
tils
::
UsageData
::
ALL_TIME_TIME_FRAME_NAME
)
aggregated_metrics_data
(
start_date:
nil
,
end_date:
nil
,
time_frame:
Gitlab
::
U
sage
::
TimeFrame
::
ALL_TIME_TIME_FRAME_NAME
)
end
def
monthly_data
aggregated_metrics_data
(
**
monthly_time_range
.
merge
(
time_frame:
Gitlab
::
U
tils
::
UsageData
::
TWENTY_EIGHT_DAYS_TIME_FRAME_NAME
))
aggregated_metrics_data
(
**
monthly_time_range
.
merge
(
time_frame:
Gitlab
::
U
sage
::
TimeFrame
::
TWENTY_EIGHT_DAYS_TIME_FRAME_NAME
))
end
def
weekly_data
aggregated_metrics_data
(
**
weekly_time_range
.
merge
(
time_frame:
Gitlab
::
U
tils
::
UsageData
::
SEVEN_DAYS_TIME_FRAME_NAME
))
aggregated_metrics_data
(
**
weekly_time_range
.
merge
(
time_frame:
Gitlab
::
U
sage
::
TimeFrame
::
SEVEN_DAYS_TIME_FRAME_NAME
))
end
private
...
...
@@ -54,7 +52,7 @@ module Gitlab
case
aggregation
[
:source
]
when
REDIS_SOURCE
if
time_frame
==
Gitlab
::
U
tils
::
UsageData
::
ALL_TIME_TIME_FRAME_NAME
if
time_frame
==
Gitlab
::
U
sage
::
TimeFrame
::
ALL_TIME_TIME_FRAME_NAME
data
[
aggregation
[
:name
]]
=
Gitlab
::
Utils
::
UsageData
::
FALLBACK
Gitlab
::
ErrorTracking
.
track_and_raise_for_dev_exception
(
...
...
lib/gitlab/usage/metrics/aggregates/sources/postgres_hll.rb
View file @
e51d2609
...
...
@@ -56,15 +56,15 @@ module Gitlab
end
def
time_period_to_human_name
(
time_period
)
return
Gitlab
::
U
tils
::
UsageData
::
ALL_TIME_TIME_FRAME_NAME
if
time_period
.
blank?
return
Gitlab
::
U
sage
::
TimeFrame
::
ALL_TIME_TIME_FRAME_NAME
if
time_period
.
blank?
start_date
=
time_period
.
first
.
to_date
end_date
=
time_period
.
last
.
to_date
if
(
end_date
-
start_date
).
to_i
>
7
Gitlab
::
U
tils
::
UsageData
::
TWENTY_EIGHT_DAYS_TIME_FRAME_NAME
Gitlab
::
U
sage
::
TimeFrame
::
TWENTY_EIGHT_DAYS_TIME_FRAME_NAME
else
Gitlab
::
U
tils
::
UsageData
::
SEVEN_DAYS_TIME_FRAME_NAME
Gitlab
::
U
sage
::
TimeFrame
::
SEVEN_DAYS_TIME_FRAME_NAME
end
end
end
...
...
lib/gitlab/usage/metrics/instrumentations/base_metric.rb
View file @
e51d2609
...
...
@@ -6,6 +6,7 @@ module Gitlab
module
Instrumentations
class
BaseMetric
include
Gitlab
::
Utils
::
UsageData
include
Gitlab
::
Usage
::
TimeFrame
attr_reader
:time_frame
attr_reader
:options
...
...
lib/gitlab/usage/metrics/instrumentations/database_metric.rb
View file @
e51d2609
...
...
@@ -52,7 +52,7 @@ module Gitlab
def
time_constraints
case
time_frame
when
'28d'
{
created_at:
30
.
days
.
ago
..
2
.
days
.
ago
}
monthly_time_range_db_params
when
'all'
{}
when
'none'
...
...
lib/gitlab/usage/metrics/instrumentations/redis_hll_metric.rb
View file @
e51d2609
...
...
@@ -35,9 +35,9 @@ module Gitlab
def
time_constraints
case
time_frame
when
'28d'
{
start_date:
4
.
weeks
.
ago
.
to_date
,
end_date:
Date
.
current
}
monthly_time_range
when
'7d'
{
start_date:
7
.
days
.
ago
.
to_date
,
end_date:
Date
.
current
}
weekly_time_range
else
raise
"Unknown time frame:
#{
time_frame
}
for RedisHLLMetric"
end
...
...
lib/gitlab/usage/time_frame.rb
0 → 100644
View file @
e51d2609
# frozen_string_literal: true
module
Gitlab
module
Usage
module
TimeFrame
ALL_TIME_TIME_FRAME_NAME
=
"all"
SEVEN_DAYS_TIME_FRAME_NAME
=
"7d"
TWENTY_EIGHT_DAYS_TIME_FRAME_NAME
=
"28d"
def
weekly_time_range
{
start_date:
7
.
days
.
ago
.
to_date
,
end_date:
Date
.
current
}
end
def
monthly_time_range
{
start_date:
4
.
weeks
.
ago
.
to_date
,
end_date:
Date
.
current
}
end
# This time range is skewed for batch counter performance.
# See https://gitlab.com/gitlab-org/gitlab/-/merge_requests/42972
def
monthly_time_range_db_params
(
column: :created_at
)
{
column
=>
30
.
days
.
ago
..
2
.
days
.
ago
}
end
end
end
end
lib/gitlab/usage_data.rb
View file @
e51d2609
...
...
@@ -33,6 +33,7 @@ module Gitlab
class
<<
self
include
Gitlab
::
Utils
::
UsageData
include
Gitlab
::
Utils
::
StrongMemoize
include
Gitlab
::
Usage
::
TimeFrame
def
data
(
force_refresh:
false
)
Rails
.
cache
.
fetch
(
'usage_data'
,
force:
force_refresh
,
expires_in:
2
.
weeks
)
do
...
...
@@ -55,7 +56,7 @@ module Gitlab
.
merge
(
object_store_usage_data
)
.
merge
(
topology_usage_data
)
.
merge
(
usage_activity_by_stage
)
.
merge
(
usage_activity_by_stage
(
:usage_activity_by_stage_monthly
,
last_28_days_time_period
))
.
merge
(
usage_activity_by_stage
(
:usage_activity_by_stage_monthly
,
monthly_time_range_db_params
))
.
merge
(
analytics_unique_visits_data
)
.
merge
(
compliance_unique_visits_data
)
.
merge
(
search_unique_visits_data
)
...
...
@@ -228,17 +229,17 @@ module Gitlab
{
counts_monthly:
{
# rubocop: disable UsageData/LargeTable:
deployments:
deployment_count
(
Deployment
.
where
(
last_28_days_time_period
)),
successful_deployments:
deployment_count
(
Deployment
.
success
.
where
(
last_28_days_time_period
)),
failed_deployments:
deployment_count
(
Deployment
.
failed
.
where
(
last_28_days_time_period
)),
deployments:
deployment_count
(
Deployment
.
where
(
monthly_time_range_db_params
)),
successful_deployments:
deployment_count
(
Deployment
.
success
.
where
(
monthly_time_range_db_params
)),
failed_deployments:
deployment_count
(
Deployment
.
failed
.
where
(
monthly_time_range_db_params
)),
# rubocop: enable UsageData/LargeTable:
projects:
count
(
Project
.
where
(
last_28_days_time_period
),
start:
minimum_id
(
Project
),
finish:
maximum_id
(
Project
)),
packages:
count
(
::
Packages
::
Package
.
where
(
last_28_days_time_period
)),
personal_snippets:
count
(
PersonalSnippet
.
where
(
last_28_days_time_period
)),
project_snippets:
count
(
ProjectSnippet
.
where
(
last_28_days_time_period
)),
projects_with_alerts_created:
distinct_count
(
::
AlertManagement
::
Alert
.
where
(
last_28_days_time_period
),
:project_id
)
projects:
count
(
Project
.
where
(
monthly_time_range_db_params
),
start:
minimum_id
(
Project
),
finish:
maximum_id
(
Project
)),
packages:
count
(
::
Packages
::
Package
.
where
(
monthly_time_range_db_params
)),
personal_snippets:
count
(
PersonalSnippet
.
where
(
monthly_time_range_db_params
)),
project_snippets:
count
(
ProjectSnippet
.
where
(
monthly_time_range_db_params
)),
projects_with_alerts_created:
distinct_count
(
::
AlertManagement
::
Alert
.
where
(
monthly_time_range_db_params
),
:project_id
)
}.
merge
(
snowplow_event_counts
(
last_28_days_time_period
(
column: :collector_tstamp
))
snowplow_event_counts
(
monthly_time_range_db_params
(
column: :collector_tstamp
))
).
tap
do
|
data
|
data
[
:snippets
]
=
add
(
data
[
:personal_snippets
],
data
[
:project_snippets
])
end
...
...
@@ -522,10 +523,6 @@ module Gitlab
"
#{
platform
}
-
#{
ohai_data
[
'platform_version'
]
}
"
end
def
last_28_days_time_period
(
column: :created_at
)
{
column
=>
batch_counter_monthly_time_range
[
:start_date
]
..
batch_counter_monthly_time_range
[
:end_date
]
}
end
# Source: https://gitlab.com/gitlab-data/analytics/blob/master/transform/snowflake-dbt/data/ping_metrics_to_stage_mapping_data.csv
def
usage_activity_by_stage
(
key
=
:usage_activity_by_stage
,
time_period
=
{})
{
...
...
lib/gitlab/usage_data_counters/hll_redis_counter.rb
View file @
e51d2609
...
...
@@ -38,6 +38,7 @@ module Gitlab
# * Get unique counts per user: Gitlab::UsageDataCounters::HLLRedisCounter.unique_events(event_names: 'g_compliance_dashboard', start_date: 28.days.ago, end_date: Date.current)
class
<<
self
include
Gitlab
::
Utils
::
UsageData
include
Gitlab
::
Usage
::
TimeFrame
# Track unique events
#
...
...
lib/gitlab/utils/usage_data.rb
View file @
e51d2609
...
...
@@ -42,25 +42,8 @@ module Gitlab
FALLBACK
=
-
1
HISTOGRAM_FALLBACK
=
{
'-1'
=>
-
1
}.
freeze
DISTRIBUTED_HLL_FALLBACK
=
-
2
ALL_TIME_TIME_FRAME_NAME
=
"all"
SEVEN_DAYS_TIME_FRAME_NAME
=
"7d"
TWENTY_EIGHT_DAYS_TIME_FRAME_NAME
=
"28d"
MAX_BUCKET_SIZE
=
100
def
weekly_time_range
{
start_date:
7
.
days
.
ago
.
to_date
,
end_date:
Date
.
current
}
end
def
monthly_time_range
{
start_date:
4
.
weeks
.
ago
.
to_date
,
end_date:
Date
.
current
}
end
# This time range is skewed for batch counter performance.
# See https://gitlab.com/gitlab-org/gitlab/-/merge_requests/42972
def
batch_counter_monthly_time_range
{
start_date:
30
.
days
.
ago
.
to_date
,
end_date:
2
.
days
.
ago
.
to_date
}
end
def
count
(
relation
,
column
=
nil
,
batch:
true
,
batch_size:
nil
,
start:
nil
,
finish:
nil
)
if
batch
Gitlab
::
Database
::
BatchCount
.
batch_count
(
relation
,
column
,
batch_size:
batch_size
,
start:
start
,
finish:
finish
)
...
...
spec/config/metrics/aggregates/aggregated_metrics_spec.rb
View file @
e51d2609
...
...
@@ -25,9 +25,9 @@ RSpec.describe 'aggregated metrics' do
RSpec
::
Matchers
.
define
:have_known_time_frame
do
allowed_time_frames
=
[
Gitlab
::
U
tils
::
UsageData
::
ALL_TIME_TIME_FRAME_NAME
,
Gitlab
::
U
tils
::
UsageData
::
TWENTY_EIGHT_DAYS_TIME_FRAME_NAME
,
Gitlab
::
U
tils
::
UsageData
::
SEVEN_DAYS_TIME_FRAME_NAME
Gitlab
::
U
sage
::
TimeFrame
::
ALL_TIME_TIME_FRAME_NAME
,
Gitlab
::
U
sage
::
TimeFrame
::
TWENTY_EIGHT_DAYS_TIME_FRAME_NAME
,
Gitlab
::
U
sage
::
TimeFrame
::
SEVEN_DAYS_TIME_FRAME_NAME
]
match
do
|
aggregate
|
...
...
@@ -63,7 +63,7 @@ RSpec.describe 'aggregated metrics' do
let_it_be
(
:events_records
)
{
known_events
.
select
{
|
event
|
aggregate
[
:events
].
include?
(
event
[
:name
])
}
}
it
"does not include 'all' time frame for Redis sourced aggregate"
do
expect
(
aggregate
[
:time_frame
]).
not_to
include
(
Gitlab
::
U
tils
::
UsageData
::
ALL_TIME_TIME_FRAME_NAME
)
expect
(
aggregate
[
:time_frame
]).
not_to
include
(
Gitlab
::
U
sage
::
TimeFrame
::
ALL_TIME_TIME_FRAME_NAME
)
end
it
"only refers to known events"
do
...
...
spec/lib/gitlab/usage_data_spec.rb
View file @
e51d2609
...
...
@@ -91,7 +91,7 @@ RSpec.describe Gitlab::UsageData, :aggregate_failures do
expect
(
described_class
.
usage_activity_by_stage_package
({})).
to
eq
(
projects_with_packages:
2
)
expect
(
described_class
.
usage_activity_by_stage_package
(
described_class
.
last_28_days_time_period
)).
to
eq
(
expect
(
described_class
.
usage_activity_by_stage_package
(
described_class
.
monthly_time_range_db_params
)).
to
eq
(
projects_with_packages:
1
)
end
...
...
@@ -135,7 +135,7 @@ RSpec.describe Gitlab::UsageData, :aggregate_failures do
project_clusters_disabled:
2
,
project_clusters_enabled:
10
)
expect
(
described_class
.
usage_activity_by_stage_configure
(
described_class
.
last_28_days_time_period
)).
to
include
(
expect
(
described_class
.
usage_activity_by_stage_configure
(
described_class
.
monthly_time_range_db_params
)).
to
include
(
clusters_applications_cert_managers:
1
,
clusters_applications_helm:
1
,
clusters_applications_ingress:
1
,
...
...
@@ -185,7 +185,7 @@ RSpec.describe Gitlab::UsageData, :aggregate_failures do
snippets:
2
,
suggestions:
2
)
expect
(
described_class
.
usage_activity_by_stage_create
(
described_class
.
last_28_days_time_period
)).
to
include
(
expect
(
described_class
.
usage_activity_by_stage_create
(
described_class
.
monthly_time_range_db_params
)).
to
include
(
deploy_keys:
1
,
keys:
1
,
merge_requests:
1
,
...
...
@@ -225,7 +225,7 @@ RSpec.describe Gitlab::UsageData, :aggregate_failures do
omniauth_providers:
[
'google_oauth2'
],
user_auth_by_provider:
{
'group_saml'
=>
2
,
'ldap'
=>
4
,
'standard'
=>
0
,
'two-factor'
=>
0
,
'two-factor-via-u2f-device'
=>
0
,
"two-factor-via-webauthn-device"
=>
0
}
)
expect
(
described_class
.
usage_activity_by_stage_manage
(
described_class
.
last_28_days_time_period
)).
to
include
(
expect
(
described_class
.
usage_activity_by_stage_manage
(
described_class
.
monthly_time_range_db_params
)).
to
include
(
events:
1
,
groups:
1
,
users_created:
3
,
...
...
@@ -252,7 +252,7 @@ RSpec.describe Gitlab::UsageData, :aggregate_failures do
unique_users_all_imports:
10
)
expect
(
described_class
.
usage_activity_by_stage_manage
(
described_class
.
last_28_days_time_period
)).
to
include
(
expect
(
described_class
.
usage_activity_by_stage_manage
(
described_class
.
monthly_time_range_db_params
)).
to
include
(
unique_users_all_imports:
5
)
end
...
...
@@ -327,7 +327,7 @@ RSpec.describe Gitlab::UsageData, :aggregate_failures do
groups_imported:
Gitlab
::
UsageData
::
DEPRECATED_VALUE
}
)
expect
(
described_class
.
usage_activity_by_stage_manage
(
described_class
.
last_28_days_time_period
)).
to
include
(
expect
(
described_class
.
usage_activity_by_stage_manage
(
described_class
.
monthly_time_range_db_params
)).
to
include
(
{
bulk_imports:
{
gitlab_v1:
1
,
...
...
@@ -411,7 +411,7 @@ RSpec.describe Gitlab::UsageData, :aggregate_failures do
projects_with_enabled_alert_integrations_histogram:
{
'1'
=>
2
}
)
data_28_days
=
described_class
.
usage_activity_by_stage_monitor
(
described_class
.
last_28_days_time_period
)
data_28_days
=
described_class
.
usage_activity_by_stage_monitor
(
described_class
.
monthly_time_range_db_params
)
expect
(
data_28_days
).
to
include
(
clusters:
1
,
clusters_applications_prometheus:
1
,
...
...
@@ -450,7 +450,7 @@ RSpec.describe Gitlab::UsageData, :aggregate_failures do
projects_jira_dvcs_cloud_active:
2
,
projects_jira_dvcs_server_active:
2
)
expect
(
described_class
.
usage_activity_by_stage_plan
(
described_class
.
last_28_days_time_period
)).
to
include
(
expect
(
described_class
.
usage_activity_by_stage_plan
(
described_class
.
monthly_time_range_db_params
)).
to
include
(
issues:
2
,
notes:
1
,
projects:
1
,
...
...
@@ -479,7 +479,7 @@ RSpec.describe Gitlab::UsageData, :aggregate_failures do
releases:
2
,
successful_deployments:
2
)
expect
(
described_class
.
usage_activity_by_stage_release
(
described_class
.
last_28_days_time_period
)).
to
include
(
expect
(
described_class
.
usage_activity_by_stage_release
(
described_class
.
monthly_time_range_db_params
)).
to
include
(
deployments:
1
,
failed_deployments:
1
,
releases:
1
,
...
...
@@ -513,7 +513,7 @@ RSpec.describe Gitlab::UsageData, :aggregate_failures do
ci_triggers:
2
,
clusters_applications_runner:
2
)
expect
(
described_class
.
usage_activity_by_stage_verify
(
described_class
.
last_28_days_time_period
)).
to
include
(
expect
(
described_class
.
usage_activity_by_stage_verify
(
described_class
.
monthly_time_range_db_params
)).
to
include
(
ci_builds:
1
,
ci_external_pipelines:
1
,
ci_internal_pipelines:
1
,
...
...
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment