Skip to content

G
gitlab-ce
Commit 3f64f5f1 authored by Dylan Griffith's avatar Dylan Griffith
Browse files
Options

Merge branch 'bypass-min-max-queries' into 'master' 

Bypass min/max queries in usage data queries

See merge request gitlab-org/gitlab!58006
parents b68cada0 e4cce2d6
Hide whitespace changes
Inline Side-by-side
Showing with 92 additions and 81 deletions
ee/lib/ee/gitlab/usage_data.rb
View file @ 3f64f5f1
......@@ -198,7 +198,7 @@ module EE
geo_nodes: count(::GeoNode),
geo_event_log_max_id: alt_usage_data { Geo::EventLog.maximum(:id) || 0 },
ldap_group_links: count(::LdapGroupLink),
issues_with_health_status: count(::Issue.with_health_status, start: issue_minimum_id, finish: issue_maximum_id),
issues_with_health_status: count(::Issue.with_health_status, start: minimum_id(::Issue), finish: maximum_id(::Issue)),
ldap_keys: count(::LDAPKey),
ldap_users: count(::User.ldap, 'users.id'),
pod_logs_usages_total: redis_usage_data { ::Gitlab::UsageCounters::PodLogs.usage_totals[:total] },
......@@ -208,7 +208,7 @@ module EE
projects_mirrored_with_pipelines_enabled: count(::Project.mirrored_with_enabled_pipelines),
projects_reporting_ci_cd_back_to_github: count(::GithubService.active),
status_page_projects: count(::StatusPage::ProjectSetting.enabled),
status_page_issues: count(::Issue.on_status_page, start: issue_minimum_id, finish: issue_maximum_id),
status_page_issues: count(::Issue.on_status_page, start: minimum_id(::Issue), finish: maximum_id(::Issue)),
template_repositories: add(count(::Project.with_repos_templates), count(::Project.with_groups_level_repos_templates))
},
requirements_counts,
......@@ -260,16 +260,16 @@ module EE
projects_imported_from_github: distinct_count(::Project.github_imported.where(time_period), :creator_id),
projects_with_repositories_enabled: distinct_count(::Project.with_repositories_enabled.where(time_period),
:creator_id,
start: user_minimum_id,
finish: user_maximum_id),
start: minimum_id(::User),
finish: maximum_id(::User)),
protected_branches: distinct_count(::Project.with_protected_branches.where(time_period),
:creator_id,
start: user_minimum_id,
finish: user_maximum_id),
start: minimum_id(::User),
finish: maximum_id(::User)),
suggestions: distinct_count(::Note.with_suggestions.where(time_period),
:author_id,
start: user_minimum_id,
finish: user_maximum_id),
start: minimum_id(::User),
finish: maximum_id(::User)),
users_using_path_locks: distinct_count(PathLock.where(time_period), :user_id),
users_using_lfs_locks: distinct_count(LfsFileLock.where(time_period), :user_id),
total_number_of_path_locks: count(::PathLock.where(time_period)),
......@@ -377,8 +377,8 @@ module EE
SECURE_PRODUCT_TYPES.each do |secure_type, attribs|
results["#{prefix}#{attribs[:name]}".to_sym] = distinct_count(::Ci::Build.where(name: secure_type).where(time_period),
:user_id,
start: user_minimum_id,
finish: user_maximum_id)
start: minimum_id(::User),
finish: maximum_id(::User))
end
results.merge!(count_secure_user_scans(time_period))
......@@ -613,8 +613,8 @@ module EE
.where.not(section: ::Gitlab::CodeOwners::Entry::DEFAULT_SECTION)
.where(time_period),
'merge_requests.target_project_id',
start: project_minimum_id,
finish: project_maximum_id
start: minimum_id(::Project),
finish: maximum_id(::Project)
)
end
# rubocop:enable CodeReuse/ActiveRecord
......
lib/gitlab/usage_data.rb
View file @ 3f64f5f1
......@@ -87,7 +87,7 @@ module Gitlab
# rubocop: disable Metrics/AbcSize
# rubocop: disable CodeReuse/ActiveRecord
def system_usage_data
issues_created_manually_from_alerts = count(Issue.with_alert_management_alerts.not_authored_by(::User.alert_bot), start: issue_minimum_id, finish: issue_maximum_id)
issues_created_manually_from_alerts = count(Issue.with_alert_management_alerts.not_authored_by(::User.alert_bot), start: minimum_id(Issue), finish: maximum_id(Issue))
{
counts: {
......@@ -138,7 +138,7 @@ module Gitlab
in_review_folder: count(::Environment.in_review_folder),
grafana_integrated_projects: count(GrafanaIntegration.enabled),
groups: count(Group),
issues: count(Issue, start: issue_minimum_id, finish: issue_maximum_id),
issues: count(Issue, start: minimum_id(Issue), finish: maximum_id(Issue)),
issues_created_from_gitlab_error_tracking_ui: count(SentryIssue),
issues_with_associated_zoom_link: count(ZoomMeeting.added_to_issue),
issues_using_zoom_quick_actions: distinct_count(ZoomMeeting, :issue_id),
......@@ -146,9 +146,9 @@ module Gitlab
issues_created_from_alerts: total_alert_issues,
issues_created_gitlab_alerts: issues_created_manually_from_alerts,
issues_created_manually_from_alerts: issues_created_manually_from_alerts,
incident_issues: count(::Issue.incident, start: issue_minimum_id, finish: issue_maximum_id),
alert_bot_incident_issues: count(::Issue.authored(::User.alert_bot), start: issue_minimum_id, finish: issue_maximum_id),
incident_labeled_issues: count(::Issue.with_label_attributes(::IncidentManagement::CreateIncidentLabelService::LABEL_PROPERTIES), start: issue_minimum_id, finish: issue_maximum_id),
incident_issues: count(::Issue.incident, start: minimum_id(Issue), finish: maximum_id(Issue)),
alert_bot_incident_issues: count(::Issue.authored(::User.alert_bot), start: minimum_id(Issue), finish: maximum_id(Issue)),
incident_labeled_issues: count(::Issue.with_label_attributes(::IncidentManagement::CreateIncidentLabelService::LABEL_PROPERTIES), start: minimum_id(Issue), finish: maximum_id(Issue)),
keys: count(Key),
label_lists: count(List.label),
lfs_objects: count(LfsObject),
......@@ -389,8 +389,8 @@ module Gitlab
# rubocop: disable CodeReuse/ActiveRecord
def container_expiration_policies_usage
results = {}
start = ::Project.minimum(:id)
finish = ::Project.maximum(:id)
start = minimum_id(Project)
finish = maximum_id(Project)
results[:projects_with_expiration_policy_disabled] = distinct_count(::ContainerExpirationPolicy.where(enabled: false), :project_id, start: start, finish: finish)
# rubocop: disable UsageData/LargeTable
......@@ -591,7 +591,7 @@ module Gitlab
{
events: distinct_count(::Event.where(time_period), :author_id),
groups: distinct_count(::GroupMember.where(time_period), :user_id),
users_created: count(::User.where(time_period), start: user_minimum_id, finish: user_maximum_id),
users_created: count(::User.where(time_period), start: minimum_id(User), finish: maximum_id(User)),
omniauth_providers: filtered_omniauth_provider_names.reject { |name| name == 'group_saml' },
user_auth_by_provider: distinct_count_user_auth_by_provider(time_period),
unique_users_all_imports: unique_users_all_imports(time_period),
......@@ -636,8 +636,8 @@ module Gitlab
clusters: distinct_count(::Clusters::Cluster.where(time_period), :user_id),
clusters_applications_prometheus: cluster_applications_user_distinct_count(::Clusters::Applications::Prometheus, time_period),
operations_dashboard_default_dashboard: count(::User.active.with_dashboard('operations').where(time_period),
start: user_minimum_id,
finish: user_maximum_id),
start: minimum_id(User),
finish: maximum_id(User)),
projects_with_tracing_enabled: distinct_count(::Project.with_tracing_enabled.where(time_period), :creator_id),
projects_with_error_tracking_enabled: distinct_count(::Project.with_enabled_error_tracking.where(time_period), :creator_id),
projects_with_incidents: distinct_count(::Issue.incident.where(time_period), :project_id),
......@@ -691,12 +691,12 @@ module Gitlab
def usage_activity_by_stage_verify(time_period)
{
ci_builds: distinct_count(::Ci::Build.where(time_period), :user_id),
ci_external_pipelines: distinct_count(::Ci::Pipeline.external.where(time_period), :user_id, start: user_minimum_id, finish: user_maximum_id),
ci_internal_pipelines: distinct_count(::Ci::Pipeline.internal.where(time_period), :user_id, start: user_minimum_id, finish: user_maximum_id),
ci_pipeline_config_auto_devops: distinct_count(::Ci::Pipeline.auto_devops_source.where(time_period), :user_id, start: user_minimum_id, finish: user_maximum_id),
ci_pipeline_config_repository: distinct_count(::Ci::Pipeline.repository_source.where(time_period), :user_id, start: user_minimum_id, finish: user_maximum_id),
ci_external_pipelines: distinct_count(::Ci::Pipeline.external.where(time_period), :user_id, start: minimum_id(User), finish: maximum_id(User)),
ci_internal_pipelines: distinct_count(::Ci::Pipeline.internal.where(time_period), :user_id, start: minimum_id(User), finish: maximum_id(User)),
ci_pipeline_config_auto_devops: distinct_count(::Ci::Pipeline.auto_devops_source.where(time_period), :user_id, start: minimum_id(User), finish: maximum_id(User)),
ci_pipeline_config_repository: distinct_count(::Ci::Pipeline.repository_source.where(time_period), :user_id, start: minimum_id(User), finish: maximum_id(User)),
ci_pipeline_schedules: distinct_count(::Ci::PipelineSchedule.where(time_period), :owner_id),
ci_pipelines: distinct_count(::Ci::Pipeline.where(time_period), :user_id, start: user_minimum_id, finish: user_maximum_id),
ci_pipelines: distinct_count(::Ci::Pipeline.where(time_period), :user_id, start: minimum_id(User), finish: maximum_id(User)),
ci_triggers: distinct_count(::Ci::Trigger.where(time_period), :owner_id),
clusters_applications_runner: cluster_applications_user_distinct_count(::Clusters::Applications::Runner, time_period)
}
......@@ -801,8 +801,8 @@ module Gitlab
end
def distinct_count_service_desk_enabled_projects(time_period)
project_creator_id_start = user_minimum_id
project_creator_id_finish = user_maximum_id
project_creator_id_start = minimum_id(User)
project_creator_id_finish = maximum_id(User)
distinct_count(::Project.service_desk_enabled.where(time_period), :creator_id, start: project_creator_id_start, finish: project_creator_id_finish) # rubocop: disable CodeReuse/ActiveRecord
end
......@@ -834,57 +834,9 @@ module Gitlab
def total_alert_issues
# Remove prometheus table queries once they are deprecated
# To be removed with https://gitlab.com/gitlab-org/gitlab/-/issues/217407.
add count(Issue.with_alert_management_alerts, start: issue_minimum_id, finish: issue_maximum_id),
count(::Issue.with_self_managed_prometheus_alert_events, start: issue_minimum_id, finish: issue_maximum_id),
count(::Issue.with_prometheus_alert_events, start: issue_minimum_id, finish: issue_maximum_id)
end
def user_minimum_id
strong_memoize(:user_minimum_id) do
::User.minimum(:id)
end
end
def user_maximum_id
strong_memoize(:user_maximum_id) do
::User.maximum(:id)
end
end
def issue_minimum_id
strong_memoize(:issue_minimum_id) do
::Issue.minimum(:id)
end
end
def issue_maximum_id
strong_memoize(:issue_maximum_id) do
::Issue.maximum(:id)
end
end
def deployment_minimum_id
strong_memoize(:deployment_minimum_id) do
::Deployment.minimum(:id)
end
end
def deployment_maximum_id
strong_memoize(:deployment_maximum_id) do
::Deployment.maximum(:id)
end
end
def project_minimum_id
strong_memoize(:project_minimum_id) do
::Project.minimum(:id)
end
end
def project_maximum_id
strong_memoize(:project_maximum_id) do
::Project.maximum(:id)
end
add count(Issue.with_alert_management_alerts, start: minimum_id(Issue), finish: maximum_id(Issue)),
count(::Issue.with_self_managed_prometheus_alert_events, start: minimum_id(Issue), finish: maximum_id(Issue)),
count(::Issue.with_prometheus_alert_events, start: minimum_id(Issue), finish: maximum_id(Issue))
end
def self_monitoring_project
......@@ -918,7 +870,7 @@ module Gitlab
end
def deployment_count(relation)
count relation, start: deployment_minimum_id, finish: deployment_maximum_id
count relation, start: minimum_id(Deployment), finish: maximum_id(Deployment)
end
def project_imports(time_period)
......
lib/gitlab/usage_data_queries.rb
View file @ 3f64f5f1
......@@ -36,6 +36,12 @@ module Gitlab
'SELECT ' + args.map {|arg| "(#{arg})" }.join(' + ')
end
def maximum_id(model)
end
def minimum_id(model)
end
private
def raw_sql(relation, column, distinct = nil)
......
lib/gitlab/utils/usage_data.rb
View file @ 3f64f5f1
......@@ -36,6 +36,7 @@
module Gitlab
module Utils
module UsageData
include Gitlab::Utils::StrongMemoize
extend self
FALLBACK = -1
......@@ -209,6 +210,20 @@ module Gitlab
Gitlab::UsageDataCounters::HLLRedisCounter.track_event(event_name.to_s, values: values)
end
def maximum_id(model)
key = :"#{model.name.downcase}_maximum_id"
strong_memoize(key) do
model.maximum(:id)
end
end
def minimum_id(model)
key = :"#{model.name.downcase}_minimum_id"
strong_memoize(key) do
model.minimum(:id)
end
end
private
def prometheus_client(verify:)
......
spec/lib/gitlab/usage_data_queries_spec.rb
View file @ 3f64f5f1
......@@ -58,4 +58,24 @@ RSpec.describe Gitlab::UsageDataQueries do
.to eq('SELECT (SELECT COUNT("users"."id") FROM "users") + (SELECT COUNT("issues"."id") FROM "issues")')
end
end
describe 'min/max methods' do
it 'returns nil' do
# user min/max
expect(described_class.minimum_id(User)).to eq(nil)
expect(described_class.maximum_id(User)).to eq(nil)
# issue min/max
expect(described_class.minimum_id(Issue)).to eq(nil)
expect(described_class.maximum_id(Issue)).to eq(nil)
# deployment min/max
expect(described_class.minimum_id(Deployment)).to eq(nil)
expect(described_class.maximum_id(Deployment)).to eq(nil)
# project min/max
expect(described_class.minimum_id(Project)).to eq(nil)
expect(described_class.maximum_id(Project)).to eq(nil)
end
end
end
spec/lib/gitlab/utils/usage_data_spec.rb
View file @ 3f64f5f1
......@@ -479,4 +479,22 @@ RSpec.describe Gitlab::Utils::UsageData do
expect { described_class.track_usage_event(unknown_event, value) }.to raise_error(Gitlab::UsageDataCounters::HLLRedisCounter::UnknownEvent)
end
end
describe 'min/max' do
let(:model) { double(:relation) }
it 'returns min from the model' do
allow(model).to receive(:minimum).and_return(2)
allow(model).to receive(:name).and_return('sample_min_model')
expect(described_class.minimum_id(model)).to eq(2)
end
it 'returns max from the model' do
allow(model).to receive(:maximum).and_return(100)
allow(model).to receive(:name).and_return('sample_max_model')
expect(described_class.maximum_id(model)).to eq(100)
end
end
end
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment
GitLab Nexedi Edition | About GitLab | About Nexedi | 沪ICP备2021021310号-2 | 沪ICP备2021021310号-7