Commit 332bfe8d authored by Adam Hegyi's avatar Adam Hegyi

Merge branch 'mwaw/add_index_to_security_scans_on_created_at_and_build_id' into 'master'

Improve usage ping metrics calculations with new index added to security scans table

See merge request gitlab-org/gitlab!48671
parents c200cda7 851bffc8
---
title: Add database index to improve performance of usage ping metrics calculation
merge_request: 48671
author:
type: performance
# frozen_string_literal: true
class AddIndexToSecurityScansOnCreatedAtAndId < ActiveRecord::Migration[6.0]
include Gitlab::Database::MigrationHelpers
DOWNTIME = false
INDEX_NAME = 'index_security_scans_on_date_created_at_and_id'
disable_ddl_transaction!
def up
add_concurrent_index :security_scans, "date(timezone('UTC', created_at)), id", name: INDEX_NAME
end
def down
remove_concurrent_index_by_name(:security_scans, INDEX_NAME)
end
end
a66f13cf62740c590aa195dd510f1e9f4a36e12d29fd0c824fbd0ace03717706
\ No newline at end of file
......@@ -22950,6 +22950,8 @@ CREATE INDEX index_security_findings_on_severity ON security_findings USING btre
CREATE UNIQUE INDEX index_security_findings_on_uuid_and_scan_id ON security_findings USING btree (uuid, scan_id);
CREATE INDEX index_security_scans_on_date_created_at_and_id ON security_scans USING btree (date(timezone('UTC'::text, created_at)), id);
CREATE INDEX index_self_managed_prometheus_alert_events_on_environment_id ON self_managed_prometheus_alert_events USING btree (environment_id);
CREATE INDEX index_sent_notifications_on_noteable_type_noteable_id ON sent_notifications USING btree (noteable_id) WHERE ((noteable_type)::text = 'Issue'::text);
......
......@@ -407,10 +407,28 @@ module EE
# time outing batch queries, to avoid that
# different join strategy is used for HLL counter
if ::Feature.enabled?(:postgres_hll_batch_counting)
relation = ::Security::Scan.where(time_period).group(:created_at)
start = relation.select('MIN(id) as min_id').order('min_id ASC').first&.min_id
finish = relation.select('MAX(id) as max_id').order('max_id DESC').first&.max_id
scans_table = ::Security::Scan.arel_table
inner_relation = ::Security::Scan.select(:id)
.where(
to_date_arel_node(Arel.sql('date_range_source'))
.eq(to_date_arel_node(scans_table[time_period.keys[0]]))
)
outer_relation = ::Security::Scan
.from("generate_series(
'#{time_period.values[0].first.to_time.to_s(:db)}'::timestamp,
'#{time_period.values[0].last.to_time.to_s(:db)}'::timestamp,
'1 day'::interval) date_range_source")
start_id = outer_relation
.select("(#{inner_relation.order(id: :asc).limit(1).to_sql})")
.order('1 ASC NULLS LAST')
.first&.id
finish_id = outer_relation
.select("(#{inner_relation.order(id: :desc).limit(1).to_sql})")
.order('1 DESC NULLS LAST')
.first&.id
::Security::Scan.scan_types.each do |name, scan_type|
relation = ::Security::Scan.joins(:build)
......@@ -419,8 +437,8 @@ module EE
.where(security_scans: time_period)
pipelines_with_secure_jobs["#{name}_pipeline".to_sym] =
if start && finish
estimate_batch_distinct_count(relation, :commit_id, batch_size: 1000, start: start, finish: finish)
if start_id && finish_id
estimate_batch_distinct_count(relation, :commit_id, batch_size: 1000, start: start_id, finish: finish_id)
else
0
end
......@@ -442,6 +460,11 @@ module EE
end
# rubocop: enable UsageData/LargeTable
def to_date_arel_node(column)
locked_timezone = Arel::Nodes::NamedFunction.new('TIMEZONE', [Arel.sql("'UTC'"), column])
Arel::Nodes::NamedFunction.new('DATE', [locked_timezone])
end
def approval_merge_request_rule_minimum_id
strong_memoize(:approval_merge_request_rule_minimum_id) do
::ApprovalMergeRequestRule.minimum(:merge_request_id)
......
......@@ -21,6 +21,8 @@ UsageData/LargeTable:
- :Gitaly::Server
- :Gitlab::UsageData
- :Gitlab::UsageDataCounters
- :Arel::Nodes::NamedFunction
- :Arel
- :License
- :Rails
- :Time
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment