Commit 97ef9b30 authored by nmilojevic1's avatar nmilojevic1

Add measurement to import_rake task

parent be711a8d
......@@ -12,7 +12,7 @@
namespace :gitlab do
namespace :import_export do
desc 'GitLab | Import/Export | EXPERIMENTAL | Import large project archives'
task :import, [:username, :namespace_path, :project_path, :archive_path] => :gitlab_environment do |_t, args|
task :import, [:username, :namespace_path, :project_path, :archive_path, :measurement_enabled] => :gitlab_environment do |_t, args|
# Load it here to avoid polluting Rake tasks with Sidekiq test warnings
require 'sidekiq/testing'
......@@ -26,7 +26,8 @@ namespace :gitlab do
namespace_path: args.namespace_path,
project_path: args.project_path,
username: args.username,
file_path: args.archive_path
file_path: args.archive_path,
measurement_enabled: args.measurement_enabled
).import
end
end
......@@ -38,6 +39,7 @@ class GitlabProjectImport
@file_path = opts.fetch(:file_path)
@namespace = Namespace.find_by_full_path(opts.fetch(:namespace_path))
@current_user = User.find_by_username(opts.fetch(:username))
@measurement_enabled = opts.fetch(:measurement_enabled || false)
end
def import
......@@ -72,6 +74,54 @@ class GitlabProjectImport
RequestStore.clear!
end
def with_count_queries(&block)
count = 0
counter_f = ->(name, started, finished, unique_id, payload) {
unless payload[:name].in? %w[CACHE SCHEMA]
count += 1
end
}
ActiveSupport::Notifications.subscribed(counter_f, "sql.active_record", &block)
puts "Number of sql calls: #{count}"
end
def with_gc_counter
gc_counts_before = GC.stat.select { |k, v| k =~ /count/ }
yield
gc_counts_after = GC.stat.select { |k, v| k =~ /count/ }
stats = gc_counts_before.merge(gc_counts_after) { |k, vb, va| va - vb }
puts "Total GC count: #{stats[:count]}"
puts "Minor GC count: #{stats[:minor_gc_count]}"
puts "Major GC count: #{stats[:major_gc_count]}"
end
def with_measure_time
timing = Benchmark.realtime do
yield
end
time = Time.at(timing).utc.strftime("%H:%M:%S")
puts "Time to finish: #{time}"
end
def with_measuring
puts "Measuring enabled..."
with_gc_counter do
with_count_queries do
with_measure_time do
yield
end
end
end
end
def measurement_enabled?
@measurement_enabled
end
# We want to ensure that all Sidekiq jobs are executed
# synchronously as part of that process.
# This ensures that all expensive operations do not escape
......@@ -80,7 +130,7 @@ class GitlabProjectImport
Sidekiq::Testing.fake! do
with_request_store do
::Gitlab::GitalyClient.allow_n_plus_1_calls do
yield
measurement_enabled? ? with_measuring { yield } : yield
end
end
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment