Commit 106a9f3d authored by Bob Van Landuyt's avatar Bob Van Landuyt

Sleep between inserting batches of refreshes

This inserts a short nap so replication can catch up before the next
batch.

This will add about 25m of doing nothing to the rake task for the 700k
records we expect to insert. But it's not like I'll be actively
waiting for this to complete.
parent fe7b5313
...@@ -28,11 +28,14 @@ namespace :gitlab do ...@@ -28,11 +28,14 @@ namespace :gitlab do
projects = Project.where(id: ids) projects = Project.where(id: ids)
Projects::BuildArtifactsSizeRefresh.enqueue_refresh(projects) Projects::BuildArtifactsSizeRefresh.enqueue_refresh(projects)
# Take a short break to allow replication to catch up
Kernel.sleep(1)
imported += projects.size imported += projects.size
missing += ids.size - projects.size missing += ids.size - projects.size
puts "#{imported}/#{project_ids.size} (missing projects: #{missing})" puts "#{imported}/#{project_ids.size} (missing projects: #{missing})"
end end
puts 'Done.'.green puts 'Done.'
else else
puts 'Project IDs must be listed in the CSV under the header PROJECT_ID'.red puts 'Project IDs must be listed in the CSV under the header PROJECT_ID'.red
end end
......
...@@ -27,6 +27,7 @@ RSpec.describe 'gitlab:refresh_project_statistics_build_artifacts_size rake task ...@@ -27,6 +27,7 @@ RSpec.describe 'gitlab:refresh_project_statistics_build_artifacts_size rake task
stub_const("BUILD_ARTIFACTS_SIZE_REFRESH_ENQUEUE_BATCH_SIZE", 2) stub_const("BUILD_ARTIFACTS_SIZE_REFRESH_ENQUEUE_BATCH_SIZE", 2)
stub_request(:get, csv_url).to_return(status: 200, body: csv_body) stub_request(:get, csv_url).to_return(status: 200, body: csv_body)
allow(Kernel).to receive(:sleep).with(1)
end end
context 'when given a list of space-separated IDs through rake argument' do context 'when given a list of space-separated IDs through rake argument' do
...@@ -35,6 +36,14 @@ RSpec.describe 'gitlab:refresh_project_statistics_build_artifacts_size rake task ...@@ -35,6 +36,14 @@ RSpec.describe 'gitlab:refresh_project_statistics_build_artifacts_size rake task
expect(Projects::BuildArtifactsSizeRefresh.all.map(&:project)).to match_array([project_1, project_2, project_3]) expect(Projects::BuildArtifactsSizeRefresh.all.map(&:project)).to match_array([project_1, project_2, project_3])
end end
it 'inserts refreshes in batches with a sleep' do
expect(Projects::BuildArtifactsSizeRefresh).to receive(:enqueue_refresh).with([project_1, project_2]).ordered
expect(Kernel).to receive(:sleep).with(1)
expect(Projects::BuildArtifactsSizeRefresh).to receive(:enqueue_refresh).with([project_3]).ordered
run_rake_task(rake_task, csv_url)
end
end end
context 'when CSV has invalid header' do context 'when CSV has invalid header' do
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment