Commit 13bdf315 authored by Grzegorz Bizon's avatar Grzegorz Bizon

Merge branch 'fix-pages-rake-concurrency' into 'master'

Fix usage of threads for pages migration

See merge request gitlab-org/gitlab!53284
parents 0ab08a4a 4d3cda69
...@@ -24,9 +24,7 @@ module Pages ...@@ -24,9 +24,7 @@ module Pages
@queue.close @queue.close
@logger.info("Waiting for threads to finish...") @logger.info("Waiting for threads to finish...")
ActiveSupport::Dependencies.interlock.permit_concurrent_loads do threads.each(&:join)
threads.each(&:join)
end
{ migrated: @migrated, errored: @errored } { migrated: @migrated, errored: @errored }
end end
...@@ -34,8 +32,8 @@ module Pages ...@@ -34,8 +32,8 @@ module Pages
def start_migration_threads def start_migration_threads
Array.new(@migration_threads) do Array.new(@migration_threads) do
Thread.new do Thread.new do
Rails.application.executor.wrap do while batch = @queue.pop
while batch = @queue.pop Rails.application.executor.wrap do
process_batch(batch) process_batch(batch)
end end
end end
...@@ -51,6 +49,11 @@ module Pages ...@@ -51,6 +49,11 @@ module Pages
end end
@logger.info("#{@migrated} projects are migrated successfully, #{@errored} projects failed to be migrated") @logger.info("#{@migrated} projects are migrated successfully, #{@errored} projects failed to be migrated")
rescue => e
# This method should never raise exception otherwise all threads might be killed
# and this will result in queue starving (and deadlock)
Gitlab::ErrorTracking.track_exception(e)
@logger.error("failed processing a batch: #{e.message}")
end end
def migrate_project(project) def migrate_project(project)
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment