Commit d2c90d74 authored by Marin Jankovski's avatar Marin Jankovski

Merge branch 'optimize-ci-tags-migration' into 'master'

CI migration rake task for GitLab CE

- Uses backup format for final GitLab CI 8.0
- Highly optimized tags migration SQL queries


See merge request !1365
parents 8bb3215d a37e591e
...@@ -131,7 +131,7 @@ ...@@ -131,7 +131,7 @@
.checkbox .checkbox
= f.label :ci_enabled do = f.label :ci_enabled do
= f.check_box :ci_enabled = f.check_box :ci_enabled
Enable Continuous Integration Disable to prevent CI usage until rake ci:migrate is run (8.0 only)
.form-actions .form-actions
= f.submit 'Save', class: 'btn btn-primary' = f.submit 'Save', class: 'btn btn-primary'
Continuous Integration has been disabled. Please ask your administrator to enable it. Continuous Integration has been disabled for time of the migration.
module Ci
module Migrate
class Builds
attr_reader :app_builds_dir, :backup_builds_tarball, :backup_dir
def initialize
@app_builds_dir = Settings.gitlab_ci.builds_path
@backup_dir = Gitlab.config.backup.path
@backup_builds_tarball = File.join(backup_dir, 'builds/builds.tar.gz')
end
def restore
backup_existing_builds_dir
FileUtils.mkdir_p(app_builds_dir, mode: 0700)
unless system('tar', '-C', app_builds_dir, '-zxvf', backup_builds_tarball)
abort 'Restore failed'.red
end
end
def backup_existing_builds_dir
timestamped_builds_path = File.join(app_builds_dir, '..', "builds.#{Time.now.to_i}")
if File.exists?(app_builds_dir)
FileUtils.mv(app_builds_dir, File.expand_path(timestamped_builds_path))
end
end
end
end
end
...@@ -9,32 +9,32 @@ module Ci ...@@ -9,32 +9,32 @@ module Ci
@config = YAML.load_file(File.join(Rails.root, 'config', 'database.yml'))[Rails.env] @config = YAML.load_file(File.join(Rails.root, 'config', 'database.yml'))[Rails.env]
end end
def restore(ci_dump) def restore
puts 'Deleting all CI related data ... ' decompress_rd, decompress_wr = IO.pipe
truncate_ci_tables decompress_pid = spawn(*%W(gzip -cd), out: decompress_wr, in: db_file_name)
decompress_wr.close
puts 'Restoring CI data ... ' restore_pid = case config["adapter"]
case config["adapter"] when /^mysql/ then
when /^mysql/ then $progress.print "Restoring MySQL database #{config['database']} ... "
print "Restoring MySQL database #{config['database']} ... " # Workaround warnings from MySQL 5.6 about passwords on cmd line
# Workaround warnings from MySQL 5.6 about passwords on cmd line ENV['MYSQL_PWD'] = config["password"].to_s if config["password"]
ENV['MYSQL_PWD'] = config["password"].to_s if config["password"] spawn('mysql', *mysql_args, config['database'], in: decompress_rd)
system('mysql', *mysql_args, config['database'], in: ci_dump) when "postgresql" then
when "postgresql" then $progress.print "Restoring PostgreSQL database #{config['database']} ... "
puts "Restoring PostgreSQL database #{config['database']} ... " pg_env
pg_env spawn('psql', config['database'], in: decompress_rd)
system('psql', config['database'], '-f', ci_dump) end
end decompress_rd.close
success = [decompress_pid, restore_pid].all? { |pid| Process.waitpid(pid); $?.success? }
abort 'Restore failed' unless success
end end
protected protected
def truncate_ci_tables def db_file_name
c = ActiveRecord::Base.connection File.join(Gitlab.config.backup.path, 'db', 'database.sql.gz')
c.tables.select { |t| t.start_with?('ci_') }.each do |table|
puts "Deleting data from #{table}..."
c.execute("DELETE FROM #{table}")
end
end end
def mysql_args def mysql_args
......
module Ci
module Migrate
class Manager
VERSION = '8.0.0.pre'
def cleanup
$progress.print "Deleting tmp directories ... "
backup_contents.each do |dir|
next unless File.exist?(File.join(Gitlab.config.backup.path, dir))
if FileUtils.rm_rf(File.join(Gitlab.config.backup.path, dir))
$progress.puts "done".green
else
puts "deleting tmp directory '#{dir}' failed".red
abort 'Backup failed'
end
end
end
def unpack
Dir.chdir(Gitlab.config.backup.path)
# check for existing backups in the backup dir
file_list = Dir.glob("*_gitlab_ci_backup.tar").each.map { |f| f.split(/_/).first.to_i }
puts "no backups found" if file_list.count == 0
if file_list.count > 1 && ENV["BACKUP"].nil?
puts "Found more than one backup, please specify which one you want to restore:"
puts "rake gitlab:backup:restore BACKUP=timestamp_of_backup"
exit 1
end
tar_file = ENV["BACKUP"].nil? ? File.join("#{file_list.first}_gitlab_ci_backup.tar") : File.join(ENV["BACKUP"] + "_gitlab_ci_backup.tar")
unless File.exists?(tar_file)
puts "The specified CI backup doesn't exist!"
exit 1
end
$progress.print "Unpacking backup ... "
unless Kernel.system(*%W(tar -xf #{tar_file}))
puts "unpacking backup failed".red
exit 1
else
$progress.puts "done".green
end
ENV["VERSION"] = "#{settings[:db_version]}" if settings[:db_version].to_i > 0
# restoring mismatching backups can lead to unexpected problems
if settings[:gitlab_version] != VERSION
puts "GitLab CI version mismatch:".red
puts " Your current GitLab CI version (#{VERSION}) differs from the GitLab CI (#{settings[:gitlab_version]}) version in the backup!".red
exit 1
end
end
private
def backup_contents
["db", "builds", "backup_information.yml"]
end
def settings
@settings ||= YAML.load_file("backup_information.yml")
end
end
end
end
...@@ -4,45 +4,38 @@ module Ci ...@@ -4,45 +4,38 @@ module Ci
module Migrate module Migrate
class Tags class Tags
def restore def restore
puts 'Migrating tags for Runners... ' puts 'Inserting tags...'
list_objects('Runner').each do |id| connection.select_all('SELECT ci_tags.name FROM ci_tags').each do |tag|
putc '.' begin
runner = Ci::Runner.find_by_id(id) connection.execute("INSERT INTO tags (name) VALUES(#{ActiveRecord::Base::sanitize(tag['name'])})")
if runner rescue ActiveRecord::RecordNotUnique
tags = list_tags('Runner', id)
runner.update_attributes(tag_list: tags)
end end
end end
puts ''
puts 'Migrating tags for Builds... ' ActiveRecord::Base.transaction do
list_objects('Build').each do |id| puts 'Deleting old taggings...'
putc '.' connection.execute "DELETE FROM taggings WHERE context = 'tags' AND taggable_type LIKE 'Ci::%'"
build = Ci::Build.find_by_id(id)
if build puts 'Inserting taggings...'
tags = list_tags('Build', id) connection.execute(
build.update_attributes(tag_list: tags) 'INSERT INTO taggings (taggable_type, taggable_id, tag_id, context) ' +
end "SELECT CONCAT('Ci::', ci_taggings.taggable_type), ci_taggings.taggable_id, tags.id, 'tags' FROM ci_taggings " +
'JOIN ci_tags ON ci_tags.id = ci_taggings.tag_id ' +
'JOIN tags ON tags.name = ci_tags.name '
)
puts 'Resetting counters... '
connection.execute(
'UPDATE tags SET ' +
'taggings_count = (SELECT COUNT(*) FROM taggings WHERE tags.id = taggings.tag_id)'
)
end end
puts ''
end end
protected protected
def list_objects(type) def connection
ids = ActiveRecord::Base.connection.select_all( ActiveRecord::Base.connection
"select distinct taggable_id from ci_taggings where taggable_type = #{ActiveRecord::Base::sanitize(type)}"
)
ids.map { |id| id['taggable_id'] }
end
def list_tags(type, id)
tags = ActiveRecord::Base.connection.select_all(
'select ci_tags.name from ci_tags ' +
'join ci_taggings on ci_tags.id = ci_taggings.tag_id ' +
"where taggable_type = #{ActiveRecord::Base::sanitize(type)} and taggable_id = #{ActiveRecord::Base::sanitize(id)} and context = 'tags'"
)
tags.map { |tag| tag['name'] }
end end
end end
end end
......
namespace :ci do namespace :ci do
desc 'GitLab | Import and migrate CI database' desc 'GitLab | Import and migrate CI database'
task migrate: :environment do task migrate: :environment do
warn_user_is_not_gitlab
configure_cron_mode
unless ENV['force'] == 'yes' unless ENV['force'] == 'yes'
puts "This will truncate all CI tables and restore it from provided backup." puts 'This will remove all CI related data and restore it from the provided backup.'
puts "You will lose any previous CI data stored in the database."
ask_to_continue ask_to_continue
puts "" puts ''
end end
Rake::Task["ci:migrate:db"].invoke # disable CI for time of migration
Rake::Task["ci:migrate:autoincrements"].invoke enable_ci(false)
Rake::Task["ci:migrate:tags"].invoke
Rake::Task["ci:migrate:services"].invoke # unpack archives
migrate = Ci::Migrate::Manager.new
migrate.unpack
Rake::Task['ci:migrate:db'].invoke
Rake::Task['ci:migrate:builds'].invoke
Rake::Task['ci:migrate:tags'].invoke
Rake::Task['ci:migrate:services'].invoke
# enable CI for time of migration
enable_ci(true)
migrate.cleanup
end end
namespace :migrate do namespace :migrate do
desc 'GitLab | Import CI database' desc 'GitLab | Import CI database'
task db: :environment do task db: :environment do
if ENV["CI_DUMP"].nil? configure_cron_mode
puts "No CI SQL dump specified:" $progress.puts 'Restoring database ... '.blue
puts "rake gitlab:backup:restore CI_DUMP=ci_dump.sql" Ci::Migrate::Database.new.restore
exit 1 $progress.puts 'done'.green
end end
ci_dump = ENV["CI_DUMP"]
unless File.exists?(ci_dump)
puts "The specified sql dump doesn't exist!"
exit 1
end
::Ci::Migrate::Database.new.restore(ci_dump) desc 'GitLab | Import CI builds'
task builds: :environment do
configure_cron_mode
$progress.puts 'Restoring builds ... '.blue
Ci::Migrate::Builds.new.restore
$progress.puts 'done'.green
end end
desc 'GitLab | Migrate CI tags' desc 'GitLab | Migrate CI tags'
task tags: :environment do task tags: :environment do
configure_cron_mode
$progress.puts 'Migrating tags ... '.blue
::Ci::Migrate::Tags.new.restore ::Ci::Migrate::Tags.new.restore
$progress.puts 'done'.green
end end
desc 'GitLab | Migrate CI auto-increments' desc 'GitLab | Migrate CI auto-increments'
...@@ -56,8 +72,16 @@ namespace :ci do ...@@ -56,8 +72,16 @@ namespace :ci do
desc 'GitLab | Migrate CI services' desc 'GitLab | Migrate CI services'
task services: :environment do task services: :environment do
$progress.puts 'Migrating services ... '.blue
c = ActiveRecord::Base.connection c = ActiveRecord::Base.connection
c.execute("UPDATE ci_services SET type=CONCAT('Ci::', type) WHERE type NOT LIKE 'Ci::%'") c.execute("UPDATE ci_services SET type=CONCAT('Ci::', type) WHERE type NOT LIKE 'Ci::%'")
$progress.puts 'done'.green
end end
end end
def enable_ci(enabled)
settings = ApplicationSetting.current || ApplicationSetting.create_from_defaults
settings.ci_enabled = enabled
settings.save!
end
end end
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment