Commit b2b6a172 authored by Marin Jankovski's avatar Marin Jankovski Committed by Marin Jankovski

Merge branch 'optimize-ci-tags-migration' into 'master'

CI migration rake task for GitLab CE

- Uses backup format for final GitLab CI 8.0
- Highly optimized tags migration SQL queries

See merge request !1365
parent b7caaab6
......@@ -131,7 +131,7 @@
.checkbox
= f.label :ci_enabled do
= f.check_box :ci_enabled
Enable Continuous Integration
Disable to prevent CI usage until rake ci:migrate is run (8.0 only)
.form-actions
= f.submit 'Save', class: 'btn btn-primary'
Continuous Integration has been disabled. Please ask your administrator to enable it.
Continuous Integration has been disabled for time of the migration.
module Ci
module Migrate
class Builds
attr_reader :app_builds_dir, :backup_builds_tarball, :backup_dir
def initialize
@app_builds_dir = Settings.gitlab_ci.builds_path
@backup_dir = Gitlab.config.backup.path
@backup_builds_tarball = File.join(backup_dir, 'builds/builds.tar.gz')
end
def restore
backup_existing_builds_dir
FileUtils.mkdir_p(app_builds_dir, mode: 0700)
unless system('tar', '-C', app_builds_dir, '-zxvf', backup_builds_tarball)
abort 'Restore failed'.red
end
end
def backup_existing_builds_dir
timestamped_builds_path = File.join(app_builds_dir, '..', "builds.#{Time.now.to_i}")
if File.exists?(app_builds_dir)
FileUtils.mv(app_builds_dir, File.expand_path(timestamped_builds_path))
end
end
end
end
end
......@@ -9,32 +9,32 @@ module Ci
@config = YAML.load_file(File.join(Rails.root, 'config', 'database.yml'))[Rails.env]
end
def restore(ci_dump)
puts 'Deleting all CI related data ... '
truncate_ci_tables
def restore
decompress_rd, decompress_wr = IO.pipe
decompress_pid = spawn(*%W(gzip -cd), out: decompress_wr, in: db_file_name)
decompress_wr.close
puts 'Restoring CI data ... '
case config["adapter"]
restore_pid = case config["adapter"]
when /^mysql/ then
print "Restoring MySQL database #{config['database']} ... "
$progress.print "Restoring MySQL database #{config['database']} ... "
# Workaround warnings from MySQL 5.6 about passwords on cmd line
ENV['MYSQL_PWD'] = config["password"].to_s if config["password"]
system('mysql', *mysql_args, config['database'], in: ci_dump)
spawn('mysql', *mysql_args, config['database'], in: decompress_rd)
when "postgresql" then
puts "Restoring PostgreSQL database #{config['database']} ... "
$progress.print "Restoring PostgreSQL database #{config['database']} ... "
pg_env
system('psql', config['database'], '-f', ci_dump)
spawn('psql', config['database'], in: decompress_rd)
end
decompress_rd.close
success = [decompress_pid, restore_pid].all? { |pid| Process.waitpid(pid); $?.success? }
abort 'Restore failed' unless success
end
protected
def truncate_ci_tables
c = ActiveRecord::Base.connection
c.tables.select { |t| t.start_with?('ci_') }.each do |table|
puts "Deleting data from #{table}..."
c.execute("DELETE FROM #{table}")
end
def db_file_name
File.join(Gitlab.config.backup.path, 'db', 'database.sql.gz')
end
def mysql_args
......
module Ci
module Migrate
class Manager
VERSION = '8.0.0.pre'
def cleanup
$progress.print "Deleting tmp directories ... "
backup_contents.each do |dir|
next unless File.exist?(File.join(Gitlab.config.backup.path, dir))
if FileUtils.rm_rf(File.join(Gitlab.config.backup.path, dir))
$progress.puts "done".green
else
puts "deleting tmp directory '#{dir}' failed".red
abort 'Backup failed'
end
end
end
def unpack
Dir.chdir(Gitlab.config.backup.path)
# check for existing backups in the backup dir
file_list = Dir.glob("*_gitlab_ci_backup.tar").each.map { |f| f.split(/_/).first.to_i }
puts "no backups found" if file_list.count == 0
if file_list.count > 1 && ENV["BACKUP"].nil?
puts "Found more than one backup, please specify which one you want to restore:"
puts "rake gitlab:backup:restore BACKUP=timestamp_of_backup"
exit 1
end
tar_file = ENV["BACKUP"].nil? ? File.join("#{file_list.first}_gitlab_ci_backup.tar") : File.join(ENV["BACKUP"] + "_gitlab_ci_backup.tar")
unless File.exists?(tar_file)
puts "The specified CI backup doesn't exist!"
exit 1
end
$progress.print "Unpacking backup ... "
unless Kernel.system(*%W(tar -xf #{tar_file}))
puts "unpacking backup failed".red
exit 1
else
$progress.puts "done".green
end
ENV["VERSION"] = "#{settings[:db_version]}" if settings[:db_version].to_i > 0
# restoring mismatching backups can lead to unexpected problems
if settings[:gitlab_version] != VERSION
puts "GitLab CI version mismatch:".red
puts " Your current GitLab CI version (#{VERSION}) differs from the GitLab CI (#{settings[:gitlab_version]}) version in the backup!".red
exit 1
end
end
private
def backup_contents
["db", "builds", "backup_information.yml"]
end
def settings
@settings ||= YAML.load_file("backup_information.yml")
end
end
end
end
......@@ -4,45 +4,38 @@ module Ci
module Migrate
class Tags
def restore
puts 'Migrating tags for Runners... '
list_objects('Runner').each do |id|
putc '.'
runner = Ci::Runner.find_by_id(id)
if runner
tags = list_tags('Runner', id)
runner.update_attributes(tag_list: tags)
puts 'Inserting tags...'
connection.select_all('SELECT ci_tags.name FROM ci_tags').each do |tag|
begin
connection.execute("INSERT INTO tags (name) VALUES(#{ActiveRecord::Base::sanitize(tag['name'])})")
rescue ActiveRecord::RecordNotUnique
end
end
puts ''
puts 'Migrating tags for Builds... '
list_objects('Build').each do |id|
putc '.'
build = Ci::Build.find_by_id(id)
if build
tags = list_tags('Build', id)
build.update_attributes(tag_list: tags)
end
end
puts ''
end
ActiveRecord::Base.transaction do
puts 'Deleting old taggings...'
connection.execute "DELETE FROM taggings WHERE context = 'tags' AND taggable_type LIKE 'Ci::%'"
protected
puts 'Inserting taggings...'
connection.execute(
'INSERT INTO taggings (taggable_type, taggable_id, tag_id, context) ' +
"SELECT CONCAT('Ci::', ci_taggings.taggable_type), ci_taggings.taggable_id, tags.id, 'tags' FROM ci_taggings " +
'JOIN ci_tags ON ci_tags.id = ci_taggings.tag_id ' +
'JOIN tags ON tags.name = ci_tags.name '
)
def list_objects(type)
ids = ActiveRecord::Base.connection.select_all(
"select distinct taggable_id from ci_taggings where taggable_type = #{ActiveRecord::Base::sanitize(type)}"
puts 'Resetting counters... '
connection.execute(
'UPDATE tags SET ' +
'taggings_count = (SELECT COUNT(*) FROM taggings WHERE tags.id = taggings.tag_id)'
)
ids.map { |id| id['taggable_id'] }
end
end
def list_tags(type, id)
tags = ActiveRecord::Base.connection.select_all(
'select ci_tags.name from ci_tags ' +
'join ci_taggings on ci_tags.id = ci_taggings.tag_id ' +
"where taggable_type = #{ActiveRecord::Base::sanitize(type)} and taggable_id = #{ActiveRecord::Base::sanitize(id)} and context = 'tags'"
)
tags.map { |tag| tag['name'] }
protected
def connection
ActiveRecord::Base.connection
end
end
end
......
namespace :ci do
desc 'GitLab | Import and migrate CI database'
task migrate: :environment do
warn_user_is_not_gitlab
configure_cron_mode
unless ENV['force'] == 'yes'
puts "This will truncate all CI tables and restore it from provided backup."
puts "You will lose any previous CI data stored in the database."
puts 'This will remove all CI related data and restore it from the provided backup.'
ask_to_continue
puts ""
puts ''
end
Rake::Task["ci:migrate:db"].invoke
Rake::Task["ci:migrate:autoincrements"].invoke
Rake::Task["ci:migrate:tags"].invoke
Rake::Task["ci:migrate:services"].invoke
# disable CI for time of migration
enable_ci(false)
# unpack archives
migrate = Ci::Migrate::Manager.new
migrate.unpack
Rake::Task['ci:migrate:db'].invoke
Rake::Task['ci:migrate:builds'].invoke
Rake::Task['ci:migrate:tags'].invoke
Rake::Task['ci:migrate:services'].invoke
# enable CI for time of migration
enable_ci(true)
migrate.cleanup
end
namespace :migrate do
desc 'GitLab | Import CI database'
task db: :environment do
if ENV["CI_DUMP"].nil?
puts "No CI SQL dump specified:"
puts "rake gitlab:backup:restore CI_DUMP=ci_dump.sql"
exit 1
configure_cron_mode
$progress.puts 'Restoring database ... '.blue
Ci::Migrate::Database.new.restore
$progress.puts 'done'.green
end
ci_dump = ENV["CI_DUMP"]
unless File.exists?(ci_dump)
puts "The specified sql dump doesn't exist!"
exit 1
end
::Ci::Migrate::Database.new.restore(ci_dump)
desc 'GitLab | Import CI builds'
task builds: :environment do
configure_cron_mode
$progress.puts 'Restoring builds ... '.blue
Ci::Migrate::Builds.new.restore
$progress.puts 'done'.green
end
desc 'GitLab | Migrate CI tags'
task tags: :environment do
configure_cron_mode
$progress.puts 'Migrating tags ... '.blue
::Ci::Migrate::Tags.new.restore
$progress.puts 'done'.green
end
desc 'GitLab | Migrate CI auto-increments'
......@@ -56,8 +72,16 @@ namespace :ci do
desc 'GitLab | Migrate CI services'
task services: :environment do
$progress.puts 'Migrating services ... '.blue
c = ActiveRecord::Base.connection
c.execute("UPDATE ci_services SET type=CONCAT('Ci::', type) WHERE type NOT LIKE 'Ci::%'")
$progress.puts 'done'.green
end
end
def enable_ci(enabled)
settings = ApplicationSetting.current || ApplicationSetting.create_from_defaults
settings.ci_enabled = enabled
settings.save!
end
end
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment