Commit ba58a66a authored by Douwe Maan's avatar Douwe Maan

Merge branch 'migrate-restore-repo-to-gitaly' into 'master'

Use gitaly repository_service.CreateRepositoryFromBundle to restore (rake task)

Closes charts/gitlab#390

See merge request gitlab-org/gitlab-ce!18644
parents 22139d81 84e6ddb4
---
title: Support restoring repositories into gitaly
merge_request:
author:
type: changed
...@@ -73,6 +73,9 @@ module Backup ...@@ -73,6 +73,9 @@ module Backup
end end
def prepare_directories def prepare_directories
# TODO: Need to find a way to do this for gitaly
# Gitaly discussion issue: https://gitlab.com/gitlab-org/gitaly/issues/1194
Gitlab.config.repositories.storages.each do |name, repository_storage| Gitlab.config.repositories.storages.each do |name, repository_storage|
path = repository_storage.legacy_disk_path path = repository_storage.legacy_disk_path
next unless File.exist?(path) next unless File.exist?(path)
...@@ -93,69 +96,64 @@ module Backup ...@@ -93,69 +96,64 @@ module Backup
end end
end end
def restore_custom_hooks(project)
# TODO: Need to find a way to do this for gitaly
# Gitaly migration issue: https://gitlab.com/gitlab-org/gitaly/issues/1195
in_path(path_to_tars(project)) do |dir|
path_to_project_repo = path_to_repo(project)
cmd = %W(tar -xf #{path_to_tars(project, dir)} -C #{path_to_project_repo} #{dir})
output, status = Gitlab::Popen.popen(cmd)
unless status.zero?
progress_warn(project, cmd.join(' '), output)
end
end
end
def restore def restore
prepare_directories prepare_directories
gitlab_shell = Gitlab::Shell.new
Project.find_each(batch_size: 1000) do |project| Project.find_each(batch_size: 1000) do |project|
progress.print " * #{display_repo_path(project)} ... " progress.print " * #{project.full_path} ... "
path_to_project_repo = path_to_repo(project)
path_to_project_bundle = path_to_bundle(project) path_to_project_bundle = path_to_bundle(project)
project.ensure_storage_path_exists project.ensure_storage_path_exists
cmd = if File.exist?(path_to_project_bundle) restore_repo_success = nil
%W(#{Gitlab.config.git.bin_path} clone --bare --mirror #{path_to_project_bundle} #{path_to_project_repo}) if File.exist?(path_to_project_bundle)
begin
gitlab_shell.remove_repository(project.repository_storage, project.disk_path) if project.repository_exists?
project.repository.create_from_bundle path_to_project_bundle
restore_repo_success = true
rescue => e
restore_repo_success = false
progress.puts "Error: #{e}".color(:red)
end
else else
%W(#{Gitlab.config.git.bin_path} init --bare #{path_to_project_repo}) restore_repo_success = gitlab_shell.create_repository(project.repository_storage, project.disk_path)
end end
output, status = Gitlab::Popen.popen(cmd) if restore_repo_success
if status.zero?
progress.puts "[DONE]".color(:green) progress.puts "[DONE]".color(:green)
else else
progress_warn(project, cmd.join(' '), output) progress.puts "[Failed] restoring #{project.full_path} repository".color(:red)
end end
in_path(path_to_tars(project)) do |dir| restore_custom_hooks(project)
cmd = %W(tar -xf #{path_to_tars(project, dir)} -C #{path_to_project_repo} #{dir})
output, status = Gitlab::Popen.popen(cmd)
unless status.zero?
progress_warn(project, cmd.join(' '), output)
end
end
wiki = ProjectWiki.new(project) wiki = ProjectWiki.new(project)
path_to_wiki_repo = path_to_repo(wiki)
path_to_wiki_bundle = path_to_bundle(wiki) path_to_wiki_bundle = path_to_bundle(wiki)
if File.exist?(path_to_wiki_bundle) if File.exist?(path_to_wiki_bundle)
progress.print " * #{display_repo_path(wiki)} ... " progress.print " * #{wiki.full_path} ... "
begin
# If a wiki bundle exists, first remove the empty repo gitlab_shell.remove_repository(wiki.repository_storage, wiki.disk_path) if wiki.repository_exists?
# that was initialized with ProjectWiki.new() and then wiki.repository.create_from_bundle(path_to_wiki_bundle)
# try to restore with 'git clone --bare'. progress.puts "[DONE]".color(:green)
FileUtils.rm_rf(path_to_wiki_repo) rescue => e
cmd = %W(#{Gitlab.config.git.bin_path} clone --bare #{path_to_wiki_bundle} #{path_to_wiki_repo}) progress.puts "[Failed] restoring #{wiki.full_path} wiki".color(:red)
progress.puts "Error #{e}".color(:red)
output, status = Gitlab::Popen.popen(cmd)
if status.zero?
progress.puts " [DONE]".color(:green)
else
progress_warn(project, cmd.join(' '), output)
end
end end
end end
progress.print 'Put GitLab hooks in repositories dirs'.color(:yellow)
cmd = %W(#{Gitlab.config.gitlab_shell.path}/bin/create-hooks) + repository_storage_paths_args
output, status = Gitlab::Popen.popen(cmd)
if status.zero?
progress.puts " [DONE]".color(:green)
else
puts " [FAILED]".color(:red)
puts "failed: #{cmd}"
puts output
end end
end end
# rubocop:enable Metrics/AbcSize # rubocop:enable Metrics/AbcSize
......
...@@ -48,14 +48,14 @@ describe Backup::Repository do ...@@ -48,14 +48,14 @@ describe Backup::Repository do
describe 'command failure' do describe 'command failure' do
before do before do
allow(Gitlab::Popen).to receive(:popen).and_return(['error', 1]) allow_any_instance_of(Gitlab::Shell).to receive(:create_repository).and_return(false)
end end
context 'hashed storage' do context 'hashed storage' do
it 'shows the appropriate error' do it 'shows the appropriate error' do
subject.restore subject.restore
expect(progress).to have_received(:puts).with("Ignoring error on #{project.full_path} (#{project.disk_path}) - error") expect(progress).to have_received(:puts).with("[Failed] restoring #{project.full_path} repository")
end end
end end
...@@ -65,33 +65,10 @@ describe Backup::Repository do ...@@ -65,33 +65,10 @@ describe Backup::Repository do
it 'shows the appropriate error' do it 'shows the appropriate error' do
subject.restore subject.restore
expect(progress).to have_received(:puts).with("Ignoring error on #{project.full_path} - error") expect(progress).to have_received(:puts).with("[Failed] restoring #{project.full_path} repository")
end end
end end
end end
describe 'folders without permissions' do
before do
allow(FileUtils).to receive(:mv).and_raise(Errno::EACCES)
end
it 'shows error message' do
expect(subject).to receive(:access_denied_error)
subject.restore
end
end
describe 'folder that is a mountpoint' do
before do
allow(FileUtils).to receive(:mv).and_raise(Errno::EBUSY)
end
it 'shows error message' do
expect(subject).to receive(:resource_busy_error).and_call_original
expect { subject.restore }.to raise_error(/is a mountpoint/)
end
end
end end
describe '#empty_repo?' do describe '#empty_repo?' do
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment