Commit 84adf8a9 authored by Alex Pooley's avatar Alex Pooley

Merge branch 'backup_manager_specs' into 'master'

Refactor backup manager specs

See merge request gitlab-org/gitlab!80578
parents 76665cad 615bbe30
...@@ -5,6 +5,7 @@ require 'spec_helper' ...@@ -5,6 +5,7 @@ require 'spec_helper'
RSpec.describe Backup::Repositories do RSpec.describe Backup::Repositories do
let(:progress) { spy(:stdout) } let(:progress) { spy(:stdout) }
let(:strategy) { spy(:strategy) } let(:strategy) { spy(:strategy) }
let(:destination) { 'repositories' }
subject { described_class.new(progress, max_concurrency: 1, max_storage_concurrency: 1, strategy: strategy) } subject { described_class.new(progress, max_concurrency: 1, max_storage_concurrency: 1, strategy: strategy) }
...@@ -16,9 +17,9 @@ RSpec.describe Backup::Repositories do ...@@ -16,9 +17,9 @@ RSpec.describe Backup::Repositories do
it 'calls enqueue for each repository type', :aggregate_failures do it 'calls enqueue for each repository type', :aggregate_failures do
create(:wiki_page, container: group) create(:wiki_page, container: group)
subject.dump subject.dump(destination)
expect(strategy).to have_received(:start).with(:create) expect(strategy).to have_received(:start).with(:create, destination)
expect(strategy).to have_received(:enqueue).with(project, Gitlab::GlRepository::PROJECT) expect(strategy).to have_received(:enqueue).with(project, Gitlab::GlRepository::PROJECT)
expect(strategy).to have_received(:enqueue).with(group, Gitlab::GlRepository::WIKI) expect(strategy).to have_received(:enqueue).with(group, Gitlab::GlRepository::WIKI)
expect(strategy).to have_received(:finish!) expect(strategy).to have_received(:finish!)
...@@ -31,38 +32,38 @@ RSpec.describe Backup::Repositories do ...@@ -31,38 +32,38 @@ RSpec.describe Backup::Repositories do
it 'creates the expected number of threads' do it 'creates the expected number of threads' do
expect(Thread).not_to receive(:new) expect(Thread).not_to receive(:new)
expect(strategy).to receive(:start).with(:create) expect(strategy).to receive(:start).with(:create, destination)
groups.each do |group| groups.each do |group|
expect(strategy).to receive(:enqueue).with(group, Gitlab::GlRepository::WIKI) expect(strategy).to receive(:enqueue).with(group, Gitlab::GlRepository::WIKI)
end end
expect(strategy).to receive(:finish!) expect(strategy).to receive(:finish!)
subject.dump subject.dump(destination)
end end
describe 'command failure' do describe 'command failure' do
it 'enqueue_group raises an error' do it 'enqueue_group raises an error' do
allow(strategy).to receive(:enqueue).with(anything, Gitlab::GlRepository::WIKI).and_raise(IOError) allow(strategy).to receive(:enqueue).with(anything, Gitlab::GlRepository::WIKI).and_raise(IOError)
expect { subject.dump }.to raise_error(IOError) expect { subject.dump(destination) }.to raise_error(IOError)
end end
it 'group query raises an error' do it 'group query raises an error' do
allow(Group).to receive_message_chain(:includes, :find_each).and_raise(ActiveRecord::StatementTimeout) allow(Group).to receive_message_chain(:includes, :find_each).and_raise(ActiveRecord::StatementTimeout)
expect { subject.dump }.to raise_error(ActiveRecord::StatementTimeout) expect { subject.dump(destination) }.to raise_error(ActiveRecord::StatementTimeout)
end end
end end
it 'avoids N+1 database queries' do it 'avoids N+1 database queries' do
control_count = ActiveRecord::QueryRecorder.new do control_count = ActiveRecord::QueryRecorder.new do
subject.dump subject.dump(destination)
end.count end.count
create_list(:group, 2, :wiki_repo) create_list(:group, 2, :wiki_repo)
expect do expect do
subject.dump subject.dump(destination)
end.not_to exceed_query_limit(control_count) end.not_to exceed_query_limit(control_count)
end end
end end
...@@ -73,9 +74,9 @@ RSpec.describe Backup::Repositories do ...@@ -73,9 +74,9 @@ RSpec.describe Backup::Repositories do
let_it_be(:group) { create(:group) } let_it_be(:group) { create(:group) }
it 'calls enqueue for each repository type', :aggregate_failures do it 'calls enqueue for each repository type', :aggregate_failures do
subject.restore subject.restore(destination)
expect(strategy).to have_received(:start).with(:restore) expect(strategy).to have_received(:start).with(:restore, destination)
expect(strategy).to have_received(:enqueue).with(project, Gitlab::GlRepository::PROJECT) expect(strategy).to have_received(:enqueue).with(project, Gitlab::GlRepository::PROJECT)
expect(strategy).to have_received(:enqueue).with(group, Gitlab::GlRepository::WIKI) expect(strategy).to have_received(:enqueue).with(group, Gitlab::GlRepository::WIKI)
expect(strategy).to have_received(:finish!) expect(strategy).to have_received(:finish!)
......
...@@ -2,14 +2,11 @@ ...@@ -2,14 +2,11 @@
module Backup module Backup
class Artifacts < Backup::Files class Artifacts < Backup::Files
attr_reader :progress
def initialize(progress) def initialize(progress)
@progress = progress super(progress, 'artifacts', JobArtifactUploader.root, excludes: ['tmp'])
super('artifacts', JobArtifactUploader.root, excludes: ['tmp'])
end end
override :human_name
def human_name def human_name
_('artifacts') _('artifacts')
end end
......
...@@ -2,14 +2,11 @@ ...@@ -2,14 +2,11 @@
module Backup module Backup
class Builds < Backup::Files class Builds < Backup::Files
attr_reader :progress
def initialize(progress) def initialize(progress)
@progress = progress super(progress, 'builds', Settings.gitlab_ci.builds_path)
super('builds', Settings.gitlab_ci.builds_path)
end end
override :human_name
def human_name def human_name
_('builds') _('builds')
end end
......
...@@ -3,10 +3,10 @@ ...@@ -3,10 +3,10 @@
require 'yaml' require 'yaml'
module Backup module Backup
class Database class Database < Task
extend ::Gitlab::Utils::Override
include Backup::Helper include Backup::Helper
attr_reader :progress attr_reader :force, :config
attr_reader :config, :db_file_name
IGNORED_ERRORS = [ IGNORED_ERRORS = [
# Ignore warnings # Ignore warnings
...@@ -18,13 +18,14 @@ module Backup ...@@ -18,13 +18,14 @@ module Backup
].freeze ].freeze
IGNORED_ERRORS_REGEXP = Regexp.union(IGNORED_ERRORS).freeze IGNORED_ERRORS_REGEXP = Regexp.union(IGNORED_ERRORS).freeze
def initialize(progress, filename: nil) def initialize(progress, force:)
@progress = progress super(progress)
@config = ActiveRecord::Base.configurations.find_db_config(Rails.env).configuration_hash @config = ActiveRecord::Base.configurations.find_db_config(Rails.env).configuration_hash
@db_file_name = filename || File.join(Gitlab.config.backup.path, 'db', 'database.sql.gz') @force = force
end end
def dump override :dump
def dump(db_file_name)
FileUtils.mkdir_p(File.dirname(db_file_name)) FileUtils.mkdir_p(File.dirname(db_file_name))
FileUtils.rm_f(db_file_name) FileUtils.rm_f(db_file_name)
compress_rd, compress_wr = IO.pipe compress_rd, compress_wr = IO.pipe
...@@ -64,12 +65,24 @@ module Backup ...@@ -64,12 +65,24 @@ module Backup
raise DatabaseBackupError.new(config, db_file_name) unless success raise DatabaseBackupError.new(config, db_file_name) unless success
end end
def restore override :restore
def restore(db_file_name)
unless force
progress.puts 'Removing all tables. Press `Ctrl-C` within 5 seconds to abort'.color(:yellow)
sleep(5)
end
# Drop all tables Load the schema to ensure we don't have any newer tables
# hanging out from a failed upgrade
puts_time 'Cleaning the database ... '.color(:blue)
Rake::Task['gitlab:db:drop_tables'].invoke
puts_time 'done'.color(:green)
decompress_rd, decompress_wr = IO.pipe decompress_rd, decompress_wr = IO.pipe
decompress_pid = spawn(*%w(gzip -cd), out: decompress_wr, in: db_file_name) decompress_pid = spawn(*%w(gzip -cd), out: decompress_wr, in: db_file_name)
decompress_wr.close decompress_wr.close
status, errors = status, @errors =
case config[:adapter] case config[:adapter]
when "postgresql" then when "postgresql" then
progress.print "Restoring PostgreSQL database #{database} ... " progress.print "Restoring PostgreSQL database #{database} ... "
...@@ -81,33 +94,47 @@ module Backup ...@@ -81,33 +94,47 @@ module Backup
Process.waitpid(decompress_pid) Process.waitpid(decompress_pid)
success = $?.success? && status.success? success = $?.success? && status.success?
if errors.present? if @errors.present?
progress.print "------ BEGIN ERRORS -----\n".color(:yellow) progress.print "------ BEGIN ERRORS -----\n".color(:yellow)
progress.print errors.join.color(:yellow) progress.print @errors.join.color(:yellow)
progress.print "------ END ERRORS -------\n".color(:yellow) progress.print "------ END ERRORS -------\n".color(:yellow)
end end
report_success(success) report_success(success)
raise Backup::Error, 'Restore failed' unless success raise Backup::Error, 'Restore failed' unless success
end
if errors.present? override :pre_restore_warning
warning = <<~MSG def pre_restore_warning
There were errors in restoring the schema. This may cause return if force
issues if this results in missing indexes, constraints, or
columns. Please record the errors above and contact GitLab <<-MSG.strip_heredoc
Support if you have questions: Be sure to stop Puma, Sidekiq, and any other process that
https://about.gitlab.com/support/ connects to the database before proceeding. For Omnibus
MSG installs, see the following link for more information:
https://docs.gitlab.com/ee/raketasks/backup_restore.html#restore-for-omnibus-gitlab-installations
warn warning.color(:red)
Gitlab::TaskHelpers.ask_to_continue Before restoring the database, we will remove all existing
end tables to avoid future upgrade problems. Be aware that if you have
custom tables in the GitLab database these tables and all data will be
removed.
MSG
end end
def enabled override :post_restore_warning
true def post_restore_warning
return unless @errors.present?
<<-MSG.strip_heredoc
There were errors in restoring the schema. This may cause
issues if this results in missing indexes, constraints, or
columns. Please record the errors above and contact GitLab
Support if you have questions:
https://about.gitlab.com/support/
MSG
end end
override :human_name
def human_name def human_name
_('database') _('database')
end end
......
# frozen_string_literal: true # frozen_string_literal: true
require 'open3' require 'open3'
require_relative 'helper'
module Backup module Backup
class Files class Files < Task
extend ::Gitlab::Utils::Override
include Backup::Helper include Backup::Helper
DEFAULT_EXCLUDE = 'lost+found' DEFAULT_EXCLUDE = 'lost+found'
attr_reader :name, :backup_tarball, :excludes attr_reader :name, :excludes
def initialize(progress, name, app_files_dir, excludes: [])
super(progress)
def initialize(name, app_files_dir, excludes: [])
@name = name @name = name
@app_files_dir = app_files_dir @app_files_dir = app_files_dir
@backup_tarball = File.join(Gitlab.config.backup.path, name + '.tar.gz')
@excludes = [DEFAULT_EXCLUDE].concat(excludes) @excludes = [DEFAULT_EXCLUDE].concat(excludes)
end end
# Copy files from public/files to backup/files # Copy files from public/files to backup/files
def dump override :dump
def dump(backup_tarball)
FileUtils.mkdir_p(Gitlab.config.backup.path) FileUtils.mkdir_p(Gitlab.config.backup.path)
FileUtils.rm_f(backup_tarball) FileUtils.rm_f(backup_tarball)
...@@ -35,7 +37,7 @@ module Backup ...@@ -35,7 +37,7 @@ module Backup
unless status == 0 unless status == 0
puts output puts output
raise_custom_error raise_custom_error(backup_tarball)
end end
tar_cmd = [tar, exclude_dirs(:tar), %W[-C #{backup_files_realpath} -cf - .]].flatten tar_cmd = [tar, exclude_dirs(:tar), %W[-C #{backup_files_realpath} -cf - .]].flatten
...@@ -47,11 +49,12 @@ module Backup ...@@ -47,11 +49,12 @@ module Backup
end end
unless pipeline_succeeded?(tar_status: status_list[0], gzip_status: status_list[1], output: output) unless pipeline_succeeded?(tar_status: status_list[0], gzip_status: status_list[1], output: output)
raise_custom_error raise_custom_error(backup_tarball)
end end
end end
def restore override :restore
def restore(backup_tarball)
backup_existing_files_dir backup_existing_files_dir
cmd_list = [%w[gzip -cd], %W[#{tar} --unlink-first --recursive-unlink -C #{app_files_realpath} -xf -]] cmd_list = [%w[gzip -cd], %W[#{tar} --unlink-first --recursive-unlink -C #{app_files_realpath} -xf -]]
...@@ -61,10 +64,6 @@ module Backup ...@@ -61,10 +64,6 @@ module Backup
end end
end end
def enabled
true
end
def tar def tar
if system(*%w[gtar --version], out: '/dev/null') if system(*%w[gtar --version], out: '/dev/null')
# It looks like we can get GNU tar by running 'gtar' # It looks like we can get GNU tar by running 'gtar'
...@@ -146,7 +145,7 @@ module Backup ...@@ -146,7 +145,7 @@ module Backup
end end
end end
def raise_custom_error def raise_custom_error(backup_tarball)
raise FileBackupError.new(app_files_realpath, backup_tarball) raise FileBackupError.new(app_files_realpath, backup_tarball)
end end
......
...@@ -15,7 +15,7 @@ module Backup ...@@ -15,7 +15,7 @@ module Backup
@storage_parallelism = storage_parallelism @storage_parallelism = storage_parallelism
end end
def start(type) def start(type, backup_repos_path)
raise Error, 'already started' if started? raise Error, 'already started' if started?
command = case type command = case type
...@@ -93,10 +93,6 @@ module Backup ...@@ -93,10 +93,6 @@ module Backup
@thread.present? @thread.present?
end end
def backup_repos_path
File.absolute_path(File.join(Gitlab.config.backup.path, 'repositories'))
end
def bin_path def bin_path
File.absolute_path(Gitlab.config.backup.gitaly_backup_path) File.absolute_path(Gitlab.config.backup.gitaly_backup_path)
end end
......
...@@ -7,10 +7,11 @@ module Backup ...@@ -7,10 +7,11 @@ module Backup
@progress = progress @progress = progress
end end
def start(type) def start(type, backup_repos_path)
raise Error, 'already started' if @type raise Error, 'already started' if @type
@type = type @type = type
@backup_repos_path = backup_repos_path
case type case type
when :create when :create
FileUtils.rm_rf(backup_repos_path) FileUtils.rm_rf(backup_repos_path)
...@@ -31,7 +32,7 @@ module Backup ...@@ -31,7 +32,7 @@ module Backup
backup_restore = BackupRestore.new( backup_restore = BackupRestore.new(
progress, progress,
repository_type.repository_for(container), repository_type.repository_for(container),
backup_repos_path @backup_repos_path
) )
case @type case @type
...@@ -52,10 +53,6 @@ module Backup ...@@ -52,10 +53,6 @@ module Backup
attr_reader :progress attr_reader :progress
def backup_repos_path
@backup_repos_path ||= File.join(Gitlab.config.backup.path, 'repositories')
end
class BackupRestore class BackupRestore
attr_accessor :progress, :repository, :backup_repos_path attr_accessor :progress, :repository, :backup_repos_path
......
...@@ -2,14 +2,11 @@ ...@@ -2,14 +2,11 @@
module Backup module Backup
class Lfs < Backup::Files class Lfs < Backup::Files
attr_reader :progress
def initialize(progress) def initialize(progress)
@progress = progress super(progress, 'lfs', Settings.lfs.storage_path)
super('lfs', Settings.lfs.storage_path)
end end
override :human_name
def human_name def human_name
_('lfs objects') _('lfs objects')
end end
......
This diff is collapsed.
...@@ -2,14 +2,11 @@ ...@@ -2,14 +2,11 @@
module Backup module Backup
class Packages < Backup::Files class Packages < Backup::Files
attr_reader :progress
def initialize(progress) def initialize(progress)
@progress = progress super(progress, 'packages', Settings.packages.storage_path, excludes: ['tmp'])
super('packages', Settings.packages.storage_path, excludes: ['tmp'])
end end
override :human_name
def human_name def human_name
_('packages') _('packages')
end end
......
...@@ -6,14 +6,11 @@ module Backup ...@@ -6,14 +6,11 @@ module Backup
# if some of these files are still there, we don't need them in the backup # if some of these files are still there, we don't need them in the backup
LEGACY_PAGES_TMP_PATH = '@pages.tmp' LEGACY_PAGES_TMP_PATH = '@pages.tmp'
attr_reader :progress
def initialize(progress) def initialize(progress)
@progress = progress super(progress, 'pages', Gitlab.config.pages.path, excludes: [LEGACY_PAGES_TMP_PATH])
super('pages', Gitlab.config.pages.path, excludes: [LEGACY_PAGES_TMP_PATH])
end end
override :human_name
def human_name def human_name
_('pages') _('pages')
end end
......
...@@ -2,18 +2,16 @@ ...@@ -2,18 +2,16 @@
module Backup module Backup
class Registry < Backup::Files class Registry < Backup::Files
attr_reader :progress
def initialize(progress) def initialize(progress)
@progress = progress super(progress, 'registry', Settings.registry.path)
super('registry', Settings.registry.path)
end end
override :human_name
def human_name def human_name
_('container registry images') _('container registry images')
end end
override :enabled
def enabled def enabled
Gitlab.config.registry.enabled Gitlab.config.registry.enabled
end end
......
...@@ -3,16 +3,20 @@ ...@@ -3,16 +3,20 @@
require 'yaml' require 'yaml'
module Backup module Backup
class Repositories class Repositories < Task
extend ::Gitlab::Utils::Override
def initialize(progress, strategy:, max_concurrency: 1, max_storage_concurrency: 1) def initialize(progress, strategy:, max_concurrency: 1, max_storage_concurrency: 1)
@progress = progress super(progress)
@strategy = strategy @strategy = strategy
@max_concurrency = max_concurrency @max_concurrency = max_concurrency
@max_storage_concurrency = max_storage_concurrency @max_storage_concurrency = max_storage_concurrency
end end
def dump override :dump
strategy.start(:create) def dump(path)
strategy.start(:create, path)
# gitaly-backup is designed to handle concurrency on its own. So we want # gitaly-backup is designed to handle concurrency on its own. So we want
# to avoid entering the buggy concurrency code here when gitaly-backup # to avoid entering the buggy concurrency code here when gitaly-backup
...@@ -50,8 +54,9 @@ module Backup ...@@ -50,8 +54,9 @@ module Backup
strategy.finish! strategy.finish!
end end
def restore override :restore
strategy.start(:restore) def restore(path)
strategy.start(:restore, path)
enqueue_consecutive enqueue_consecutive
ensure ensure
...@@ -61,17 +66,14 @@ module Backup ...@@ -61,17 +66,14 @@ module Backup
restore_object_pools restore_object_pools
end end
def enabled override :human_name
true
end
def human_name def human_name
_('repositories') _('repositories')
end end
private private
attr_reader :progress, :strategy, :max_concurrency, :max_storage_concurrency attr_reader :strategy, :max_concurrency, :max_storage_concurrency
def check_valid_storages! def check_valid_storages!
repository_storage_klasses.each do |klass| repository_storage_klasses.each do |klass|
......
# frozen_string_literal: true
module Backup
class Task
def initialize(progress)
@progress = progress
end
# human readable task name used for logging
def human_name
raise NotImplementedError
end
# dump task backup to `path`
def dump(path)
raise NotImplementedError
end
# restore task backup from `path`
def restore(path)
raise NotImplementedError
end
# a string returned here will be displayed to the user before calling #restore
def pre_restore_warning
end
# a string returned here will be displayed to the user after calling #restore
def post_restore_warning
end
# returns `true` when the task should be used
def enabled
true
end
private
attr_reader :progress
def puts_time(msg)
progress.puts "#{Time.zone.now} -- #{msg}"
Gitlab::BackupLogger.info(message: "#{Rainbow.uncolor(msg)}")
end
end
end
...@@ -2,14 +2,11 @@ ...@@ -2,14 +2,11 @@
module Backup module Backup
class TerraformState < Backup::Files class TerraformState < Backup::Files
attr_reader :progress
def initialize(progress) def initialize(progress)
@progress = progress super(progress, 'terraform_state', Settings.terraform_state.storage_path, excludes: ['tmp'])
super('terraform_state', Settings.terraform_state.storage_path, excludes: ['tmp'])
end end
override :human_name
def human_name def human_name
_('terraform states') _('terraform states')
end end
......
...@@ -2,14 +2,11 @@ ...@@ -2,14 +2,11 @@
module Backup module Backup
class Uploads < Backup::Files class Uploads < Backup::Files
attr_reader :progress
def initialize(progress) def initialize(progress)
@progress = progress super(progress, 'uploads', File.join(Gitlab.config.uploads.storage_path, "uploads"), excludes: ['tmp'])
super('uploads', File.join(Gitlab.config.uploads.storage_path, "uploads"), excludes: ['tmp'])
end end
override :human_name
def human_name def human_name
_('uploads') _('uploads')
end end
......
...@@ -18,7 +18,7 @@ RSpec.describe Backup::Artifacts do ...@@ -18,7 +18,7 @@ RSpec.describe Backup::Artifacts do
expect(backup).to receive(:tar).and_return('blabla-tar') expect(backup).to receive(:tar).and_return('blabla-tar')
expect(backup).to receive(:run_pipeline!).with([%w(blabla-tar --exclude=lost+found --exclude=./tmp -C /var/gitlab-artifacts -cf - .), 'gzip -c -1'], any_args).and_return([[true, true], '']) expect(backup).to receive(:run_pipeline!).with([%w(blabla-tar --exclude=lost+found --exclude=./tmp -C /var/gitlab-artifacts -cf - .), 'gzip -c -1'], any_args).and_return([[true, true], ''])
expect(backup).to receive(:pipeline_succeeded?).and_return(true) expect(backup).to receive(:pipeline_succeeded?).and_return(true)
backup.dump backup.dump('artifacts.tar.gz')
end end
end end
end end
...@@ -6,25 +6,49 @@ RSpec.describe Backup::Database do ...@@ -6,25 +6,49 @@ RSpec.describe Backup::Database do
let(:progress) { StringIO.new } let(:progress) { StringIO.new }
let(:output) { progress.string } let(:output) { progress.string }
before do before(:all) do
allow(Gitlab::TaskHelpers).to receive(:ask_to_continue) Rake.application.rake_require 'active_record/railties/databases'
Rake.application.rake_require 'tasks/gitlab/backup'
Rake.application.rake_require 'tasks/gitlab/shell'
Rake.application.rake_require 'tasks/gitlab/db'
Rake.application.rake_require 'tasks/cache'
end end
describe '#restore' do describe '#restore' do
let(:cmd) { %W[#{Gem.ruby} -e $stdout.puts(1)] } let(:cmd) { %W[#{Gem.ruby} -e $stdout.puts(1)] }
let(:data) { Rails.root.join("spec/fixtures/pages_empty.tar.gz").to_s } let(:data) { Rails.root.join("spec/fixtures/pages_empty.tar.gz").to_s }
let(:force) { true }
subject { described_class.new(progress, filename: data) } subject { described_class.new(progress, force: force) }
before do before do
allow(subject).to receive(:pg_restore_cmd).and_return(cmd) allow(subject).to receive(:pg_restore_cmd).and_return(cmd)
end end
context 'when not forced' do
let(:force) { false }
it 'warns the user and waits' do
expect(subject).to receive(:sleep)
expect(Rake::Task['gitlab:db:drop_tables']).to receive(:invoke)
subject.restore(data)
expect(output).to include('Removing all tables. Press `Ctrl-C` within 5 seconds to abort')
end
it 'has a pre restore warning' do
expect(subject.pre_restore_warning).not_to be_nil
end
end
context 'with an empty .gz file' do context 'with an empty .gz file' do
let(:data) { Rails.root.join("spec/fixtures/pages_empty.tar.gz").to_s } let(:data) { Rails.root.join("spec/fixtures/pages_empty.tar.gz").to_s }
it 'returns successfully' do it 'returns successfully' do
subject.restore expect(Rake::Task['gitlab:db:drop_tables']).to receive(:invoke)
subject.restore(data)
expect(output).to include("Restoring PostgreSQL database") expect(output).to include("Restoring PostgreSQL database")
expect(output).to include("[DONE]") expect(output).to include("[DONE]")
...@@ -36,7 +60,9 @@ RSpec.describe Backup::Database do ...@@ -36,7 +60,9 @@ RSpec.describe Backup::Database do
let(:data) { Rails.root.join("spec/fixtures/big-image.png").to_s } let(:data) { Rails.root.join("spec/fixtures/big-image.png").to_s }
it 'raises a backup error' do it 'raises a backup error' do
expect { subject.restore }.to raise_error(Backup::Error) expect(Rake::Task['gitlab:db:drop_tables']).to receive(:invoke)
expect { subject.restore(data) }.to raise_error(Backup::Error)
end end
end end
...@@ -45,12 +71,15 @@ RSpec.describe Backup::Database do ...@@ -45,12 +71,15 @@ RSpec.describe Backup::Database do
let(:noise) { "Table projects does not exist\nmust be owner of extension pg_trgm\nWARNING: no privileges could be revoked for public\n" } let(:noise) { "Table projects does not exist\nmust be owner of extension pg_trgm\nWARNING: no privileges could be revoked for public\n" }
let(:cmd) { %W[#{Gem.ruby} -e $stderr.write("#{noise}#{visible_error}")] } let(:cmd) { %W[#{Gem.ruby} -e $stderr.write("#{noise}#{visible_error}")] }
it 'filters out noise from errors' do it 'filters out noise from errors and has a post restore warning' do
subject.restore expect(Rake::Task['gitlab:db:drop_tables']).to receive(:invoke)
subject.restore(data)
expect(output).to include("ERRORS") expect(output).to include("ERRORS")
expect(output).not_to include(noise) expect(output).not_to include(noise)
expect(output).to include(visible_error) expect(output).to include(visible_error)
expect(subject.post_restore_warning).not_to be_nil
end end
end end
...@@ -66,7 +95,9 @@ RSpec.describe Backup::Database do ...@@ -66,7 +95,9 @@ RSpec.describe Backup::Database do
end end
it 'overrides default config values' do it 'overrides default config values' do
subject.restore expect(Rake::Task['gitlab:db:drop_tables']).to receive(:invoke)
subject.restore(data)
expect(output).to include(%("PGHOST"=>"test.example.com")) expect(output).to include(%("PGHOST"=>"test.example.com"))
expect(output).to include(%("PGPASSWORD"=>"donotchange")) expect(output).to include(%("PGPASSWORD"=>"donotchange"))
......
...@@ -39,7 +39,7 @@ RSpec.describe Backup::Files do ...@@ -39,7 +39,7 @@ RSpec.describe Backup::Files do
end end
describe '#restore' do describe '#restore' do
subject { described_class.new('registry', '/var/gitlab-registry') } subject { described_class.new(progress, 'registry', '/var/gitlab-registry') }
let(:timestamp) { Time.utc(2017, 3, 22) } let(:timestamp) { Time.utc(2017, 3, 22) }
...@@ -58,11 +58,11 @@ RSpec.describe Backup::Files do ...@@ -58,11 +58,11 @@ RSpec.describe Backup::Files do
it 'moves all necessary files' do it 'moves all necessary files' do
allow(subject).to receive(:backup_existing_files).and_call_original allow(subject).to receive(:backup_existing_files).and_call_original
expect(FileUtils).to receive(:mv).with(["/var/gitlab-registry/sample1"], File.join(Gitlab.config.backup.path, "tmp", "registry.#{Time.now.to_i}")) expect(FileUtils).to receive(:mv).with(["/var/gitlab-registry/sample1"], File.join(Gitlab.config.backup.path, "tmp", "registry.#{Time.now.to_i}"))
subject.restore subject.restore('registry.tar.gz')
end end
it 'raises no errors' do it 'raises no errors' do
expect { subject.restore }.not_to raise_error expect { subject.restore('registry.tar.gz') }.not_to raise_error
end end
it 'calls tar command with unlink' do it 'calls tar command with unlink' do
...@@ -70,13 +70,13 @@ RSpec.describe Backup::Files do ...@@ -70,13 +70,13 @@ RSpec.describe Backup::Files do
expect(subject).to receive(:run_pipeline!).with([%w(gzip -cd), %w(blabla-tar --unlink-first --recursive-unlink -C /var/gitlab-registry -xf -)], any_args) expect(subject).to receive(:run_pipeline!).with([%w(gzip -cd), %w(blabla-tar --unlink-first --recursive-unlink -C /var/gitlab-registry -xf -)], any_args)
expect(subject).to receive(:pipeline_succeeded?).and_return(true) expect(subject).to receive(:pipeline_succeeded?).and_return(true)
subject.restore subject.restore('registry.tar.gz')
end end
it 'raises an error on failure' do it 'raises an error on failure' do
expect(subject).to receive(:pipeline_succeeded?).and_return(false) expect(subject).to receive(:pipeline_succeeded?).and_return(false)
expect { subject.restore }.to raise_error(/Restore operation failed:/) expect { subject.restore('registry.tar.gz') }.to raise_error(/Restore operation failed:/)
end end
end end
...@@ -89,7 +89,7 @@ RSpec.describe Backup::Files do ...@@ -89,7 +89,7 @@ RSpec.describe Backup::Files do
it 'shows error message' do it 'shows error message' do
expect(subject).to receive(:access_denied_error).with("/var/gitlab-registry") expect(subject).to receive(:access_denied_error).with("/var/gitlab-registry")
subject.restore subject.restore('registry.tar.gz')
end end
end end
...@@ -104,13 +104,13 @@ RSpec.describe Backup::Files do ...@@ -104,13 +104,13 @@ RSpec.describe Backup::Files do
expect(subject).to receive(:resource_busy_error).with("/var/gitlab-registry") expect(subject).to receive(:resource_busy_error).with("/var/gitlab-registry")
.and_call_original .and_call_original
expect { subject.restore }.to raise_error(/is a mountpoint/) expect { subject.restore('registry.tar.gz') }.to raise_error(/is a mountpoint/)
end end
end end
end end
describe '#dump' do describe '#dump' do
subject { described_class.new('pages', '/var/gitlab-pages', excludes: ['@pages.tmp']) } subject { described_class.new(progress, 'pages', '/var/gitlab-pages', excludes: ['@pages.tmp']) }
before do before do
allow(subject).to receive(:run_pipeline!).and_return([[true, true], '']) allow(subject).to receive(:run_pipeline!).and_return([[true, true], ''])
...@@ -118,14 +118,14 @@ RSpec.describe Backup::Files do ...@@ -118,14 +118,14 @@ RSpec.describe Backup::Files do
end end
it 'raises no errors' do it 'raises no errors' do
expect { subject.dump }.not_to raise_error expect { subject.dump('registry.tar.gz') }.not_to raise_error
end end
it 'excludes tmp dirs from archive' do it 'excludes tmp dirs from archive' do
expect(subject).to receive(:tar).and_return('blabla-tar') expect(subject).to receive(:tar).and_return('blabla-tar')
expect(subject).to receive(:run_pipeline!).with([%w(blabla-tar --exclude=lost+found --exclude=./@pages.tmp -C /var/gitlab-pages -cf - .), 'gzip -c -1'], any_args) expect(subject).to receive(:run_pipeline!).with([%w(blabla-tar --exclude=lost+found --exclude=./@pages.tmp -C /var/gitlab-pages -cf - .), 'gzip -c -1'], any_args)
subject.dump subject.dump('registry.tar.gz')
end end
it 'raises an error on failure' do it 'raises an error on failure' do
...@@ -133,7 +133,7 @@ RSpec.describe Backup::Files do ...@@ -133,7 +133,7 @@ RSpec.describe Backup::Files do
expect(subject).to receive(:pipeline_succeeded?).and_return(false) expect(subject).to receive(:pipeline_succeeded?).and_return(false)
expect do expect do
subject.dump subject.dump('registry.tar.gz')
end.to raise_error(/Failed to create compressed file/) end.to raise_error(/Failed to create compressed file/)
end end
...@@ -149,7 +149,7 @@ RSpec.describe Backup::Files do ...@@ -149,7 +149,7 @@ RSpec.describe Backup::Files do
.with(%w(rsync -a --delete --exclude=lost+found --exclude=/gitlab-pages/@pages.tmp /var/gitlab-pages /var/gitlab-backup)) .with(%w(rsync -a --delete --exclude=lost+found --exclude=/gitlab-pages/@pages.tmp /var/gitlab-pages /var/gitlab-backup))
.and_return(['', 0]) .and_return(['', 0])
subject.dump subject.dump('registry.tar.gz')
end end
it 'retries if rsync fails due to vanishing files' do it 'retries if rsync fails due to vanishing files' do
...@@ -158,7 +158,7 @@ RSpec.describe Backup::Files do ...@@ -158,7 +158,7 @@ RSpec.describe Backup::Files do
.and_return(['rsync failed', 24], ['', 0]) .and_return(['rsync failed', 24], ['', 0])
expect do expect do
subject.dump subject.dump('registry.tar.gz')
end.to output(/files vanished during rsync, retrying/).to_stdout end.to output(/files vanished during rsync, retrying/).to_stdout
end end
...@@ -168,7 +168,7 @@ RSpec.describe Backup::Files do ...@@ -168,7 +168,7 @@ RSpec.describe Backup::Files do
.and_return(['rsync failed', 1]) .and_return(['rsync failed', 1])
expect do expect do
subject.dump subject.dump('registry.tar.gz')
end.to output(/rsync failed/).to_stdout end.to output(/rsync failed/).to_stdout
.and raise_error(/Failed to create compressed file/) .and raise_error(/Failed to create compressed file/)
end end
...@@ -176,7 +176,7 @@ RSpec.describe Backup::Files do ...@@ -176,7 +176,7 @@ RSpec.describe Backup::Files do
end end
describe '#exclude_dirs' do describe '#exclude_dirs' do
subject { described_class.new('pages', '/var/gitlab-pages', excludes: ['@pages.tmp']) } subject { described_class.new(progress, 'pages', '/var/gitlab-pages', excludes: ['@pages.tmp']) }
it 'prepends a leading dot slash to tar excludes' do it 'prepends a leading dot slash to tar excludes' do
expect(subject.exclude_dirs(:tar)).to eq(['--exclude=lost+found', '--exclude=./@pages.tmp']) expect(subject.exclude_dirs(:tar)).to eq(['--exclude=lost+found', '--exclude=./@pages.tmp'])
...@@ -188,7 +188,7 @@ RSpec.describe Backup::Files do ...@@ -188,7 +188,7 @@ RSpec.describe Backup::Files do
end end
describe '#run_pipeline!' do describe '#run_pipeline!' do
subject { described_class.new('registry', '/var/gitlab-registry') } subject { described_class.new(progress, 'registry', '/var/gitlab-registry') }
it 'executes an Open3.pipeline for cmd_list' do it 'executes an Open3.pipeline for cmd_list' do
expect(Open3).to receive(:pipeline).with(%w[whew command], %w[another cmd], any_args) expect(Open3).to receive(:pipeline).with(%w[whew command], %w[another cmd], any_args)
...@@ -222,7 +222,7 @@ RSpec.describe Backup::Files do ...@@ -222,7 +222,7 @@ RSpec.describe Backup::Files do
end end
describe '#pipeline_succeeded?' do describe '#pipeline_succeeded?' do
subject { described_class.new('registry', '/var/gitlab-registry') } subject { described_class.new(progress, 'registry', '/var/gitlab-registry') }
it 'returns true if both tar and gzip succeeeded' do it 'returns true if both tar and gzip succeeeded' do
expect( expect(
...@@ -262,7 +262,7 @@ RSpec.describe Backup::Files do ...@@ -262,7 +262,7 @@ RSpec.describe Backup::Files do
end end
describe '#tar_ignore_non_success?' do describe '#tar_ignore_non_success?' do
subject { described_class.new('registry', '/var/gitlab-registry') } subject { described_class.new(progress, 'registry', '/var/gitlab-registry') }
context 'if `tar` command exits with 1 exitstatus' do context 'if `tar` command exits with 1 exitstatus' do
it 'returns true' do it 'returns true' do
...@@ -310,7 +310,7 @@ RSpec.describe Backup::Files do ...@@ -310,7 +310,7 @@ RSpec.describe Backup::Files do
end end
describe '#noncritical_warning?' do describe '#noncritical_warning?' do
subject { described_class.new('registry', '/var/gitlab-registry') } subject { described_class.new(progress, 'registry', '/var/gitlab-registry') }
it 'returns true if given text matches noncritical warnings list' do it 'returns true if given text matches noncritical warnings list' do
expect( expect(
......
...@@ -5,6 +5,7 @@ require 'spec_helper' ...@@ -5,6 +5,7 @@ require 'spec_helper'
RSpec.describe Backup::GitalyBackup do RSpec.describe Backup::GitalyBackup do
let(:max_parallelism) { nil } let(:max_parallelism) { nil }
let(:storage_parallelism) { nil } let(:storage_parallelism) { nil }
let(:destination) { File.join(Gitlab.config.backup.path, 'repositories') }
let(:progress) do let(:progress) do
Tempfile.new('progress').tap do |progress| Tempfile.new('progress').tap do |progress|
...@@ -27,7 +28,7 @@ RSpec.describe Backup::GitalyBackup do ...@@ -27,7 +28,7 @@ RSpec.describe Backup::GitalyBackup do
context 'unknown' do context 'unknown' do
it 'fails to start unknown' do it 'fails to start unknown' do
expect { subject.start(:unknown) }.to raise_error(::Backup::Error, 'unknown backup type: unknown') expect { subject.start(:unknown, destination) }.to raise_error(::Backup::Error, 'unknown backup type: unknown')
end end
end end
...@@ -42,7 +43,7 @@ RSpec.describe Backup::GitalyBackup do ...@@ -42,7 +43,7 @@ RSpec.describe Backup::GitalyBackup do
expect(Open3).to receive(:popen2).with(expected_env, anything, 'create', '-path', anything).and_call_original expect(Open3).to receive(:popen2).with(expected_env, anything, 'create', '-path', anything).and_call_original
subject.start(:create) subject.start(:create, destination)
subject.enqueue(project, Gitlab::GlRepository::PROJECT) subject.enqueue(project, Gitlab::GlRepository::PROJECT)
subject.enqueue(project, Gitlab::GlRepository::WIKI) subject.enqueue(project, Gitlab::GlRepository::WIKI)
subject.enqueue(project, Gitlab::GlRepository::DESIGN) subject.enqueue(project, Gitlab::GlRepository::DESIGN)
...@@ -50,11 +51,11 @@ RSpec.describe Backup::GitalyBackup do ...@@ -50,11 +51,11 @@ RSpec.describe Backup::GitalyBackup do
subject.enqueue(project_snippet, Gitlab::GlRepository::SNIPPET) subject.enqueue(project_snippet, Gitlab::GlRepository::SNIPPET)
subject.finish! subject.finish!
expect(File).to exist(File.join(Gitlab.config.backup.path, 'repositories', project.disk_path + '.bundle')) expect(File).to exist(File.join(destination, project.disk_path + '.bundle'))
expect(File).to exist(File.join(Gitlab.config.backup.path, 'repositories', project.disk_path + '.wiki.bundle')) expect(File).to exist(File.join(destination, project.disk_path + '.wiki.bundle'))
expect(File).to exist(File.join(Gitlab.config.backup.path, 'repositories', project.disk_path + '.design.bundle')) expect(File).to exist(File.join(destination, project.disk_path + '.design.bundle'))
expect(File).to exist(File.join(Gitlab.config.backup.path, 'repositories', personal_snippet.disk_path + '.bundle')) expect(File).to exist(File.join(destination, personal_snippet.disk_path + '.bundle'))
expect(File).to exist(File.join(Gitlab.config.backup.path, 'repositories', project_snippet.disk_path + '.bundle')) expect(File).to exist(File.join(destination, project_snippet.disk_path + '.bundle'))
end end
context 'parallel option set' do context 'parallel option set' do
...@@ -63,7 +64,7 @@ RSpec.describe Backup::GitalyBackup do ...@@ -63,7 +64,7 @@ RSpec.describe Backup::GitalyBackup do
it 'passes parallel option through' do it 'passes parallel option through' do
expect(Open3).to receive(:popen2).with(expected_env, anything, 'create', '-path', anything, '-parallel', '3').and_call_original expect(Open3).to receive(:popen2).with(expected_env, anything, 'create', '-path', anything, '-parallel', '3').and_call_original
subject.start(:create) subject.start(:create, destination)
subject.finish! subject.finish!
end end
end end
...@@ -74,7 +75,7 @@ RSpec.describe Backup::GitalyBackup do ...@@ -74,7 +75,7 @@ RSpec.describe Backup::GitalyBackup do
it 'passes parallel option through' do it 'passes parallel option through' do
expect(Open3).to receive(:popen2).with(expected_env, anything, 'create', '-path', anything, '-parallel-storage', '3').and_call_original expect(Open3).to receive(:popen2).with(expected_env, anything, 'create', '-path', anything, '-parallel-storage', '3').and_call_original
subject.start(:create) subject.start(:create, destination)
subject.finish! subject.finish!
end end
end end
...@@ -82,7 +83,7 @@ RSpec.describe Backup::GitalyBackup do ...@@ -82,7 +83,7 @@ RSpec.describe Backup::GitalyBackup do
it 'raises when the exit code not zero' do it 'raises when the exit code not zero' do
expect(subject).to receive(:bin_path).and_return(Gitlab::Utils.which('false')) expect(subject).to receive(:bin_path).and_return(Gitlab::Utils.which('false'))
subject.start(:create) subject.start(:create, destination)
expect { subject.finish! }.to raise_error(::Backup::Error, 'gitaly-backup exit status 1') expect { subject.finish! }.to raise_error(::Backup::Error, 'gitaly-backup exit status 1')
end end
end end
...@@ -114,7 +115,7 @@ RSpec.describe Backup::GitalyBackup do ...@@ -114,7 +115,7 @@ RSpec.describe Backup::GitalyBackup do
it 'passes through SSL envs' do it 'passes through SSL envs' do
expect(Open3).to receive(:popen2).with(ssl_env, anything, 'create', '-path', anything).and_call_original expect(Open3).to receive(:popen2).with(ssl_env, anything, 'create', '-path', anything).and_call_original
subject.start(:create) subject.start(:create, destination)
subject.finish! subject.finish!
end end
end end
...@@ -139,7 +140,7 @@ RSpec.describe Backup::GitalyBackup do ...@@ -139,7 +140,7 @@ RSpec.describe Backup::GitalyBackup do
expect(Open3).to receive(:popen2).with(expected_env, anything, 'restore', '-path', anything).and_call_original expect(Open3).to receive(:popen2).with(expected_env, anything, 'restore', '-path', anything).and_call_original
subject.start(:restore) subject.start(:restore, destination)
subject.enqueue(project, Gitlab::GlRepository::PROJECT) subject.enqueue(project, Gitlab::GlRepository::PROJECT)
subject.enqueue(project, Gitlab::GlRepository::WIKI) subject.enqueue(project, Gitlab::GlRepository::WIKI)
subject.enqueue(project, Gitlab::GlRepository::DESIGN) subject.enqueue(project, Gitlab::GlRepository::DESIGN)
...@@ -162,7 +163,7 @@ RSpec.describe Backup::GitalyBackup do ...@@ -162,7 +163,7 @@ RSpec.describe Backup::GitalyBackup do
it 'passes parallel option through' do it 'passes parallel option through' do
expect(Open3).to receive(:popen2).with(expected_env, anything, 'restore', '-path', anything, '-parallel', '3').and_call_original expect(Open3).to receive(:popen2).with(expected_env, anything, 'restore', '-path', anything, '-parallel', '3').and_call_original
subject.start(:restore) subject.start(:restore, destination)
subject.finish! subject.finish!
end end
end end
...@@ -173,7 +174,7 @@ RSpec.describe Backup::GitalyBackup do ...@@ -173,7 +174,7 @@ RSpec.describe Backup::GitalyBackup do
it 'passes parallel option through' do it 'passes parallel option through' do
expect(Open3).to receive(:popen2).with(expected_env, anything, 'restore', '-path', anything, '-parallel-storage', '3').and_call_original expect(Open3).to receive(:popen2).with(expected_env, anything, 'restore', '-path', anything, '-parallel-storage', '3').and_call_original
subject.start(:restore) subject.start(:restore, destination)
subject.finish! subject.finish!
end end
end end
...@@ -181,7 +182,7 @@ RSpec.describe Backup::GitalyBackup do ...@@ -181,7 +182,7 @@ RSpec.describe Backup::GitalyBackup do
it 'raises when the exit code not zero' do it 'raises when the exit code not zero' do
expect(subject).to receive(:bin_path).and_return(Gitlab::Utils.which('false')) expect(subject).to receive(:bin_path).and_return(Gitlab::Utils.which('false'))
subject.start(:restore) subject.start(:restore, destination)
expect { subject.finish! }.to raise_error(::Backup::Error, 'gitaly-backup exit status 1') expect { subject.finish! }.to raise_error(::Backup::Error, 'gitaly-backup exit status 1')
end end
end end
......
...@@ -4,6 +4,7 @@ require 'spec_helper' ...@@ -4,6 +4,7 @@ require 'spec_helper'
RSpec.describe Backup::GitalyRpcBackup do RSpec.describe Backup::GitalyRpcBackup do
let(:progress) { spy(:stdout) } let(:progress) { spy(:stdout) }
let(:destination) { File.join(Gitlab.config.backup.path, 'repositories') }
subject { described_class.new(progress) } subject { described_class.new(progress) }
...@@ -14,7 +15,7 @@ RSpec.describe Backup::GitalyRpcBackup do ...@@ -14,7 +15,7 @@ RSpec.describe Backup::GitalyRpcBackup do
context 'unknown' do context 'unknown' do
it 'fails to start unknown' do it 'fails to start unknown' do
expect { subject.start(:unknown) }.to raise_error(::Backup::Error, 'unknown backup type: unknown') expect { subject.start(:unknown, destination) }.to raise_error(::Backup::Error, 'unknown backup type: unknown')
end end
end end
...@@ -27,7 +28,7 @@ RSpec.describe Backup::GitalyRpcBackup do ...@@ -27,7 +28,7 @@ RSpec.describe Backup::GitalyRpcBackup do
project_snippet = create(:project_snippet, :repository, project: project) project_snippet = create(:project_snippet, :repository, project: project)
personal_snippet = create(:personal_snippet, :repository, author: project.first_owner) personal_snippet = create(:personal_snippet, :repository, author: project.first_owner)
subject.start(:create) subject.start(:create, destination)
subject.enqueue(project, Gitlab::GlRepository::PROJECT) subject.enqueue(project, Gitlab::GlRepository::PROJECT)
subject.enqueue(project, Gitlab::GlRepository::WIKI) subject.enqueue(project, Gitlab::GlRepository::WIKI)
subject.enqueue(project, Gitlab::GlRepository::DESIGN) subject.enqueue(project, Gitlab::GlRepository::DESIGN)
...@@ -35,11 +36,11 @@ RSpec.describe Backup::GitalyRpcBackup do ...@@ -35,11 +36,11 @@ RSpec.describe Backup::GitalyRpcBackup do
subject.enqueue(project_snippet, Gitlab::GlRepository::SNIPPET) subject.enqueue(project_snippet, Gitlab::GlRepository::SNIPPET)
subject.finish! subject.finish!
expect(File).to exist(File.join(Gitlab.config.backup.path, 'repositories', project.disk_path + '.bundle')) expect(File).to exist(File.join(destination, project.disk_path + '.bundle'))
expect(File).to exist(File.join(Gitlab.config.backup.path, 'repositories', project.disk_path + '.wiki.bundle')) expect(File).to exist(File.join(destination, project.disk_path + '.wiki.bundle'))
expect(File).to exist(File.join(Gitlab.config.backup.path, 'repositories', project.disk_path + '.design.bundle')) expect(File).to exist(File.join(destination, project.disk_path + '.design.bundle'))
expect(File).to exist(File.join(Gitlab.config.backup.path, 'repositories', personal_snippet.disk_path + '.bundle')) expect(File).to exist(File.join(destination, personal_snippet.disk_path + '.bundle'))
expect(File).to exist(File.join(Gitlab.config.backup.path, 'repositories', project_snippet.disk_path + '.bundle')) expect(File).to exist(File.join(destination, project_snippet.disk_path + '.bundle'))
end end
context 'failure' do context 'failure' do
...@@ -50,7 +51,7 @@ RSpec.describe Backup::GitalyRpcBackup do ...@@ -50,7 +51,7 @@ RSpec.describe Backup::GitalyRpcBackup do
end end
it 'logs an appropriate message', :aggregate_failures do it 'logs an appropriate message', :aggregate_failures do
subject.start(:create) subject.start(:create, destination)
subject.enqueue(project, Gitlab::GlRepository::PROJECT) subject.enqueue(project, Gitlab::GlRepository::PROJECT)
subject.finish! subject.finish!
...@@ -90,7 +91,7 @@ RSpec.describe Backup::GitalyRpcBackup do ...@@ -90,7 +91,7 @@ RSpec.describe Backup::GitalyRpcBackup do
copy_bundle_to_backup_path('personal_snippet_repo.bundle', personal_snippet.disk_path + '.bundle') copy_bundle_to_backup_path('personal_snippet_repo.bundle', personal_snippet.disk_path + '.bundle')
copy_bundle_to_backup_path('project_snippet_repo.bundle', project_snippet.disk_path + '.bundle') copy_bundle_to_backup_path('project_snippet_repo.bundle', project_snippet.disk_path + '.bundle')
subject.start(:restore) subject.start(:restore, destination)
subject.enqueue(project, Gitlab::GlRepository::PROJECT) subject.enqueue(project, Gitlab::GlRepository::PROJECT)
subject.enqueue(project, Gitlab::GlRepository::WIKI) subject.enqueue(project, Gitlab::GlRepository::WIKI)
subject.enqueue(project, Gitlab::GlRepository::DESIGN) subject.enqueue(project, Gitlab::GlRepository::DESIGN)
...@@ -123,7 +124,7 @@ RSpec.describe Backup::GitalyRpcBackup do ...@@ -123,7 +124,7 @@ RSpec.describe Backup::GitalyRpcBackup do
repository repository
end end
subject.start(:restore) subject.start(:restore, destination)
subject.enqueue(project, Gitlab::GlRepository::PROJECT) subject.enqueue(project, Gitlab::GlRepository::PROJECT)
subject.enqueue(project, Gitlab::GlRepository::WIKI) subject.enqueue(project, Gitlab::GlRepository::WIKI)
subject.enqueue(project, Gitlab::GlRepository::DESIGN) subject.enqueue(project, Gitlab::GlRepository::DESIGN)
...@@ -141,7 +142,7 @@ RSpec.describe Backup::GitalyRpcBackup do ...@@ -141,7 +142,7 @@ RSpec.describe Backup::GitalyRpcBackup do
end end
it 'logs an appropriate message', :aggregate_failures do it 'logs an appropriate message', :aggregate_failures do
subject.start(:restore) subject.start(:restore, destination)
subject.enqueue(project, Gitlab::GlRepository::PROJECT) subject.enqueue(project, Gitlab::GlRepository::PROJECT)
subject.finish! subject.finish!
......
...@@ -20,7 +20,7 @@ RSpec.describe Backup::Lfs do ...@@ -20,7 +20,7 @@ RSpec.describe Backup::Lfs do
expect(backup).to receive(:run_pipeline!).with([%w(blabla-tar --exclude=lost+found -C /var/lfs-objects -cf - .), 'gzip -c -1'], any_args).and_return([[true, true], '']) expect(backup).to receive(:run_pipeline!).with([%w(blabla-tar --exclude=lost+found -C /var/lfs-objects -cf - .), 'gzip -c -1'], any_args).and_return([[true, true], ''])
expect(backup).to receive(:pipeline_succeeded?).and_return(true) expect(backup).to receive(:pipeline_succeeded?).and_return(true)
backup.dump backup.dump('lfs.tar.gz')
end end
end end
end end
This diff is collapsed.
...@@ -21,7 +21,7 @@ RSpec.shared_examples 'backup object' do |setting| ...@@ -21,7 +21,7 @@ RSpec.shared_examples 'backup object' do |setting|
expect(backup).to receive(:run_pipeline!).with([%W(blabla-tar --exclude=lost+found --exclude=./tmp -C #{backup_path} -cf - .), 'gzip -c -1'], any_args).and_return([[true, true], '']) expect(backup).to receive(:run_pipeline!).with([%W(blabla-tar --exclude=lost+found --exclude=./tmp -C #{backup_path} -cf - .), 'gzip -c -1'], any_args).and_return([[true, true], ''])
expect(backup).to receive(:pipeline_succeeded?).and_return(true) expect(backup).to receive(:pipeline_succeeded?).and_return(true)
backup.dump backup.dump('backup_object.tar.gz')
end end
end end
end end
......
...@@ -19,7 +19,7 @@ RSpec.describe Backup::Pages do ...@@ -19,7 +19,7 @@ RSpec.describe Backup::Pages do
expect(subject).to receive(:tar).and_return('blabla-tar') expect(subject).to receive(:tar).and_return('blabla-tar')
expect(subject).to receive(:run_pipeline!).with([%w(blabla-tar --exclude=lost+found --exclude=./@pages.tmp -C /var/gitlab-pages -cf - .), 'gzip -c -1'], any_args).and_return([[true, true], '']) expect(subject).to receive(:run_pipeline!).with([%w(blabla-tar --exclude=lost+found --exclude=./@pages.tmp -C /var/gitlab-pages -cf - .), 'gzip -c -1'], any_args).and_return([[true, true], ''])
expect(subject).to receive(:pipeline_succeeded?).and_return(true) expect(subject).to receive(:pipeline_succeeded?).and_return(true)
subject.dump subject.dump('pages.tar.gz')
end end
end end
end end
...@@ -8,6 +8,7 @@ RSpec.describe Backup::Repositories do ...@@ -8,6 +8,7 @@ RSpec.describe Backup::Repositories do
let(:strategy) { spy(:strategy, parallel_enqueue?: parallel_enqueue) } let(:strategy) { spy(:strategy, parallel_enqueue?: parallel_enqueue) }
let(:max_concurrency) { 1 } let(:max_concurrency) { 1 }
let(:max_storage_concurrency) { 1 } let(:max_storage_concurrency) { 1 }
let(:destination) { 'repositories' }
subject do subject do
described_class.new( described_class.new(
...@@ -26,9 +27,9 @@ RSpec.describe Backup::Repositories do ...@@ -26,9 +27,9 @@ RSpec.describe Backup::Repositories do
project_snippet = create(:project_snippet, :repository, project: project) project_snippet = create(:project_snippet, :repository, project: project)
personal_snippet = create(:personal_snippet, :repository, author: project.first_owner) personal_snippet = create(:personal_snippet, :repository, author: project.first_owner)
subject.dump subject.dump(destination)
expect(strategy).to have_received(:start).with(:create) expect(strategy).to have_received(:start).with(:create, destination)
expect(strategy).to have_received(:enqueue).with(project, Gitlab::GlRepository::PROJECT) expect(strategy).to have_received(:enqueue).with(project, Gitlab::GlRepository::PROJECT)
expect(strategy).to have_received(:enqueue).with(project, Gitlab::GlRepository::WIKI) expect(strategy).to have_received(:enqueue).with(project, Gitlab::GlRepository::WIKI)
expect(strategy).to have_received(:enqueue).with(project, Gitlab::GlRepository::DESIGN) expect(strategy).to have_received(:enqueue).with(project, Gitlab::GlRepository::DESIGN)
...@@ -54,38 +55,38 @@ RSpec.describe Backup::Repositories do ...@@ -54,38 +55,38 @@ RSpec.describe Backup::Repositories do
it 'creates the expected number of threads' do it 'creates the expected number of threads' do
expect(Thread).not_to receive(:new) expect(Thread).not_to receive(:new)
expect(strategy).to receive(:start).with(:create) expect(strategy).to receive(:start).with(:create, destination)
projects.each do |project| projects.each do |project|
expect(strategy).to receive(:enqueue).with(project, Gitlab::GlRepository::PROJECT) expect(strategy).to receive(:enqueue).with(project, Gitlab::GlRepository::PROJECT)
end end
expect(strategy).to receive(:finish!) expect(strategy).to receive(:finish!)
subject.dump subject.dump(destination)
end end
describe 'command failure' do describe 'command failure' do
it 'enqueue_project raises an error' do it 'enqueue_project raises an error' do
allow(strategy).to receive(:enqueue).with(anything, Gitlab::GlRepository::PROJECT).and_raise(IOError) allow(strategy).to receive(:enqueue).with(anything, Gitlab::GlRepository::PROJECT).and_raise(IOError)
expect { subject.dump }.to raise_error(IOError) expect { subject.dump(destination) }.to raise_error(IOError)
end end
it 'project query raises an error' do it 'project query raises an error' do
allow(Project).to receive_message_chain(:includes, :find_each).and_raise(ActiveRecord::StatementTimeout) allow(Project).to receive_message_chain(:includes, :find_each).and_raise(ActiveRecord::StatementTimeout)
expect { subject.dump }.to raise_error(ActiveRecord::StatementTimeout) expect { subject.dump(destination) }.to raise_error(ActiveRecord::StatementTimeout)
end end
end end
it 'avoids N+1 database queries' do it 'avoids N+1 database queries' do
control_count = ActiveRecord::QueryRecorder.new do control_count = ActiveRecord::QueryRecorder.new do
subject.dump subject.dump(destination)
end.count end.count
create_list(:project, 2, :repository) create_list(:project, 2, :repository)
expect do expect do
subject.dump subject.dump(destination)
end.not_to exceed_query_limit(control_count) end.not_to exceed_query_limit(control_count)
end end
end end
...@@ -98,13 +99,13 @@ RSpec.describe Backup::Repositories do ...@@ -98,13 +99,13 @@ RSpec.describe Backup::Repositories do
it 'enqueues all projects sequentially' do it 'enqueues all projects sequentially' do
expect(Thread).not_to receive(:new) expect(Thread).not_to receive(:new)
expect(strategy).to receive(:start).with(:create) expect(strategy).to receive(:start).with(:create, destination)
projects.each do |project| projects.each do |project|
expect(strategy).to receive(:enqueue).with(project, Gitlab::GlRepository::PROJECT) expect(strategy).to receive(:enqueue).with(project, Gitlab::GlRepository::PROJECT)
end end
expect(strategy).to receive(:finish!) expect(strategy).to receive(:finish!)
subject.dump subject.dump(destination)
end end
end end
...@@ -122,13 +123,13 @@ RSpec.describe Backup::Repositories do ...@@ -122,13 +123,13 @@ RSpec.describe Backup::Repositories do
.exactly(storage_keys.length * (max_storage_concurrency + 1)).times .exactly(storage_keys.length * (max_storage_concurrency + 1)).times
.and_call_original .and_call_original
expect(strategy).to receive(:start).with(:create) expect(strategy).to receive(:start).with(:create, destination)
projects.each do |project| projects.each do |project|
expect(strategy).to receive(:enqueue).with(project, Gitlab::GlRepository::PROJECT) expect(strategy).to receive(:enqueue).with(project, Gitlab::GlRepository::PROJECT)
end end
expect(strategy).to receive(:finish!) expect(strategy).to receive(:finish!)
subject.dump subject.dump(destination)
end end
context 'with extra max concurrency' do context 'with extra max concurrency' do
...@@ -139,13 +140,13 @@ RSpec.describe Backup::Repositories do ...@@ -139,13 +140,13 @@ RSpec.describe Backup::Repositories do
.exactly(storage_keys.length * (max_storage_concurrency + 1)).times .exactly(storage_keys.length * (max_storage_concurrency + 1)).times
.and_call_original .and_call_original
expect(strategy).to receive(:start).with(:create) expect(strategy).to receive(:start).with(:create, destination)
projects.each do |project| projects.each do |project|
expect(strategy).to receive(:enqueue).with(project, Gitlab::GlRepository::PROJECT) expect(strategy).to receive(:enqueue).with(project, Gitlab::GlRepository::PROJECT)
end end
expect(strategy).to receive(:finish!) expect(strategy).to receive(:finish!)
subject.dump subject.dump(destination)
end end
end end
...@@ -153,33 +154,33 @@ RSpec.describe Backup::Repositories do ...@@ -153,33 +154,33 @@ RSpec.describe Backup::Repositories do
it 'enqueue_project raises an error' do it 'enqueue_project raises an error' do
allow(strategy).to receive(:enqueue).and_raise(IOError) allow(strategy).to receive(:enqueue).and_raise(IOError)
expect { subject.dump }.to raise_error(IOError) expect { subject.dump(destination) }.to raise_error(IOError)
end end
it 'project query raises an error' do it 'project query raises an error' do
allow(Project).to receive_message_chain(:for_repository_storage, :includes, :find_each).and_raise(ActiveRecord::StatementTimeout) allow(Project).to receive_message_chain(:for_repository_storage, :includes, :find_each).and_raise(ActiveRecord::StatementTimeout)
expect { subject.dump }.to raise_error(ActiveRecord::StatementTimeout) expect { subject.dump(destination) }.to raise_error(ActiveRecord::StatementTimeout)
end end
context 'misconfigured storages' do context 'misconfigured storages' do
let(:storage_keys) { %w[test_second_storage] } let(:storage_keys) { %w[test_second_storage] }
it 'raises an error' do it 'raises an error' do
expect { subject.dump }.to raise_error(Backup::Error, 'repositories.storages in gitlab.yml is misconfigured') expect { subject.dump(destination) }.to raise_error(Backup::Error, 'repositories.storages in gitlab.yml is misconfigured')
end end
end end
end end
it 'avoids N+1 database queries' do it 'avoids N+1 database queries' do
control_count = ActiveRecord::QueryRecorder.new do control_count = ActiveRecord::QueryRecorder.new do
subject.dump subject.dump(destination)
end.count end.count
create_list(:project, 2, :repository) create_list(:project, 2, :repository)
expect do expect do
subject.dump subject.dump(destination)
end.not_to exceed_query_limit(control_count) end.not_to exceed_query_limit(control_count)
end end
end end
...@@ -192,9 +193,9 @@ RSpec.describe Backup::Repositories do ...@@ -192,9 +193,9 @@ RSpec.describe Backup::Repositories do
let_it_be(:project_snippet) { create(:project_snippet, project: project, author: project.first_owner) } let_it_be(:project_snippet) { create(:project_snippet, project: project, author: project.first_owner) }
it 'calls enqueue for each repository type', :aggregate_failures do it 'calls enqueue for each repository type', :aggregate_failures do
subject.restore subject.restore(destination)
expect(strategy).to have_received(:start).with(:restore) expect(strategy).to have_received(:start).with(:restore, destination)
expect(strategy).to have_received(:enqueue).with(project, Gitlab::GlRepository::PROJECT) expect(strategy).to have_received(:enqueue).with(project, Gitlab::GlRepository::PROJECT)
expect(strategy).to have_received(:enqueue).with(project, Gitlab::GlRepository::WIKI) expect(strategy).to have_received(:enqueue).with(project, Gitlab::GlRepository::WIKI)
expect(strategy).to have_received(:enqueue).with(project, Gitlab::GlRepository::DESIGN) expect(strategy).to have_received(:enqueue).with(project, Gitlab::GlRepository::DESIGN)
...@@ -208,7 +209,7 @@ RSpec.describe Backup::Repositories do ...@@ -208,7 +209,7 @@ RSpec.describe Backup::Repositories do
pool_repository = create(:pool_repository, :failed) pool_repository = create(:pool_repository, :failed)
pool_repository.delete_object_pool pool_repository.delete_object_pool
subject.restore subject.restore(destination)
pool_repository.reload pool_repository.reload
expect(pool_repository).not_to be_failed expect(pool_repository).not_to be_failed
...@@ -219,7 +220,7 @@ RSpec.describe Backup::Repositories do ...@@ -219,7 +220,7 @@ RSpec.describe Backup::Repositories do
pool_repository = create(:pool_repository, state: :obsolete) pool_repository = create(:pool_repository, state: :obsolete)
pool_repository.update_column(:source_project_id, nil) pool_repository.update_column(:source_project_id, nil)
subject.restore subject.restore(destination)
pool_repository.reload pool_repository.reload
expect(pool_repository).to be_obsolete expect(pool_repository).to be_obsolete
...@@ -236,14 +237,14 @@ RSpec.describe Backup::Repositories do ...@@ -236,14 +237,14 @@ RSpec.describe Backup::Repositories do
end end
it 'shows the appropriate error' do it 'shows the appropriate error' do
subject.restore subject.restore(destination)
expect(progress).to have_received(:puts).with("Snippet #{personal_snippet.full_path} can't be restored: Repository has more than one branch") expect(progress).to have_received(:puts).with("Snippet #{personal_snippet.full_path} can't be restored: Repository has more than one branch")
expect(progress).to have_received(:puts).with("Snippet #{project_snippet.full_path} can't be restored: Repository has more than one branch") expect(progress).to have_received(:puts).with("Snippet #{project_snippet.full_path} can't be restored: Repository has more than one branch")
end end
it 'removes the snippets from the DB' do it 'removes the snippets from the DB' do
expect { subject.restore }.to change(PersonalSnippet, :count).by(-1) expect { subject.restore(destination) }.to change(PersonalSnippet, :count).by(-1)
.and change(ProjectSnippet, :count).by(-1) .and change(ProjectSnippet, :count).by(-1)
.and change(SnippetRepository, :count).by(-2) .and change(SnippetRepository, :count).by(-2)
end end
...@@ -253,7 +254,7 @@ RSpec.describe Backup::Repositories do ...@@ -253,7 +254,7 @@ RSpec.describe Backup::Repositories do
shard_name = personal_snippet.repository.shard shard_name = personal_snippet.repository.shard
path = personal_snippet.disk_path + '.git' path = personal_snippet.disk_path + '.git'
subject.restore subject.restore(destination)
expect(gitlab_shell.repository_exists?(shard_name, path)).to eq false expect(gitlab_shell.repository_exists?(shard_name, path)).to eq false
end end
......
# frozen_string_literal: true
require 'spec_helper'
RSpec.describe Backup::Task do
let(:progress) { StringIO.new }
subject { described_class.new(progress) }
describe '#human_name' do
it 'must be implemented by the subclass' do
expect { subject.human_name }.to raise_error(NotImplementedError)
end
end
describe '#dump' do
it 'must be implemented by the subclass' do
expect { subject.dump('some/path') }.to raise_error(NotImplementedError)
end
end
describe '#restore' do
it 'must be implemented by the subclass' do
expect { subject.restore('some/path') }.to raise_error(NotImplementedError)
end
end
end
...@@ -19,7 +19,7 @@ RSpec.describe Backup::Uploads do ...@@ -19,7 +19,7 @@ RSpec.describe Backup::Uploads do
expect(backup).to receive(:tar).and_return('blabla-tar') expect(backup).to receive(:tar).and_return('blabla-tar')
expect(backup).to receive(:run_pipeline!).with([%w(blabla-tar --exclude=lost+found --exclude=./tmp -C /var/uploads -cf - .), 'gzip -c -1'], any_args).and_return([[true, true], '']) expect(backup).to receive(:run_pipeline!).with([%w(blabla-tar --exclude=lost+found --exclude=./tmp -C /var/uploads -cf - .), 'gzip -c -1'], any_args).and_return([[true, true], ''])
expect(backup).to receive(:pipeline_succeeded?).and_return(true) expect(backup).to receive(:pipeline_succeeded?).and_return(true)
backup.dump backup.dump('uploads.tar.gz')
end end
end end
end end
...@@ -72,7 +72,6 @@ RSpec.describe 'gitlab:app namespace rake task', :delete do ...@@ -72,7 +72,6 @@ RSpec.describe 'gitlab:app namespace rake task', :delete do
before do before do
allow(YAML).to receive(:load_file) allow(YAML).to receive(:load_file)
.and_return({ gitlab_version: gitlab_version }) .and_return({ gitlab_version: gitlab_version })
expect(Rake::Task['gitlab:db:drop_tables']).to receive(:invoke)
expect_next_instance_of(::Backup::Manager) do |instance| expect_next_instance_of(::Backup::Manager) do |instance|
backup_types.each do |subtask| backup_types.each do |subtask|
expect(instance).to receive(:run_restore_task).with(subtask).ordered expect(instance).to receive(:run_restore_task).with(subtask).ordered
...@@ -85,10 +84,6 @@ RSpec.describe 'gitlab:app namespace rake task', :delete do ...@@ -85,10 +84,6 @@ RSpec.describe 'gitlab:app namespace rake task', :delete do
it 'invokes restoration on match' do it 'invokes restoration on match' do
expect { run_rake_task('gitlab:backup:restore') }.to output.to_stdout_from_any_process expect { run_rake_task('gitlab:backup:restore') }.to output.to_stdout_from_any_process
end end
it 'prints timestamps on messages' do
expect { run_rake_task('gitlab:backup:restore') }.to output(/.*\d{4}-\d{2}-\d{2}\s\d{2}:\d{2}:\d{2}\s[-+]\d{4}\s--\s.*/).to_stdout_from_any_process
end
end end
end end
...@@ -131,8 +126,6 @@ RSpec.describe 'gitlab:app namespace rake task', :delete do ...@@ -131,8 +126,6 @@ RSpec.describe 'gitlab:app namespace rake task', :delete do
allow(YAML).to receive(:load_file) allow(YAML).to receive(:load_file)
.and_return({ gitlab_version: Gitlab::VERSION }) .and_return({ gitlab_version: Gitlab::VERSION })
expect(Rake::Task['gitlab:db:drop_tables']).to receive(:invoke)
expect_next_instance_of(::Backup::Manager) do |instance| expect_next_instance_of(::Backup::Manager) do |instance|
backup_types.each do |subtask| backup_types.each do |subtask|
expect(instance).to receive(:run_restore_task).with(subtask).ordered expect(instance).to receive(:run_restore_task).with(subtask).ordered
...@@ -486,7 +479,6 @@ RSpec.describe 'gitlab:app namespace rake task', :delete do ...@@ -486,7 +479,6 @@ RSpec.describe 'gitlab:app namespace rake task', :delete do
allow(Rake::Task['gitlab:shell:setup']) allow(Rake::Task['gitlab:shell:setup'])
.to receive(:invoke).and_return(true) .to receive(:invoke).and_return(true)
expect(Rake::Task['gitlab:db:drop_tables']).to receive :invoke
expect_next_instance_of(::Backup::Manager) do |instance| expect_next_instance_of(::Backup::Manager) do |instance|
(backup_types - %w{repositories uploads}).each do |subtask| (backup_types - %w{repositories uploads}).each do |subtask|
expect(instance).to receive(:run_restore_task).with(subtask).ordered expect(instance).to receive(:run_restore_task).with(subtask).ordered
...@@ -531,7 +523,6 @@ RSpec.describe 'gitlab:app namespace rake task', :delete do ...@@ -531,7 +523,6 @@ RSpec.describe 'gitlab:app namespace rake task', :delete do
allow(Rake::Task['gitlab:shell:setup']) allow(Rake::Task['gitlab:shell:setup'])
.to receive(:invoke).and_return(true) .to receive(:invoke).and_return(true)
expect(Rake::Task['gitlab:db:drop_tables']).to receive :invoke
expect_next_instance_of(::Backup::Manager) do |instance| expect_next_instance_of(::Backup::Manager) do |instance|
backup_types.each do |subtask| backup_types.each do |subtask|
expect(instance).to receive(:run_restore_task).with(subtask).ordered expect(instance).to receive(:run_restore_task).with(subtask).ordered
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment