Merge branch 'use-backup-custom-hooks-gitaly' into 'master'

Use gitaly for backup rake task

See merge request gitlab-org/gitlab-ce!19742

GITALY_SERVER_VERSION

-0.105.1
+0.107.0

Gemfile

@@ -419,7 +419,7 @@ group :ed25519 do
 end
 
 # Gitaly GRPC client
-gem 'gitaly-proto', '~> 0.101.0', require: 'gitaly'
+gem 'gitaly-proto', '~> 0.102.0', require: 'gitaly'
 gem 'grpc', '~> 1.11.0'
 
 # Locked until https://github.com/google/protobuf/issues/4210 is closed

Gemfile.lock

@@ -283,7 +283,7 @@ GEM
       gettext_i18n_rails (>= 0.7.1)
       po_to_json (>= 1.0.0)
       rails (>= 3.2.0)
-    gitaly-proto (0.101.0)
+    gitaly-proto (0.102.0)
       google-protobuf (~> 3.1)
       grpc (~> 1.10)
     github-linguist (5.3.3)
@@ -1039,7 +1039,7 @@ DEPENDENCIES
   gettext (~> 3.2.2)
   gettext_i18n_rails (~> 1.8.0)
   gettext_i18n_rails_js (~> 1.3)
-  gitaly-proto (~> 0.101.0)
+  gitaly-proto (~> 0.102.0)
   github-linguist (~> 5.3.3)
   gitlab-flowdock-git-hook (~> 1.0.1)
   gitlab-gollum-lib (~> 4.2)

Gemfile.rails5.lock

@@ -286,7 +286,7 @@ GEM
       gettext_i18n_rails (>= 0.7.1)
       po_to_json (>= 1.0.0)
       rails (>= 3.2.0)
-    gitaly-proto (0.101.0)
+    gitaly-proto (0.102.0)
       google-protobuf (~> 3.1)
       grpc (~> 1.10)
     github-linguist (5.3.3)
@@ -1049,7 +1049,7 @@ DEPENDENCIES
   gettext (~> 3.2.2)
   gettext_i18n_rails (~> 1.8.0)
   gettext_i18n_rails_js (~> 1.3)
-  gitaly-proto (~> 0.101.0)
+  gitaly-proto (~> 0.102.0)
   github-linguist (~> 5.3.3)
   gitlab-flowdock-git-hook (~> 1.0.1)
   gitlab-gollum-lib (~> 4.2)

changelogs/unreleased/use-backup-custom-hooks-gitaly.yml

+---
+title: migrate backup rake task to gitaly
+merge_request:
+author:
+type: added

lib/backup/repository.rb

@@ -4,7 +4,6 @@ require_relative 'helper'
 module Backup
   class Repository
     include Backup::Helper
-    # rubocop:disable Metrics/AbcSize
 
     attr_reader :progress
 
@@ -18,69 +17,66 @@ module Backup
       Project.find_each(batch_size: 1000) do |project|
         progress.print " * #{display_repo_path(project)} ... "
 
-        path_to_project_repo = Gitlab::GitalyClient::StorageSettings.allow_disk_access do
-          path_to_repo(project)
-        end
-        path_to_project_bundle = path_to_bundle(project)
-
-        # Create namespace dir or hashed path if missing
         if project.hashed_storage?(:repository)
           FileUtils.mkdir_p(File.dirname(File.join(backup_repos_path, project.disk_path)))
         else
           FileUtils.mkdir_p(File.join(backup_repos_path, project.namespace.full_path)) if project.namespace
         end
 
-        if empty_repo?(project)
-          progress.puts "[SKIPPED]".color(:cyan)
+        if !empty_repo?(project)
+          backup_project(project)
+          progress.puts "[DONE]".color(:green)
         else
-          in_path(path_to_project_repo) do |dir|
-            FileUtils.mkdir_p(path_to_tars(project))
-            cmd = %W(tar -cf #{path_to_tars(project, dir)} -C #{path_to_project_repo} #{dir})
-            output, status = Gitlab::Popen.popen(cmd)
-
-            unless status.zero?
-              progress_warn(project, cmd.join(' '), output)
-            end
+          progress.puts "[SKIPPED]".color(:cyan)
         end
 
-          cmd = %W(#{Gitlab.config.git.bin_path} --git-dir=#{path_to_project_repo} bundle create #{path_to_project_bundle} --all)
-          output, status = Gitlab::Popen.popen(cmd)
+        wiki = ProjectWiki.new(project)
 
-          if status.zero?
-            progress.puts "[DONE]".color(:green)
+        if !empty_repo?(wiki)
+          backup_project(wiki)
+          progress.puts "[DONE] Wiki".color(:green)
         else
-            progress_warn(project, cmd.join(' '), output)
+          progress.puts "[SKIPPED] Wiki".color(:cyan)
         end
       end
-
-        wiki = ProjectWiki.new(project)
-        path_to_wiki_repo = Gitlab::GitalyClient::StorageSettings.allow_disk_access do
-          path_to_repo(wiki)
     end
-        path_to_wiki_bundle = path_to_bundle(wiki)
 
-        if File.exist?(path_to_wiki_repo)
-          progress.print " * #{display_repo_path(wiki)} ... "
+    def prepare_directories
+      Gitlab.config.repositories.storages.each do |name, repository_storage|
+        delete_all_repositories(name, repository_storage)
+      end
+    end
 
-          if empty_repo?(wiki)
-            progress.puts " [SKIPPED]".color(:cyan)
-          else
-            cmd = %W(#{Gitlab.config.git.bin_path} --git-dir=#{path_to_wiki_repo} bundle create #{path_to_wiki_bundle} --all)
-            output, status = Gitlab::Popen.popen(cmd)
-            if status.zero?
-              progress.puts " [DONE]".color(:green)
+    def backup_project(project)
+      gitaly_migrate(:repository_backup) do |is_enabled|
+        if is_enabled
+          backup_project_gitaly(project)
         else
-              progress_warn(wiki, cmd.join(' '), output)
-            end
+          backup_project_local(project)
         end
       end
+
+      backup_custom_hooks(project)
+    rescue => e
+      progress_warn(project, e, 'Failed to backup repo')
     end
+
+    def backup_project_gitaly(project)
+      path_to_project_bundle = path_to_bundle(project)
+      Gitlab::GitalyClient::RepositoryService.new(project.repository)
+        .create_bundle(path_to_project_bundle)
     end
 
-    def prepare_directories
-      Gitlab.config.repositories.storages.each do |name, repository_storage|
-        delete_all_repositories(name, repository_storage)
+    def backup_project_local(project)
+      path_to_project_repo = Gitlab::GitalyClient::StorageSettings.allow_disk_access do
+        path_to_repo(project)
       end
+
+      path_to_project_bundle = path_to_bundle(project)
+
+      cmd = %W(#{Gitlab.config.git.bin_path} --git-dir=#{path_to_project_repo} bundle create #{path_to_project_bundle} --all)
+      output, status = Gitlab::Popen.popen(cmd)
+      progress_warn(project, cmd.join(' '), output) unless status.zero?
     end
 
     def delete_all_repositories(name, repository_storage)
@@ -97,8 +93,6 @@ module Backup
       path = repository_storage.legacy_disk_path
       return unless File.exist?(path)
 
-      # Move all files in the existing repos directory except . and .. to
-      # repositories.old.<timestamp> directory
       bk_repos_path = File.join(Gitlab.config.backup.path, "tmp", "#{name}-repositories.old." + Time.now.to_i.to_s)
       FileUtils.mkdir_p(bk_repos_path, mode: 0700)
       files = Dir.glob(File.join(path, "*"), File::FNM_DOTMATCH) - [File.join(path, "."), File.join(path, "..")]
@@ -129,13 +123,47 @@ module Backup
         .restore_custom_hooks(custom_hooks_path)
     end
 
+    def local_backup_custom_hooks(project)
+      in_path(path_to_tars(project)) do |dir|
+        path_to_project_repo = Gitlab::GitalyClient::StorageSettings.allow_disk_access do
+          path_to_repo(project)
+        end
+        break unless File.exist?(File.join(path_to_project_repo, dir))
+
+        FileUtils.mkdir_p(path_to_tars(project))
+        cmd = %W(tar -cf #{path_to_tars(project, dir)} -c #{path_to_project_repo} #{dir})
+        output, status = Gitlab::Popen.popen(cmd)
+
+        unless status.zero?
+          progress_warn(project, cmd.join(' '), output)
+        end
+      end
+    end
+
+    def gitaly_backup_custom_hooks(project)
+      FileUtils.mkdir_p(path_to_tars(project))
+      custom_hooks_path = path_to_tars(project, 'custom_hooks')
+      Gitlab::GitalyClient::RepositoryService.new(project.repository)
+        .backup_custom_hooks(custom_hooks_path)
+    end
+
+    def backup_custom_hooks(project)
+      gitaly_migrate(:backup_custom_hooks) do |is_enabled|
+        if is_enabled
+          gitaly_backup_custom_hooks(project)
+        else
+          local_backup_custom_hooks(project)
+        end
+      end
+    end
+
     def restore_custom_hooks(project)
       in_path(path_to_tars(project)) do |dir|
         gitaly_migrate(:restore_custom_hooks) do |is_enabled|
           if is_enabled
-            local_restore_custom_hooks(project, dir)
-          else
             gitaly_restore_custom_hooks(project, dir)
+          else
+            local_restore_custom_hooks(project, dir)
           end
         end
       end
@@ -186,7 +214,6 @@ module Backup
         end
       end
     end
-    # rubocop:enable Metrics/AbcSize
 
     protected
 
@@ -224,9 +251,7 @@ module Backup
 
     def prepare
       FileUtils.rm_rf(backup_repos_path)
-      # Ensure the parent dir of backup_repos_path exists
       FileUtils.mkdir_p(Gitlab.config.backup.path)
-      # Fail if somebody raced to create backup_repos_path before us
       FileUtils.mkdir(backup_repos_path, mode: 0700)
     end
 
@@ -242,7 +267,6 @@ module Backup
     end
 
     def empty_repo?(project_or_wiki)
-      # Protect against stale caches
       project_or_wiki.repository.expire_emptiness_caches
       project_or_wiki.repository.empty?
     end

lib/gitlab/gitaly_client/repository_service.rb

@@ -196,20 +196,21 @@ module Gitlab
       end
 
       def create_bundle(save_path)
-        request = Gitaly::CreateBundleRequest.new(repository: @gitaly_repo)
-        response = GitalyClient.call(
-          @storage,
-          :repository_service,
+        gitaly_fetch_stream_to_file(
+          save_path,
           :create_bundle,
-          request,
-          timeout: GitalyClient.default_timeout
+          Gitaly::CreateBundleRequest,
+          GitalyClient.default_timeout
         )
-
-        File.open(save_path, 'wb') do |f|
-          response.each do |message|
-            f.write(message.data)
-          end
       end
+
+      def backup_custom_hooks(save_path)
+        gitaly_fetch_stream_to_file(
+          save_path,
+          :backup_custom_hooks,
+          Gitaly::BackupCustomHooksRequest,
+          GitalyClient.default_timeout
+        )
       end
 
       def create_from_bundle(bundle_path)
@@ -309,6 +310,25 @@ module Gitlab
 
       private
 
+      def gitaly_fetch_stream_to_file(save_path, rpc_name, request_class, timeout)
+        request = request_class.new(repository: @gitaly_repo)
+        response = GitalyClient.call(
+          @storage,
+          :repository_service,
+          rpc_name,
+          request,
+          timeout: timeout
+        )
+
+        File.open(save_path, 'wb') do |f|
+          response.each do |message|
+            f.write(message.data)
+          end
+        end
+        # If the file is empty means that we recieved an empty stream, we delete the file
+        FileUtils.rm(save_path) if File.zero?(save_path)
+      end
+
       def gitaly_repo_stream_request(file_path, rpc_name, request_class, timeout)
         request = request_class.new(repository: @gitaly_repo)
         enum = Enumerator.new do |y|