Merge branch 'ce-gitlab-git-backport' into 'master'

CE repository mirroring changes backport

See merge request gitlab-org/gitlab-ce!15184

app/models/concerns/repository_mirroring.rb

-module RepositoryMirroring
-  IMPORT_HEAD_REFS = '+refs/heads/*:refs/heads/*'.freeze
-  IMPORT_TAG_REFS = '+refs/tags/*:refs/tags/*'.freeze
-
-  def set_remote_as_mirror(name)
-    # This is used to define repository as equivalent as "git clone --mirror"
-    raw_repository.rugged.config["remote.#{name}.fetch"] = 'refs/*:refs/*'
-    raw_repository.rugged.config["remote.#{name}.mirror"] = true
-    raw_repository.rugged.config["remote.#{name}.prune"] = true
-  end
-
-  def set_import_remote_as_mirror(remote_name)
-    # Add first fetch with Rugged so it does not create its own.
-    raw_repository.rugged.config["remote.#{remote_name}.fetch"] = IMPORT_HEAD_REFS
-
-    add_remote_fetch_config(remote_name, IMPORT_TAG_REFS)
-
-    raw_repository.rugged.config["remote.#{remote_name}.mirror"] = true
-    raw_repository.rugged.config["remote.#{remote_name}.prune"] = true
-  end
-
-  def add_remote_fetch_config(remote_name, refspec)
-    run_git(%W[config --add remote.#{remote_name}.fetch #{refspec}])
-  end
-
-  def fetch_mirror(remote, url)
-    add_remote(remote, url)
-    set_remote_as_mirror(remote)
-    fetch_remote(remote, forced: true)
-    remove_remote(remote)
-  end
-end

app/models/project.rb

@@ -1684,6 +1684,10 @@ class Project < ActiveRecord::Base
     Gitlab::GlRepository.gl_repository(self, is_wiki)
   end
 
+  def reference_counter(wiki: false)
+    Gitlab::ReferenceCounter.new(gl_repository(is_wiki: wiki))
+  end
+
   private
 
   def storage
@@ -1702,11 +1706,11 @@ class Project < ActiveRecord::Base
   end
 
   def repo_reference_count
-    Gitlab::ReferenceCounter.new(gl_repository(is_wiki: false)).value
+    reference_counter.value
   end
 
   def wiki_reference_count
-    Gitlab::ReferenceCounter.new(gl_repository(is_wiki: true)).value
+    reference_counter(wiki: true).value
   end
 
   def check_repository_absence!

app/models/project_wiki.rb

@@ -135,7 +135,7 @@ class ProjectWiki
   end
 
   def repository
-    @repository ||= Repository.new(full_path, @project, disk_path: disk_path)
+    @repository ||= Repository.new(full_path, @project, disk_path: disk_path, is_wiki: true)
   end
 
   def default_branch

app/models/repository.rb

@@ -15,9 +15,8 @@ class Repository
   ].freeze
 
   include Gitlab::ShellAdapter
-  include RepositoryMirroring
 
-  attr_accessor :full_path, :disk_path, :project
+  attr_accessor :full_path, :disk_path, :project, :is_wiki
 
   delegate :ref_name_for_sha, to: :raw_repository
 
@@ -72,11 +71,12 @@ class Repository
     end
   end
 
-  def initialize(full_path, project, disk_path: nil)
+  def initialize(full_path, project, disk_path: nil, is_wiki: false)
     @full_path = full_path
     @disk_path = disk_path || full_path
     @project = project
     @commit_cache = {}
+    @is_wiki = is_wiki
   end
 
   def ==(other)
@@ -965,21 +965,8 @@ class Repository
     run_git(args).first.lines.map(&:strip)
   end
 
-  def add_remote(name, url)
-    raw_repository.remote_add(name, url)
-  rescue Rugged::ConfigError
-    raw_repository.remote_update(name, url: url)
-  end
-
-  def remove_remote(name)
-    raw_repository.remote_delete(name)
-    true
-  rescue Rugged::ConfigError
-    false
-  end
-
-  def fetch_remote(remote, forced: false, no_tags: false)
-    gitlab_shell.fetch_remote(raw_repository, remote, forced: forced, no_tags: no_tags)
+  def fetch_remote(remote, forced: false, ssh_auth: nil, no_tags: false)
+    gitlab_shell.fetch_remote(raw_repository, remote, ssh_auth: ssh_auth, forced: forced, no_tags: no_tags)
   end
 
   def fetch_source_branch(source_repository, source_branch, local_ref)
@@ -1141,7 +1128,7 @@ class Repository
   end
 
   def initialize_raw_repository
-    Gitlab::Git::Repository.new(project.repository_storage, disk_path + '.git', Gitlab::GlRepository.gl_repository(project, false))
+    Gitlab::Git::Repository.new(project.repository_storage, disk_path + '.git', Gitlab::GlRepository.gl_repository(project, is_wiki))
   end
 
   def find_commits_by_message_by_shelling_out(query, ref, path, limit, offset)

app/services/projects/import_service.rb

@@ -44,7 +44,7 @@ module Projects
         else
           clone_repository
         end
-      rescue Gitlab::Shell::Error => e
+      rescue Gitlab::Shell::Error, Gitlab::Git::RepositoryMirroring::RemoteError => e
         # Expire cache to prevent scenarios such as:
         # 1. First import failed, but the repo was imported successfully, so +exists?+ returns true
         # 2. Retried import, repo is broken or not imported but +exists?+ still returns true

lib/github/import.rb

@@ -60,7 +60,9 @@ module Github
         project.repository.set_import_remote_as_mirror('github')
         project.repository.add_remote_fetch_config('github', '+refs/pull/*/head:refs/merge-requests/*/head')
         fetch_remote(forced: true)
-      rescue Gitlab::Git::Repository::NoRepository, Gitlab::Shell::Error => e
+      rescue Gitlab::Git::Repository::NoRepository,
+             Gitlab::Git::RepositoryMirroring::RemoteError,
+             Gitlab::Shell::Error => e
         error(:project, repo_url, e.message)
         raise Github::RepositoryFetchError
       end

lib/gitlab/git/repository.rb

@@ -6,6 +6,7 @@ require "rubygems/package"
 module Gitlab
   module Git
     class Repository
+      include Gitlab::Git::RepositoryMirroring
       include Gitlab::Git::Popen
 
       ALLOWED_OBJECT_DIRECTORIES_VARIABLES = %w[
@@ -898,16 +899,25 @@ module Gitlab
         end
       end
 
-      # Delete the specified remote from this repository.
-      def remote_delete(remote_name)
-        rugged.remotes.delete(remote_name)
-        nil
+      def add_remote(remote_name, url)
+        rugged.remotes.create(remote_name, url)
+      rescue Rugged::ConfigError
+        remote_update(remote_name, url: url)
       end
 
-      # Add a new remote to this repository.
-      def remote_add(remote_name, url)
-        rugged.remotes.create(remote_name, url)
-        nil
+      def remove_remote(remote_name)
+        # When a remote is deleted all its remote refs are deleted too, but in
+        # the case of mirrors we map its refs (that would usualy go under
+        # [remote_name]/) to the top level namespace. We clean the mapping so
+        # those don't get deleted.
+        if rugged.config["remote.#{remote_name}.mirror"]
+          rugged.config.delete("remote.#{remote_name}.fetch")
+        end
+
+        rugged.remotes.delete(remote_name)
+        true
+      rescue Rugged::ConfigError
+        false
       end
 
       # Update the specified remote using the values in the +options+ hash

lib/gitlab/git/repository_mirroring.rb

+module Gitlab
+  module Git
+    module RepositoryMirroring
+      IMPORT_HEAD_REFS = '+refs/heads/*:refs/heads/*'.freeze
+      IMPORT_TAG_REFS = '+refs/tags/*:refs/tags/*'.freeze
+      MIRROR_REMOTE = 'mirror'.freeze
+
+      RemoteError = Class.new(StandardError)
+
+      def set_remote_as_mirror(remote_name)
+        # This is used to define repository as equivalent as "git clone --mirror"
+        rugged.config["remote.#{remote_name}.fetch"] = 'refs/*:refs/*'
+        rugged.config["remote.#{remote_name}.mirror"] = true
+        rugged.config["remote.#{remote_name}.prune"] = true
+      end
+
+      def set_import_remote_as_mirror(remote_name)
+        # Add first fetch with Rugged so it does not create its own.
+        rugged.config["remote.#{remote_name}.fetch"] = IMPORT_HEAD_REFS
+
+        add_remote_fetch_config(remote_name, IMPORT_TAG_REFS)
+
+        rugged.config["remote.#{remote_name}.mirror"] = true
+        rugged.config["remote.#{remote_name}.prune"] = true
+      end
+
+      def add_remote_fetch_config(remote_name, refspec)
+        run_git(%W[config --add remote.#{remote_name}.fetch #{refspec}])
+      end
+
+      def fetch_mirror(url)
+        add_remote(MIRROR_REMOTE, url)
+        set_remote_as_mirror(MIRROR_REMOTE)
+        fetch(MIRROR_REMOTE)
+        remove_remote(MIRROR_REMOTE)
+      end
+
+      def remote_tags(remote)
+        # Each line has this format: "dc872e9fa6963f8f03da6c8f6f264d0845d6b092\trefs/tags/v1.10.0\n"
+        # We want to convert it to: [{ 'v1.10.0' => 'dc872e9fa6963f8f03da6c8f6f264d0845d6b092' }, ...]
+        list_remote_tags(remote).map do |line|
+          target, path = line.strip.split("\t")
+
+          # When the remote repo does not have tags.
+          if target.nil? || path.nil?
+            Rails.logger.info "Empty or invalid list of tags for remote: #{remote}. Output: #{output}"
+            return []
+          end
+
+          name = path.split('/', 3).last
+          # We're only interested in tag references
+          # See: http://stackoverflow.com/questions/15472107/when-listing-git-ls-remote-why-theres-after-the-tag-name
+          next if name =~ /\^\{\}\Z/
+
+          target_commit = Gitlab::Git::Commit.find(self, target)
+          Gitlab::Git::Tag.new(self, name, target, target_commit)
+        end.compact
+      end
+
+      def remote_branches(remote_name)
+        branches = []
+
+        rugged.references.each("refs/remotes/#{remote_name}/*").map do |ref|
+          name = ref.name.sub(/\Arefs\/remotes\/#{remote_name}\//, '')
+
+          begin
+            target_commit = Gitlab::Git::Commit.find(self, ref.target)
+            branches << Gitlab::Git::Branch.new(self, name, ref.target, target_commit)
+          rescue Rugged::ReferenceError
+            # Omit invalid branch
+          end
+        end
+
+        branches
+      end
+
+      private
+
+      def list_remote_tags(remote)
+        tag_list, exit_code, error = nil
+        cmd = %W(#{Gitlab.config.git.bin_path} --git-dir=#{full_path} ls-remote --tags #{remote})
+
+        Open3.popen3(*cmd) do |stdin, stdout, stderr, wait_thr|
+          tag_list  = stdout.read
+          error     = stderr.read
+          exit_code = wait_thr.value.exitstatus
+        end
+
+        raise RemoteError, error unless exit_code.zero?
+
+        tag_list.split('\n')
+      end
+    end
+  end
+end

lib/gitlab/git/wiki.rb

@@ -10,6 +10,8 @@ module Gitlab
       end
       PageBlob = Struct.new(:name)
 
+      attr_reader :repository
+
       def self.default_ref
         'master'
       end

spec/lib/gitlab/git/repository_spec.rb

@@ -559,10 +559,10 @@ describe Gitlab::Git::Repository, seed_helper: true do
     end
   end
 
-  describe "#remote_delete" do
+  describe "#remove_remote" do
     before(:all) do
       @repo = Gitlab::Git::Repository.new('default', TEST_MUTABLE_REPO_PATH, '')
-      @repo.remote_delete("expendable")
+      @repo.remove_remote("expendable")
     end
 
     it "should remove the remote" do
@@ -575,14 +575,16 @@ describe Gitlab::Git::Repository, seed_helper: true do
     end
   end
 
-  describe "#remote_add" do
+  describe "#remote_update" do
     before(:all) do
       @repo = Gitlab::Git::Repository.new('default', TEST_MUTABLE_REPO_PATH, '')
-      @repo.remote_add("new_remote", SeedHelper::GITLAB_GIT_TEST_REPO_URL)
+      @repo.remote_update("expendable", url: TEST_NORMAL_REPO_PATH)
     end
 
     it "should add the remote" do
-      expect(@repo.rugged.remotes.each_name.to_a).to include("new_remote")
+      expect(@repo.rugged.remotes["expendable"].url).to(
+        eq(TEST_NORMAL_REPO_PATH)
+      )
     end
 
     after(:all) do
@@ -591,21 +593,58 @@ describe Gitlab::Git::Repository, seed_helper: true do
     end
   end
 
-  describe "#remote_update" do
-    before(:all) do
-      @repo = Gitlab::Git::Repository.new('default', TEST_MUTABLE_REPO_PATH, '')
-      @repo.remote_update("expendable", url: TEST_NORMAL_REPO_PATH)
+  describe '#fetch_mirror' do
+    let(:new_repository) do
+      Gitlab::Git::Repository.new('default', 'my_project.git', '')
     end
 
-    it "should add the remote" do
-      expect(@repo.rugged.remotes["expendable"].url).to(
-        eq(TEST_NORMAL_REPO_PATH)
-      )
+    subject { new_repository.fetch_mirror(repository.path) }
+
+    before do
+      Gitlab::Shell.new.add_repository('default', 'my_project')
     end
 
-    after(:all) do
-      FileUtils.rm_rf(TEST_MUTABLE_REPO_PATH)
-      ensure_seeds
+    after do
+      Gitlab::Shell.new.remove_repository(TestEnv.repos_path, 'my_project')
+    end
+
+    it 'fetches a url as a mirror remote' do
+      subject
+
+      expect(refs(new_repository.path)).to eq(refs(repository.path))
+    end
+
+    context 'with keep-around refs' do
+      let(:sha) { SeedRepo::Commit::ID }
+      let(:keep_around_ref) { "refs/keep-around/#{sha}" }
+      let(:tmp_ref) { "refs/tmp/#{SecureRandom.hex}" }
+
+      before do
+        repository.rugged.references.create(keep_around_ref, sha, force: true)
+        repository.rugged.references.create(tmp_ref, sha, force: true)
+      end
+
+      it 'includes the temporary and keep-around refs' do
+        subject
+
+        expect(refs(new_repository.path)).to include(keep_around_ref)
+        expect(refs(new_repository.path)).to include(tmp_ref)
+      end
+    end
+  end
+
+  describe '#remote_tags' do
+    let(:target_commit_id) { SeedRepo::Commit::ID }
+
+    subject { repository.remote_tags('upstream') }
+
+    it 'gets the remote tags' do
+      expect(repository).to receive(:list_remote_tags).with('upstream')
+        .and_return(["#{target_commit_id}\trefs/tags/v0.0.1\n"])
+
+      expect(subject.first).to be_an_instance_of(Gitlab::Git::Tag)
+      expect(subject.first.name).to eq('v0.0.1')
+      expect(subject.first.dereferenced_target.id).to eq(target_commit_id)
     end
   end
 
@@ -1685,6 +1724,21 @@ describe Gitlab::Git::Repository, seed_helper: true do
     end
   end
 
+  describe '#fetch' do
+    let(:git_path) { Gitlab.config.git.bin_path }
+    let(:remote_name) { 'my_remote' }
+
+    subject { repository.fetch(remote_name) }
+
+    it 'fetches the remote and returns true if the command was successful' do
+      expect(repository).to receive(:popen)
+        .with(%W(#{git_path} fetch #{remote_name}), repository.path)
+        .and_return(['', 0])
+
+      expect(subject).to be(true)
+    end
+  end
+
   def create_remote_branch(repository, remote_name, branch_name, source_branch_name)
     source_branch = repository.branches.find { |branch| branch.name == source_branch_name }
     rugged = repository.rugged
@@ -1760,4 +1814,10 @@ describe Gitlab::Git::Repository, seed_helper: true do
     sha = Rugged::Commit.create(repo, options)
     repo.lookup(sha)
   end
+
+  def refs(dir)
+    IO.popen(%W[git -C #{dir} for-each-ref], &:read).split("\n").map do |line|
+      line.split("\t").last
+    end
+  end
 end

spec/models/repository_spec.rb

@@ -2298,4 +2298,24 @@ describe Repository do
       project.commit_by(oid: '1' * 40)
     end
   end
+
+  describe '#raw_repository' do
+    subject { repository.raw_repository }
+
+    it 'returns a Gitlab::Git::Repository representation of the repository' do
+      expect(subject).to be_a(Gitlab::Git::Repository)
+      expect(subject.relative_path).to eq(project.disk_path + '.git')
+      expect(subject.gl_repository).to eq("project-#{project.id}")
+    end
+
+    context 'with a wiki repository' do
+      let(:repository) { project.wiki.repository }
+
+      it 'creates a Gitlab::Git::Repository with the proper attributes' do
+        expect(subject).to be_a(Gitlab::Git::Repository)
+        expect(subject.relative_path).to eq(project.disk_path + '.wiki.git')
+        expect(subject.gl_repository).to eq("wiki-#{project.id}")
+      end
+    end
+  end
 end