Commit 5ee78765 authored by John Cai's avatar John Cai

Add client methods for FetchIntoObjectPool RPC

Gitaly's FetchIntoObjectPool RPC will idempotently fetch objects into an
object pool. If the pool doesn't exist, it will create an empty pool
before attempting the fetch. This change adds client code as well as
specs to cover this behavior.
parent fbaafdd7
...@@ -417,7 +417,7 @@ group :ed25519 do ...@@ -417,7 +417,7 @@ group :ed25519 do
end end
# Gitaly GRPC client # Gitaly GRPC client
gem 'gitaly-proto', '~> 1.22.0', require: 'gitaly' gem 'gitaly-proto', '~> 1.26.0', require: 'gitaly'
gem 'grpc', '~> 1.19.0' gem 'grpc', '~> 1.19.0'
......
...@@ -283,7 +283,7 @@ GEM ...@@ -283,7 +283,7 @@ GEM
gettext_i18n_rails (>= 0.7.1) gettext_i18n_rails (>= 0.7.1)
po_to_json (>= 1.0.0) po_to_json (>= 1.0.0)
rails (>= 3.2.0) rails (>= 3.2.0)
gitaly-proto (1.22.0) gitaly-proto (1.26.0)
grpc (~> 1.0) grpc (~> 1.0)
github-markup (1.7.0) github-markup (1.7.0)
gitlab-default_value_for (3.1.1) gitlab-default_value_for (3.1.1)
...@@ -1056,7 +1056,7 @@ DEPENDENCIES ...@@ -1056,7 +1056,7 @@ DEPENDENCIES
gettext (~> 3.2.2) gettext (~> 3.2.2)
gettext_i18n_rails (~> 1.8.0) gettext_i18n_rails (~> 1.8.0)
gettext_i18n_rails_js (~> 1.3) gettext_i18n_rails_js (~> 1.3)
gitaly-proto (~> 1.22.0) gitaly-proto (~> 1.26.0)
github-markup (~> 1.7.0) github-markup (~> 1.7.0)
gitlab-default_value_for (~> 3.1.1) gitlab-default_value_for (~> 3.1.1)
gitlab-labkit (~> 0.1.2) gitlab-labkit (~> 0.1.2)
......
...@@ -40,6 +40,10 @@ module Gitlab ...@@ -40,6 +40,10 @@ module Gitlab
@repository ||= Gitlab::Git::Repository.new(storage, relative_path, GL_REPOSITORY, gl_project_path) @repository ||= Gitlab::Git::Repository.new(storage, relative_path, GL_REPOSITORY, gl_project_path)
end end
def fetch
object_pool_service.fetch(source_repository)
end
private private
def object_pool_service def object_pool_service
......
...@@ -33,6 +33,15 @@ module Gitlab ...@@ -33,6 +33,15 @@ module Gitlab
GitalyClient.call(storage, :object_pool_service, :link_repository_to_object_pool, GitalyClient.call(storage, :object_pool_service, :link_repository_to_object_pool,
request, timeout: GitalyClient.fast_timeout) request, timeout: GitalyClient.fast_timeout)
end end
def fetch(repository)
request = Gitaly::FetchIntoObjectPoolRequest.new(
object_pool: object_pool,
origin: repository.gitaly_repository
)
GitalyClient.call(storage, :object_pool_service, :fetch_into_object_pool, request)
end
end end
end end
end end
...@@ -3,8 +3,12 @@ ...@@ -3,8 +3,12 @@
require 'spec_helper' require 'spec_helper'
describe Gitlab::Git::ObjectPool do describe Gitlab::Git::ObjectPool do
include RepoHelpers
let(:pool_repository) { create(:pool_repository) } let(:pool_repository) { create(:pool_repository) }
let(:source_repository) { pool_repository.source_project.repository } let(:source_repository) { pool_repository.source_project.repository }
let(:source_repository_path) { File.join(TestEnv.repos_path, source_repository.relative_path) }
let(:source_repository_rugged) { Rugged::Repository.new(source_repository_path) }
subject { pool_repository.object_pool } subject { pool_repository.object_pool }
...@@ -76,4 +80,41 @@ describe Gitlab::Git::ObjectPool do ...@@ -76,4 +80,41 @@ describe Gitlab::Git::ObjectPool do
end end
end end
end end
describe '#fetch' do
let(:commit_count) { source_repository.commit_count }
context "when the object's pool repository exists" do
it 'does not raise an error' do
expect { subject.fetch }.not_to raise_error
end
end
context "when the object's pool repository does not exist" do
before do
subject.delete
end
it "re-creates the object pool's repository" do
subject.fetch
expect(subject.repository.exists?).to be(true)
end
it 'does not raise an error' do
expect { subject.fetch }.not_to raise_error
end
it 'fetches objects from the source repository' do
new_commit_id = new_commit_edit_old_file(source_repository_rugged).oid
expect(subject.repository.exists?).to be false
subject.fetch
expect(subject.repository.commit_count('refs/remotes/origin/master')).to eq(commit_count)
expect(subject.repository.commit(new_commit_id).id).to eq(new_commit_id)
end
end
end
end end
...@@ -3,6 +3,7 @@ require "spec_helper" ...@@ -3,6 +3,7 @@ require "spec_helper"
describe Gitlab::Git::Repository, :seed_helper do describe Gitlab::Git::Repository, :seed_helper do
include Gitlab::EncodingHelper include Gitlab::EncodingHelper
include RepoHelpers
using RSpec::Parameterized::TableSyntax using RSpec::Parameterized::TableSyntax
shared_examples 'wrapping gRPC errors' do |gitaly_client_class, gitaly_client_method| shared_examples 'wrapping gRPC errors' do |gitaly_client_class, gitaly_client_method|
...@@ -2209,83 +2210,6 @@ describe Gitlab::Git::Repository, :seed_helper do ...@@ -2209,83 +2210,6 @@ describe Gitlab::Git::Repository, :seed_helper do
repository_rugged.references.create("refs/remotes/#{remote_name}/#{branch_name}", source_branch.dereferenced_target.sha) repository_rugged.references.create("refs/remotes/#{remote_name}/#{branch_name}", source_branch.dereferenced_target.sha)
end end
# Build the options hash that's passed to Rugged::Commit#create
def commit_options(repo, index, target, ref, message)
options = {}
options[:tree] = index.write_tree(repo)
options[:author] = {
email: "test@example.com",
name: "Test Author",
time: Time.gm(2014, "mar", 3, 20, 15, 1)
}
options[:committer] = {
email: "test@example.com",
name: "Test Author",
time: Time.gm(2014, "mar", 3, 20, 15, 1)
}
options[:message] ||= message
options[:parents] = repo.empty? ? [] : [target].compact
options[:update_ref] = ref
options
end
# Writes a new commit to the repo and returns a Rugged::Commit. Replaces the
# contents of CHANGELOG with a single new line of text.
def new_commit_edit_old_file(repo)
oid = repo.write("I replaced the changelog with this text", :blob)
index = repo.index
index.read_tree(repo.head.target.tree)
index.add(path: "CHANGELOG", oid: oid, mode: 0100644)
options = commit_options(
repo,
index,
repo.head.target,
"HEAD",
"Edit CHANGELOG in its original location"
)
sha = Rugged::Commit.create(repo, options)
repo.lookup(sha)
end
# Writes a new commit to the repo and returns a Rugged::Commit. Replaces the
# contents of the specified file_path with new text.
def new_commit_edit_new_file(repo, file_path, commit_message, text, branch = repo.head)
oid = repo.write(text, :blob)
index = repo.index
index.read_tree(branch.target.tree)
index.add(path: file_path, oid: oid, mode: 0100644)
options = commit_options(repo, index, branch.target, branch.canonical_name, commit_message)
sha = Rugged::Commit.create(repo, options)
repo.lookup(sha)
end
# Writes a new commit to the repo and returns a Rugged::Commit. Replaces the
# contents of encoding/CHANGELOG with new text.
def new_commit_edit_new_file_on_branch(repo, file_path, branch_name, commit_message, text)
branch = repo.branches[branch_name]
new_commit_edit_new_file(repo, file_path, commit_message, text, branch)
end
# Writes a new commit to the repo and returns a Rugged::Commit. Moves the
# CHANGELOG file to the encoding/ directory.
def new_commit_move_file(repo)
blob_oid = repo.head.target.tree.detect { |i| i[:name] == "CHANGELOG" }[:oid]
file_content = repo.lookup(blob_oid).content
oid = repo.write(file_content, :blob)
index = repo.index
index.read_tree(repo.head.target.tree)
index.add(path: "encoding/CHANGELOG", oid: oid, mode: 0100644)
index.remove("CHANGELOG")
options = commit_options(repo, index, repo.head.target, "HEAD", "Move CHANGELOG to encoding/")
sha = Rugged::Commit.create(repo, options)
repo.lookup(sha)
end
def refs(dir) def refs(dir)
IO.popen(%W[git -C #{dir} for-each-ref], &:read).split("\n").map do |line| IO.popen(%W[git -C #{dir} for-each-ref], &:read).split("\n").map do |line|
line.split("\t").last line.split("\t").last
......
...@@ -43,4 +43,24 @@ describe Gitlab::GitalyClient::ObjectPoolService do ...@@ -43,4 +43,24 @@ describe Gitlab::GitalyClient::ObjectPoolService do
end end
end end
end end
describe '#fetch' do
before do
subject.delete
end
it 'restores the pool repository objects' do
subject.fetch(project.repository)
expect(object_pool.repository.exists?).to be(true)
end
context 'when called twice' do
it "doesn't raise an error" do
subject.delete
expect { subject.fetch(project.repository) }.not_to raise_error
end
end
end
end end
...@@ -11,6 +11,8 @@ module RepoHelpers ...@@ -11,6 +11,8 @@ module RepoHelpers
# blob.path # => 'files/js/commit.js.coffee' # blob.path # => 'files/js/commit.js.coffee'
# blob.data # => 'class Commit...' # blob.data # => 'class Commit...'
# #
# Build the options hash that's passed to Rugged::Commit#create
def sample_blob def sample_blob
OpenStruct.new( OpenStruct.new(
oid: '5f53439ca4b009096571d3c8bc3d09d30e7431b3', oid: '5f53439ca4b009096571d3c8bc3d09d30e7431b3',
...@@ -129,4 +131,80 @@ eos ...@@ -129,4 +131,80 @@ eos
file_content: content file_content: content
).execute ).execute
end end
def commit_options(repo, index, target, ref, message)
options = {}
options[:tree] = index.write_tree(repo)
options[:author] = {
email: "test@example.com",
name: "Test Author",
time: Time.gm(2014, "mar", 3, 20, 15, 1)
}
options[:committer] = {
email: "test@example.com",
name: "Test Author",
time: Time.gm(2014, "mar", 3, 20, 15, 1)
}
options[:message] ||= message
options[:parents] = repo.empty? ? [] : [target].compact
options[:update_ref] = ref
options
end
# Writes a new commit to the repo and returns a Rugged::Commit. Replaces the
# contents of CHANGELOG with a single new line of text.
def new_commit_edit_old_file(repo)
oid = repo.write("I replaced the changelog with this text", :blob)
index = repo.index
index.read_tree(repo.head.target.tree)
index.add(path: "CHANGELOG", oid: oid, mode: 0100644)
options = commit_options(
repo,
index,
repo.head.target,
"HEAD",
"Edit CHANGELOG in its original location"
)
sha = Rugged::Commit.create(repo, options)
repo.lookup(sha)
end
# Writes a new commit to the repo and returns a Rugged::Commit. Replaces the
# contents of the specified file_path with new text.
def new_commit_edit_new_file(repo, file_path, commit_message, text, branch = repo.head)
oid = repo.write(text, :blob)
index = repo.index
index.read_tree(branch.target.tree)
index.add(path: file_path, oid: oid, mode: 0100644)
options = commit_options(repo, index, branch.target, branch.canonical_name, commit_message)
sha = Rugged::Commit.create(repo, options)
repo.lookup(sha)
end
# Writes a new commit to the repo and returns a Rugged::Commit. Replaces the
# contents of encoding/CHANGELOG with new text.
def new_commit_edit_new_file_on_branch(repo, file_path, branch_name, commit_message, text)
branch = repo.branches[branch_name]
new_commit_edit_new_file(repo, file_path, commit_message, text, branch)
end
# Writes a new commit to the repo and returns a Rugged::Commit. Moves the
# CHANGELOG file to the encoding/ directory.
def new_commit_move_file(repo)
blob_oid = repo.head.target.tree.detect { |i| i[:name] == "CHANGELOG" }[:oid]
file_content = repo.lookup(blob_oid).content
oid = repo.write(file_content, :blob)
index = repo.index
index.read_tree(repo.head.target.tree)
index.add(path: "encoding/CHANGELOG", oid: oid, mode: 0100644)
index.remove("CHANGELOG")
options = commit_options(repo, index, repo.head.target, "HEAD", "Move CHANGELOG to encoding/")
sha = Rugged::Commit.create(repo, options)
repo.lookup(sha)
end
end end
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment