Commit 0241c182 authored by Rémy Coutable's avatar Rémy Coutable

Improve the ee_compat_check_job to handle gitlab-ee forks

Signed-off-by: default avatarRémy Coutable <remy@rymai.me>
parent 9b76d851
...@@ -2,8 +2,8 @@ ...@@ -2,8 +2,8 @@
module Gitlab module Gitlab
# Checks if a set of migrations requires downtime or not. # Checks if a set of migrations requires downtime or not.
class EeCompatCheck class EeCompatCheck
DEFAULT_CE_PROJECT_URL = 'https://gitlab.com/gitlab-org/gitlab-ce'.freeze CANONICAL_CE_PROJECT_URL = 'https://gitlab.com/gitlab-org/gitlab-ce'.freeze
EE_REPO_URL = 'https://gitlab.com/gitlab-org/gitlab-ee.git'.freeze CANONICAL_EE_REPO_URL = 'https://gitlab.com/gitlab-org/gitlab-ee.git'.freeze
CHECK_DIR = Rails.root.join('ee_compat_check') CHECK_DIR = Rails.root.join('ee_compat_check')
IGNORED_FILES_REGEX = %r{VERSION|CHANGELOG\.md|db/schema\.rb}i.freeze IGNORED_FILES_REGEX = %r{VERSION|CHANGELOG\.md|db/schema\.rb}i.freeze
PLEASE_READ_THIS_BANNER = %Q{ PLEASE_READ_THIS_BANNER = %Q{
...@@ -11,57 +11,81 @@ module Gitlab ...@@ -11,57 +11,81 @@ module Gitlab
===================== PLEASE READ THIS ===================== ===================== PLEASE READ THIS =====================
============================================================ ============================================================
}.freeze }.freeze
STAY_STRONG_LINK_TO_DOCS = %Q{
Stay 💪! For more information, see
https://docs.gitlab.com/ce/development/automatic_ce_ee_merge.html
}.freeze
THANKS_FOR_READING_BANNER = %Q{ THANKS_FOR_READING_BANNER = %Q{
============================================================ ============================================================
==================== THANKS FOR READING ==================== ==================== THANKS FOR READING ====================
============================================================\n ============================================================\n
}.freeze }.freeze
attr_reader :ee_repo_dir, :patches_dir, :ce_project_url, :ce_repo_url, :ce_branch, :ee_branch_found attr_reader :ee_repo_dir, :patches_dir
attr_reader :ce_project_url, :ee_repo_url
attr_reader :ce_branch, :ee_remote_with_branch, :ee_branch_found
attr_reader :job_id, :failed_files attr_reader :job_id, :failed_files
def initialize(branch:, ce_project_url: DEFAULT_CE_PROJECT_URL, job_id: nil) def initialize(branch:, ce_project_url: CANONICAL_CE_PROJECT_URL, job_id: nil)
@ee_repo_dir = CHECK_DIR.join('ee-repo') @ee_repo_dir = CHECK_DIR.join('ee-repo')
@patches_dir = CHECK_DIR.join('patches') @patches_dir = CHECK_DIR.join('patches')
@ce_branch = branch @ce_branch = branch
@ce_project_url = ce_project_url @ce_project_url = ce_project_url
@ce_repo_url = "#{ce_project_url}.git" @ee_repo_url = ce_public_repo_url.sub('gitlab-ce', 'gitlab-ee')
@job_id = job_id @job_id = job_id
end end
def check def check
ensure_patches_dir ensure_patches_dir
add_remote('canonical-ce', "#{DEFAULT_CE_PROJECT_URL}.git") # We're generating the patch against the canonical-ce remote since forks'
generate_patch(branch: ce_branch, patch_path: ce_patch_full_path, remote: 'canonical-ce') # master branch are not necessarily up-to-date.
add_remote('canonical-ce', "#{CANONICAL_CE_PROJECT_URL}.git")
generate_patch(branch: ce_branch, patch_path: ce_patch_full_path, branch_remote: 'origin', master_remote: 'canonical-ce')
ensure_ee_repo ensure_ee_repo
Dir.chdir(ee_repo_dir) do Dir.chdir(ee_repo_dir) do
step("In the #{ee_repo_dir} directory") step("In the #{ee_repo_dir} directory")
add_remote('canonical-ee', EE_REPO_URL) ee_remotes.each do |key, url|
add_remote(key, url)
end
fetch(branch: 'master', depth: 20, remote: 'canonical-ee')
status = catch(:halt_check) do status = catch(:halt_check) do
ce_branch_compat_check! ce_branch_compat_check!
delete_ee_branches_locally! delete_ee_branches_locally!
ee_branch_presence_check! ee_branch_presence_check!
step("Checking out #{ee_branch_found}", %W[git checkout -b #{ee_branch_found} canonical-ee/#{ee_branch_found}]) step("Checking out #{ee_remote_with_branch}/#{ee_branch_found}", %W[git checkout -b #{ee_branch_found} #{ee_remote_with_branch}/#{ee_branch_found}])
generate_patch(branch: ee_branch_found, patch_path: ee_patch_full_path, remote: 'canonical-ee') generate_patch(branch: ee_branch_found, patch_path: ee_patch_full_path, branch_remote: ee_remote_with_branch, master_remote: 'canonical-ee')
ee_branch_compat_check! ee_branch_compat_check!
end end
delete_ee_branches_locally! delete_ee_branches_locally!
if status.nil? status.nil?
true
else
false
end
end end
end end
private private
def fork?
ce_project_url != CANONICAL_CE_PROJECT_URL
end
def ee_remotes
return @ee_remotes if defined?(@ee_remotes)
remotes =
{
'ee' => ee_repo_url,
'canonical-ee' => CANONICAL_EE_REPO_URL
}
remotes.delete('ee') unless fork?
@ee_remotes = remotes
end
def add_remote(name, url) def add_remote(name, url)
step( step(
"Adding the #{name} remote (#{url})", "Adding the #{name} remote (#{url})",
...@@ -70,28 +94,32 @@ module Gitlab ...@@ -70,28 +94,32 @@ module Gitlab
end end
def ensure_ee_repo def ensure_ee_repo
if Dir.exist?(ee_repo_dir) unless clone_repo(ee_repo_url, ee_repo_dir)
step("#{ee_repo_dir} already exists") # Fallback to using the canonical EE if there is no forked EE
else clone_repo(CANONICAL_EE_REPO_URL, ee_repo_dir)
step( end
"Cloning #{EE_REPO_URL} into #{ee_repo_dir}",
%W[git clone --branch master --single-branch --depth=200 #{EE_REPO_URL} #{ee_repo_dir}]
)
end end
def clone_repo(url, dir)
_, status = step(
"Cloning #{url} into #{dir}",
%W[git clone --branch master --single-branch --depth=200 #{url} #{dir}]
)
status.zero?
end end
def ensure_patches_dir def ensure_patches_dir
FileUtils.mkdir_p(patches_dir) FileUtils.mkdir_p(patches_dir)
end end
def generate_patch(branch:, patch_path:, remote:) def generate_patch(branch:, patch_path:, branch_remote:, master_remote:)
FileUtils.rm(patch_path, force: true) FileUtils.rm(patch_path, force: true)
find_merge_base_with_master(branch: branch, master_remote: remote) find_merge_base_with_master(branch: branch, branch_remote: branch_remote, master_remote: master_remote)
step( step(
"Generating the patch against #{remote}/master in #{patch_path}", "Generating the patch against #{master_remote}/master in #{patch_path}",
%W[git diff --binary #{remote}/master...origin/#{branch}] %W[git diff --binary #{master_remote}/master...#{branch_remote}/#{branch}]
) do |output, status| ) do |output, status|
throw(:halt_check, :ko) unless status.zero? throw(:halt_check, :ko) unless status.zero?
...@@ -109,24 +137,23 @@ module Gitlab ...@@ -109,24 +137,23 @@ module Gitlab
end end
def ee_branch_presence_check! def ee_branch_presence_check!
_, status = step("Fetching origin/#{ee_branch_prefix}", %W[git fetch canonical-ee #{ee_branch_prefix}]) ee_remotes.keys.each do |remote|
[ee_branch_prefix, ee_branch_suffix].each do |branch|
_, status = step("Fetching #{remote}/#{ee_branch_prefix}", %W[git fetch #{remote} #{branch}])
if status.zero? if status.zero?
@ee_branch_found = ee_branch_prefix @ee_remote_with_branch = remote
return @ee_branch_found = branch
return true
end
end
end end
_, status = step("Fetching origin/#{ee_branch_suffix}", %W[git fetch canonical-ee #{ee_branch_suffix}])
if status.zero?
@ee_branch_found = ee_branch_suffix
else
puts puts
puts ce_branch_doesnt_apply_cleanly_and_no_ee_branch_msg puts ce_branch_doesnt_apply_cleanly_and_no_ee_branch_msg
throw(:halt_check, :ko) throw(:halt_check, :ko)
end end
end
def ee_branch_compat_check! def ee_branch_compat_check!
unless check_patch(ee_patch_full_path).zero? unless check_patch(ee_patch_full_path).zero?
...@@ -181,10 +208,10 @@ module Gitlab ...@@ -181,10 +208,10 @@ module Gitlab
command(%W[git branch --delete --force #{ee_branch_suffix}]) command(%W[git branch --delete --force #{ee_branch_suffix}])
end end
def merge_base_found?(master_remote:, branch:) def merge_base_found?(branch:, branch_remote:, master_remote:)
step( step(
"Finding merge base with #{master_remote}/master", "Finding merge base with #{master_remote}/master",
%W[git merge-base #{master_remote}/master origin/#{branch}] %W[git merge-base #{master_remote}/master #{branch_remote}/#{branch}]
) do |output, status| ) do |output, status|
if status.zero? if status.zero?
puts "Merge base was found: #{output}" puts "Merge base was found: #{output}"
...@@ -193,7 +220,7 @@ module Gitlab ...@@ -193,7 +220,7 @@ module Gitlab
end end
end end
def find_merge_base_with_master(branch:, master_remote:) def find_merge_base_with_master(branch:, branch_remote:, master_remote:)
# Start with (Math.exp(3).to_i = 20) until (Math.exp(6).to_i = 403) # Start with (Math.exp(3).to_i = 20) until (Math.exp(6).to_i = 403)
# In total we go (20 + 54 + 148 + 403 = 625) commits deeper # In total we go (20 + 54 + 148 + 403 = 625) commits deeper
depth = 20 depth = 20
...@@ -202,10 +229,10 @@ module Gitlab ...@@ -202,10 +229,10 @@ module Gitlab
depth += Math.exp(factor).to_i depth += Math.exp(factor).to_i
# Repository is initially cloned with a depth of 20 so we need to fetch # Repository is initially cloned with a depth of 20 so we need to fetch
# deeper in the case the branch has more than 20 commits on top of master # deeper in the case the branch has more than 20 commits on top of master
fetch(branch: branch, depth: depth, remote: 'origin') fetch(branch: branch, depth: depth, remote: branch_remote)
fetch(branch: 'master', depth: depth, remote: master_remote) fetch(branch: 'master', depth: depth, remote: master_remote)
merge_base_found?(master_remote: master_remote, branch: branch) merge_base_found?(branch: branch, branch_remote: branch_remote, master_remote: master_remote)
end end
raise "\n#{branch} is too far behind #{master_remote}/master, please rebase it!\n" unless success raise "\n#{branch} is too far behind #{master_remote}/master, please rebase it!\n" unless success
...@@ -274,6 +301,13 @@ module Gitlab ...@@ -274,6 +301,13 @@ module Gitlab
Gitlab::Popen.popen(cmd) Gitlab::Popen.popen(cmd)
end end
# We're "re-creating" the repo URL because ENV['CI_REPOSITORY_URL'] contains
# redacted credentials (e.g. "***:****") which are useless in instructions
# the job gives.
def ce_public_repo_url
"#{ce_project_url}.git"
end
def applies_cleanly_msg(branch) def applies_cleanly_msg(branch)
%Q{ %Q{
#{PLEASE_READ_THIS_BANNER} #{PLEASE_READ_THIS_BANNER}
...@@ -288,13 +322,15 @@ module Gitlab ...@@ -288,13 +322,15 @@ module Gitlab
end end
def ce_branch_doesnt_apply_cleanly_and_no_ee_branch_msg def ce_branch_doesnt_apply_cleanly_and_no_ee_branch_msg
ee_repos = ee_remotes.values.uniq
%Q{ %Q{
#{PLEASE_READ_THIS_BANNER} #{PLEASE_READ_THIS_BANNER}
💥 Oh no! 💥 💥 Oh no! 💥
The `#{ce_branch}` branch does not apply cleanly to the current The `#{ce_branch}` branch does not apply cleanly to the current
EE/master, and no `#{ee_branch_prefix}` or `#{ee_branch_suffix}` branch EE/master, and no `#{ee_branch_prefix}` or `#{ee_branch_suffix}` branch
was found in the EE repository. was found in #{ee_repos.join(' nor in ')}.
If you're a community contributor, don't worry, someone from If you're a community contributor, don't worry, someone from
GitLab Inc. will take care of this, and you don't have to do anything. GitLab Inc. will take care of this, and you don't have to do anything.
...@@ -314,17 +350,17 @@ module Gitlab ...@@ -314,17 +350,17 @@ module Gitlab
1. Create a new branch from master and cherry-pick your CE commits 1. Create a new branch from master and cherry-pick your CE commits
# In the EE repo # In the EE repo
$ git fetch #{EE_REPO_URL} master $ git fetch #{CANONICAL_EE_REPO_URL} master
$ git checkout -b #{ee_branch_prefix} FETCH_HEAD $ git checkout -b #{ee_branch_prefix} FETCH_HEAD
$ git fetch #{ce_repo_url} #{ce_branch} $ git fetch #{ce_public_repo_url} #{ce_branch}
$ git cherry-pick SHA # Repeat for all the commits you want to pick $ git cherry-pick SHA # Repeat for all the commits you want to pick
You can squash the `#{ce_branch}` commits into a single "Port of #{ce_branch} to EE" commit. Note: You can squash the `#{ce_branch}` commits into a single "Port of #{ce_branch} to EE" commit.
2. Apply your branch's patch to EE 2. Apply your branch's patch to EE
# In the EE repo # In the EE repo
$ git fetch #{EE_REPO_URL} master $ git fetch #{CANONICAL_EE_REPO_URL} master
$ git checkout -b #{ee_branch_prefix} FETCH_HEAD $ git checkout -b #{ee_branch_prefix} FETCH_HEAD
$ wget #{patch_url} && git apply --3way #{ce_patch_name} $ wget #{patch_url} && git apply --3way #{ce_patch_name}
...@@ -356,10 +392,9 @@ module Gitlab ...@@ -356,10 +392,9 @@ module Gitlab
⚠️ Also, don't forget to create a new merge request on gitlab-ee and ⚠️ Also, don't forget to create a new merge request on gitlab-ee and
cross-link it with the CE merge request. cross-link it with the CE merge request.
Once this is done, you can retry this failed build, and it should pass. Once this is done, you can retry this failed job, and it should pass.
Stay 💪 ! For more information, see #{STAY_STRONG_LINK_TO_DOCS}
https://docs.gitlab.com/ce/development/automatic_ce_ee_merge.html
#{THANKS_FOR_READING_BANNER} #{THANKS_FOR_READING_BANNER}
} }
end end
...@@ -371,16 +406,15 @@ module Gitlab ...@@ -371,16 +406,15 @@ module Gitlab
The `#{ce_branch}` does not apply cleanly to the current EE/master, and The `#{ce_branch}` does not apply cleanly to the current EE/master, and
even though a `#{ee_branch_found}` branch even though a `#{ee_branch_found}` branch
exists in the EE repository, it does not apply cleanly either to exists in #{ee_repo_url}, it does not apply cleanly either to
EE/master! EE/master!
#{conflicting_files_msg} #{conflicting_files_msg}
Please update the `#{ee_branch_found}`, push it again to gitlab-ee, and Please update the `#{ee_branch_found}`, push it again to gitlab-ee, and
retry this build. retry this job.
Stay 💪 ! For more information, see #{STAY_STRONG_LINK_TO_DOCS}
https://docs.gitlab.com/ce/development/automatic_ce_ee_merge.html
#{THANKS_FOR_READING_BANNER} #{THANKS_FOR_READING_BANNER}
} }
end end
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment