Commit 843a29b0 authored by Grzegorz Bizon's avatar Grzegorz Bizon

Merge branch '1499-api-endpoint-for-configuring-pull-mirroring-via-http' into 'master'

API endpoint for configuring repository pull mirroring via HTTP

Closes #1499

See merge request gitlab-org/gitlab-ee!6485
parents 0f04ceec 429b9388
......@@ -15,7 +15,9 @@ class Projects::MirrorsController < Projects::ApplicationController
end
def update
if project.update(mirror_params)
result = ::Projects::UpdateService.new(project, current_user, mirror_params).execute
if result[:status] == :success
flash[:notice] = 'Mirroring settings were successfully updated.'
else
flash[:alert] = project.errors.full_messages.join(', ').html_safe
......
......@@ -669,6 +669,12 @@ POST /projects
| `ci_config_path` | string | no | The path to CI config file |
| `repository_storage` | string | no | Which storage shard the repository is on. Available only to admins |
| `approvals_before_merge` | integer | no | How many approvers should approve merge request by default |
| `mirror` | boolean | no | Enables pull mirroring in a project |
| `mirror_trigger_builds` | boolean | no | Pull mirroring triggers builds |
>**Note**: If your HTTP repository is not publicly accessible,
add authentication information to the URL: `https://username:password@gitlab.company.com/group/project.git`
where `password` is a public access key with the `api` scope enabled.
## Create project for user
......@@ -708,6 +714,12 @@ POST /projects/user/:user_id
| `repository_storage` | string | no | Which storage shard the repository is on. Available only to admins |
| `approvals_before_merge` | integer | no | How many approvers should approve merge request by default |
| `external_authorization_classification_label` | string | no | The classification label for the project |
| `mirror` | boolean | no | Enables pull mirroring in a project |
| `mirror_trigger_builds` | boolean | no | Pull mirroring triggers builds |
>**Note**: If your HTTP repository is not publicly accessible,
add authentication information to the URL: `https://username:password@gitlab.company.com/group/project.git`
where `password` is a public access key with the `api` scope enabled.
## Edit project
......@@ -746,6 +758,15 @@ PUT /projects/:id
| `repository_storage` | string | no | Which storage shard the repository is on. Available only to admins |
| `approvals_before_merge` | integer | no | How many approvers should approve merge request by default |
| `external_authorization_classification_label` | string | no | The classification label for the project |
| `mirror` | boolean | no | Enables pull mirroring in a project |
| `mirror_user_id` | integer | no | User responsible for all the activity surrounding a pull mirror event |
| `mirror_trigger_builds` | boolean | no | Pull mirroring triggers builds |
| `only_mirror_protected_branches` | boolean | no | Only mirror protected branches |
| `mirror_overwrites_diverged_branches` | boolean | no | Pull mirror overwrites diverged branches |
>**Note**: If your HTTP repository is not publicly accessible,
add authentication information to the URL: `https://username:password@gitlab.company.com/group/project.git`
where `password` is a public access key with the `api` scope enabled.
## Fork project
......
......@@ -4,10 +4,6 @@ module EE
extend ::Gitlab::Utils::Override
extend ActiveSupport::Concern
prepended do
include SafeMirrorParams
end
def ssh_host_keys
lookup = SshHostKey.new(project: project, url: params[:ssh_url])
......@@ -26,16 +22,17 @@ module EE
override :update
def update
if project.update(safe_mirror_params)
if project.mirror?
project.force_import_job!
flash[:notice] = "Mirroring settings were successfully updated. The project is being updated."
elsif project.previous_changes.key?('mirror')
flash[:notice] = "Mirroring was successfully disabled."
else
flash[:notice] = "Mirroring settings were successfully updated."
end
result = ::Projects::UpdateService.new(project, current_user, safe_mirror_params).execute
if result[:status] == :success
flash[:notice] =
if project.mirror?
"Mirroring settings were successfully updated. The project is being updated."
elsif project.previous_changes.key?('mirror')
"Mirroring was successfully disabled."
else
"Mirroring settings were successfully updated."
end
else
flash[:alert] = project.errors.full_messages.join(', ').html_safe
end
......@@ -98,8 +95,6 @@ module EE
def safe_mirror_params
params = mirror_params
params[:mirror_user_id] = current_user.id unless valid_mirror_user?(params)
import_data = params[:import_data_attributes]
if import_data.present?
# Prevent Rails from destroying the existing import data
......
......@@ -9,7 +9,7 @@ module EE
private
def project_params_ee
%i[
attrs = %i[
approvals_before_merge
approver_group_ids
approver_ids
......@@ -19,11 +19,22 @@ module EE
repository_size_limit
reset_approvals_on_push
service_desk_enabled
external_authorization_classification_label
ci_cd_only
]
if allow_mirror_params?
attrs + mirror_params
else
attrs
end
end
def mirror_params
%i[
mirror
mirror_trigger_builds
mirror_user_id
external_authorization_classification_label
ci_cd_only
]
end
......@@ -40,5 +51,13 @@ module EE
def active_new_project_tab
project_params[:ci_cd_only] == 'true' ? 'ci_cd_only' : super
end
def allow_mirror_params?
if @project # rubocop:disable Gitlab/ModuleWithInstanceVariables
can?(current_user, :admin_mirror, @project) # rubocop:disable Gitlab/ModuleWithInstanceVariables
else
::Gitlab::CurrentSettings.current_application_settings.mirror_available || current_user&.admin?
end
end
end
end
......@@ -7,8 +7,8 @@ module EE
override :execute
def execute
limit = params.delete(:repository_size_limit)
mirror = params.delete(:mirror)
mirror_user_id = params.delete(:mirror_user_id)
mirror = ::Gitlab::Utils.to_boolean(params.delete(:mirror))
mirror_user_id = current_user.id if mirror
mirror_trigger_builds = params.delete(:mirror_trigger_builds)
ci_cd_only = ::Gitlab::Utils.to_boolean(params.delete(:ci_cd_only))
......@@ -16,7 +16,7 @@ module EE
# Repository size limit comes as MB from the view
project.repository_size_limit = ::Gitlab::Utils.try_megabytes_to_bytes(limit) if limit
if mirror && project.feature_available?(:repository_mirrors)
if mirror && can?(current_user, :admin_mirror, project)
project.mirror = mirror unless mirror.nil?
project.mirror_trigger_builds = mirror_trigger_builds unless mirror_trigger_builds.nil?
project.mirror_user_id = mirror_user_id
......
......@@ -7,16 +7,16 @@ module EE
override :execute
def execute
unless project.feature_available?(:repository_mirrors)
params.delete(:mirror)
params.delete(:mirror_user_id)
params.delete(:mirror_trigger_builds)
end
should_remove_old_approvers = params.delete(:remove_old_approvers)
wiki_was_enabled = project.wiki_enabled?
limit = params.delete(:repository_size_limit)
unless valid_mirror_user?
project.errors.add(:mirror_user_id, 'is invalid')
return project
end
result = super do
# Repository size limit comes as MB from the view
project.repository_size_limit = ::Gitlab::Utils.try_megabytes_to_bytes(limit) if limit
......@@ -34,6 +34,7 @@ module EE
log_audit_events
sync_wiki_on_enable if !wiki_was_enabled && project.wiki_enabled?
project.force_import_job! if params[:mirror].present? && project.mirror?
end
result
......@@ -48,6 +49,15 @@ module EE
private
def valid_mirror_user?
return true unless params[:mirror_user_id].present?
mirror_user_id = params[:mirror_user_id].to_i
mirror_user_id == current_user.id ||
mirror_user_id == project.mirror_user&.id
end
def log_audit_events
EE::Audit::ProjectChangesAuditor.new(current_user, project).execute
end
......
......@@ -10,5 +10,3 @@
- if Gitlab::CurrentSettings.should_check_namespace_plan?
.form-text.text-muted
Mirroring will only be available if the feature is included in the plan of the selected group or user.
= f.hidden_field :mirror_user_id, value: current_user.id
---
title: Enables configuration of pull mirroring through API
merge_request: 6485
author:
type: added
......@@ -18,6 +18,11 @@ module EE
prepended do
expose :repository_storage, if: ->(_project, options) { options[:current_user].try(:admin?) }
expose :approvals_before_merge, if: ->(project, _) { project.feature_available?(:merge_request_approvers) }
expose :mirror, if: ->(project, _) { project.feature_available?(:repository_mirrors) }
expose :mirror_user_id, if: ->(project, _) { project.mirror? }
expose :mirror_trigger_builds, if: ->(project, _) { project.mirror? }
expose :only_mirror_protected_branches, if: ->(project, _) { project.mirror? }
expose :mirror_overwrites_diverged_branches, if: ->(project, _) { project.mirror? }
end
end
......
# frozen_string_literal: true
module EE
module API
module Projects
......@@ -5,11 +7,20 @@ module EE
prepended do
helpers do
extend ::Gitlab::Utils::Override
params :optional_filter_params_ee do
optional :wiki_checksum_failed, type: Grape::API::Boolean, default: false, desc: 'Limit by projects where wiki checksum is failed'
optional :repository_checksum_failed, type: Grape::API::Boolean, default: false, desc: 'Limit by projects where repository checksum is failed'
end
params :optional_update_params_ee do
optional :mirror_user_id, type: Integer, desc: 'User responsible for all the activity surrounding a pull mirror event'
optional :only_mirror_protected_branches, type: Grape::API::Boolean, desc: 'Only mirror protected branches'
optional :mirror_overwrites_diverged_branches, type: Grape::API::Boolean, desc: 'Pull mirror overwrites diverged branches'
optional :import_url, type: String, desc: 'URL from which the project is imported'
end
def apply_filters(projects)
projects = super(projects)
projects = projects.verification_failed_wikis if params[:wiki_checksum_failed]
......@@ -17,6 +28,38 @@ module EE
projects
end
override :verify_update_project_attrs!
def verify_update_project_attrs!(project, attrs)
super
verify_mirror_attrs!(project, attrs)
end
def verify_mirror_attrs!(project, attrs)
unless can?(current_user, :admin_mirror, project)
attrs.delete(:mirror)
attrs.delete(:mirror_user_id)
attrs.delete(:mirror_trigger_builds)
attrs.delete(:only_mirror_protected_branches)
attrs.delete(:mirror_overwrites_diverged_branches)
attrs.delete(:import_data_attributes)
end
end
end
end
class_methods do
extend ::Gitlab::Utils::Override
override :update_params_at_least_one_of
def update_params_at_least_one_of
super.concat [
:approvals_before_merge,
:repository_storage,
:external_authorization_classification_label,
:import_url
]
end
end
end
......
......@@ -42,6 +42,8 @@ describe Admin::GroupsController do
context 'PUT update' do
context 'no license' do
it 'does not update the project_creation_level successfully' do
stub_licensed_features(project_creation_level: false)
expect do
post :update, id: group.to_param, group: { project_creation_level: ::EE::Gitlab::Access::NO_ONE_PROJECT_ACCESS }
end.not_to change { group.reload.project_creation_level }
......
......@@ -15,28 +15,10 @@ describe Projects::MirrorsController do
end
it 'allows to create a remote mirror' do
expect_any_instance_of(EE::Project).to receive(:force_import_job!)
expect do
do_put(project, remote_mirrors_attributes: { '0' => { 'enabled' => 1, 'url' => url } })
end.to change { RemoteMirror.count }.to(1)
end
context 'when remote mirror has the same URL' do
it 'does not allow to create the remote mirror' do
expect do
do_put(project, remote_mirrors_attributes: { '0' => { 'enabled' => 1, 'url' => project.import_url } })
end.not_to change { RemoteMirror.count }
end
context 'with disabled local mirror' do
it 'allows to create a remote mirror' do
expect do
do_put(project, mirror: 0, remote_mirrors_attributes: { '0' => { 'enabled' => 1, 'url' => project.import_url } })
end.to change { RemoteMirror.count }.to(1)
end
end
end
end
context 'when the current project has a remote mirror' do
......@@ -47,7 +29,7 @@ describe Projects::MirrorsController do
end
context 'when trying to create a mirror with the same URL' do
it 'should not setup the mirror' do
it 'does not setup the mirror' do
do_put(project, mirror: true, import_url: remote_mirror.url)
expect(project.reload.mirror).to be_falsey
......@@ -56,9 +38,7 @@ describe Projects::MirrorsController do
end
context 'when trying to create a mirror with a different URL' do
it 'should setup the mirror' do
expect_any_instance_of(EE::Project).to receive(:force_import_job!)
it 'sets up the mirror' do
do_put(project, mirror: true, mirror_user_id: project.owner.id, import_url: 'http://local.dev')
expect(project.reload.mirror).to eq(true)
......@@ -66,16 +46,14 @@ describe Projects::MirrorsController do
end
context 'mirror user is not the current user' do
it 'should only assign the current user' do
expect_any_instance_of(EE::Project).to receive(:force_import_job!)
it 'does not setup the mirror' do
new_user = create(:user)
project.add_maintainer(new_user)
do_put(project, mirror: true, mirror_user_id: new_user.id, import_url: 'http://local.dev')
expect(project.reload.mirror).to eq(true)
expect(project.reload.mirror_user.id).to eq(project.owner.id)
expect(project.reload.mirror).to be_falsey
expect(project.reload.import_url).to be_blank
end
end
end
......@@ -96,7 +74,6 @@ describe Projects::MirrorsController do
it 'creates a new mirror' do
sign_in(admin)
expect_any_instance_of(EE::Project).to receive(:force_import_job!)
expect do
do_put(project, mirror: true, mirror_user_id: admin.id, import_url: url)
......@@ -122,8 +99,6 @@ describe Projects::MirrorsController do
context 'when project does not have a mirror' do
it 'allows to create a mirror' do
expect_any_instance_of(EE::Project).to receive(:force_import_job!)
expect do
do_put(project, mirror: true, mirror_user_id: project.owner.id, import_url: url)
end.to change { Project.mirror.count }.to(1)
......@@ -210,15 +185,13 @@ describe Projects::MirrorsController do
end
it 'only allows the current user to be the mirror user' do
mirror_user = project.mirror_user
other_user = create(:user)
project.add_maintainer(other_user)
do_put(project, { mirror_user_id: other_user.id }, format: :json)
expect(response).to have_gitlab_http_status(200)
expect(project.mirror_user(true)).to eq(mirror_user)
expect(response).to have_gitlab_http_status(422)
expect(json_response['mirror_user_id'].first).to eq("is invalid")
end
end
......
......@@ -20,7 +20,6 @@ describe ProjectsController do
namespace_id: user.namespace.id,
visibility_level: Gitlab::VisibilityLevel::PUBLIC,
mirror: true,
mirror_user_id: user.id,
mirror_trigger_builds: true
}
end
......@@ -144,6 +143,8 @@ describe ProjectsController do
end
it 'updates repository mirror attributes' do
expect_any_instance_of(EE::Project).to receive(:force_import_job!).once
put :update,
namespace_id: project.namespace,
id: project,
......
......@@ -6,7 +6,77 @@ describe API::Projects do
let(:user) { create(:user) }
let(:project) { create(:project, namespace: user.namespace) }
describe 'POST /projects' do
context 'when importing with mirror attributes' do
let(:import_url) { generate(:url) }
let(:mirror_params) do
{
name: "Foo",
mirror: true,
import_url: import_url,
mirror_trigger_builds: true
}
end
it 'creates new project with pull mirroring setup' do
post api('/projects', user), mirror_params
expect(response).to have_gitlab_http_status(201)
expect(Project.first).to have_attributes(
mirror: true,
import_url: import_url,
mirror_user_id: user.id,
mirror_trigger_builds: true
)
end
it 'creates project without mirror settings when repository mirroring feature is disabled' do
stub_licensed_features(repository_mirrors: false)
expect { post api('/projects', user), mirror_params }
.to change { Project.count }.by(1)
expect(response).to have_gitlab_http_status(201)
expect(Project.first).to have_attributes(
mirror: false,
import_url: import_url,
mirror_user_id: nil,
mirror_trigger_builds: false
)
end
context 'when pull mirroring is not available' do
before do
stub_ee_application_setting(mirror_available: false)
end
it 'ignores the mirroring options' do
post api('/projects', user), mirror_params
expect(response).to have_gitlab_http_status(201)
expect(Project.first.mirror?).to be false
end
it 'creates project with mirror settings' do
admin = create(:admin)
post api('/projects', admin), mirror_params
expect(response).to have_gitlab_http_status(201)
expect(Project.first).to have_attributes(
mirror: true,
import_url: import_url,
mirror_user_id: admin.id,
mirror_trigger_builds: true
)
end
end
end
end
describe 'PUT /projects/:id' do
let(:project) { create(:project, namespace: user.namespace) }
before do
enable_external_authorization_service_check
end
......@@ -18,6 +88,98 @@ describe API::Projects do
expect(project.reload.external_authorization_classification_label).to eq('new label')
end
context 'when updating mirror related attributes' do
let(:import_url) { generate(:url) }
let(:mirror_params) do
{
mirror: true,
import_url: import_url,
mirror_user_id: user.id,
mirror_trigger_builds: true,
only_mirror_protected_branches: true,
mirror_overwrites_diverged_branches: true
}
end
context 'when pull mirroring is not available' do
before do
stub_ee_application_setting(mirror_available: false)
end
it 'does not update mirror related attributes' do
put(api("/projects/#{project.id}", user), mirror_params)
expect(response).to have_gitlab_http_status(200)
expect(project.reload.mirror).to be false
end
it 'updates mirror related attributes when user is admin' do
admin = create(:admin)
mirror_params[:mirror_user_id] = admin.id
project.add_maintainer(admin)
expect_any_instance_of(EE::Project).to receive(:force_import_job!).once
put(api("/projects/#{project.id}", admin), mirror_params)
expect(response).to have_gitlab_http_status(200)
expect(project.reload).to have_attributes(
mirror: true,
import_url: import_url,
mirror_user_id: admin.id,
mirror_trigger_builds: true,
only_mirror_protected_branches: true,
mirror_overwrites_diverged_branches: true
)
end
end
it 'updates mirror related attributes' do
expect_any_instance_of(EE::Project).to receive(:force_import_job!).once
put(api("/projects/#{project.id}", user), mirror_params)
expect(response).to have_gitlab_http_status(200)
expect(project.reload).to have_attributes(
mirror: true,
import_url: import_url,
mirror_user_id: user.id,
mirror_trigger_builds: true,
only_mirror_protected_branches: true,
mirror_overwrites_diverged_branches: true
)
end
it 'updates project without mirror attributes when the project is unable to setup repository mirroring' do
stub_licensed_features(repository_mirrors: false)
put(api("/projects/#{project.id}", user), mirror_params)
expect(response).to have_gitlab_http_status(200)
expect(project.reload.mirror).to be false
end
it 'renders an API error when mirror user is invalid' do
invalid_mirror_user = create(:user)
project.add_developer(invalid_mirror_user)
mirror_params[:mirror_user_id] = invalid_mirror_user.id
put(api("/projects/#{project.id}", user), mirror_params)
expect(response).to have_gitlab_http_status(400)
expect(json_response["message"]["mirror_user_id"].first).to eq("is invalid")
end
it 'returns 403 when the user does not have access to mirror settings' do
developer = create(:user)
project.add_developer(developer)
put(api("/projects/#{project.id}", developer), mirror_params)
expect(response).to have_gitlab_http_status(:forbidden)
end
end
end
describe 'GET /projects' do
......
......@@ -20,7 +20,7 @@ describe Projects::MirrorsController do
project: {
mirror: '1',
import_url: '',
mirror_user_id: '1',
mirror_user_id: user.id,
mirror_trigger_builds: '0'
}
......
......@@ -85,8 +85,7 @@ describe Projects::CreateService, '#execute' do
context 'with repository mirror' do
before do
opts.merge!(import_url: 'http://foo.com',
mirror: true,
mirror_user_id: user.id)
mirror: true)
end
context 'when licensed' do
......@@ -229,7 +228,6 @@ describe Projects::CreateService, '#execute' do
visibility_level: Gitlab::VisibilityLevel::PRIVATE,
namespace_id: user.namespace.id,
mirror: true,
mirror_user_id: user.id,
mirror_trigger_builds: true
}
......
......@@ -10,43 +10,21 @@ describe Projects::UpdateService, '#execute' do
context 'repository mirror' do
let!(:opts) do
{
}
end
it 'forces an import job' do
opts = {
import_url: 'http://foo.com',
mirror: true,
mirror_user_id: user.id,
mirror_trigger_builds: true
}
end
context 'when licensed' do
before do
stub_licensed_features(repository_mirrors: true)
end
stub_licensed_features(repository_mirrors: true)
expect(project).to receive(:force_import_job!).once
it 'updates the correct attributes' do
update_project(project, user, opts)
updated_project = project.reload
expect(updated_project).to be_valid
expect(updated_project.mirror).to be true
expect(updated_project.mirror_user_id).to eq(user.id)
expect(updated_project.mirror_trigger_builds).to be true
end
end
context 'when unlicensed' do
before do
stub_licensed_features(repository_mirrors: false)
end
it 'does not update mirror attributes' do
update_project(project, user, opts)
updated_project = project.reload
expect(updated_project).to be_valid
expect(updated_project.mirror).to be false
expect(updated_project.mirror_user_id).to be_nil
expect(updated_project.mirror_trigger_builds).to be false
end
update_project(project, user, opts)
end
end
......
......@@ -32,6 +32,8 @@ module API
optional :repository_storage, type: String, desc: 'Which storage shard the repository is on. Available only to admins'
optional :approvals_before_merge, type: Integer, desc: 'How many approvers should approve merge request by default'
optional :external_authorization_classification_label, type: String, desc: 'The classification label for the project'
optional :mirror, type: Boolean, desc: 'Enables pull mirroring in a project'
optional :mirror_trigger_builds, type: Boolean, desc: 'Pull mirroring triggers builds'
end
params :optional_project_params do
......
......@@ -13,6 +13,10 @@ module API
# EE::API::Projects would override this helper
end
params :optional_update_params_ee do
# EE::API::Projects would override this helper
end
# EE::API::Projects would override this method
def apply_filters(projects)
projects = projects.with_issues_available_for_user(current_user) if params[:with_issues_enabled]
......@@ -21,10 +25,41 @@ module API
projects
end
def verify_update_project_attrs!(project, attrs)
end
end
prepend EE::API::Projects
def self.update_params_at_least_one_of
[
:jobs_enabled,
:resolve_outdated_diff_discussions,
:ci_config_path,
:container_registry_enabled,
:default_branch,
:description,
:issues_enabled,
:lfs_enabled,
:merge_requests_enabled,
:merge_method,
:name,
:only_allow_merge_if_all_discussions_are_resolved,
:only_allow_merge_if_pipeline_succeeds,
:path,
:printing_merge_request_link_enabled,
:public_builds,
:request_access_enabled,
:shared_runners_enabled,
:snippets_enabled,
:tag_list,
:visibility,
:wiki_enabled,
:avatar
]
end
helpers do
params :statistics_params do
optional :statistics, type: Boolean, default: false, desc: 'Include project statistics'
......@@ -254,46 +289,14 @@ module API
success Entities::Project
end
params do
# CE
at_least_one_of_ce =
[
:jobs_enabled,
:resolve_outdated_diff_discussions,
:ci_config_path,
:container_registry_enabled,
:default_branch,
:description,
:issues_enabled,
:lfs_enabled,
:merge_requests_enabled,
:merge_method,
:name,
:only_allow_merge_if_all_discussions_are_resolved,
:only_allow_merge_if_pipeline_succeeds,
:path,
:printing_merge_request_link_enabled,
:public_builds,
:request_access_enabled,
:shared_runners_enabled,
:snippets_enabled,
:tag_list,
:visibility,
:wiki_enabled,
:avatar
]
optional :name, type: String, desc: 'The name of the project'
optional :default_branch, type: String, desc: 'The default branch of the project'
optional :path, type: String, desc: 'The path of the repository'
# EE
at_least_one_of_ee = [
:approvals_before_merge,
:repository_storage,
:external_authorization_classification_label
]
use :optional_project_params
at_least_one_of(*(at_least_one_of_ce + at_least_one_of_ee))
use :optional_update_params_ee
at_least_one_of(*::API::Projects.update_params_at_least_one_of)
end
put ':id' do
authorize_admin_project
......@@ -303,6 +306,8 @@ module API
attrs = translate_params_for_compatibility(attrs)
verify_update_project_attrs!(user_project, attrs)
result = ::Projects::UpdateService.new(user_project, current_user, attrs).execute
if result[:status] == :success
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment