Commit 043c17ad authored by Rubén Dávila's avatar Rubén Dávila

BE changes for creating Ci/CD projects from GitHub

parent eb091439
class Import::GithubController < Import::BaseController
prepend ::EE::Import::GithubController
before_action :verify_import_enabled
before_action :provider_auth, only: [:status, :jobs, :create]
......@@ -42,7 +44,9 @@ class Import::GithubController < Import::BaseController
target_namespace = find_or_create_namespace(namespace_path, current_user.namespace_path)
if can?(current_user, :create_projects, target_namespace)
project = Gitlab::LegacyGithubImport::ProjectCreator.new(repo, project_name, target_namespace, current_user, access_params, type: provider).execute
project = Gitlab::LegacyGithubImport::ProjectCreator
.new(repo, project_name, target_namespace, current_user, access_params, type: provider)
.execute(extra_project_attrs)
if project.persisted?
render json: ProjectSerializer.new.represent(project)
......@@ -73,15 +77,15 @@ class Import::GithubController < Import::BaseController
end
def new_import_url
public_send("new_import_#{provider}_url") # rubocop:disable GitlabSecurity/PublicSend
public_send("new_import_#{provider}_url", extra_import_params) # rubocop:disable GitlabSecurity/PublicSend
end
def status_import_url
public_send("status_import_#{provider}_url") # rubocop:disable GitlabSecurity/PublicSend
public_send("status_import_#{provider}_url", extra_import_params) # rubocop:disable GitlabSecurity/PublicSend
end
def callback_import_url
public_send("callback_import_#{provider}_url") # rubocop:disable GitlabSecurity/PublicSend
public_send("callback_import_#{provider}_url", extra_import_params) # rubocop:disable GitlabSecurity/PublicSend
end
def provider_unauthorized
......@@ -116,4 +120,12 @@ class Import::GithubController < Import::BaseController
def client_options
{}
end
def extra_project_attrs
{}
end
def extra_import_params
{}
end
end
......@@ -30,6 +30,10 @@
= text_field_tag :personal_access_token, '', class: 'form-control', placeholder: "Personal Access Token", size: 40
= submit_tag 'List your GitHub repositories', class: 'btn btn-success'
-# EE-specific start
= hidden_field_tag :ci_cd_only, params[:ci_cd_only]
-# EE-specific end
- unless github_import_configured?
%hr
%p
......
......@@ -134,6 +134,7 @@
- object_storage:object_storage_migrate_uploads
- admin_emails
- create_github_webhook
- elastic_batch_project_indexer
- elastic_commit_indexer
- elastic_indexer
......
......@@ -73,6 +73,7 @@
# EE-specific queues
- [ldap_group_sync, 2]
- [create_github_webhook, 2]
- [geo, 1]
- [repository_remove_remote, 1]
- [repository_update_mirror, 1]
......
......@@ -1955,6 +1955,7 @@ ActiveRecord::Schema.define(version: 20180306074045) do
t.integer "jobs_cache_index"
t.boolean "mirror_overwrites_diverged_branches"
t.string "external_authorization_classification_label"
t.string "external_webhook_token"
end
add_index "projects", ["ci_id"], name: "index_projects_on_ci_id", using: :btree
......
module EE
module Import
module GithubController
extend ::Gitlab::Utils::Override
override :extra_project_attrs
def extra_project_attrs
super.merge(ci_cd_only: params[:ci_cd_only])
end
override :extra_import_params
def extra_import_params
extra_params = super
extra_params[:ci_cd_only] = true if params[:ci_cd_only].present?
extra_params
end
end
end
end
......@@ -85,6 +85,12 @@ module EE
end
end
def ensure_external_webhook_token
return if external_webhook_token.present?
self.external_webhook_token = Devise.friendly_token
end
def shared_runners_limit_namespace
if Feature.enabled?(:shared_runner_minutes_on_root_namespace)
root_namespace
......
module CiCd
class GithubSetupService
attr_reader :project
def initialize(project)
@project = project
end
def execute
create_webhook
end
private
def create_webhook
::CreateGithubWebhookWorker.perform_async(project.id)
end
end
end
module CiCd
class SetupProject < ::BaseService
def execute
return if project.import_url.blank?
update_project
disable_project_features
setup_external_service
end
private
def update_project
project.update_attributes(
container_registry_enabled: false,
mirror: true,
mirror_trigger_builds: true,
mirror_overwrites_diverged_branches: true,
only_mirror_protected_branches: false,
mirror_user_id: current_user.id
)
end
def disable_project_features
project.project_feature.update_attributes(
issues_access_level: ProjectFeature::DISABLED,
merge_requests_access_level: ProjectFeature::DISABLED,
wiki_access_level: ProjectFeature::DISABLED,
snippets_access_level: ProjectFeature::DISABLED
)
end
def setup_external_service
return unless requires_extra_setup?
service_class.new(@project).execute
end
def requires_extra_setup?
return false if project.import_type.blank?
Gitlab::ImportSources.importer(project.import_type).try(:requires_ci_cd_setup?)
end
def service_class
"CiCd::#{@project.import_type.classify}SetupService".constantize
end
end
end
......@@ -61,7 +61,7 @@ module EE
def setup_ci_cd_project
return unless ::License.feature_available?(:ci_cd_projects)
::Projects::SetupCiCd.new(project, current_user).execute
::CiCd::SetupProject.new(project, current_user).execute
end
def log_audit_event(project)
......
class CreateGithubWebhookWorker
include ApplicationWorker
include GrapeRouteHelpers::NamedRouteMatcher
attr_reader :project
def perform(project_id)
@project = Project.find(project_id)
create_webhook
end
def create_webhook
client.create_hook(
project.import_source,
'web',
{
url: webhook_url,
content_type: 'json',
secret: webhook_token,
insecure_ssl: 1
},
{
events: ['push'],
active: true
}
)
end
private
def client
@client ||= Gitlab::LegacyGithubImport::Client.new(access_token)
end
def access_token
@access_token ||= project.import_data.credentials[:user]
end
def webhook_url
"#{Settings.gitlab.url}#{api_v4_projects_mirror_pull_path(id: project.id)}"
end
def webhook_token
project.ensure_external_webhook_token
project.save if project.changed?
project.external_webhook_token
end
end
# See http://doc.gitlab.com/ce/development/migration_style_guide.html
# for more information on how to write migrations for GitLab.
class AddExternalWebhookTokenToProjects < ActiveRecord::Migration
DOWNTIME = false
def change
add_column :projects, :external_webhook_token, :string
end
end
require_dependency 'declarative_policy'
module API
class ProjectMirror < Grape::API
helpers do
def github_webhook_signature
@github_webhook_signature ||= headers['X-Hub-Signature']
end
def authenticate_from_github_webhook!
return unless github_webhook_signature
unless valid_github_signature?
Guest.can?(:read_project, project) ? unauthorized! : not_found!
end
end
def valid_github_signature?
request.body.rewind
token = project.external_webhook_token
payload_body = request.body.read
signature = 'sha1=' + OpenSSL::HMAC.hexdigest(OpenSSL::Digest.new('sha1'), token, payload_body)
Rack::Utils.secure_compare(signature, github_webhook_signature)
end
def authenticate_with_webhook_token!
if github_webhook_signature
not_found! unless project
authenticate_from_github_webhook!
else
authenticate!
authorize_admin_project
end
end
def project
@project ||= github_webhook_signature ? find_project(params[:id]) : user_project
end
end
params do
requires :id, type: String, desc: 'The ID of a project'
end
resource :projects, requirements: API::PROJECT_ENDPOINT_REQUIREMENTS do
desc 'Triggers a pull mirror operation'
post ":id/mirror/pull" do
authenticate_with_webhook_token!
return render_api_error!('The project is not mirrored', 400) unless project.mirror?
project.force_import_job!
status 200
end
end
end
end
module EE
module Gitlab
module GithubImport
module ParallelImporter
extend ActiveSupport::Concern
class_methods do
def requires_ci_cd_setup?
true
end
end
end
end
end
end
require 'spec_helper'
describe Gitlab::LegacyGithubImport::ProjectCreator do
let(:user) { create(:user) }
let(:namespace) { create(:group, owner: user) }
let(:repo) do
OpenStruct.new(
login: 'vim',
name: 'vim',
full_name: 'asd/vim',
clone_url: 'https://gitlab.com/asd/vim.git'
)
end
subject(:service) do
described_class.new(repo, repo.name, namespace, user, github_access_token: 'asdffg')
end
before do
namespace.add_owner(user)
stub_licensed_features(ci_cd_projects: true)
allow_any_instance_of(EE::Project).to receive(:add_import_job)
allow_any_instance_of(CiCd::SetupProject).to receive(:setup_external_service)
end
describe '#execute' do
context 'creating a CI/CD only project' do
let(:params) { { ci_cd_only: true } }
it 'creates a project' do
expect { service.execute(params) }.to change(Project, :count).by(1)
end
it 'calls the service to setup the project' do
expect(CiCd::SetupProject).to receive_message_chain(:new, :execute)
service.execute(params)
end
end
context 'creating a regular project' do
let(:params) { {} }
it 'creates a project' do
expect { service.execute(params) }.to change(Project, :count).by(1)
end
it "doesn't apply any special setup" do
expect(CiCd::SetupProject).not_to receive(:new)
service.execute(params)
end
end
end
end
......@@ -102,6 +102,18 @@ describe Project do
end
end
describe '#ensure_external_webhook_token' do
let(:project) { create(:project, :repository) }
it "sets external_webhook_token when it's missing" do
project.update_attribute(:external_webhook_token, nil)
expect(project.external_webhook_token).to be_blank
project.ensure_external_webhook_token
expect(project.external_webhook_token).to be_present
end
end
describe 'hard failing a mirror' do
it 'sends a notification' do
project = create(:project, :mirror, :import_started)
......
# -*- coding: utf-8 -*-
require 'spec_helper'
describe API::ProjectMirror do
let(:user) { create(:user) }
let(:user2) { create(:user) }
let(:project) { create(:project, namespace: user.namespace) }
describe 'POST /projects/:id/mirror/pull' do
context 'when the project is not mirrored' do
it 'returns error' do
allow(project).to receive(:mirror?).and_return(false)
post api("/projects/#{project.id}/mirror/pull", user)
expect(response).to have_gitlab_http_status(400)
end
end
context 'when the project is mirrored' do
before do
allow_any_instance_of(Projects::UpdateMirrorService).to receive(:execute).and_return(status: :success)
end
context 'when import state is' do
def project_in_state(state)
project = create(:project, :repository, :mirror, state, namespace: user.namespace)
project.mirror_data.update_attributes(next_execution_timestamp: 10.minutes.from_now)
project
end
it 'none it triggers the pull mirroring operation' do
project = project_in_state(:import_none)
expect(UpdateAllMirrorsWorker).to receive(:perform_async).once
post api("/projects/#{project.id}/mirror/pull", user)
expect(response).to have_gitlab_http_status(200)
end
it 'failed it triggers the pull mirroring operation' do
project = project_in_state(:import_failed)
expect(UpdateAllMirrorsWorker).to receive(:perform_async).once
post api("/projects/#{project.id}/mirror/pull", user)
expect(response).to have_gitlab_http_status(200)
end
it 'finished it triggers the pull mirroring operation' do
project = project_in_state(:import_finished)
expect(UpdateAllMirrorsWorker).to receive(:perform_async).once
post api("/projects/#{project.id}/mirror/pull", user)
expect(response).to have_gitlab_http_status(200)
end
it 'scheduled does not trigger the pull mirroring operation and returns 200' do
project = project_in_state(:import_scheduled)
expect(UpdateAllMirrorsWorker).not_to receive(:perform_async)
post api("/projects/#{project.id}/mirror/pull", user)
expect(response).to have_gitlab_http_status(200)
end
it 'started does not trigger the pull mirroring operation and returns 200' do
project = project_in_state(:import_started)
expect(UpdateAllMirrorsWorker).not_to receive(:perform_async)
post api("/projects/#{project.id}/mirror/pull", user)
expect(response).to have_gitlab_http_status(200)
end
end
context 'when user' do
let(:project_mirrored) { create(:project, :repository, :mirror, :import_finished, namespace: user.namespace) }
def project_member(role, user)
create(:project_member, role, user: user, project: project_mirrored)
end
context 'is unauthenticated' do
it 'returns authentication error' do
post api("/projects/#{project_mirrored.id}/mirror/pull")
expect(response).to have_gitlab_http_status(401)
end
end
context 'is authenticated as developer' do
it 'returns forbidden error' do
project_member(:developer, user2)
post api("/projects/#{project_mirrored.id}/mirror/pull", user2)
expect(response).to have_gitlab_http_status(403)
end
end
context 'is authenticated as reporter' do
it 'returns forbidden error' do
project_member(:reporter, user2)
post api("/projects/#{project_mirrored.id}/mirror/pull", user2)
expect(response).to have_gitlab_http_status(403)
end
end
context 'is authenticated as guest' do
it 'returns forbidden error' do
project_member(:guest, user2)
post api("/projects/#{project_mirrored.id}/mirror/pull", user2)
expect(response).to have_gitlab_http_status(403)
end
end
context 'is authenticated as master' do
it 'triggers the pull mirroring operation' do
project_member(:master, user2)
post api("/projects/#{project_mirrored.id}/mirror/pull", user2)
expect(response).to have_gitlab_http_status(200)
end
end
context 'is authenticated as owner' do
it 'triggers the pull mirroring operation' do
post api("/projects/#{project_mirrored.id}/mirror/pull", user)
expect(response).to have_gitlab_http_status(200)
end
end
end
context 'authenticating from GitHub signature' do
let(:visibility) { Gitlab::VisibilityLevel::PUBLIC }
let(:project_mirrored) { create(:project, :repository, :mirror, :import_finished, visibility: visibility) }
def do_post
post api("/projects/#{project_mirrored.id}/mirror/pull"), {}, { 'X-Hub-Signature' => 'signature' }
end
context "when it's valid" do
before do
Grape::Endpoint.before_each do |endpoint|
allow(endpoint).to receive(:project).and_return(project_mirrored)
allow(endpoint).to receive(:valid_github_signature?).and_return(true)
end
end
it 'syncs the mirror' do
expect(project_mirrored).to receive(:force_import_job!)
do_post
end
end
context "when it's invalid" do
before do
Grape::Endpoint.before_each do |endpoint|
allow(endpoint).to receive(:project).and_return(project_mirrored)
allow(endpoint).to receive(:valid_github_signature?).and_return(false)
end
end
after do
Grape::Endpoint.before_each nil
end
it "doesn't sync the mirror" do
expect(project_mirrored).not_to receive(:force_import_job!)
post api("/projects/#{project_mirrored.id}/mirror/pull"), {}, { 'X-Hub-Signature' => 'signature' }
end
context 'with a public project' do
let(:visibility) { Gitlab::VisibilityLevel::PUBLIC }
it 'returns a 401 status' do
do_post
expect(response).to have_gitlab_http_status(401)
end
end
context 'with an internal project' do
let(:visibility) { Gitlab::VisibilityLevel::INTERNAL }
it 'returns a 404 status' do
do_post
expect(response).to have_gitlab_http_status(404)
end
end
context 'with a private project' do
let(:visibility) { Gitlab::VisibilityLevel::PRIVATE }
it 'returns a 404 status' do
do_post
expect(response).to have_gitlab_http_status(404)
end
end
end
end
end
end
end
require 'spec_helper'
describe CiCd::GithubSetupService do
let(:project) { create(:project) }
subject do
described_class.new(project)
end
describe '#execute' do
it 'creates the webhook in the background' do
expect(CreateGithubWebhookWorker).to receive(:perform_async).with(project.id)
subject.execute
end
end
end
require 'spec_helper'
describe CiCd::SetupProject do
let(:user) { create(:user) }
let(:project) { create(:project, creator: user, import_type: 'github', import_url: 'http://foo.com') }
subject do
described_class.new(project, project.creator)
end
before do
allow(CiCd::GithubSetupService).to receive_message_chain(:new, :execute)
end
it 'sets up pull mirroring on the project' do
subject.execute
expect(project.mirror).to be_truthy
expect(project.mirror_trigger_builds).to be_truthy
expect(project.mirror_user_id).to eq(user.id)
end
it 'disables some features' do
subject.execute
project_feature = project.project_feature
expect(project.container_registry_enabled).to be_falsey
expect(project_feature).not_to be_issues_enabled
expect(project_feature).not_to be_merge_requests_enabled
expect(project_feature).not_to be_wiki_enabled
expect(project_feature.snippets_access_level).to eq(ProjectFeature::DISABLED)
end
context 'when import_url is blank' do
before do
project.update_attribute(:import_url, nil)
end
it "doesn't update the project" do
expect(project).not_to receive(:update_project)
expect(project).not_to receive(:disable_project_features)
subject.execute
end
end
describe '#setup_external_service' do
context 'when import_type is missing' do
it "does not invoke the service class" do
project.update_attribute(:import_type, nil)
expect(CiCd::GithubSetupService).not_to receive(:new)
subject.execute
end
end
context "when importer doesn't require extra setup" do
it "does not invoke the service class" do
allow(Gitlab::GithubImport::ParallelImporter).to receive(:requires_ci_cd_setup?).and_return(false)
expect(CiCd::GithubSetupService).not_to receive(:new)
subject.execute
end
end
context 'whem importer requires extra setup' do
it 'invokes the custom service class' do
expect(CiCd::GithubSetupService).to receive_message_chain(:new, :execute)
subject.execute
end
end
end
end
......@@ -23,7 +23,7 @@ describe Projects::CreateService, '#execute' do
end
it 'calls the service to setup CI/CD on the project' do
expect(Projects::SetupCiCd).to receive_message_chain(:new, :execute)
expect(CiCd::SetupProject).to receive_message_chain(:new, :execute)
create_project(user, opts)
end
......@@ -35,7 +35,7 @@ describe Projects::CreateService, '#execute' do
end
it "doesn't call the service to setup CI/CD on the project" do
expect(Projects::SetupCiCd).not_to receive(:new)
expect(CiCd::SetupProject).not_to receive(:new)
create_project(user, opts)
end
......
require 'spec_helper'
describe CreateGithubWebhookWorker do
include GrapeRouteHelpers::NamedRouteMatcher
let(:project) do
create(:project,
import_source: 'foo/bar',
import_type: 'github',
import_data_attributes: { credentials: { user: 'gh_token' } })
end
subject do
described_class.new
end
describe '#perform' do
before do
project.ensure_external_webhook_token
project.save
end
it 'creates the webhook' do
expect_any_instance_of(Gitlab::LegacyGithubImport::Client).to receive(:create_hook)
.with(
'foo/bar',
'web',
{
url: "http://localhost#{api_v4_projects_mirror_pull_path(id: project.id)}",
content_type: 'json',
secret: project.external_webhook_token,
insecure_ssl: 1
},
{
events: ['push'],
active: true
}
)
subject.perform(project.id)
end
end
end
......@@ -193,6 +193,7 @@ module API
mount ::API::Ldap
mount ::API::LdapGroupLinks
mount ::API::License
mount ::API::ProjectMirror
mount ::API::ProjectPushRule
## EE-specific API V4 endpoints END
......
......@@ -495,17 +495,6 @@ module API
conflict!(error.message)
end
end
desc 'Triggers a pull mirror operation'
post ":id/mirror/pull" do
authorize_admin_project
return render_api_error!('The project is not mirrored', 400) unless user_project.mirror?
user_project.force_import_job!
status 200
end
end
end
end
......@@ -5,6 +5,8 @@ module Gitlab
# The ParallelImporter schedules the importing of a GitHub project using
# Sidekiq.
class ParallelImporter
prepend ::EE::Gitlab::GithubImport::ParallelImporter
attr_reader :project
def self.async?
......
......@@ -12,9 +12,8 @@ module Gitlab
@type = type
end
def execute
::Projects::CreateService.new(
current_user,
def execute(extra_attrs = {})
attrs = {
name: name,
path: name,
description: repo.description,
......@@ -24,7 +23,9 @@ module Gitlab
import_source: repo.full_name,
import_url: import_url,
skip_wiki: skip_wiki
).execute
}.merge!(extra_attrs)
::Projects::CreateService.new(current_user, attrs).execute
end
private
......
......@@ -474,6 +474,7 @@ Project:
- merge_requests_rebase_enabled
- jobs_cache_index
- external_authorization_classification_label
- external_webhook_token
Author:
- name
ProjectFeature:
......
......@@ -1952,146 +1952,6 @@ describe API::Projects do
end
end
describe 'POST /projects/:id/mirror/pull' do
context 'when the project is not mirrored' do
it 'returns error' do
allow(project).to receive(:mirror?).and_return(false)
post api("/projects/#{project.id}/mirror/pull", user)
expect(response).to have_gitlab_http_status(400)
end
end
context 'when the project is mirrored' do
before do
allow_any_instance_of(Projects::UpdateMirrorService).to receive(:execute).and_return(status: :success)
end
context 'when import state is' do
def project_in_state(state)
project = create(:project, :repository, :mirror, state, namespace: user.namespace)
project.mirror_data.update_attributes(next_execution_timestamp: 10.minutes.from_now)
project
end
it 'none it triggers the pull mirroring operation' do
project = project_in_state(:import_none)
expect(UpdateAllMirrorsWorker).to receive(:perform_async).once
post api("/projects/#{project.id}/mirror/pull", user)
expect(response).to have_gitlab_http_status(200)
end
it 'failed it triggers the pull mirroring operation' do
project = project_in_state(:import_failed)
expect(UpdateAllMirrorsWorker).to receive(:perform_async).once
post api("/projects/#{project.id}/mirror/pull", user)
expect(response).to have_gitlab_http_status(200)
end
it 'finished it triggers the pull mirroring operation' do
project = project_in_state(:import_finished)
expect(UpdateAllMirrorsWorker).to receive(:perform_async).once
post api("/projects/#{project.id}/mirror/pull", user)
expect(response).to have_gitlab_http_status(200)
end
it 'scheduled does not trigger the pull mirroring operation and returns 200' do
project = project_in_state(:import_scheduled)
expect(UpdateAllMirrorsWorker).not_to receive(:perform_async)
post api("/projects/#{project.id}/mirror/pull", user)
expect(response).to have_gitlab_http_status(200)
end
it 'started does not trigger the pull mirroring operation and returns 200' do
project = project_in_state(:import_started)
expect(UpdateAllMirrorsWorker).not_to receive(:perform_async)
post api("/projects/#{project.id}/mirror/pull", user)
expect(response).to have_gitlab_http_status(200)
end
end
context 'when user' do
let(:project_mirrored) { create(:project, :repository, :mirror, :import_finished, namespace: user.namespace) }
def project_member(role, user)
create(:project_member, role, user: user, project: project_mirrored)
end
context 'is unauthenticated' do
it 'returns authentication error' do
post api("/projects/#{project_mirrored.id}/mirror/pull")
expect(response).to have_gitlab_http_status(401)
end
end
context 'is authenticated as developer' do
it 'returns forbidden error' do
project_member(:developer, user3)
post api("/projects/#{project_mirrored.id}/mirror/pull", user3)
expect(response).to have_gitlab_http_status(403)
end
end
context 'is authenticated as reporter' do
it 'returns forbidden error' do
project_member(:reporter, user3)
post api("/projects/#{project_mirrored.id}/mirror/pull", user3)
expect(response).to have_gitlab_http_status(403)
end
end
context 'is authenticated as guest' do
it 'returns forbidden error' do
project_member(:guest, user3)
post api("/projects/#{project_mirrored.id}/mirror/pull", user3)
expect(response).to have_gitlab_http_status(403)
end
end
context 'is authenticated as master' do
it 'triggers the pull mirroring operation' do
project_member(:master, user3)
post api("/projects/#{project_mirrored.id}/mirror/pull", user3)
expect(response).to have_gitlab_http_status(200)
end
end
context 'is authenticated as owner' do
it 'triggers the pull mirroring operation' do
post api("/projects/#{project_mirrored.id}/mirror/pull", user)
expect(response).to have_gitlab_http_status(200)
end
end
end
end
end
it_behaves_like 'custom attributes endpoints', 'projects' do
let(:attributable) { project }
let(:other_attributable) { project2 }
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment