Commit 0d7986a8 authored by Dmitriy Zaporozhets's avatar Dmitriy Zaporozhets

Merge branch 'import-timeout' of https://dev.gitlab.org/dzaporozhets/gitlabhq...

Merge branch 'import-timeout' of https://dev.gitlab.org/dzaporozhets/gitlabhq into dzaporozhets/gitlabhq-import-timeout
Signed-off-by: default avatarDmitriy Zaporozhets <dmitriy.zaporozhets@gmail.com>

Conflicts:
	CHANGELOG
	db/schema.rb
parents 58e4e6b0 9064fba0
...@@ -17,6 +17,7 @@ v 6.7.0 ...@@ -17,6 +17,7 @@ v 6.7.0
- Add GFM autocompletion for MergeRequests (Robert Speicher) - Add GFM autocompletion for MergeRequests (Robert Speicher)
- Add webhook when a new tag is pushed (Jeroen van Baarsen) - Add webhook when a new tag is pushed (Jeroen van Baarsen)
- Add button for toggling inline comments in diff view - Add button for toggling inline comments in diff view
- Add retry feature for repository import
v 6.6.2 v 6.6.2
- Fix 500 error on branch/tag create or remove via UI - Fix 500 error on branch/tag create or remove via UI
......
...@@ -5,7 +5,7 @@ class ProjectsController < ApplicationController ...@@ -5,7 +5,7 @@ class ProjectsController < ApplicationController
# Authorize # Authorize
before_filter :authorize_read_project!, except: [:index, :new, :create] before_filter :authorize_read_project!, except: [:index, :new, :create]
before_filter :authorize_admin_project!, only: [:edit, :update, :destroy, :transfer, :archive, :unarchive] before_filter :authorize_admin_project!, only: [:edit, :update, :destroy, :transfer, :archive, :unarchive, :retry_import]
before_filter :require_non_empty_project, only: [:blob, :tree, :graph] before_filter :require_non_empty_project, only: [:blob, :tree, :graph]
layout 'navless', only: [:new, :create, :fork] layout 'navless', only: [:new, :create, :fork]
...@@ -21,16 +21,9 @@ class ProjectsController < ApplicationController ...@@ -21,16 +21,9 @@ class ProjectsController < ApplicationController
def create def create
@project = ::Projects::CreateService.new(current_user, params[:project]).execute @project = ::Projects::CreateService.new(current_user, params[:project]).execute
flash[:notice] = 'Project was successfully created.' if @project.saved?
respond_to do |format| respond_to do |format|
flash[:notice] = 'Project was successfully created.' if @project.saved?
format.html do
if @project.saved?
redirect_to @project
else
render "new"
end
end
format.js format.js
end end
end end
...@@ -55,6 +48,11 @@ class ProjectsController < ApplicationController ...@@ -55,6 +48,11 @@ class ProjectsController < ApplicationController
end end
def show def show
if @project.import_in_progress?
redirect_to import_project_path(@project)
return
end
return authenticate_user! unless @project.public? || current_user return authenticate_user! unless @project.public? || current_user
limit = (params[:limit] || 20).to_i limit = (params[:limit] || 20).to_i
...@@ -67,9 +65,7 @@ class ProjectsController < ApplicationController ...@@ -67,9 +65,7 @@ class ProjectsController < ApplicationController
if @project.empty_repo? if @project.empty_repo?
render "projects/empty", layout: user_layout render "projects/empty", layout: user_layout
else else
if current_user @last_push = current_user.recent_push(@project.id) if current_user
@last_push = current_user.recent_push(@project.id)
end
render :show, layout: user_layout render :show, layout: user_layout
end end
end end
...@@ -77,6 +73,28 @@ class ProjectsController < ApplicationController ...@@ -77,6 +73,28 @@ class ProjectsController < ApplicationController
end end
end end
def import
if project.import_finished?
redirect_to @project
return
end
end
def retry_import
unless @project.import_failed?
redirect_to import_project_path(@project)
end
@project.import_url = params[:project][:import_url]
if @project.save
@project.reload
@project.import_retry
end
redirect_to import_project_path(@project)
end
def destroy def destroy
return access_denied! unless can?(current_user, :remove_project, project) return access_denied! unless can?(current_user, :remove_project, project)
......
...@@ -28,7 +28,6 @@ class Project < ActiveRecord::Base ...@@ -28,7 +28,6 @@ class Project < ActiveRecord::Base
include Gitlab::VisibilityLevel include Gitlab::VisibilityLevel
extend Enumerize extend Enumerize
default_value_for :imported, false
default_value_for :archived, false default_value_for :archived, false
ActsAsTaggableOn.strict_case_match = true ActsAsTaggableOn.strict_case_match = true
...@@ -59,13 +58,10 @@ class Project < ActiveRecord::Base ...@@ -59,13 +58,10 @@ class Project < ActiveRecord::Base
has_one :gemnasium_service, dependent: :destroy has_one :gemnasium_service, dependent: :destroy
has_one :forked_project_link, dependent: :destroy, foreign_key: "forked_to_project_id" has_one :forked_project_link, dependent: :destroy, foreign_key: "forked_to_project_id"
has_one :forked_from_project, through: :forked_project_link has_one :forked_from_project, through: :forked_project_link
# Merge Requests for target project should be removed with it # Merge Requests for target project should be removed with it
has_many :merge_requests, dependent: :destroy, foreign_key: "target_project_id" has_many :merge_requests, dependent: :destroy, foreign_key: "target_project_id"
# Merge requests from source project should be kept when source project was removed # Merge requests from source project should be kept when source project was removed
has_many :fork_merge_requests, foreign_key: "source_project_id", class_name: MergeRequest has_many :fork_merge_requests, foreign_key: "source_project_id", class_name: MergeRequest
has_many :issues, -> { order "state DESC, created_at DESC" }, dependent: :destroy has_many :issues, -> { order "state DESC, created_at DESC" }, dependent: :destroy
has_many :services, dependent: :destroy has_many :services, dependent: :destroy
has_many :events, dependent: :destroy has_many :events, dependent: :destroy
...@@ -74,10 +70,8 @@ class Project < ActiveRecord::Base ...@@ -74,10 +70,8 @@ class Project < ActiveRecord::Base
has_many :snippets, dependent: :destroy, class_name: "ProjectSnippet" has_many :snippets, dependent: :destroy, class_name: "ProjectSnippet"
has_many :hooks, dependent: :destroy, class_name: "ProjectHook" has_many :hooks, dependent: :destroy, class_name: "ProjectHook"
has_many :protected_branches, dependent: :destroy has_many :protected_branches, dependent: :destroy
has_many :users_projects, dependent: :destroy has_many :users_projects, dependent: :destroy
has_many :users, through: :users_projects has_many :users, through: :users_projects
has_many :deploy_keys_projects, dependent: :destroy has_many :deploy_keys_projects, dependent: :destroy
has_many :deploy_keys, through: :deploy_keys_projects has_many :deploy_keys, through: :deploy_keys_projects
...@@ -97,15 +91,12 @@ class Project < ActiveRecord::Base ...@@ -97,15 +91,12 @@ class Project < ActiveRecord::Base
validates :issues_enabled, :wall_enabled, :merge_requests_enabled, validates :issues_enabled, :wall_enabled, :merge_requests_enabled,
:wiki_enabled, inclusion: { in: [true, false] } :wiki_enabled, inclusion: { in: [true, false] }
validates :issues_tracker_id, length: { maximum: 255 }, allow_blank: true validates :issues_tracker_id, length: { maximum: 255 }, allow_blank: true
validates :namespace, presence: true validates :namespace, presence: true
validates_uniqueness_of :name, scope: :namespace_id validates_uniqueness_of :name, scope: :namespace_id
validates_uniqueness_of :path, scope: :namespace_id validates_uniqueness_of :path, scope: :namespace_id
validates :import_url, validates :import_url,
format: { with: URI::regexp(%w(git http https)), message: "should be a valid url" }, format: { with: URI::regexp(%w(git http https)), message: "should be a valid url" },
if: :import? if: :import?
validate :check_limit, on: :create validate :check_limit, on: :create
# Scopes # Scopes
...@@ -118,14 +109,36 @@ class Project < ActiveRecord::Base ...@@ -118,14 +109,36 @@ class Project < ActiveRecord::Base
scope :sorted_by_activity, -> { reorder("projects.last_activity_at DESC") } scope :sorted_by_activity, -> { reorder("projects.last_activity_at DESC") }
scope :personal, ->(user) { where(namespace_id: user.namespace_id) } scope :personal, ->(user) { where(namespace_id: user.namespace_id) }
scope :joined, ->(user) { where("namespace_id != ?", user.namespace_id) } scope :joined, ->(user) { where("namespace_id != ?", user.namespace_id) }
scope :public_only, -> { where(visibility_level: Project::PUBLIC) } scope :public_only, -> { where(visibility_level: Project::PUBLIC) }
scope :public_and_internal_only, -> { where(visibility_level: Project.public_and_internal_levels) } scope :public_and_internal_only, -> { where(visibility_level: Project.public_and_internal_levels) }
scope :non_archived, -> { where(archived: false) } scope :non_archived, -> { where(archived: false) }
enumerize :issues_tracker, in: (Gitlab.config.issues_tracker.keys).append(:gitlab), default: :gitlab enumerize :issues_tracker, in: (Gitlab.config.issues_tracker.keys).append(:gitlab), default: :gitlab
state_machine :import_status, initial: :none do
event :import_start do
transition :none => :started
end
event :import_finish do
transition :started => :finished
end
event :import_fail do
transition :started => :failed
end
event :import_retry do
transition :failed => :started
end
state :started
state :finished
state :failed
after_transition any => :started, :do => :add_import_job
end
class << self class << self
def public_and_internal_levels def public_and_internal_levels
[Project::PUBLIC, Project::INTERNAL] [Project::PUBLIC, Project::INTERNAL]
...@@ -202,12 +215,28 @@ class Project < ActiveRecord::Base ...@@ -202,12 +215,28 @@ class Project < ActiveRecord::Base
id && persisted? id && persisted?
end end
def add_import_job
RepositoryImportWorker.perform_in(2.seconds, id)
end
def import? def import?
import_url.present? import_url.present?
end end
def imported? def imported?
imported import_finished?
end
def import_in_progress?
import? && import_status == 'started'
end
def import_failed?
import_status == 'failed'
end
def import_finished?
import_status == 'finished'
end end
def check_limit def check_limit
......
class ProjectObserver < BaseObserver class ProjectObserver < BaseObserver
def after_create(project) def after_create(project)
project.update_column(:last_activity_at, project.created_at)
return true if project.forked?
if project.import?
RepositoryImportWorker.perform_in(5.seconds, project.id)
else
GitlabShellWorker.perform_async(
:add_repository,
project.path_with_namespace
)
log_info("#{project.owner.name} created a new project \"#{project.name_with_namespace}\"") log_info("#{project.owner.name} created a new project \"#{project.name_with_namespace}\"")
end end
if project.wiki_enabled?
begin
# force the creation of a wiki,
GollumWiki.new(project, project.owner).wiki
rescue GollumWiki::CouldNotCreateWikiError => ex
# Prevent project observer crash
# if failed to create wiki
nil
end
end
end
def after_update(project) def after_update(project)
project.send_move_instructions if project.namespace_id_changed? project.send_move_instructions if project.namespace_id_changed?
project.rename_repo if project.path_changed? project.rename_repo if project.path_changed?
......
...@@ -58,6 +58,29 @@ module Projects ...@@ -58,6 +58,29 @@ module Projects
user: current_user user: current_user
) )
end end
@project.update_column(:last_activity_at, @project.created_at)
if @project.import?
@project.import_start
else
GitlabShellWorker.perform_async(
:add_repository,
@project.path_with_namespace
)
end
if @project.wiki_enabled?
begin
# force the creation of a wiki,
GollumWiki.new(@project, @project.owner).wiki
rescue GollumWiki::CouldNotCreateWikiError => ex
# Prevent project observer crash
# if failed to create wiki
nil
end
end
end end
@project @project
......
- if @project.saved? - if @project.saved?
- if @project.import?
:plain
location.href = "#{import_project_path(@project)}";
- else
:plain :plain
location.href = "#{project_path(@project)}"; location.href = "#{project_path(@project)}";
- else - else
......
= render "home_panel" = render "home_panel"
- if @project.import? && !@project.imported %div.git-empty
.save-project-loader
%center
%h2
%i.icon-spinner.icon-spin
Importing repository.
%p.monospace git clone --bare #{@project.import_url}
%p Please wait while we import the repository for you. Refresh at will.
:javascript
new ProjectImport();
- else
%div.git-empty
%fieldset %fieldset
%legend Git global setup: %legend Git global setup:
%pre.dark %pre.dark
...@@ -41,6 +29,6 @@ ...@@ -41,6 +29,6 @@
git remote add origin #{ content_tag(:span, default_url_to_repo, class: 'clone')} git remote add origin #{ content_tag(:span, default_url_to_repo, class: 'clone')}
git push -u origin master git push -u origin master
- if can? current_user, :remove_project, @project - if can? current_user, :remove_project, @project
.prepend-top-20 .prepend-top-20
= link_to 'Remove project', @project, data: { confirm: remove_project_message(@project)}, method: :delete, class: "btn btn-remove pull-right" = link_to 'Remove project', @project, data: { confirm: remove_project_message(@project)}, method: :delete, class: "btn btn-remove pull-right"
- if @project.import_in_progress?
.save-project-loader
%center
%h2
%i.icon-spinner.icon-spin
Import in progress.
%p.monospace git clone --bare #{@project.import_url}
%p Please wait while we import the repository for you. Refresh at will.
:javascript
new ProjectImport();
- elsif @project.import_failed?
.save-project-loader
%center
%h2
Import failed. Retry?
%hr
- if can?(current_user, :admin_project, @project)
= form_for @project, url: retry_import_project_path(@project), method: :put, html: { class: 'form-horizontal' } do |f|
.form-group.import-url-data
= f.label :import_url, class: 'control-label' do
%span Import existing repo
.col-sm-10
= f.text_field :import_url, class: 'form-control', placeholder: 'https://github.com/randx/six.git'
.bs-callout.bs-callout-info
This url must be publicly accessible or you can add a username and password like this: https://username:password@gitlab.com/company/project.git.
%br
The import will time out after 2 minutes. For big repositories, use a clone/push combination.
.form-actions
= f.submit 'Retry import', class: "btn btn-create", tabindex: 4
...@@ -23,6 +23,7 @@ ...@@ -23,6 +23,7 @@
.col-sm-2 .col-sm-2
.col-sm-10 .col-sm-10
= link_to "#", class: 'js-toggle-button' do = link_to "#", class: 'js-toggle-button' do
%i.icon-edit
%span Customize repository name? %span Customize repository name?
.js-toggle-content.hide .js-toggle-content.hide
.form-group .form-group
...@@ -46,8 +47,10 @@ ...@@ -46,8 +47,10 @@
%span Import existing repo %span Import existing repo
.col-sm-10 .col-sm-10
= f.text_field :import_url, class: 'form-control', placeholder: 'https://github.com/randx/six.git' = f.text_field :import_url, class: 'form-control', placeholder: 'https://github.com/randx/six.git'
.light .bs-callout.bs-callout-info
URL must be cloneable This url must be publicly accessible or you can add a username and password like this: https://username:password@gitlab.com/company/project.git.
%br
The import will time out after 2 minutes. For big repositories, use a clone/push combination.
%hr %hr
.form-group .form-group
......
...@@ -11,11 +11,11 @@ class RepositoryImportWorker ...@@ -11,11 +11,11 @@ class RepositoryImportWorker
project.import_url) project.import_url)
if result if result
project.imported = true project.import_finish
project.save project.save
project.satellite.create unless project.satellite.exists? project.satellite.create unless project.satellite.exists?
else else
project.imported = false project.import_fail
end end
end end
end end
...@@ -179,6 +179,8 @@ Gitlab::Application.routes.draw do ...@@ -179,6 +179,8 @@ Gitlab::Application.routes.draw do
post :archive post :archive
post :unarchive post :unarchive
get :autocomplete_sources get :autocomplete_sources
get :import
put :retry_import
end end
scope module: :projects do scope module: :projects do
......
class AddImportStatusToProject < ActiveRecord::Migration
def change
add_column :projects, :import_status, :string
end
end
class MigrateAlreadyImportedProjects < ActiveRecord::Migration
def up
Project.where(imported: true).update_all(import_status: "finished")
Project.where(imported: false).update_all(import_status: "none")
remove_column :projects, :imported
end
def down
add_column :projects, :imported, :boolean, default: false
Project.where(import_status: 'finished').update_all(imported: true)
end
end
...@@ -11,7 +11,7 @@ ...@@ -11,7 +11,7 @@
# #
# It's strongly recommended that you check this file into your version control system. # It's strongly recommended that you check this file into your version control system.
ActiveRecord::Schema.define(version: 20140305193308) do ActiveRecord::Schema.define(version: 20140313092127) do
# These are extensions that must be enabled in order to support this database # These are extensions that must be enabled in order to support this database
enable_extension "plpgsql" enable_extension "plpgsql"
...@@ -214,10 +214,10 @@ ActiveRecord::Schema.define(version: 20140305193308) do ...@@ -214,10 +214,10 @@ ActiveRecord::Schema.define(version: 20140305193308) do
t.string "issues_tracker_id" t.string "issues_tracker_id"
t.boolean "snippets_enabled", default: true, null: false t.boolean "snippets_enabled", default: true, null: false
t.datetime "last_activity_at" t.datetime "last_activity_at"
t.boolean "imported", default: false, null: false
t.string "import_url" t.string "import_url"
t.integer "visibility_level", default: 0, null: false t.integer "visibility_level", default: 0, null: false
t.boolean "archived", default: false, null: false t.boolean "archived", default: false, null: false
t.string "import_status"
end end
add_index "projects", ["creator_id"], name: "index_projects_on_creator_id", using: :btree add_index "projects", ["creator_id"], name: "index_projects_on_creator_id", using: :btree
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment