Merge branch 'feature/project-export' into 'master'

Export project functionality

This is a MR for the export functionality of https://gitlab.com/gitlab-org/gitlab-ce/issues/3050, which adds the ability to export single projects.

- [x] members
- DB data
  - [x] issues
  - [x] issue comments
  - [x] merge requests
  - [x] merge request diff
  - [x] merge request comments
  - [x] labels
  - [x] milestones
  - [x] snippets
  - [x] releases
  - [x] events
  - [x] commit statuses
  - [x] CI builds
- File system data
  - [x] Git repository
  - [x] wiki
  - [x] uploads
  - [ ] ~~CI build traces~~
  - [ ] ~~CI build artifacts~~
  - [ ] ~~LFS objects~~
- DB configuration
  - [x] services
  - [x] web hooks
  - [x] protected branches
  - [x] deploy keys
  - [x] CI variables
  - [x] CI triggers

See merge request !3114

CHANGELOG

@@ -99,6 +99,7 @@ v 8.9.0 (unreleased)
   - Improved UX of date pickers on issue & milestone forms
   - Cache on the database if a project has an active external issue tracker.
   - Put project Labels and Milestones pages links under Issues and Merge Requests tabs as subnav
+  - GitLab project import and export functionality
   - All classes in the Banzai::ReferenceParser namespace are now instrumented
   - Remove deprecated issues_tracker and issues_tracker_id from project model
   - Allow users to create confidential issues in private projects

Gemfile

@@ -221,7 +221,7 @@ gem 'jquery-turbolinks', '~> 2.1.0'
 
 gem 'addressable',        '~> 2.3.8'
 gem 'bootstrap-sass',     '~> 3.3.0'
-gem 'font-awesome-rails', '~> 4.2'
+gem 'font-awesome-rails', '~> 4.6.1'
 gem 'gitlab_emoji',       '~> 0.3.0'
 gem 'gon',                '~> 6.0.1'
 gem 'jquery-atwho-rails', '~> 1.3.2'

Gemfile.lock

@@ -246,7 +246,7 @@ GEM
     fog-xml (0.1.2)
       fog-core
       nokogiri (~> 1.5, >= 1.5.11)
-    font-awesome-rails (4.5.0.1)
+    font-awesome-rails (4.6.1.0)
       railties (>= 3.2, < 5.1)
     foreman (0.78.0)
       thor (~> 0.19.1)
@@ -866,7 +866,7 @@ DEPENDENCIES
   fog-google (~> 0.3)
   fog-local (~> 0.3)
   fog-openstack (~> 0.1)
-  font-awesome-rails (~> 4.2)
+  font-awesome-rails (~> 4.6.1)
   foreman
   fuubar (~> 2.0.0)
   gemnasium-gitlab-service (~> 0.2)

app/controllers/application_controller.rb

@@ -24,7 +24,7 @@ class ApplicationController < ActionController::Base
   protect_from_forgery with: :exception
 
   helper_method :abilities, :can?, :current_application_settings
-  helper_method :import_sources_enabled?, :github_import_enabled?, :github_import_configured?, :gitlab_import_enabled?, :gitlab_import_configured?, :bitbucket_import_enabled?, :bitbucket_import_configured?, :gitorious_import_enabled?, :google_code_import_enabled?, :fogbugz_import_enabled?, :git_import_enabled?
+  helper_method :import_sources_enabled?, :github_import_enabled?, :github_import_configured?, :gitlab_import_enabled?, :gitlab_import_configured?, :bitbucket_import_enabled?, :bitbucket_import_configured?, :gitorious_import_enabled?, :google_code_import_enabled?, :fogbugz_import_enabled?, :git_import_enabled?, :gitlab_project_import_enabled?
 
   rescue_from Encoding::CompatibilityError do |exception|
     log_exception(exception)
@@ -326,6 +326,10 @@ class ApplicationController < ActionController::Base
     current_application_settings.import_sources.include?('git')
   end
 
+  def gitlab_project_import_enabled?
+    current_application_settings.import_sources.include?('gitlab_project')
+  end
+
   def two_factor_authentication_required?
     current_application_settings.require_two_factor_authentication
   end

app/controllers/import/gitlab_projects_controller.rb

+class Import::GitlabProjectsController < Import::BaseController
+  before_action :verify_gitlab_project_import_enabled
+
+  def new
+    @namespace_id = project_params[:namespace_id]
+    @namespace_name = Namespace.find(project_params[:namespace_id]).name
+    @path = project_params[:path]
+  end
+
+  def create
+    unless file_is_valid?
+      return redirect_back_or_default(options: { alert: "You need to upload a GitLab project export archive." })
+    end
+
+    @project = Gitlab::ImportExport::ProjectCreator.new(project_params[:namespace_id],
+                                                        current_user,
+                                                        File.expand_path(project_params[:file].path),
+                                                        project_params[:path]).execute
+
+    if @project.saved?
+      redirect_to(
+        project_path(@project),
+        notice: "Project '#{@project.name}' is being imported."
+      )
+    else
+      redirect_to(
+        new_import_gitlab_project_path,
+        alert: "Project could not be imported: #{@project.errors.full_messages.join(', ')}"
+      )
+    end
+  end
+
+  private
+
+  def file_is_valid?
+    project_params[:file] && project_params[:file].respond_to?(:read)
+  end
+
+  def verify_gitlab_project_import_enabled
+    render_404 unless gitlab_project_import_enabled?
+  end
+
+  def project_params
+    params.permit(
+      :path, :namespace_id, :file
+    )
+  end
+end

app/controllers/projects_controller.rb

@@ -7,7 +7,7 @@ class ProjectsController < Projects::ApplicationController
   before_action :assign_ref_vars, :tree, only: [:show], if: :repo_exists?
 
   # Authorize
-  before_action :authorize_admin_project!, only: [:edit, :update, :housekeeping]
+  before_action :authorize_admin_project!, only: [:edit, :update, :housekeeping, :download_export, :export, :remove_export, :generate_new_export]
   before_action :event_filter, only: [:show, :activity]
 
   layout :determine_layout
@@ -186,6 +186,48 @@ class ProjectsController < Projects::ApplicationController
     )
   end
 
+  def export
+    @project.add_export_job(current_user: current_user)
+
+    redirect_to(
+      edit_project_path(@project),
+      notice: "Project export started. A download link will be sent by email."
+    )
+  end
+
+  def download_export
+    export_project_path = @project.export_project_path
+
+    if export_project_path
+      send_file export_project_path, disposition: 'attachment'
+    else
+      redirect_to(
+        edit_project_path(@project),
+        alert: "Project export link has expired. Please generate a new export from your project settings."
+      )
+    end
+  end
+
+  def remove_export
+    if @project.remove_exports
+      flash[:notice] = "Project export has been deleted."
+    else
+      flash[:alert] = "Project export could not be deleted."
+    end
+    redirect_to(edit_project_path(@project))
+  end
+
+  def generate_new_export
+    if @project.remove_exports
+      export
+    else
+      redirect_to(
+        edit_project_path(@project),
+        alert: "Project export could not be deleted."
+      )
+    end
+  end
+
   def toggle_star
     current_user.toggle_star(@project)
     @project.reload

app/mailers/emails/projects.rb

@@ -9,6 +9,19 @@ module Emails
            subject: subject("Project was moved"))
     end
 
+    def project_was_exported_email(current_user, project)
+      @project = project
+      mail(to: current_user.notification_email,
+           subject: subject("Project was exported"))
+    end
+
+    def project_was_not_exported_email(current_user, project, errors)
+      @project = project
+      @errors = errors
+      mail(to: current_user.notification_email,
+           subject: subject("Project export error"))
+    end
+
     def repository_push_email(project_id, opts = {})
       @message =
         Gitlab::Email::Message::RepositoryPush.new(self, project_id, opts)

app/models/application_setting.rb

@@ -123,7 +123,7 @@ class ApplicationSetting < ActiveRecord::Base
       default_project_visibility: Settings.gitlab.default_projects_features['visibility_level'],
       default_snippet_visibility: Settings.gitlab.default_projects_features['visibility_level'],
       restricted_signup_domains: Settings.gitlab['restricted_signup_domains'],
-      import_sources: ['github','bitbucket','gitlab','gitorious','google_code','fogbugz','git'],
+      import_sources: %w[github bitbucket gitlab gitorious google_code fogbugz git gitlab_project],
       shared_runners_enabled: Settings.gitlab_ci['shared_runners_enabled'],
       max_artifacts_size: Settings.artifacts['max_size'],
       require_two_factor_authentication: false,

app/models/ci/pipeline.rb

@@ -170,6 +170,10 @@ module Ci
       builds.where.not(environment: nil).success.pluck(:environment).uniq
     end
 
+    def notes
+      Note.for_commit_id(sha)
+    end
+
     private
 
     def build_builds_for_stages(stages, user, status, trigger_request)

app/models/commit_status.rb

 class CommitStatus < ActiveRecord::Base
   include Statuseable
+  include Importable
 
   self.table_name = 'ci_builds'
 
@@ -7,7 +8,7 @@ class CommitStatus < ActiveRecord::Base
   belongs_to :pipeline, class_name: 'Ci::Pipeline', foreign_key: :commit_id, touch: true
   belongs_to :user
 
-  validates :pipeline, presence: true
+  validates :pipeline, presence: true, unless: :importing?
 
   validates_presence_of :name
 

app/models/concerns/importable.rb

+module Importable
+  extend ActiveSupport::Concern
+
+  attr_accessor :importing
+  alias_method :importing?, :importing
+end

app/models/member.rb

 class Member < ActiveRecord::Base
   include Sortable
+  include Importable
   include Gitlab::Access
 
   attr_accessor :raw_invite_token
@@ -41,11 +42,11 @@ class Member < ActiveRecord::Base
 
   before_validation :generate_invite_token, on: :create, if: -> (member) { member.invite_email.present? }
 
-  after_create :send_invite, if: :invite?
-  after_create :send_request, if: :request?
-  after_create :create_notification_setting, unless: :pending?
-  after_create :post_create_hook, unless: :pending?
-  after_update :post_update_hook, unless: :pending?
+  after_create :send_invite, if: :invite?, unless: :importing?
+  after_create :send_request, if: :request?, unless: :importing?
+  after_create :create_notification_setting, unless: [:pending?, :importing?]
+  after_create :post_create_hook, unless: [:pending?, :importing?]
+  after_update :post_update_hook, unless: [:pending?, :importing?]
   after_destroy :post_destroy_hook, unless: :pending?
   after_destroy :post_decline_request, if: :request?
 

app/models/merge_request.rb

@@ -4,6 +4,7 @@ class MergeRequest < ActiveRecord::Base
   include Referable
   include Sortable
   include Taskable
+  include Importable
 
   belongs_to :target_project, foreign_key: :target_project_id, class_name: "Project"
   belongs_to :source_project, foreign_key: :source_project_id, class_name: "Project"
@@ -13,7 +14,7 @@ class MergeRequest < ActiveRecord::Base
 
   serialize :merge_params, Hash
 
-  after_create :create_merge_request_diff
+  after_create :create_merge_request_diff, unless: :importing
   after_update :update_merge_request_diff
 
   delegate :commits, :diffs, :real_size, to: :merge_request_diff, prefix: nil
@@ -95,12 +96,12 @@ class MergeRequest < ActiveRecord::Base
     end
   end
 
-  validates :source_project, presence: true, unless: :allow_broken
+  validates :source_project, presence: true, unless: [:allow_broken, :importing?]
   validates :source_branch, presence: true
   validates :target_project, presence: true
   validates :target_branch, presence: true
   validates :merge_user, presence: true, if: :merge_when_build_succeeds?
-  validate :validate_branches, unless: :allow_broken
+  validate :validate_branches, unless: [:allow_broken, :importing?]
   validate :validate_fork
 
   scope :by_branch, ->(branch_name) { where("(source_branch LIKE :branch) OR (target_branch LIKE :branch)", branch: branch_name) }

app/models/merge_request_diff.rb

 class MergeRequestDiff < ActiveRecord::Base
   include Sortable
+  include Importable
 
   # Prevent store of diff if commits amount more then 500
   COMMITS_SAFE_SIZE = 100
@@ -22,7 +23,7 @@ class MergeRequestDiff < ActiveRecord::Base
   serialize :st_commits
   serialize :st_diffs
 
-  after_create :reload_content
+  after_create :reload_content, unless: :importing?
 
   def reload_content
     reload_commits

app/models/note.rb

@@ -4,6 +4,7 @@ class Note < ActiveRecord::Base
   include Participable
   include Mentionable
   include Awardable
+  include Importable
 
   default_value_for :system, false
 
@@ -28,11 +29,11 @@ class Note < ActiveRecord::Base
   validates :attachment, file_size: { maximum: :max_attachment_size }
 
   validates :noteable_type, presence: true
-  validates :noteable_id, presence: true, unless: :for_commit?
+  validates :noteable_id, presence: true, unless: [:for_commit?, :importing?]
   validates :commit_id, presence: true, if: :for_commit?
   validates :author, presence: true
 
-  validate unless: :for_commit? do |note|
+  validate unless: [:for_commit?, :importing?] do |note|
     unless note.noteable.try(:project) == note.project
       errors.add(:invalid_project, 'Note and noteable project mismatch')
     end

app/models/project.rb

@@ -367,6 +367,11 @@ class Project < ActiveRecord::Base
 
       joins(join_body).reorder('join_note_counts.amount DESC')
     end
+
+    # Deletes gitlab project export files older than 24 hours
+    def remove_gitlab_exports!
+      Gitlab::Popen.popen(%W(find #{Gitlab::ImportExport.storage_path} -not -path #{Gitlab::ImportExport.storage_path} -mmin +1440 -delete))
+    end
   end
 
   def team
@@ -470,7 +475,7 @@ class Project < ActiveRecord::Base
   end
 
   def import?
-    external_import? || forked?
+    external_import? || forked? || gitlab_project_import?
   end
 
   def no_import?
@@ -501,6 +506,10 @@ class Project < ActiveRecord::Base
     Gitlab::UrlSanitizer.new(import_url).masked_url
   end
 
+  def gitlab_project_import?
+    import_type == 'gitlab_project'
+  end
+
   def check_limit
     unless creator.can_create_project? or namespace.kind == 'group'
       projects_limit = creator.projects_limit
@@ -1096,4 +1105,27 @@ class Project < ActiveRecord::Base
   ensure
     @errors = original_errors
   end
+
+  def add_export_job(current_user:)
+    job_id = ProjectExportWorker.perform_async(current_user.id, self.id)
+
+    if job_id
+      Rails.logger.info "Export job started for project ID #{self.id} with job ID #{job_id}"
+    else
+      Rails.logger.error "Export job failed to start for project ID #{self.id}"
+    end
+  end
+
+  def export_path
+    File.join(Gitlab::ImportExport.storage_path, path_with_namespace)
+  end
+
+  def export_project_path
+    Dir.glob("#{export_path}/*export.tar.gz").max_by { |f| File.ctime(f) }
+  end
+
+  def remove_exports
+    _, status = Gitlab::Popen.popen(%W(find #{export_path} -not -path #{export_path} -delete))
+    status.zero?
+  end
 end

app/services/notification_service.rb

@@ -266,6 +266,14 @@ class NotificationService
     end
   end
 
+  def project_exported(project, current_user)
+    mailer.project_was_exported_email(current_user, project).deliver_later
+  end
+
+  def project_not_exported(project, current_user, errors)
+    mailer.project_was_not_exported_email(current_user, project, errors).deliver_later
+  end
+
   protected
 
   # Get project users with WATCH notification level

app/services/projects/create_service.rb

@@ -80,16 +80,18 @@ module Projects
     def after_create_actions
       log_info("#{@project.owner.name} created a new project \"#{@project.name_with_namespace}\"")
 
-      @project.create_wiki if @project.wiki_enabled?
+      unless @project.gitlab_project_import?
+        @project.create_wiki if @project.wiki_enabled?
 
-      @project.build_missing_services
+        @project.build_missing_services
 
-      @project.create_labels
+        @project.create_labels
+      end
 
       event_service.create_project(@project, current_user)
       system_hook_service.execute_hooks_for(@project, :create)
 
-      unless @project.group
+      unless @project.group || @project.gitlab_project_import?
         @project.team << [current_user, :master, current_user]
       end
     end

app/services/projects/import_export/export_service.rb

+module Projects
+  module ImportExport
+    class ExportService < BaseService
+
+      def execute(_options = {})
+        @shared = Gitlab::ImportExport::Shared.new(relative_path: File.join(project.path_with_namespace, 'work'))
+        save_all
+      end
+
+      private
+
+      def save_all
+        if [version_saver, project_tree_saver, uploads_saver, repo_saver, wiki_repo_saver].all?(&:save)
+          Gitlab::ImportExport::Saver.save(shared: @shared)
+          notify_success
+        else
+          cleanup_and_notify
+        end
+      end
+
+      def version_saver
+        Gitlab::ImportExport::VersionSaver.new(shared: @shared)
+      end
+
+      def project_tree_saver
+        Gitlab::ImportExport::ProjectTreeSaver.new(project: project, shared: @shared)
+      end
+
+      def uploads_saver
+        Gitlab::ImportExport::UploadsSaver.new(project: project, shared: @shared)
+      end
+
+      def repo_saver
+        Gitlab::ImportExport::RepoSaver.new(project: project, shared: @shared)
+      end
+
+      def wiki_repo_saver
+        Gitlab::ImportExport::WikiRepoSaver.new(project: project, shared: @shared)
+      end
+
+      def cleanup_and_notify
+        FileUtils.rm_rf(@shared.export_path)
+
+        notify_error
+        raise Gitlab::ImportExport::Error.new(@shared.errors.join(', '))
+      end
+
+      def notify_success
+        notification_service.project_exported(@project, @current_user)
+      end
+
+      def notify_error
+        notification_service.project_not_exported(@project, @current_user, @shared.errors.join(', '))
+      end
+    end
+  end
+end

app/services/projects/import_service.rb

@@ -9,26 +9,31 @@ module Projects
       'fogbugz',
       'gitlab',
       'github',
-      'google_code'
+      'google_code',
+      'gitlab_project'
     ]
 
     def execute
-      if unknown_url?
-        # In this case, we only want to import issues, not a repository.
-        create_repository
-      else
-        import_repository
-      end
+      add_repository_to_project unless project.gitlab_project_import?
 
       import_data
 
       success
-    rescue Error => e
+    rescue => e
       error(e.message)
     end
 
     private
 
+    def add_repository_to_project
+      if unknown_url?
+        # In this case, we only want to import issues, not a repository.
+        create_repository
+      else
+        import_repository
+      end
+    end
+
     def create_repository
       unless project.create_repository
         raise Error, 'The repository could not be created.'
@@ -46,7 +51,7 @@ module Projects
     def import_data
       return unless has_importer?
 
-      project.repository.before_import
+      project.repository.before_import unless project.gitlab_project_import?
 
       unless importer.execute
         raise Error, 'The remote data could not be imported.'
@@ -58,6 +63,8 @@ module Projects
     end
 
     def importer
+      return Gitlab::ImportExport::Importer.new(project) if @project.gitlab_project_import?
+
       class_name = "Gitlab::#{project.import_type.camelize}Import::Importer"
       class_name.constantize.new(project)
     end

app/views/import/gitlab_projects/new.html.haml

+- page_title "GitLab Import"
+- header_title "Projects", root_path
+%h3.page-title
+  = icon('gitlab')
+  Import an exported GitLab project
+%hr
+
+= form_tag import_gitlab_project_path, class: 'form-horizontal', multipart: true do
+  %p
+    Project will be imported as
+    %strong
+      #{@namespace_name}/#{@path}
+
+  %p
+    To move or copy an entire GitLab project from another GitLab installation to this one, navigate to the original project's settings page, generate an export file, and upload it here.
+  .form-group
+    = hidden_field_tag :namespace_id, @namespace_id
+    = hidden_field_tag :path, @path
+    = label_tag :file, class: 'control-label' do
+      %span GitLab project export
+    .col-sm-10
+      = file_field_tag :file, class: ''
+
+  .form-actions
+    = submit_tag 'Import project', class: 'btn btn-create'

app/views/notify/project_was_exported_email.html.haml

+%p
+  Project #{@project.name} was exported successfully.
+%p
+  The project export can be downloaded from:
+  = link_to download_export_namespace_project_url(@project.namespace, @project) do
+    = @project.name_with_namespace + " export"
+%p
+  The download link will expire in 24 hours.

app/views/notify/project_was_exported_email.text.erb

+Project <%= @project.name %> was exported successfully.
+
+The project export can be downloaded from:
+<%= download_export_namespace_project_url(@project.namespace, @project) %>
+
+The download link will expire in 24 hours.

app/views/notify/project_was_not_exported_email.html.haml

+%p
+  Project #{@project.name} couldn't be exported.
+%p
+  The errors we encountered were:
+
+  %ul
+  - @errors.each do |error|
+    %li
+      error

app/views/notify/project_was_not_exported_email.text.erb

+Project <%= @project.name %> couldn't be exported.
+
+The errors we encountered were:
+
+- @errors.each do |error|
+<%= error %>
\ No newline at end of file

app/views/projects/edit.html.haml

@@ -120,6 +120,42 @@
       = link_to 'Housekeeping', housekeeping_namespace_project_path(@project.namespace, @project),
           method: :post, class: "btn btn-save"
   %hr
+  .row.prepend-top-default
+    .col-lg-3
+      %h4.prepend-top-0
+        Export project
+      %p.append-bottom-0
+        %p
+          Export this project with all its related data in order to move your project to a new GitLab instance. Once the export is finished, you can import the file from the "New Project" page.
+        %p
+          Once the exported file is ready, you will receive a notification email with a download link.
+
+    .col-lg-9
+
+      - if @project.export_project_path
+        = link_to 'Download export',  download_export_namespace_project_path(@project.namespace, @project),
+                method: :get, class: "btn btn-default"
+        = link_to 'Generate new export',  generate_new_export_namespace_project_path(@project.namespace, @project),
+                method: :post, class: "btn btn-default"
+      - else
+        = link_to 'Export project', export_namespace_project_path(@project.namespace, @project),
+                method: :post, class: "btn btn-default"
+
+      .bs-callout.bs-callout-info
+        %p.append-bottom-0
+          %p
+            The following items will be exported:
+          %ul
+            %li Project and wiki repositories
+            %li Project uploads
+            %li Project configuration including web hooks and services
+            %li Issues with comments, merge requests with diffs and comments, labels, milestones, snippets, and other project entities
+          %p
+            The following items will NOT be exported:
+          %ul
+            %li Build traces and artifacts
+            %li LFS objects
+  %hr
   - if can? current_user, :archive_project, @project
     .row.prepend-top-default
       .col-lg-3

app/views/projects/imports/show.html.haml

@@ -7,7 +7,7 @@
         Forking in progress.
       - else
         Import in progress.
-    - unless @project.forked?
+    - if @project.external_import?
       %p.monospace git clone --bare #{@project.safe_import_url}
     %p Please wait while we import the repository for you. Refresh at will.
     :javascript

app/views/projects/new.html.haml

@@ -88,7 +88,12 @@
               - if git_import_enabled?
                 = link_to "#", class: 'btn js-toggle-button import_git' do
                   %i.fa.fa-git
-                  %span Any repo by URL
+                  %span Repo by URL
+
+              - if gitlab_project_import_enabled?
+                = link_to new_import_gitlab_project_path, class: 'btn import_gitlab_project project-submit' do
+                  %i.fa.fa-gitlab
+                  %span GitLab export
 
           .js-toggle-content.hide
             = render "shared/import_form", f: f
@@ -119,6 +124,33 @@
     e.preventDefault();
     var import_modal = $(this).next(".modal").show();
   });
+
   $('.modal-header .close').bind('click', function() {
     $(".modal").hide();
   });
+
+  $('.import_gitlab_project').bind('click', function() {
+    var _href = $("a.import_gitlab_project").attr("href");
+    $(".import_gitlab_project").attr("href", _href + '?namespace_id=' + $("#project_namespace_id").val() + '&path=' + $("#project_path").val());
+  });
+
+  $('.import_gitlab_project').attr('disabled',true)
+  $('.import_gitlab_project').attr('title', 'Project path required.');
+
+  $('.import_gitlab_project').click(function( event ) {
+    if($('.import_gitlab_project').attr('disabled')) {
+      event.preventDefault();
+      new Flash("Please enter a path for the project to be imported to.");
+    }
+  });
+
+  $('#project_path').keyup(function(){
+    if($(this).val().length !=0) {
+      $('.import_gitlab_project').attr('disabled', false);
+      $('.import_gitlab_project').attr('title','');
+      $(".flash-container").html("")
+    } else {
+      $('.import_gitlab_project').attr('disabled',true);
+      $('.import_gitlab_project').attr('title', 'Project path required.');
+    }
+  })

app/workers/gitlab_remove_project_export_worker.rb

+class GitlabRemoveProjectExportWorker
+  include Sidekiq::Worker
+
+  sidekiq_options queue: :default
+
+  def perform
+    Project.remove_gitlab_exports!
+  end
+end

app/workers/project_export_worker.rb

+class ProjectExportWorker
+  include Sidekiq::Worker
+
+  sidekiq_options queue: :gitlab_shell, retry: true
+
+  def perform(current_user_id, project_id)
+    current_user = User.find(current_user_id)
+    project = Project.find(project_id)
+
+    ::Projects::ImportExport::ExportService.new(project, current_user).execute
+  end
+end

config/initializers/1_settings.rb

@@ -291,6 +291,9 @@ Settings.cron_jobs['admin_email_worker']['job_class'] = 'AdminEmailWorker'
 Settings.cron_jobs['repository_archive_cache_worker'] ||= Settingslogic.new({})
 Settings.cron_jobs['repository_archive_cache_worker']['cron'] ||= '0 * * * *'
 Settings.cron_jobs['repository_archive_cache_worker']['job_class'] = 'RepositoryArchiveCacheWorker'
+Settings.cron_jobs['gitlab_remove_project_export_worker'] ||= Settingslogic.new({})
+Settings.cron_jobs['gitlab_remove_project_export_worker']['cron'] ||= '0 * * * *'
+Settings.cron_jobs['gitlab_remove_project_export_worker']['job_class'] = 'GitlabRemoveProjectExportWorker'
 
 #
 # GitLab Shell

config/routes.rb

@@ -171,6 +171,10 @@ Rails.application.routes.draw do
       get   :new_user_map,    path: :user_map
       post  :create_user_map, path: :user_map
     end
+
+    resource :gitlab_project, only: [:create, :new] do
+      post :create
+    end
   end
 
   #
@@ -455,6 +459,10 @@ Rails.application.routes.draw do
         post :housekeeping
         post :toggle_star
         post :markdown_preview
+        post :export
+        post :remove_export
+        post :generate_new_export
+        get :download_export
         get :autocomplete_sources
         get :activity
       end

features/dashboard/new_project.feature

@@ -14,7 +14,7 @@ Background:
   @javascript
   Scenario: I should see instructions on how to import from Git URL
   Given I see "New Project" page
-  When I click on "Any repo by URL"
+  When I click on "Repo by URL"
   Then I see instructions on how to import from Git URL
 
   @javascript

features/steps/dashboard/new_project.rb

@@ -19,7 +19,8 @@ class Spinach::Features::NewProject < Spinach::FeatureSteps
     expect(page).to have_link('GitLab.com')
     expect(page).to have_link('Gitorious.org')
     expect(page).to have_link('Google Code')
-    expect(page).to have_link('Any repo by URL')
+    expect(page).to have_link('Repo by URL')
+    expect(page).to have_link('GitLab export')
   end
 
   step 'I click on "Import project from GitHub"' do
@@ -36,7 +37,7 @@ class Spinach::Features::NewProject < Spinach::FeatureSteps
     end
   end
 
-  step 'I click on "Any repo by URL"' do
+  step 'I click on "Repo by URL"' do
     first('.import_git').click
   end
 

lib/gitlab/current_settings.rb

@@ -36,7 +36,7 @@ module Gitlab
         default_project_visibility: Settings.gitlab.default_projects_features['visibility_level'],
         default_snippet_visibility: Settings.gitlab.default_projects_features['visibility_level'],
         restricted_signup_domains: Settings.gitlab['restricted_signup_domains'],
-        import_sources: ['github','bitbucket','gitlab','gitorious','google_code','fogbugz','git'],
+        import_sources: %w[github bitbucket gitlab gitorious google_code fogbugz git gitlab_project],
         shared_runners_enabled: Settings.gitlab_ci['shared_runners_enabled'],
         max_artifacts_size: Settings.artifacts['max_size'],
         require_two_factor_authentication: false,

lib/gitlab/gitlab_import/project_creator.rb

@@ -11,7 +11,7 @@ module Gitlab
       end
 
       def execute
-        project = ::Projects::CreateService.new(
+        ::Projects::CreateService.new(
           current_user,
           name: repo["name"],
           path: repo["path"],
@@ -22,8 +22,6 @@ module Gitlab
           import_source: repo["path_with_namespace"],
           import_url: repo["http_url_to_repo"].sub("://", "://oauth2:#{@session_data[:gitlab_access_token]}@")
         ).execute
-
-        project
       end
     end
   end

lib/gitlab/import_export.rb

+module Gitlab
+  module ImportExport
+    extend self
+
+    VERSION = '0.1.0'
+
+    def export_path(relative_path:)
+      File.join(storage_path, relative_path)
+    end
+
+    def storage_path
+      File.join(Settings.shared['path'], 'tmp/project_exports')
+    end
+
+    def project_filename
+      "project.json"
+    end
+
+    def project_bundle_filename
+      "project.bundle"
+    end
+
+    def config_file
+      'lib/gitlab/import_export/import_export.yml'
+    end
+
+    def version_filename
+      'VERSION'
+    end
+
+    def version
+      VERSION
+    end
+
+    def reset_tokens?
+      true
+    end
+  end
+end

lib/gitlab/import_export/attributes_finder.rb

+module Gitlab
+  module ImportExport
+    class AttributesFinder
+    
+      def initialize(included_attributes:, excluded_attributes:, methods:)
+        @included_attributes = included_attributes || {}
+        @excluded_attributes = excluded_attributes || {}
+        @methods = methods || {}
+      end
+
+      def find(model_object)
+        parsed_hash = find_attributes_only(model_object)
+        parsed_hash.empty? ? model_object : { model_object => parsed_hash }
+      end
+
+      def parse(model_object)
+        parsed_hash = find_attributes_only(model_object)
+        yield parsed_hash unless parsed_hash.empty?
+      end
+
+      def find_included(value)
+        key = key_from_hash(value)
+        @included_attributes[key].nil? ? {} : { only: @included_attributes[key] }
+      end
+
+      def find_excluded(value)
+        key = key_from_hash(value)
+        @excluded_attributes[key].nil? ? {} : { except: @excluded_attributes[key] }
+      end
+
+      def find_method(value)
+        key = key_from_hash(value)
+        @methods[key].nil? ? {} : { methods: @methods[key] }
+      end
+
+      private
+
+      def find_attributes_only(value)
+        find_included(value).merge(find_excluded(value)).merge(find_method(value))
+      end
+
+      def key_from_hash(value)
+        value.is_a?(Hash) ? value.keys.first : value
+      end
+    end
+  end
+end

lib/gitlab/import_export/command_line_util.rb

+module Gitlab
+  module ImportExport
+    module CommandLineUtil
+      def tar_czf(archive:, dir:)
+        tar_with_options(archive: archive, dir: dir, options: 'czf')
+      end
+
+      def untar_zxf(archive:, dir:)
+        untar_with_options(archive: archive, dir: dir, options: 'zxf')
+      end
+
+      def git_bundle(repo_path:, bundle_path:)
+        execute(%W(#{git_bin_path} --git-dir=#{repo_path} bundle create #{bundle_path} --all))
+      end
+
+      def git_unbundle(repo_path:, bundle_path:)
+        execute(%W(#{git_bin_path} clone --bare #{bundle_path} #{repo_path}))
+      end
+
+      private
+
+      def tar_with_options(archive:, dir:, options:)
+        execute(%W(tar -#{options} #{archive} -C #{dir} .))
+      end
+
+      def untar_with_options(archive:, dir:, options:)
+        execute(%W(tar -#{options} #{archive} -C #{dir}))
+      end
+
+      def execute(cmd)
+        _output, status = Gitlab::Popen.popen(cmd)
+        status.zero?
+      end
+
+      def git_bin_path
+        Gitlab.config.git.bin_path
+      end
+    end
+  end
+end

lib/gitlab/import_export/error.rb

+module Gitlab
+  module ImportExport
+    class Error < StandardError; end
+  end
+end

lib/gitlab/import_export/file_importer.rb

+module Gitlab
+  module ImportExport
+    class FileImporter
+      include Gitlab::ImportExport::CommandLineUtil
+
+      def self.import(*args)
+        new(*args).import
+      end
+
+      def initialize(archive_file:, shared:)
+        @archive_file = archive_file
+        @shared = shared
+      end
+
+      def import
+        FileUtils.mkdir_p(@shared.export_path)
+        decompress_archive
+      rescue => e
+        @shared.error(e)
+        false
+      end
+
+      private
+
+      def decompress_archive
+        untar_zxf(archive: @archive_file, dir: @shared.export_path)
+      end
+    end
+  end
+end

lib/gitlab/import_export/import_export.yml

+# Model relationships to be included in the project import/export
+project_tree:
+  - issues:
+    - notes:
+        :author
+  - :labels
+  - :milestones
+  - snippets:
+    - notes:
+        :author
+  - :releases
+  - :events
+  - project_members:
+    - :user
+  - merge_requests:
+    - notes:
+        :author
+    - :merge_request_diff
+  - pipelines:
+    - notes:
+        :author
+    - :statuses
+  - :variables
+  - :triggers
+  - :deploy_keys
+  - :services
+  - :hooks
+  - :protected_branches
+
+# Only include the following attributes for the models specified.
+included_attributes:
+  project:
+    - :description
+    - :issues_enabled
+    - :merge_requests_enabled
+    - :wiki_enabled
+    - :snippets_enabled
+    - :visibility_level
+    - :archived
+  user:
+    - :id
+    - :email
+    - :username
+  author:
+    - :name
+
+# Do not include the following attributes for the models specified.
+excluded_attributes:
+  snippets:
+    - :expired_at
+
+methods:
+  statuses:
+    - :type
\ No newline at end of file

lib/gitlab/import_export/importer.rb

+module Gitlab
+  module ImportExport
+    class Importer
+
+      def initialize(project)
+        @archive_file = project.import_source
+        @current_user = project.creator
+        @project = project
+        @shared = Gitlab::ImportExport::Shared.new(relative_path: path_with_namespace)
+      end
+
+      def execute
+        Gitlab::ImportExport::FileImporter.import(archive_file: @archive_file,
+                                                  shared: @shared)
+        if check_version! && [project_tree, repo_restorer, wiki_restorer, uploads_restorer].all?(&:restore)
+          project_tree.restored_project
+        else
+          raise Projects::ImportService::Error.new(@shared.errors.join(', '))
+        end
+      end
+
+      private
+
+      def check_version!
+        Gitlab::ImportExport::VersionChecker.check!(shared: @shared)
+      end
+
+      def project_tree
+        @project_tree ||= Gitlab::ImportExport::ProjectTreeRestorer.new(user: @current_user,
+                                                                        shared: @shared,
+                                                                        project: @project)
+      end
+
+      def repo_restorer
+        Gitlab::ImportExport::RepoRestorer.new(path_to_bundle: repo_path,
+                                               shared: @shared,
+                                               project: project_tree.restored_project)
+      end
+
+      def wiki_restorer
+        Gitlab::ImportExport::RepoRestorer.new(path_to_bundle: wiki_repo_path,
+                                               shared: @shared,
+                                               project: ProjectWiki.new(project_tree.restored_project),
+                                               wiki: true)
+      end
+
+      def uploads_restorer
+        Gitlab::ImportExport::UploadsRestorer.new(project: project_tree.restored_project, shared: @shared)
+      end
+
+      def path_with_namespace
+        File.join(@project.namespace.path, @project.path)
+      end
+
+      def repo_path
+        File.join(@shared.export_path, 'project.bundle')
+      end
+
+      def wiki_repo_path
+        File.join(@shared.export_path, 'project.wiki.bundle')
+      end
+    end
+  end
+end

lib/gitlab/import_export/members_mapper.rb

+module Gitlab
+  module ImportExport
+    class MembersMapper
+
+      attr_reader :missing_author_ids
+
+      def initialize(exported_members:, user:, project:)
+        @exported_members = exported_members
+        @user = user
+        @project = project
+        @missing_author_ids = []
+
+        # This needs to run first, as second call would be from #map
+        # which means project members already exist.
+        ensure_default_member!
+      end
+
+      def map
+        @map ||=
+          begin
+            @exported_members.inject(missing_keys_tracking_hash) do |hash, member|
+              existing_user = User.where(find_project_user_query(member)).first
+              old_user_id = member['user']['id']
+              if existing_user && add_user_as_team_member(existing_user, member)
+                hash[old_user_id] = existing_user.id
+              end
+              hash
+            end
+          end
+      end
+
+      def default_user_id
+        @user.id
+      end
+
+      private
+
+      def missing_keys_tracking_hash
+        Hash.new do |_, key|
+          @missing_author_ids << key
+          default_user_id
+        end
+      end
+
+      def ensure_default_member!
+        ProjectMember.create!(user: @user, access_level: ProjectMember::MASTER, source_id: @project.id, importing: true)
+      end
+
+      def add_user_as_team_member(existing_user, member)
+        member['user'] = existing_user
+
+        ProjectMember.create(member_hash(member)).persisted?
+      end
+
+      def member_hash(member)
+        member.except('id').merge(source_id: @project.id, importing: true)
+      end
+
+      def find_project_user_query(member)
+        user_arel[:username].eq(member['user']['username']).or(user_arel[:email].eq(member['user']['email']))
+      end
+
+      def user_arel
+        @user_arel ||= User.arel_table
+      end
+    end
+  end
+end

lib/gitlab/import_export/project_creator.rb

+module Gitlab
+  module ImportExport
+    class ProjectCreator
+
+      def initialize(namespace_id, current_user, file, project_path)
+        @namespace_id = namespace_id
+        @current_user = current_user
+        @file = file
+        @project_path = project_path
+      end
+
+      def execute
+        ::Projects::CreateService.new(
+          @current_user,
+          name: @project_path,
+          path: @project_path,
+          namespace_id: @namespace_id,
+          import_type: "gitlab_project",
+          import_source: @file
+        ).execute
+      end
+    end
+  end
+end

lib/gitlab/import_export/project_tree_restorer.rb

+module Gitlab
+  module ImportExport
+    class ProjectTreeRestorer
+
+      def initialize(user:, shared:, project:)
+        @path = File.join(shared.export_path, 'project.json')
+        @user = user
+        @shared = shared
+        @project = project
+      end
+
+      def restore
+        json = IO.read(@path)
+        @tree_hash = ActiveSupport::JSON.decode(json)
+        @project_members = @tree_hash.delete('project_members')
+        create_relations
+      rescue => e
+        @shared.error(e)
+        false
+      end
+
+      def restored_project
+        @restored_project ||= restore_project
+      end
+
+      private
+
+      def members_mapper
+        @members_mapper ||= Gitlab::ImportExport::MembersMapper.new(exported_members: @project_members,
+                                                                    user: @user,
+                                                                    project: restored_project)
+      end
+
+      # Loops through the tree of models defined in import_export.yml and
+      # finds them in the imported JSON so they can be instantiated and saved
+      # in the DB. The structure and relationships between models are guessed from
+      # the configuration yaml file too.
+      # Finally, it updates each attribute in the newly imported project.
+      def create_relations
+        saved = []
+        default_relation_list.each do |relation|
+          next unless relation.is_a?(Hash) || @tree_hash[relation.to_s].present?
+
+          create_sub_relations(relation, @tree_hash) if relation.is_a?(Hash)
+
+          relation_key = relation.is_a?(Hash) ? relation.keys.first : relation
+          relation_hash = create_relation(relation_key, @tree_hash[relation_key.to_s])
+          saved << restored_project.update_attribute(relation_key, relation_hash)
+        end
+        saved.all?
+      end
+
+      def default_relation_list
+        Gitlab::ImportExport::Reader.new(shared: @shared).tree.reject do |model|
+          model.is_a?(Hash) && model[:project_members]
+        end
+      end
+
+      def restore_project
+        return @project unless @tree_hash
+
+        project_params = @tree_hash.reject { |_key, value| value.is_a?(Array) }
+        @project.update(project_params)
+        @project
+      end
+
+      # Given a relation hash containing one or more models and its relationships,
+      # loops through each model and each object from a model type and
+      # and assigns its correspondent attributes hash from +tree_hash+
+      # Example:
+      # +relation_key+ issues, loops through the list of *issues* and for each individual
+      # issue, finds any subrelations such as notes, creates them and assign them back to the hash
+      def create_sub_relations(relation, tree_hash)
+        relation_key = relation.keys.first.to_s
+        tree_hash[relation_key].each do |relation_item|
+          relation.values.flatten.each do |sub_relation|
+            relation_hash, sub_relation = assign_relation_hash(relation_item, sub_relation)
+            relation_item[sub_relation.to_s] = create_relation(sub_relation, relation_hash) unless relation_hash.blank?
+          end
+        end
+      end
+
+      def assign_relation_hash(relation_item, sub_relation)
+        if sub_relation.is_a?(Hash)
+          relation_hash = relation_item[sub_relation.keys.first.to_s]
+          sub_relation = sub_relation.keys.first
+        else
+          relation_hash = relation_item[sub_relation.to_s]
+        end
+        [relation_hash, sub_relation]
+      end
+
+      def create_relation(relation, relation_hash_list)
+        relation_array = [relation_hash_list].flatten.map do |relation_hash|
+          Gitlab::ImportExport::RelationFactory.create(relation_sym: relation.to_sym,
+                                                       relation_hash: relation_hash.merge('project_id' => restored_project.id),
+                                                       members_mapper: members_mapper,
+                                                       user: @user)
+        end
+
+        relation_hash_list.is_a?(Array) ? relation_array : relation_array.first
+      end
+    end
+  end
+end

lib/gitlab/import_export/project_tree_saver.rb

+module Gitlab
+  module ImportExport
+    class ProjectTreeSaver
+      attr_reader :full_path
+
+      def initialize(project:, shared:)
+        @project = project
+        @shared = shared
+        @full_path = File.join(@shared.export_path, ImportExport.project_filename)
+      end
+
+      def save
+        FileUtils.mkdir_p(@shared.export_path)
+
+        File.write(full_path, project_json_tree)
+        true
+      rescue => e
+        @shared.error(e)
+        false
+      end
+
+      private
+
+      def project_json_tree
+        @project.to_json(Gitlab::ImportExport::Reader.new(shared: @shared).project_tree)
+      end
+    end
+  end
+end

lib/gitlab/import_export/reader.rb

+module Gitlab
+  module ImportExport
+    class Reader
+
+      attr_reader :tree
+
+      def initialize(shared:)
+        @shared = shared
+        config_hash = YAML.load_file(Gitlab::ImportExport.config_file).deep_symbolize_keys
+        @tree = config_hash[:project_tree]
+        @attributes_finder = Gitlab::ImportExport::AttributesFinder.new(included_attributes: config_hash[:included_attributes],
+                                                                        excluded_attributes: config_hash[:excluded_attributes],
+                                                                        methods: config_hash[:methods])
+      end
+
+      # Outputs a hash in the format described here: http://api.rubyonrails.org/classes/ActiveModel/Serializers/JSON.html
+      # for outputting a project in JSON format, including its relations and sub relations.
+      def project_tree
+        @attributes_finder.find_included(:project).merge(include: build_hash(@tree))
+      rescue => e
+        @shared.error(e)
+        false
+      end
+
+      private
+
+      # Builds a hash in the format described here: http://api.rubyonrails.org/classes/ActiveModel/Serializers/JSON.html
+      #
+      # +model_list+ - List of models as a relation tree to be included in the generated JSON, from the _import_export.yml_ file
+      def build_hash(model_list)
+        model_list.map do |model_objects|
+          if model_objects.is_a?(Hash)
+            build_json_config_hash(model_objects)
+          else
+            @attributes_finder.find(model_objects)
+          end
+        end
+      end
+
+      # Called when the model is actually a hash containing other relations (more models)
+      # Returns the config in the right format for calling +to_json+
+      # +model_object_hash+ - A model relationship such as:
+      #   {:merge_requests=>[:merge_request_diff, :notes]}
+      def build_json_config_hash(model_object_hash)
+        @json_config_hash = {}
+
+        model_object_hash.values.flatten.each do |model_object|
+          current_key = model_object_hash.keys.first
+
+          @attributes_finder.parse(current_key) { |hash| @json_config_hash[current_key] ||= hash }
+
+          handle_model_object(current_key, model_object)
+          process_sub_model(current_key, model_object) if model_object.is_a?(Hash)
+        end
+        @json_config_hash
+      end
+
+
+      # If the model is a hash, process the sub_models, which could also be hashes
+      # If there is a list, add to an existing array, otherwise use hash syntax
+      # +current_key+ main model that will be a key in the hash
+      # +model_object+ model or list of models to include in the hash
+      def process_sub_model(current_key, model_object)
+        sub_model_json = build_json_config_hash(model_object).dup
+        @json_config_hash.slice!(current_key)
+
+        if @json_config_hash[current_key] && @json_config_hash[current_key][:include]
+          @json_config_hash[current_key][:include] << sub_model_json
+        else
+          @json_config_hash[current_key] = { include: sub_model_json }
+        end
+      end
+
+      # Creates or adds to an existing hash an individual model or list
+      # +current_key+ main model that will be a key in the hash
+      # +model_object+ model or list of models to include in the hash
+      def handle_model_object(current_key, model_object)
+        if @json_config_hash[current_key]
+          add_model_value(current_key, model_object)
+        else
+          create_model_value(current_key, model_object)
+        end
+      end
+
+      # Constructs a new hash that will hold the configuration for that particular object
+      # It may include exceptions or other attribute detail configuration, parsed by +@attributes_finder+
+      # +current_key+ main model that will be a key in the hash
+      # +value+ existing model to be included in the hash
+      def create_model_value(current_key, value)
+        parsed_hash = { include: value }
+
+        @attributes_finder.parse(value) do |hash|
+          parsed_hash = { include: hash_or_merge(value, hash) }
+        end
+        @json_config_hash[current_key] = parsed_hash
+      end
+
+      # Adds new model configuration to an existing hash with key +current_key+
+      # It may include exceptions or other attribute detail configuration, parsed by +@attributes_finder+
+      # +current_key+ main model that will be a key in the hash
+      # +value+ existing model to be included in the hash
+      def add_model_value(current_key, value)
+        @attributes_finder.parse(value) { |hash| value = { value => hash } }
+        old_values = @json_config_hash[current_key][:include]
+        @json_config_hash[current_key][:include] = ([old_values] + [value]).compact.flatten
+      end
+
+      # Construct a new hash or merge with an existing one a model configuration
+      # This is to fulfil +to_json+ requirements.
+      # +value+ existing model to be included in the hash
+      # +hash+ hash containing configuration generated mainly from +@attributes_finder+
+      def hash_or_merge(value, hash)
+        value.is_a?(Hash) ? value.merge(hash) : { value => hash }
+      end
+    end
+  end
+end

lib/gitlab/import_export/relation_factory.rb

+module Gitlab
+  module ImportExport
+    class RelationFactory
+
+      OVERRIDES = { snippets: :project_snippets,
+                    pipelines: 'Ci::Pipeline',
+                    statuses: 'commit_status',
+                    variables: 'Ci::Variable',
+                    triggers: 'Ci::Trigger',
+                    builds: 'Ci::Build',
+                    hooks: 'ProjectHook' }.freeze
+
+      USER_REFERENCES = %w[author_id assignee_id updated_by_id user_id].freeze
+
+      def self.create(*args)
+        new(*args).create
+      end
+
+      def initialize(relation_sym:, relation_hash:, members_mapper:, user:)
+        @relation_name = OVERRIDES[relation_sym] || relation_sym
+        @relation_hash = relation_hash.except('id', 'noteable_id')
+        @members_mapper = members_mapper
+        @user = user
+      end
+
+      # Creates an object from an actual model with name "relation_sym" with params from
+      # the relation_hash, updating references with new object IDs, mapping users using
+      # the "members_mapper" object, also updating notes if required.
+      def create
+        set_note_author if @relation_name == :notes
+        update_user_references
+        update_project_references
+        reset_ci_tokens if @relation_name == 'Ci::Trigger'
+
+        generate_imported_object
+      end
+
+      private
+
+      def update_user_references
+        USER_REFERENCES.each do |reference|
+          if @relation_hash[reference]
+            @relation_hash[reference] = @members_mapper.map[@relation_hash[reference]]
+          end
+        end
+      end
+
+      # Sets the author for a note. If the user importing the project
+      # has admin access, an actual mapping with new project members
+      # will be used. Otherwise, a note stating the original author name
+      # is left.
+      def set_note_author
+        old_author_id = @relation_hash['author_id']
+
+        # Users with admin access can map users
+        @relation_hash['author_id'] = admin_user? ? @members_mapper.map[old_author_id] : @members_mapper.default_user_id
+
+        author = @relation_hash.delete('author')
+
+        update_note_for_missing_author(author['name']) if missing_author?(old_author_id)
+      end
+
+      def missing_author?(old_author_id)
+        !admin_user? || @members_mapper.missing_author_ids.include?(old_author_id)
+      end
+
+      def missing_author_note(updated_at, author_name)
+        timestamp = updated_at.split('.').first
+        "\n\n *By #{author_name} on #{timestamp} (imported from GitLab project)*"
+      end
+
+      def generate_imported_object
+        if @relation_sym == 'commit_status' # call #trace= method after assigning the other attributes
+          trace = @relation_hash.delete('trace')
+          imported_object do |object|
+            object.trace = trace
+            object.commit_id = nil
+          end
+        else
+          imported_object
+        end
+      end
+
+      def update_project_references
+        project_id = @relation_hash.delete('project_id')
+
+        # project_id may not be part of the export, but we always need to populate it if required.
+        @relation_hash['project_id'] = project_id if relation_class.column_names.include?('project_id')
+        @relation_hash['gl_project_id'] = project_id if @relation_hash['gl_project_id']
+        @relation_hash['target_project_id'] = project_id if @relation_hash['target_project_id']
+        @relation_hash['source_project_id'] = -1 if @relation_hash['source_project_id']
+
+        # If source and target are the same, populate them with the new project ID.
+        if @relation_hash['source_project_id'] && @relation_hash['target_project_id'] &&
+          @relation_hash['target_project_id'] == @relation_hash['source_project_id']
+          @relation_hash['source_project_id'] = project_id
+        end
+      end
+
+      def reset_ci_tokens
+        return unless Gitlab::ImportExport.reset_tokens?
+
+        # If we import/export a project to the same instance, tokens will have to be reset.
+        @relation_hash['token'] = nil
+      end
+
+      def relation_class
+        @relation_class ||= @relation_name.to_s.classify.constantize
+      end
+
+      def imported_object
+        imported_object = relation_class.new(@relation_hash)
+        yield(imported_object) if block_given?
+        imported_object.importing = true if imported_object.respond_to?(:importing)
+        imported_object
+      end
+
+      def update_note_for_missing_author(author_name)
+        @relation_hash['note'] = '*Blank note*' if @relation_hash['note'].blank?
+        @relation_hash['note'] += missing_author_note(@relation_hash['updated_at'], author_name)
+      end
+
+      def admin_user?
+        @user.is_admin?
+      end
+    end
+  end
+end

lib/gitlab/import_export/repo_restorer.rb

+module Gitlab
+  module ImportExport
+    class RepoRestorer
+      include Gitlab::ImportExport::CommandLineUtil
+
+      def initialize(project:, shared:, path_to_bundle:, wiki: false)
+        @project = project
+        @path_to_bundle = path_to_bundle
+        @shared = shared
+        @wiki = wiki
+      end
+
+      def restore
+        return wiki? unless File.exist?(@path_to_bundle)
+
+        FileUtils.mkdir_p(path_to_repo)
+
+        git_unbundle(repo_path: path_to_repo, bundle_path: @path_to_bundle)
+      rescue => e
+        @shared.error(e)
+        false
+      end
+
+      private
+
+      def repos_path
+        Gitlab.config.gitlab_shell.repos_path
+      end
+
+      def path_to_repo
+        @project.repository.path_to_repo
+      end
+
+      def wiki?
+        @wiki
+      end
+    end
+  end
+end

lib/gitlab/import_export/repo_saver.rb

+module Gitlab
+  module ImportExport
+    class RepoSaver
+      include Gitlab::ImportExport::CommandLineUtil
+
+      attr_reader :full_path
+
+      def initialize(project:, shared:)
+        @project = project
+        @shared = shared
+      end
+
+      def save
+        return false if @project.empty_repo?
+
+        @full_path = File.join(@shared.export_path, ImportExport.project_bundle_filename)
+        bundle_to_disk
+      end
+
+      private
+
+      def bundle_to_disk
+        FileUtils.mkdir_p(@shared.export_path)
+        git_bundle(repo_path: path_to_repo, bundle_path: @full_path)
+      rescue => e
+        @shared.error(e)
+        false
+      end
+
+      def path_to_repo
+        @project.repository.path_to_repo
+      end
+    end
+  end
+end

lib/gitlab/import_export/saver.rb

+module Gitlab
+  module ImportExport
+    class Saver
+      include Gitlab::ImportExport::CommandLineUtil
+
+      def self.save(*args)
+        new(*args).save
+      end
+
+      def initialize(shared:)
+        @shared = shared
+      end
+
+      def save
+        if compress_and_save
+          remove_export_path
+          Rails.logger.info("Saved project export #{archive_file}")
+          archive_file
+        else
+          false
+        end
+      rescue => e
+        @shared.error(e)
+        false
+      end
+
+      private
+
+      def compress_and_save
+        tar_czf(archive: archive_file, dir: @shared.export_path)
+      end
+
+      def remove_export_path
+        FileUtils.rm_rf(@shared.export_path)
+      end
+
+      def archive_file
+        @archive_file ||= File.join(@shared.export_path, '..', "#{Time.now.strftime('%Y-%m-%d_%H-%M-%3N')}_project_export.tar.gz")
+      end
+    end
+  end
+end

lib/gitlab/import_export/shared.rb

+module Gitlab
+  module ImportExport
+    class Shared
+
+      attr_reader :errors, :opts
+
+      def initialize(opts)
+        @opts = opts
+        @errors = []
+      end
+
+      def export_path
+        @export_path ||= Gitlab::ImportExport.export_path(relative_path: opts[:relative_path])
+      end
+
+      def error(error)
+        error_out(error.message, caller[0].dup)
+        @errors << error.message
+        # Debug:
+        Rails.logger.error(error.backtrace)
+      end
+
+      private
+
+      def error_out(message, caller)
+        Rails.logger.error("Import/Export error raised on #{caller}: #{message}")
+      end
+    end
+  end
+end

lib/gitlab/import_export/uploads_restorer.rb

+module Gitlab
+  module ImportExport
+    class UploadsRestorer < UploadsSaver
+      def restore
+        return true unless File.directory?(uploads_export_path)
+
+        copy_files(uploads_export_path, uploads_path)
+      rescue => e
+        @shared.error(e)
+        false
+      end
+    end
+  end
+end

lib/gitlab/import_export/uploads_saver.rb

+module Gitlab
+  module ImportExport
+    class UploadsSaver
+
+      def initialize(project:, shared:)
+        @project = project
+        @shared = shared
+      end
+
+      def save
+        return true unless File.directory?(uploads_path)
+
+        copy_files(uploads_path, uploads_export_path)
+      rescue => e
+        @shared.error(e)
+        false
+      end
+
+      private
+
+      def copy_files(source, destination)
+        FileUtils.mkdir_p(destination)
+        FileUtils.copy_entry(source, destination)
+        true
+      end
+
+      def uploads_export_path
+        File.join(@shared.export_path, 'uploads')
+      end
+
+      def uploads_path
+        File.join(Rails.root.join('public/uploads'), @project.path_with_namespace)
+      end
+    end
+  end
+end

lib/gitlab/import_export/version_checker.rb

+module Gitlab
+  module ImportExport
+    class VersionChecker
+
+      def self.check!(*args)
+        new(*args).check!
+      end
+
+      def initialize(shared:)
+        @shared = shared
+      end
+
+      def check!
+        version = File.open(version_file, &:readline)
+        verify_version!(version)
+      rescue => e
+        @shared.error(e)
+        false
+      end
+
+      private
+
+      def version_file
+        File.join(@shared.export_path, Gitlab::ImportExport.version_filename)
+      end
+
+      def verify_version!(version)
+        if Gem::Version.new(version) > Gem::Version.new(Gitlab::ImportExport.version)
+          raise Gitlab::ImportExport::Error("Import version mismatch: Required <= #{Gitlab::ImportExport.version} but was #{version}")
+        else
+          true
+        end
+      end
+    end
+  end
+end

lib/gitlab/import_export/version_saver.rb

+module Gitlab
+  module ImportExport
+    class VersionSaver
+
+      def initialize(shared:)
+        @shared = shared
+      end
+
+      def save
+        FileUtils.mkdir_p(@shared.export_path)
+
+        File.write(version_file, Gitlab::ImportExport.version, mode: 'w')
+      rescue => e
+        @shared.error(e)
+        false
+      end
+
+      private
+
+      def version_file
+        File.join(@shared.export_path, Gitlab::ImportExport.version_filename)
+      end
+    end
+  end
+end

lib/gitlab/import_export/wiki_repo_saver.rb

+module Gitlab
+  module ImportExport
+    class WikiRepoSaver < RepoSaver
+      def save
+        @wiki = ProjectWiki.new(@project)
+        return true unless wiki_repository_exists? # it's okay to have no Wiki
+        bundle_to_disk(File.join(@shared.export_path, project_filename))
+      end
+
+      def bundle_to_disk(full_path)
+        FileUtils.mkdir_p(@shared.export_path)
+        git_bundle(repo_path: path_to_repo, bundle_path: full_path)
+      rescue => e
+        @shared.error(e)
+        false
+      end
+
+      private
+
+      def project_filename
+        "project.wiki.bundle"
+      end
+
+      def path_to_repo
+        @wiki.repository.path_to_repo
+      end
+
+      def wiki_repository_exists?
+        File.exist?(@wiki.repository.path_to_repo) && !@wiki.repository.empty?
+      end
+    end
+  end
+end

lib/gitlab/import_sources.rb

@@ -20,7 +20,8 @@ module Gitlab
           'Gitorious.org'   => 'gitorious',
           'Google Code'     => 'google_code',
           'FogBugz'         => 'fogbugz',
-          'Any repo by URL' => 'git',
+          'Repo by URL'     => 'git',
+          'GitLab export' => 'gitlab_project'
         }
       end
 

spec/features/projects/import_export/import_file_spec.rb

+require 'spec_helper'
+
+feature 'project import', feature: true, js: true do
+  include Select2Helper
+
+  let(:user) { create(:admin) }
+  let!(:namespace) { create(:namespace, name: "asd", owner: user) }
+  let(:file) { File.join(Rails.root, 'spec', 'features', 'projects', 'import_export', 'test_project_export.tar.gz') }
+  let(:export_path) { "#{Dir::tmpdir}/import_file_spec" }
+  let(:project) { Project.last }
+
+  background do
+    allow_any_instance_of(Gitlab::ImportExport).to receive(:storage_path).and_return(export_path)
+    login_as(user)
+  end
+
+  after(:each) do
+    FileUtils.rm_rf(export_path, secure: true)
+  end
+
+  scenario 'user imports an exported project successfully' do
+    expect(Project.all.count).to be_zero
+
+    visit new_project_path
+
+    select2('2', from: '#project_namespace_id')
+    fill_in :project_path, with:'test-project-path', visible: true
+    click_link 'GitLab export'
+
+    expect(page).to have_content('GitLab project export')
+    expect(URI.parse(current_url).query).to eq('namespace_id=2&path=test-project-path')
+
+    attach_file('file', file)
+
+    click_on 'Import project' # import starts
+
+    expect(project).not_to be_nil
+    expect(project.issues).not_to be_empty
+    expect(project.merge_requests).not_to be_empty
+    expect(project.repo_exists?).to be true
+    expect(wiki_exists?).to be true
+    expect(project.import_status).to eq('finished')
+  end
+
+  def wiki_exists?
+    wiki = ProjectWiki.new(project)
+    File.exist?(wiki.repository.path_to_repo) && !wiki.repository.empty?
+  end
+end

spec/lib/gitlab/import_export/members_mapper_spec.rb

+require 'spec_helper'
+
+describe Gitlab::ImportExport::MembersMapper, services: true do
+  describe 'map members' do
+
+    let(:user) { create(:user) }
+    let(:project) { create(:project, :public, name: 'searchable_project') }
+    let(:user2) { create(:user) }
+    let(:exported_user_id) { 99 }
+    let(:exported_members) do
+      [{
+         "id" => 2,
+         "access_level" => 40,
+         "source_id" => 14,
+         "source_type" => "Project",
+         "user_id" => 19,
+         "notification_level" => 3,
+         "created_at" => "2016-03-11T10:21:44.822Z",
+         "updated_at" => "2016-03-11T10:21:44.822Z",
+         "created_by_id" => nil,
+         "invite_email" => nil,
+         "invite_token" => nil,
+         "invite_accepted_at" => nil,
+         "user" =>
+           {
+             "id" => exported_user_id,
+             "email" => user2.email,
+             "username" => user2.username
+           }
+       }]
+    end
+
+    let(:members_mapper) do
+      described_class.new(
+        exported_members: exported_members, user: user, project: project)
+    end
+
+    it 'maps a project member' do
+      expect(members_mapper.map[exported_user_id]).to eq(user2.id)
+    end
+
+    it 'defaults to importer project member if it does not exist' do
+      expect(members_mapper.map[-1]).to eq(user.id)
+    end
+
+    it 'updates missing author IDs on missing project member' do
+      members_mapper.map[-1]
+
+      expect(members_mapper.missing_author_ids.first).to eq(-1)
+    end
+  end
+end

spec/lib/gitlab/import_export/project_tree_restorer_spec.rb

+require 'spec_helper'
+
+describe Gitlab::ImportExport::ProjectTreeRestorer, services: true do
+  describe 'restore project tree' do
+
+    let(:user) { create(:user) }
+    let(:namespace) { create(:namespace, owner: user) }
+    let(:shared) { Gitlab::ImportExport::Shared.new(relative_path: "", project_path: 'path') }
+    let(:project) { create(:empty_project, name: 'project', path: 'project') }
+    let(:project_tree_restorer) { described_class.new(user: user, shared: shared, project: project) }
+    let(:restored_project_json) { project_tree_restorer.restore }
+
+    before do
+      allow(shared).to receive(:export_path).and_return('spec/lib/gitlab/import_export/')
+    end
+
+    context 'JSON' do
+      it 'restores models based on JSON' do
+        expect(restored_project_json).to be true
+      end
+    end
+  end
+end

spec/lib/gitlab/import_export/project_tree_saver_spec.rb

+require 'spec_helper'
+
+describe Gitlab::ImportExport::ProjectTreeSaver, services: true do
+  describe 'saves the project tree into a json object' do
+
+    let(:shared) { Gitlab::ImportExport::Shared.new(relative_path: project.path_with_namespace) }
+    let(:project_tree_saver) { described_class.new(project: project, shared: shared) }
+    let(:export_path) { "#{Dir::tmpdir}/project_tree_saver_spec" }
+    let(:user) { create(:user) }
+    let(:project) { setup_project }
+
+    before do
+      project.team << [user, :master]
+      allow_any_instance_of(Gitlab::ImportExport).to receive(:storage_path).and_return(export_path)
+    end
+
+    after do
+      FileUtils.rm_rf(export_path)
+    end
+
+    it 'saves project successfully' do
+      expect(project_tree_saver.save).to be true
+    end
+
+    context 'JSON' do
+
+      let(:saved_project_json) do
+        project_tree_saver.save
+        project_json(project_tree_saver.full_path)
+      end
+
+      it 'saves the correct json' do
+        expect(saved_project_json).to include({ "visibility_level" => 20 })
+      end
+
+      it 'has events' do
+        expect(saved_project_json['events']).not_to be_empty
+      end
+
+      it 'has milestones' do
+        expect(saved_project_json['milestones']).not_to be_empty
+      end
+
+      it 'has merge requests' do
+        expect(saved_project_json['merge_requests']).not_to be_empty
+      end
+
+      it 'has labels' do
+        expect(saved_project_json['labels']).not_to be_empty
+      end
+
+      it 'has snippets' do
+        expect(saved_project_json['snippets']).not_to be_empty
+      end
+
+      it 'has snippet notes' do
+        expect(saved_project_json['snippets'].first['notes']).not_to be_empty
+      end
+
+      it 'has releases' do
+        expect(saved_project_json['releases']).not_to be_empty
+      end
+
+      it 'has issues' do
+        expect(saved_project_json['issues']).not_to be_empty
+      end
+
+      it 'has issue comments' do
+        expect(saved_project_json['issues'].first['notes']).not_to be_empty
+      end
+
+      it 'has author on issue comments' do
+        expect(saved_project_json['issues'].first['notes'].first['author']).not_to be_empty
+      end
+
+      it 'has project members' do
+        expect(saved_project_json['project_members']).not_to be_empty
+      end
+
+      it 'has merge requests diffs' do
+        expect(saved_project_json['merge_requests'].first['merge_request_diff']).not_to be_empty
+      end
+
+      it 'has merge requests comments' do
+        expect(saved_project_json['merge_requests'].first['notes']).not_to be_empty
+      end
+
+      it 'has author on merge requests comments' do
+        expect(saved_project_json['merge_requests'].first['notes'].first['author']).not_to be_empty
+      end
+
+      it 'has pipeline statuses' do
+        expect(saved_project_json['pipelines'].first['statuses']).not_to be_empty
+      end
+
+      it 'has pipeline builds' do
+        expect(saved_project_json['pipelines'].first['statuses'].count { |hash| hash['type'] == 'Ci::Build'}).to eq(1)
+      end
+
+      it 'has pipeline commits' do
+        expect(saved_project_json['pipelines']).not_to be_empty
+      end
+
+      it 'has ci pipeline notes' do
+        expect(saved_project_json['pipelines'].first['notes']).not_to be_empty
+      end
+    end
+  end
+
+  def setup_project
+    issue = create(:issue, assignee: user)
+    merge_request = create(:merge_request)
+    label = create(:label)
+    snippet = create(:project_snippet)
+    release = create(:release)
+
+    project = create(:project,
+                     :public,
+                     issues: [issue],
+                     merge_requests: [merge_request],
+                     labels: [label],
+                     snippets: [snippet],
+                     releases: [release]
+                    )
+
+    commit_status = create(:commit_status, project: project)
+
+    ci_pipeline = create(:ci_pipeline,
+                       project: project,
+                       sha: merge_request.last_commit.id,
+                       ref: merge_request.source_branch,
+                       statuses: [commit_status])
+
+    create(:ci_build, pipeline: ci_pipeline, project: project)
+    create(:milestone, project: project)
+    create(:note, noteable: issue, project: project)
+    create(:note, noteable: merge_request, project: project)
+    create(:note, noteable: snippet, project: project)
+    create(:note_on_commit,
+           author: user,
+           project: project,
+           commit_id: ci_pipeline.sha)
+    project
+  end
+
+  def project_json(filename)
+    JSON.parse(IO.read(filename))
+  end
+end

spec/lib/gitlab/import_export/reader_spec.rb

+require 'spec_helper'
+
+describe Gitlab::ImportExport::Reader, lib: true  do
+  let(:shared) { Gitlab::ImportExport::Shared.new(relative_path:'') }
+  let(:test_config) { 'spec/support/import_export/import_export.yml' }
+  let(:project_tree_hash) do
+    {
+      only: [:name, :path],
+      include: [:issues, :labels,
+                { merge_requests: {
+                  only: [:id],
+                  except: [:iid],
+                  include: [:merge_request_diff, :merge_request_test]
+                } },
+                { commit_statuses: { include: :commit } }]
+    }
+  end
+
+  before do
+    allow_any_instance_of(Gitlab::ImportExport).to receive(:config_file).and_return(test_config)
+  end
+
+  it 'generates hash from project tree config' do
+    expect(described_class.new(shared: shared).project_tree).to match(project_tree_hash)
+  end
+
+  context 'individual scenarios' do
+
+    it 'generates the correct hash for a single project relation' do
+      setup_yaml(project_tree: [:issues])
+
+      expect(described_class.new(shared: shared).project_tree).to match(include: [:issues])
+    end
+
+    it 'generates the correct hash for a multiple project relation' do
+      setup_yaml(project_tree: [:issues, :snippets])
+
+      expect(described_class.new(shared: shared).project_tree).to match(include: [:issues, :snippets])
+    end
+
+    it 'generates the correct hash for a single sub-relation' do
+      setup_yaml(project_tree: [issues: [:notes]])
+
+      expect(described_class.new(shared: shared).project_tree).to match(include: [{ issues: { include: :notes } }])
+    end
+
+    it 'generates the correct hash for a multiple sub-relation' do
+      setup_yaml(project_tree: [merge_requests: [:notes, :merge_request_diff]])
+
+      expect(described_class.new(shared: shared).project_tree).to match(include: [{ merge_requests: { include: [:notes, :merge_request_diff] } }])
+    end
+
+    it 'generates the correct hash for a sub-relation with another sub-relation' do
+      setup_yaml(project_tree: [merge_requests: [notes: :author]])
+
+      expect(described_class.new(shared: shared).project_tree).to match(include: [{ merge_requests: { include: { notes: { include: :author } } } }])
+    end
+
+    it 'generates the correct hash for a relation with included attributes' do
+      setup_yaml(project_tree: [:issues], included_attributes: { issues: [:name, :description] })
+
+      expect(described_class.new(shared: shared).project_tree).to match(include: [{ issues: { only: [:name, :description] } }])
+    end
+
+    it 'generates the correct hash for a relation with excluded attributes' do
+      setup_yaml(project_tree: [:issues], excluded_attributes: { issues: [:name] })
+
+      expect(described_class.new(shared: shared).project_tree).to match(include: [{ issues: { except: [:name] } }])
+    end
+
+    it 'generates the correct hash for a relation with both excluded and included attributes' do
+      setup_yaml(project_tree: [:issues], excluded_attributes: { issues: [:name] }, included_attributes: { issues: [:description] })
+
+      expect(described_class.new(shared: shared).project_tree).to match(include: [{ issues: { except: [:name], only: [:description] } }])
+    end
+
+    it 'generates the correct hash for a relation with custom methods' do
+      setup_yaml(project_tree: [:issues], methods: { issues: [:name] })
+
+      expect(described_class.new(shared: shared).project_tree).to match(include: [{ issues: { methods: [:name] } }])
+    end
+
+    def setup_yaml(hash)
+      allow(YAML).to receive(:load_file).with(test_config).and_return(hash)
+    end
+  end
+end

spec/lib/gitlab/import_export/repo_bundler_spec.rb

+require 'spec_helper'
+
+describe Gitlab::ImportExport::RepoSaver, services: true do
+  describe 'bundle a project Git repo' do
+
+    let(:user) { create(:user) }
+    let!(:project) { create(:project, :public, name: 'searchable_project') }
+    let(:export_path) { "#{Dir::tmpdir}/project_tree_saver_spec" }
+    let(:shared) { Gitlab::ImportExport::Shared.new(relative_path: project.path_with_namespace) }
+    let(:bundler) { described_class.new(project: project, shared: shared) }
+
+    before do
+      project.team << [user, :master]
+      allow_any_instance_of(Gitlab::ImportExport).to receive(:storage_path).and_return(export_path)
+    end
+
+    after do
+      FileUtils.rm_rf(export_path)
+    end
+
+    it 'bundles the repo successfully' do
+      expect(bundler.save).to be true
+    end
+  end
+end

spec/lib/gitlab/import_export/wiki_repo_bundler_spec.rb

+require 'spec_helper'
+
+describe Gitlab::ImportExport::WikiRepoSaver, services: true do
+  describe 'bundle a wiki Git repo' do
+
+    let(:user) { create(:user) }
+    let!(:project) { create(:project, :public, name: 'searchable_project') }
+    let(:export_path) { "#{Dir::tmpdir}/project_tree_saver_spec" }
+    let(:shared) { Gitlab::ImportExport::Shared.new(relative_path: project.path_with_namespace) }
+    let(:wiki_bundler) { described_class.new(project: project, shared: shared) }
+    let!(:project_wiki) { ProjectWiki.new(project, user) }
+
+    before do
+      project.team << [user, :master]
+      allow_any_instance_of(Gitlab::ImportExport).to receive(:storage_path).and_return(export_path)
+      project_wiki.wiki
+      project_wiki.create_page("index", "test content")
+    end
+
+    after do
+      FileUtils.rm_rf(export_path)
+    end
+
+    it 'bundles the repo successfully' do
+      expect(wiki_bundler.save).to be true
+    end
+  end
+end

spec/support/import_export/import_export.yml

+# Class relationships to be included in the project import/export
+project_tree:
+  - :issues
+  - :labels
+  - merge_requests:
+    - :merge_request_diff
+    - :merge_request_test
+  - commit_statuses:
+    - :commit
+
+included_attributes:
+  project:
+    - :name
+    - :path
+  merge_requests:
+    - :id
+
+excluded_attributes:
+  merge_requests:
+    - :iid
\ No newline at end of file