Commit 7b5bc32c authored by Douwe Maan's avatar Douwe Maan

Allow projects to be imported from Google Code.

parent 9157985c
......@@ -119,6 +119,22 @@
li {
line-height: 1.5;
}
a[href*="/uploads/"], a[href*="storage.googleapis.com/google-code-attachments/"] {
&:before {
margin-right: 4px;
font: normal normal normal 14px/1 FontAwesome;
font-size: inherit;
text-rendering: auto;
-webkit-font-smoothing: antialiased;
content: "\f0c6";
}
&:hover:before {
text-decoration: none;
}
}
}
@mixin str-truncated($max_width: 82%) {
......
......@@ -62,20 +62,8 @@ ul.notes {
word-wrap: break-word;
@include md-typography;
a[href*="/uploads/"] {
&:before {
margin-right: 4px;
font: normal normal normal 14px/1 FontAwesome;
font-size: inherit;
text-rendering: auto;
-webkit-font-smoothing: antialiased;
content: "\f0c6";
}
&:hover:before {
text-decoration: none;
}
hr {
margin: 10px 0;
}
}
}
......
class Import::GoogleCodeController < Import::BaseController
def new
end
def callback
dump_file = params[:dump_file]
unless dump_file.respond_to?(:read)
return redirect_to :back, alert: "You need to upload a Google Takeout JSON file."
end
begin
dump = JSON.parse(dump_file.read)
rescue
return redirect_to :back, alert: "The uploaded file is not a valid Google Takeout JSON file."
end
unless Gitlab::GoogleCodeImport::Client.new(dump).valid?
return redirect_to :back, alert: "The uploaded file is not a valid Google Takeout JSON file."
end
session[:google_code_dump] = dump
redirect_to status_import_google_code_path
end
def status
unless client.valid?
return redirect_to new_import_google_path
end
@repos = client.repos
@already_added_projects = current_user.created_projects.where(import_type: "google_code")
already_added_projects_names = @already_added_projects.pluck(:import_source)
@repos.reject! { |repo| already_added_projects_names.include? repo.name }
end
def jobs
jobs = current_user.created_projects.where(import_type: "google_code").to_json(only: [:id, :import_status])
render json: jobs
end
def create
@repo_id = params[:repo_id]
repo = client.repo(@repo_id)
@target_namespace = current_user.namespace
@project_name = repo.name
namespace = @target_namespace
@project = Gitlab::GoogleCodeImport::ProjectCreator.new(repo, namespace, current_user).execute
end
private
def client
@client ||= Gitlab::GoogleCodeImport::Client.new(session[:google_code_dump])
end
end
......@@ -27,6 +27,7 @@
# import_type :string(255)
# import_source :string(255)
# avatar :string(255)
# import_data :text
#
require 'carrierwave/orm/activerecord'
......@@ -50,6 +51,8 @@ class Project < ActiveRecord::Base
default_value_for :wall_enabled, false
default_value_for :snippets_enabled, gitlab_config_features.snippets
serialize :import_data, JSON
# set last_activity_at to the same as created_at
after_create :set_last_activity_at
def set_last_activity_at
......@@ -185,6 +188,7 @@ class Project < ActiveRecord::Base
state :failed
after_transition any => :started, do: :add_import_job
after_transition any => :finished, do: :clear_import_data
end
class << self
......@@ -262,6 +266,11 @@ class Project < ActiveRecord::Base
RepositoryImportWorker.perform_in(2.seconds, id)
end
def clear_import_data
self.import_data = nil
self.save
end
def import?
import_url.present?
end
......
%h3.page-title
%i.fa.fa-google
Import projects from Google Code
%hr
= form_tag callback_import_google_code_path, class: 'form-horizontal', multipart: true do
%ul
%li
Use Google Takeout etc
.form-group
= label_tag :dump_file, "Google Takeout JSON file", class: 'control-label'
.col-sm-10
%input{type: "file", name: "dump_file", id: "dump_file"}
.form-actions
= submit_tag 'Select projects to import', class: "btn btn-create"
%h3.page-title
%i.fa.fa-google
Import projects from Google Code
%p.light
Select projects you want to import.
%hr
%p
= button_tag 'Import all projects', class: "btn btn-success js-import-all"
%table.table.import-jobs
%thead
%tr
%th From Google Code
%th To GitLab
%th Status
%tbody
- @already_added_projects.each do |project|
%tr{id: "project_#{project.id}", class: "#{project_status_css_class(project.import_status)}"}
%td
= link_to project.import_source, "https://code.google.com/p/#{project.import_source}", target: "_blank"
%td
%strong= link_to project.path_with_namespace, [project.namespace.becomes(Namespace), project]
%td.job-status
- if project.import_status == 'finished'
%span
%i.fa.fa-check
done
- elsif project.import_status == 'started'
%i.fa.fa-spinner.fa-spin
started
- else
= project.human_import_status_name
- @repos.each do |repo|
%tr{id: "repo_#{repo.id}"}
%td
= link_to repo.name, "https://code.google.com/p/#{repo.name}", target: "_blank"
%td.import-target
= "#{current_user.username}/#{repo.name}"
%td.import-actions.job-status
= button_tag "Import", class: "btn js-add-to-import"
:coffeescript
new ImporterStatus("#{jobs_import_google_code_path}", "#{import_google_code_path}")
......@@ -62,6 +62,10 @@
%i.icon-gitorious.icon-gitorious-small
Gitorious.org
= link_to new_import_google_code_path, class: 'btn' do
%i.fa.fa-google
Google Code
= link_to "#", class: 'btn js-toggle-button' do
%i.fa.fa-git
%span Any repo by URL
......
......@@ -18,6 +18,8 @@ class RepositoryImportWorker
Gitlab::GitlabImport::Importer.new(project).execute
elsif project.import_type == 'bitbucket'
Gitlab::BitbucketImport::Importer.new(project).execute
elsif project.import_type == 'google_code'
Gitlab::GoogleCodeImport::Importer.new(project).execute
else
true
end
......
......@@ -81,6 +81,12 @@ Gitlab::Application.routes.draw do
get :callback
get :jobs
end
resource :google_code, only: [:create, :new], controller: :google_code do
get :status
post :callback
get :jobs
end
end
#
......
class AddImportDataToProject < ActiveRecord::Migration
def change
add_column :projects, :import_data, :text
end
end
......@@ -342,6 +342,7 @@ ActiveRecord::Schema.define(version: 20150328132231) do
t.integer "star_count", default: 0, null: false
t.string "import_type"
t.string "import_source"
t.text "import_data"
end
add_index "projects", ["created_at", "id"], name: "index_projects_on_created_at_and_id", using: :btree
......
module Gitlab
module GoogleCodeImport
class Client
attr_reader :raw_data
def initialize(raw_data)
@raw_data = raw_data
end
def valid?
raw_data.is_a?(Hash) && raw_data["kind"] == "projecthosting#user" && raw_data.has_key?("projects")
end
def repos
@repos ||= raw_data["projects"].map { |raw_repo| GoogleCodeImport::Repository.new(raw_repo) }.select(&:git?)
end
def repo(id)
repos.find { |repo| repo.id == id }
end
end
end
end
module Gitlab
module GoogleCodeImport
class Importer
attr_reader :project, :repo
def initialize(project)
@project = project
@repo = GoogleCodeImport::Repository.new(project.import_data)
@closed_statuses = []
@known_labels = Set.new
end
def execute
return true unless repo.valid?
import_status_labels
import_labels
import_issues
true
end
private
def import_status_labels
repo.raw_data["issuesConfig"]["statuses"].each do |status|
closed = !status["meansOpen"]
@closed_statuses << status["status"] if closed
name = nice_status_name(status["status"])
create_label(name)
@known_labels << name
end
end
def import_labels
repo.raw_data["issuesConfig"]["labels"].each do |label|
name = nice_label_name(label["label"])
create_label(name)
@known_labels << name
end
end
def import_issues
return unless repo.raw_data["issues"]
last_id = 0
deleted_issues = []
issues = repo.raw_data["issues"]["items"]
issues.each_with_index do |raw_issue, i|
while raw_issue["id"] > last_id + 1
last_id += 1
issue = project.issues.create!(
title: "Deleted issue",
description: "*This issue has been deleted*",
author_id: project.creator_id,
state: "closed"
)
deleted_issues << issue
end
last_id = raw_issue["id"]
author = mask_email(raw_issue["author"]["name"])
author_link = raw_issue["author"]["htmlLink"]
date = DateTime.parse(raw_issue["published"]).to_formatted_s(:long)
body = []
body << "*By [#{author}](#{author_link}) on #{date}*"
body << "---"
comments = raw_issue["comments"]["items"]
issue_comment = comments.shift
content = format_content(issue_comment["content"])
if content.blank?
content = "*(No description has been entered for this issue)*"
end
body << content
attachments = format_attachments(raw_issue["id"], 0, issue_comment["attachments"])
if attachments.any?
body << "---"
body += attachments
end
labels = []
raw_issue["labels"].each do |label|
name = nice_label_name(label)
labels << name
unless @known_labels.include?(name)
create_label(name)
@known_labels << name
end
end
labels << nice_status_name(raw_issue["status"])
issue = project.issues.create!(
title: raw_issue["title"],
description: body.join("\n\n"),
author_id: project.creator_id,
state: raw_issue["state"] == "closed" ? "closed" : "opened"
)
issue.add_labels_by_names(labels)
import_issue_comments(issue, comments)
end
deleted_issues.each(&:destroy!)
end
def import_issue_comments(issue, comments)
comments.each_with_index do |raw_comment, i|
next if raw_comment.has_key?("deletedBy")
author = mask_email(raw_comment["author"]["name"])
author_link = raw_comment["author"]["htmlLink"]
date = DateTime.parse(raw_comment["published"]).to_formatted_s(:long)
body = []
body << "*By [#{author}](#{author_link}) on #{date}*"
body << "---"
content = format_content(raw_comment["content"])
if content.blank?
content = "*(No comment has been entered for this change)*"
end
body << content
updates = format_updates(raw_comment["updates"])
if updates.any?
body << "---"
body += updates
end
attachments = format_attachments(issue.iid, raw_comment["id"], raw_comment["attachments"])
if attachments.any?
body << "---"
body += attachments
end
comment = issue.notes.create!(
project_id: project.id,
author_id: project.creator_id,
note: body.join("\n\n")
)
end
end
def nice_label_color(name)
case name
when /\AComponent:/
"#fff39e"
when /\AOpSys:/
"#e2e2e2"
when /\AMilestone:/
"#fee3ff"
when *@closed_statuses.map { |s| nice_status_name(s) }
"#cfcfcf"
when "Status: New"
"#428bca"
when "Status: Accepted"
"#5cb85c"
when "Status: NeedInfo"
"#f0ad4e"
when "Status: Started"
"#8e44ad"
when "Status: Wishlist"
"#a8d695"
#
when "Priority: Critical"
"#ffcfcf"
when "Priority: High"
"#deffcf"
when "Priority: Medium"
"#fff5cc"
when "Priority: Low"
"#cfe9ff"
#
when "Type: Defect"
"#d9534f"
when "Type: Enhancement"
"#44ad8e"
when "Type: Other"
"#7f8c8d"
when "Type: Review"
"#8e44ad"
when "Type: Task"
"#4b6dd0"
else
"#e2e2e2"
end
end
def nice_label_name(name)
name.sub("-", ": ")
end
def nice_status_name(name)
"Status: #{name}"
end
def mask_email(author)
parts = author.split("@", 2)
parts[0] = "#{parts[0][0...-3]}..."
parts.join("@")
end
def linkify_issues(s)
s.gsub(/([Ii]ssue) ([0-9]+)/, '\1 #\2')
end
def escape_for_markdown(s)
s = s.gsub("*", "\\*")
s = s.gsub("#", "\\#")
s = s.gsub("`", "\\`")
s = s.gsub(":", "\\:")
s = s.gsub("-", "\\-")
s = s.gsub("+", "\\+")
s = s.gsub("_", "\\_")
s = s.gsub("(", "\\(")
s = s.gsub(")", "\\)")
s = s.gsub("[", "\\[")
s = s.gsub("]", "\\]")
s = s.gsub("<", "\\<")
s = s.gsub(">", "\\>")
s = s.gsub("\r", "")
s = s.gsub("\n", " \n")
s
end
def create_label(name)
color = nice_label_color(name)
project.labels.create!(name: name, color: color)
end
def format_content(raw_content)
linkify_issues(escape_for_markdown(raw_content))
end
def format_updates(raw_updates)
updates = []
if raw_updates.has_key?("status")
updates << "*Status: #{raw_updates["status"]}*"
end
if raw_updates.has_key?("cc")
cc = raw_updates["cc"].map do |l|
deleted = l.start_with?("-")
l = l[1..-1] if deleted
l = mask_email(l)
l = "~~#{l}~~" if deleted
l
end
updates << "*Cc: #{cc.join(", ")}*"
end
if raw_updates.has_key?("labels")
labels = raw_updates["labels"].map do |l|
deleted = l.start_with?("-")
l = l[1..-1] if deleted
l = nice_label_name(l)
l = "~~#{l}~~" if deleted
l
end
updates << "*Labels: #{labels.join(", ")}*"
end
if raw_updates.has_key?("owner")
updates << "*Owner: #{raw_updates["owner"]}*"
end
if raw_updates.has_key?("mergedInto")
updates << "*Merged into: ##{raw_updates["mergedInto"]}*"
end
if raw_updates.has_key?("blockedOn")
blocked_ons = raw_updates["blockedOn"].map do |raw_blocked_on|
name, id = raw_blocked_on.split(":", 2)
if name == project.import_source
"##{id}"
else
"#{project.namespace.path}/#{name}##{id}"
end
end
updates << "*Blocked on: #{blocked_ons.join(", ")}*"
end
if raw_updates.has_key?("blocking")
blockings = raw_updates["blocking"].map do |raw_blocked_on|
name, id = raw_blocked_on.split(":", 2)
if name == project.import_source
"##{id}"
else
"#{project.namespace.path}/#{name}##{id}"
end
end
updates << "*Blocking: #{blockings.join(", ")}*"
end
updates
end
def format_attachments(issue_id, comment_id, raw_attachments)
return [] unless raw_attachments
raw_attachments.map do |attachment|
next if attachment["isDeleted"]
link = "https://storage.googleapis.com/google-code-attachments/#{@repo.name}/issue-#{issue_id}/comment-#{comment_id}/#{attachment["fileName"]}"
"[#{attachment["fileName"]}](#{link})"
end.compact
end
end
end
end
module Gitlab
module GoogleCodeImport
class ProjectCreator
attr_reader :repo, :namespace, :current_user
def initialize(repo, namespace, current_user)
@repo = repo
@namespace = namespace
@current_user = current_user
end
def execute
@project = Project.new(
name: repo.name,
path: repo.name,
description: repo.summary,
namespace: namespace,
creator: current_user,
visibility_level: Gitlab::VisibilityLevel::PUBLIC,
import_type: "google_code",
import_source: repo.name,
import_url: repo.import_url,
import_data: repo.raw_data
)
if @project.save!
@project.reload
if @project.import_failed?
@project.import_retry
else
@project.import_start
end
end
@project
end
end
end
end
module Gitlab
module GoogleCodeImport
class Repository
attr_accessor :raw_data
def initialize(raw_data)
@raw_data = raw_data
end
def valid?
raw_data.is_a?(Hash) && raw_data["kind"] == "projecthosting#project"
end
def id
raw_data["externalId"]
end
def name
raw_data["name"]
end
def summary
raw_data["summary"]
end
def description
raw_data["description"]
end
def git?
raw_data["versionControlSystem"] == "git"
end
def import_url
raw_data["repositoryUrls"].first
end
end
end
end
require 'spec_helper'
describe Import::GoogleCodeController do
let(:user) { create(:user) }
let(:dump_file) { fixture_file_upload(Rails.root + 'spec/fixtures/GoogleCodeProjectHosting.json', 'application/json') }
before do
sign_in(user)
end
describe "POST callback" do
it "stores Google Takeout dump list in session" do
post :callback, dump_file: dump_file
expect(session[:google_code_dump]).to be_a(Hash)
expect(session[:google_code_dump]["kind"]).to eq("projecthosting#user")
expect(session[:google_code_dump]).to have_key("projects")
end
end
describe "GET status" do
before do
@repo = OpenStruct.new(name: 'vim')
controller.stub_chain(:client, :valid?).and_return(true)
end
it "assigns variables" do
@project = create(:project, import_type: 'google_code', creator_id: user.id)
controller.stub_chain(:client, :repos).and_return([@repo])
get :status
expect(assigns(:already_added_projects)).to eq([@project])
expect(assigns(:repos)).to eq([@repo])
end
it "does not show already added project" do
@project = create(:project, import_type: 'google_code', creator_id: user.id, import_source: 'vim')
controller.stub_chain(:client, :repos).and_return([@repo])
get :status
expect(assigns(:already_added_projects)).to eq([@project])
expect(assigns(:repos)).to eq([])
end
end
end
This diff is collapsed.
require "spec_helper"
describe Gitlab::GoogleCodeImport::Client do
let(:raw_data) { JSON.parse(File.read(Rails.root.join("spec/fixtures/GoogleCodeProjectHosting.json"))) }
subject { described_class.new(raw_data) }
describe "#valid?" do
context "when the data is valid" do
it "returns true" do
expect(subject).to be_valid
end
end
context "when the data is invalid" do
let(:raw_data) { "No clue" }
it "returns true" do
expect(subject).to_not be_valid
end
end
end
describe "#repos" do
it "returns only Git repositories" do
expect(subject.repos.length).to eq(1)
end
end
describe "#repo" do
it "returns the referenced repository" do
expect(subject.repo("tint2").name).to eq("tint2")
end
end
end
require "spec_helper"
describe Gitlab::GoogleCodeImport::Importer do
let(:raw_data) { JSON.parse(File.read(Rails.root.join("spec/fixtures/GoogleCodeProjectHosting.json"))) }
let(:client) { Gitlab::GoogleCodeImport::Client.new(raw_data) }
let(:import_data) { client.repo("tint2").raw_data }
let(:project) { create(:project, import_data: import_data) }
subject { described_class.new(project) }
describe "#execute" do
it "imports status labels" do
subject.execute
%w(New NeedInfo Accepted Wishlist Started Fixed Invalid Duplicate WontFix Incomplete).each do |status|
expect(project.labels.find_by(name: "Status: #{status}")).to_not be_nil
end
end
it "imports labels" do
subject.execute
%w(
Type-Defect Type-Enhancement Type-Task Type-Review Type-Other Milestone-0.12 Priority-Critical
Priority-High Priority-Medium Priority-Low OpSys-All OpSys-Windows OpSys-Linux OpSys-OSX Security
Performance Usability Maintainability Component-Panel Component-Taskbar Component-Battery
Component-Systray Component-Clock Component-Launcher Component-Tint2conf Component-Docs Component-New
).each do |label|
label.sub!("-", ": ")
expect(project.labels.find_by(name: label)).to_not be_nil
end
end
it "imports issues" do
subject.execute
issue = project.issues.first
expect(issue).to_not be_nil
expect(issue.iid).to eq(169)
expect(issue.state).to eq("closed")
expect(issue.label_names).to include("Priority: Medium")
expect(issue.label_names).to include("Status: Fixed")
expect(issue.label_names).to include("Type: Enhancement")
expect(issue.title).to eq("Scrolling through tasks")
expect(issue.state).to eq("closed")
expect(issue.description).to include("schattenpr...")
expect(issue.description).to include("https://code.google.com/u/106498139506637530000/")
expect(issue.description).to include("November 18, 2009 00:20")
expect(issue.description).to include('I like to scroll through the tasks with my scrollwheel \(like in fluxbox\).')
expect(issue.description).to include('Patch is attached that adds two new mouse\-actions \(next\_taskprev\_task\)')
expect(issue.description).to include('that can be used for exactly that purpose.')
expect(issue.description).to include('all the best!')
expect(issue.description).to include('https://storage.googleapis.com/google-code-attachments/tint2/issue-169/comment-0/tint2_task_scrolling.diff')
end
it "imports issue comments" do
subject.execute
note = project.issues.first.notes.first
expect(note).to_not be_nil
expect(note.note).to include("thilo...")
expect(note.note).to include("https://code.google.com/u/104224918623172014000/")
expect(note.note).to include("November 18, 2009 05:14")
expect(note.note).to include("applied, thanks.")
expect(note.note).to include("Status: Fixed")
expect(note.note).to include("~~Type: Defect~~")
expect(note.note).to include("Type: Enhancement")
end
end
end
require 'spec_helper'
describe Gitlab::GoogleCodeImport::ProjectCreator do
let(:user) { create(:user) }
let(:repo) {
Gitlab::GoogleCodeImport::Repository.new(
"name" => 'vim',
"summary" => 'VI Improved',
"repositoryUrls" => [ "https://vim.googlecode.com/git/" ]
)
}
let(:namespace) { create(:namespace) }
it 'creates project' do
allow_any_instance_of(Project).to receive(:add_import_job)
project_creator = Gitlab::GoogleCodeImport::ProjectCreator.new(repo, namespace, user)
project_creator.execute
project = Project.last
expect(project.import_url).to eq("https://vim.googlecode.com/git/")
expect(project.visibility_level).to eq(Gitlab::VisibilityLevel::PUBLIC)
end
end
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment