From 4d894a7a953f18d8a71f003677b5f8ea5bbc1779 Mon Sep 17 00:00:00 2001 From: James Lopez <james@jameslopez.es> Date: Wed, 18 May 2016 14:32:30 +0200 Subject: [PATCH] added commits mapper and DB config import stuff --- lib/gitlab/import_export/commit_mapper.rb | 49 ++++++++++++++ .../import_export/project_tree_restorer.rb | 14 +++- lib/gitlab/import_export/relation_factory.rb | 66 +++++++++++++------ 3 files changed, 106 insertions(+), 23 deletions(-) create mode 100644 lib/gitlab/import_export/commit_mapper.rb diff --git a/lib/gitlab/import_export/commit_mapper.rb b/lib/gitlab/import_export/commit_mapper.rb new file mode 100644 index 0000000000..405149d3cd --- /dev/null +++ b/lib/gitlab/import_export/commit_mapper.rb @@ -0,0 +1,49 @@ +module Gitlab + module ImportExport + class CommitMapper + def initialize(commits:, members_map:, project_id:, relation_factory: Gitlab::ImportExport::RelationFactory, user_admin:) + @commits = commits + @members_map = members_map + @project_id = project_id + @relation_factory = relation_factory + @user_admin = user_admin + end + + def ids_map + @ids_map ||= map_commits + end + + def map_commits + @id_map = Hash.new(-1) + + @commits.each do |commit_hash| + @relation_factory.update_user_references(commit_hash, @members_map) + + commit_hash['project_id'] = @project_id + @relation_factory.update_project_references(commit_hash, Ci::Commit) + create_commit_statuses(commit_hash) + create_commit(commit_hash) + end + @id_map + end + + def create_commit(commit_hash) + old_id = commit_hash.delete('id') + commit = Ci::Commit.new(commit_hash) + commit.save! + @id_map[old_id] = commit.id + end + + def create_commit_statuses(commit_hash) + commit_hash['statuses'].map! do |status_hash| + @relation_factory.create(relation_sym: :statuses, + relation_hash: status_hash.merge('project_id' => @project_id, + 'commit_id' => nil), + members_mapper: @members_map, + commits_mapper: nil, + user_admin: @user_admin) + end + end + end + end +end diff --git a/lib/gitlab/import_export/project_tree_restorer.rb b/lib/gitlab/import_export/project_tree_restorer.rb index 911ba06e74..750b69caaf 100644 --- a/lib/gitlab/import_export/project_tree_restorer.rb +++ b/lib/gitlab/import_export/project_tree_restorer.rb @@ -14,6 +14,7 @@ module Gitlab json = IO.read(@path) @tree_hash = ActiveSupport::JSON.decode(json) @project_members = @tree_hash.delete('project_members') + @commits = @tree_hash.delete('ci_commits') create_relations rescue => e @shared.error(e) @@ -32,6 +33,13 @@ module Gitlab project: project) end + def commits_mapper + @commits_mapper ||= Gitlab::ImportExport::CommitMapper.new(commits: @commits, + members_map: members_mapper.map, + project_id: project.id, + user_admin: @user.is_admin?) + end + def create_relations(relation_list = default_relation_list, tree_hash = @tree_hash) saved = [] relation_list.each do |relation| @@ -47,7 +55,7 @@ module Gitlab def default_relation_list Gitlab::ImportExport::ImportExportReader.new(shared: @shared).tree.reject do |model| - model.is_a?(Hash) && model[:project_members] + model.is_a?(Hash) && (model[:project_members] || model[:ci_commits]) end end @@ -65,9 +73,8 @@ module Gitlab relation_key = relation.keys.first.to_s tree_hash[relation_key].each do |relation_item| relation.values.flatten.each do |sub_relation| - if sub_relation.is_a?(Hash) - relation_hash = relation_item[sub_relation.keys.first.to_s] + relation_hash = relation_item[sub_relation.keys.first.to_s] sub_relation = sub_relation.keys.first else relation_hash = relation_item[sub_relation.to_s] @@ -97,6 +104,7 @@ module Gitlab Gitlab::ImportExport::RelationFactory.create(relation_sym: relation.to_sym, relation_hash: relation_hash.merge('project_id' => project.id), members_mapper: members_mapper, + commits_mapper: commits_mapper, user_admin: @user.is_admin?) end end diff --git a/lib/gitlab/import_export/relation_factory.rb b/lib/gitlab/import_export/relation_factory.rb index 082398d1f0..7f0f8f8077 100644 --- a/lib/gitlab/import_export/relation_factory.rb +++ b/lib/gitlab/import_export/relation_factory.rb @@ -3,21 +3,27 @@ module Gitlab module RelationFactory extend self - OVERRIDES = { snippets: :project_snippets, ci_commits: 'Ci::Commit', statuses: 'commit_status' }.freeze - USER_REFERENCES = %w(author_id assignee_id updated_by_id).freeze - - def create(relation_sym:, relation_hash:, members_mapper:, user_admin:) + OVERRIDES = { snippets: :project_snippets, + ci_commits: 'Ci::Commit', + statuses: 'commit_status', + variables: 'Ci::Variable', + triggers: 'Ci::Trigger', + builds: 'Ci::Build', + hooks: 'ProjectHook' }.freeze + USER_REFERENCES = %w(author_id assignee_id updated_by_id user_id).freeze + + def create(relation_sym:, relation_hash:, members_mapper:, commits_mapper:, user_admin:) relation_sym = parse_relation_sym(relation_sym) klass = parse_relation(relation_hash, relation_sym) update_missing_author(relation_hash, members_mapper, user_admin) if relation_sym == :notes update_user_references(relation_hash, members_mapper.map) update_project_references(relation_hash, klass) + update_commit_references(relation_hash, commits_mapper.ids_map) if commits_mapper - imported_object(klass, relation_hash) + generate_imported_object(klass, relation_hash, relation_sym) end - private def update_user_references(relation_hash, members_map) USER_REFERENCES.each do |reference| @@ -27,6 +33,32 @@ module Gitlab end end + def update_project_references(relation_hash, klass) + project_id = relation_hash.delete('project_id') + + if relation_hash['source_project_id'] && relation_hash['target_project_id'] + # If source and target are the same, populate them with the new project ID. + if relation_hash['target_project_id'] == relation_hash['source_project_id'] + relation_hash['source_project_id'] = project_id + else + relation_hash['source_project_id'] = -1 + end + end + relation_hash['target_project_id'] = project_id if relation_hash['target_project_id'] + + # project_id may not be part of the export, but we always need to populate it if required. + relation_hash['project_id'] = project_id if klass.column_names.include?('project_id') + relation_hash['gl_project_id'] = project_id if relation_hash['gl_project_id'] + end + + private + + def update_commit_references(relation_hash, commit_ids_map) + return unless relation_hash['commit_id'] + old_commit_id = relation_hash['commit_id'] + relation_hash['commit_id'] = commit_ids_map[old_commit_id] + end + def update_missing_author(relation_hash, members_map, user_admin) old_author_id = relation_hash['author_id'] @@ -50,22 +82,15 @@ module Gitlab "\n\n *By #{author_name} on #{timestamp} (imported from GitLab project)*" end - def update_project_references(relation_hash, klass) - project_id = relation_hash.delete('project_id') - - if relation_hash['source_project_id'] && relation_hash['target_project_id'] - # If source and target are the same, populate them with the new project ID. - if relation_hash['target_project_id'] == relation_hash['source_project_id'] - relation_hash['source_project_id'] = project_id - else - relation_hash['source_project_id'] = -1 + def generate_imported_object(klass, relation_hash, relation_sym) + if relation_sym == 'Ci::Build' # call #trace= method after assigning the other attributes + trace = relation_hash.delete('trace') + imported_object(klass, relation_hash) do |imported_object| + imported_object.trace = trace end + else + imported_object(klass, relation_hash) end - relation_hash['target_project_id'] = project_id if relation_hash['target_project_id'] - - # project_id may not be part of the export, but we always need to populate it if required. - relation_hash['project_id'] = project_id if klass.column_names.include?('project_id') - relation_hash['gl_project_id'] = project_id if relation_hash ['gl_project_id'] end def relation_class(relation_sym) @@ -78,6 +103,7 @@ module Gitlab def imported_object(klass, relation_hash) imported_object = klass.new(relation_hash) + yield(imported_object) if block_given? imported_object.importing = true if imported_object.respond_to?(:importing) imported_object end -- 2.30.9