Merge branch 'optimise-import-performance' into 'master'

Optimise import performance

Closes #64924

See merge request gitlab-org/gitlab-ce!31045

app/models/project.rb

@@ -1862,16 +1862,24 @@ class Project < ApplicationRecord
   end
 
   def append_or_update_attribute(name, value)
-    old_values = public_send(name.to_s) # rubocop:disable GitlabSecurity/PublicSend
+    if Project.reflect_on_association(name).try(:macro) == :has_many
+      # if this is 1-to-N relation, update the parent object
+      value.each do |item|
+        item.update!(
+          Project.reflect_on_association(name).foreign_key => id)
+      end
+
+      # force to drop relation cache
+      public_send(name).reset # rubocop:disable GitlabSecurity/PublicSend
 
-    if Project.reflect_on_association(name).try(:macro) == :has_many && old_values.any?
-      update_attribute(name, old_values + value)
+      # succeeded
+      true
     else
+      # if this is another relation or attribute, update just object
       update_attribute(name, value)
     end
-
-  rescue ActiveRecord::RecordNotSaved => e
-    handle_update_attribute_error(e, value)
+  rescue ActiveRecord::RecordInvalid => e
+    raise e, "Failed to set #{name}: #{e.message}"
   end
 
   # Tries to set repository as read_only, checking for existing Git transfers in progress beforehand
@@ -2260,18 +2268,6 @@ class Project < ApplicationRecord
     ContainerRepository.build_root_repository(self).has_tags?
   end
 
-  def handle_update_attribute_error(ex, value)
-    if ex.message.start_with?('Failed to replace')
-      if value.respond_to?(:each)
-        invalid = value.detect(&:invalid?)
-
-        raise ex, ([ex.message] + invalid.errors.full_messages).join(' ') if invalid
-      end
-    end
-
-    raise ex
-  end
-
   def fetch_branch_allows_collaboration(user, branch_name = nil)
     return false unless user
 

changelogs/unreleased/optimise-import-performance.yml

+---
+title: Optimise import performance
+merge_request: 31045
+author:
+type: performance

lib/gitlab/import_export/attributes_finder.rb

@@ -45,7 +45,7 @@ module Gitlab
       end
 
       def key_from_hash(value)
-        value.is_a?(Hash) ? value.keys.first : value
+        value.is_a?(Hash) ? value.first.first : value
       end
     end
   end

lib/gitlab/import_export/json_hash_builder.rb

@@ -27,7 +27,7 @@ module Gitlab
       #   {:merge_requests=>[:merge_request_diff, :notes]}
       def process_model_objects(model_object_hash)
         json_config_hash = {}
-        current_key = model_object_hash.keys.first
+        current_key = model_object_hash.first.first
 
         model_object_hash.values.flatten.each do |model_object|
           @attributes_finder.parse(current_key) { |hash| json_config_hash[current_key] ||= hash }

lib/gitlab/import_export/members_mapper.rb

@@ -35,7 +35,7 @@ module Gitlab
       end
 
       def include?(old_author_id)
-        map.keys.include?(old_author_id) && map[old_author_id] != default_user_id
+        map.has_key?(old_author_id) && map[old_author_id] != default_user_id
       end
 
       private

lib/gitlab/import_export/relation_factory.rb

@@ -185,7 +185,7 @@ module Gitlab
         return unless EXISTING_OBJECT_CHECK.include?(@relation_name)
         return unless @relation_hash['group_id']
 
-        @relation_hash['group_id'] = @project.group&.id
+        @relation_hash['group_id'] = @project.namespace_id
       end
 
       def reset_tokens!

spec/controllers/user_callouts_controller_spec.rb

@@ -13,7 +13,7 @@ describe UserCalloutsController do
     subject { post :create, params: { feature_name: feature_name }, format: :json }
 
     context 'with valid feature name' do
-      let(:feature_name) { UserCallout.feature_names.keys.first }
+      let(:feature_name) { UserCallout.feature_names.first.first }
 
       context 'when callout entry does not exist' do
         it 'creates a callout entry with dismissed state' do
@@ -28,7 +28,7 @@ describe UserCalloutsController do
       end
 
       context 'when callout entry already exists' do
-        let!(:callout) { create(:user_callout, feature_name: UserCallout.feature_names.keys.first, user: user) }
+        let!(:callout) { create(:user_callout, feature_name: UserCallout.feature_names.first.first, user: user) }
 
         it 'returns success' do
           subject

spec/lib/gitlab/import_export/project.group.json

@@ -19,7 +19,7 @@
   "labels": [
     {
       "id": 2,
-      "title": "project label",
+      "title": "A project label",
       "color": "#428bca",
       "project_id": 8,
       "created_at": "2016-07-22T08:55:44.161Z",
@@ -105,7 +105,7 @@
           "updated_at": "2017-08-15T18:37:40.795Z",
           "label": {
             "id": 6,
-            "title": "project label",
+            "title": "A project label",
             "color": "#A8D695",
             "project_id": null,
             "created_at": "2017-08-15T18:37:19.698Z",
@@ -162,7 +162,7 @@
           "updated_at": "2017-08-15T18:37:40.795Z",
           "label": {
             "id": 2,
-            "title": "project label",
+            "title": "A project label",
             "color": "#A8D695",
             "project_id": null,
             "created_at": "2017-08-15T18:37:19.698Z",

spec/lib/gitlab/import_export/project_tree_restorer_spec.rb

@@ -272,7 +272,7 @@ describe Gitlab::ImportExport::ProjectTreeRestorer do
     end
 
     it 'has label priorities' do
-      expect(project.labels.first.priorities).not_to be_empty
+      expect(project.labels.find_by(title: 'A project label').priorities).not_to be_empty
     end
 
     it 'has milestones' do
@@ -325,7 +325,7 @@ describe Gitlab::ImportExport::ProjectTreeRestorer do
 
       it_behaves_like 'restores project correctly',
                       issues: 1,
-                      labels: 1,
+                      labels: 2,
                       milestones: 1,
                       first_issue_labels: 1,
                       services: 1
@@ -402,7 +402,7 @@ describe Gitlab::ImportExport::ProjectTreeRestorer do
       it_behaves_like 'restores project successfully'
       it_behaves_like 'restores project correctly',
                       issues: 2,
-                      labels: 1,
+                      labels: 2,
                       milestones: 2,
                       first_issue_labels: 1
 

spec/models/project_spec.rb

@@ -3117,11 +3117,8 @@ describe Project do
     let(:project) { create(:project) }
 
     it 'shows full error updating an invalid MR' do
-      error_message = 'Failed to replace merge_requests because one or more of the new records could not be saved.'\
-        ' Validate fork Source project is not a fork of the target project'
-
       expect { project.append_or_update_attribute(:merge_requests, [create(:merge_request)]) }
-        .to raise_error(ActiveRecord::RecordNotSaved, error_message)
+        .to raise_error(ActiveRecord::RecordInvalid, /Failed to set merge_requests:/)
     end
 
     it 'updates the project successfully' do