Merge branch '331733-githubimporter-optimize-pull-request-mergedby-importer-2' into 'master'

GithubImporter: Optimize Pull Request MergedBy Importer

See merge request gitlab-org/gitlab!69478

lib/gitlab/github_import/importer/pull_requests_merged_by_importer.rb

@@ -31,9 +31,7 @@ module Gitlab
         end
 
         def each_object_to_import
-          project.merge_requests.with_state(:merged).find_each do |merge_request|
-            next if already_imported?(merge_request)
-
+          merge_requests_to_import.find_each do |merge_request|
             Gitlab::GithubImport::ObjectCounter.increment(project, object_type, :fetched)
 
             pull_request = client.pull_request(project.import_source, merge_request.iid)
@@ -42,6 +40,17 @@ module Gitlab
             mark_as_imported(merge_request)
           end
         end
+
+        private
+
+        # Returns only the merge requests that still have merged_by to be imported.
+        def merge_requests_to_import
+          project.merge_requests.id_not_in(already_imported_objects).with_state(:merged)
+        end
+
+        def already_imported_objects
+          Gitlab::Cache::Import::Caching.values_from_set(already_imported_cache_key)
+        end
       end
     end
   end

lib/gitlab/github_import/importer/pull_requests_reviews_importer.rb

@@ -86,7 +86,7 @@ module Gitlab
 
         # Returns only the merge requests that still have reviews to be imported.
         def merge_requests_to_import
-          project.merge_requests.where.not(id: already_imported_merge_requests) # rubocop: disable CodeReuse/ActiveRecord
+          project.merge_requests.id_not_in(already_imported_merge_requests)
         end
 
         def already_imported_merge_requests

spec/lib/gitlab/github_import/importer/pull_requests_merged_by_importer_spec.rb

@@ -4,7 +4,8 @@ require 'spec_helper'
 
 RSpec.describe Gitlab::GithubImport::Importer::PullRequestsMergedByImporter do
   let(:client) { double }
-  let(:project) { create(:project, import_source: 'http://somegithub.com') }
+
+  let_it_be(:project) { create(:project, import_source: 'http://somegithub.com') }
 
   subject { described_class.new(project, client) }
 
@@ -27,14 +28,11 @@ RSpec.describe Gitlab::GithubImport::Importer::PullRequestsMergedByImporter do
   end
 
   describe '#each_object_to_import', :clean_gitlab_redis_cache do
-    it 'fetchs the merged pull requests data' do
-      create(
-        :merged_merge_request,
-        iid: 999,
-        source_project: project,
-        target_project: project
-      )
+    let!(:merge_request) do
+      create(:merged_merge_request, iid: 999, source_project: project, target_project: project)
+    end
 
+    it 'fetches the merged pull requests data' do
       pull_request = double
 
       allow(client)
@@ -48,5 +46,16 @@ RSpec.describe Gitlab::GithubImport::Importer::PullRequestsMergedByImporter do
 
       subject.each_object_to_import {}
     end
+
+    it 'skips cached merge requests' do
+      Gitlab::Cache::Import::Caching.set_add(
+        "github-importer/already-imported/#{project.id}/pull_requests_merged_by",
+        merge_request.id
+      )
+
+      expect(client).not_to receive(:pull_request)
+
+      subject.each_object_to_import {}
+    end
   end
 end