Process each page of GitHub resources instead of concating them then processing

This should avoid having large memory growth when importing GitHub repos
with lots of resources.

lib/gitlab/github_import/client.rb

@@ -52,7 +52,7 @@ module Gitlab
 
       def method_missing(method, *args, &block)
         if api.respond_to?(method)
-          request { api.send(method, *args, &block) }
+          request(method, *args, &block)
         else
           super(method, *args, &block)
         end
@@ -99,20 +99,19 @@ module Gitlab
         rate_limit.resets_in + GITHUB_SAFE_SLEEP_TIME
       end
 
-      def request
+      def request(method, *args, &block)
         sleep rate_limit_sleep_time if rate_limit_exceed?
 
-        data = yield
+        data = api.send(method, *args, &block)
+        yield data
 
         last_response = api.last_response
 
         while last_response.rels[:next]
           sleep rate_limit_sleep_time if rate_limit_exceed?
           last_response = last_response.rels[:next].get
-          data.concat(last_response.data) if last_response.data.is_a?(Array)
+          yield last_response.data if last_response.data.is_a?(Array)
         end
-
-        data
       end
     end
   end

lib/gitlab/github_import/importer.rb

@@ -46,8 +46,7 @@ module Gitlab
       end
 
       def import_labels
-        labels = client.labels(repo, per_page: 100)
-
+        client.labels(repo, per_page: 100) do |labels|
           labels.each do |raw|
             begin
               LabelFormatter.new(project, raw).create!
@@ -56,10 +55,10 @@ module Gitlab
             end
           end
         end
+      end
 
       def import_milestones
-        milestones = client.milestones(repo, state: :all, per_page: 100)
-
+        client.milestones(repo, state: :all, per_page: 100) do |milestones|
           milestones.each do |raw|
             begin
               MilestoneFormatter.new(project, raw).create!
@@ -68,10 +67,10 @@ module Gitlab
             end
           end
         end
+      end
 
       def import_issues
-        issues = client.issues(repo, state: :all, sort: :created, direction: :asc, per_page: 100)
-
+        client.issues(repo, state: :all, sort: :created, direction: :asc, per_page: 100) do |issues|
           issues.each do |raw|
             gh_issue = IssueFormatter.new(project, raw)
 
@@ -86,12 +85,14 @@ module Gitlab
             end
           end
         end
+      end
 
       def import_pull_requests
-        pull_requests = client.pull_requests(repo, state: :all, sort: :created, direction: :asc, per_page: 100)
-        pull_requests = pull_requests.map { |raw| PullRequestFormatter.new(project, raw) }.select(&:valid?)
+        client.pull_requests(repo, state: :all, sort: :created, direction: :asc, per_page: 100) do |pull_requests|
+          pull_requests.each do |raw|
+            pull_request = PullRequestFormatter.new(project, raw)
+            next unless pull_request.valid?
 
-        pull_requests.each do |pull_request|
             begin
               restore_source_branch(pull_request) unless pull_request.source_branch_exists?
               restore_target_branch(pull_request) unless pull_request.target_branch_exists?
@@ -107,6 +108,7 @@ module Gitlab
             end
           end
         end
+      end
 
       def restore_source_branch(pull_request)
         project.repository.fetch_ref(repo_url, "pull/#{pull_request.number}/head", pull_request.source_branch_name)
@@ -180,7 +182,7 @@ module Gitlab
       end
 
       def import_releases
-        releases = client.releases(repo, per_page: 100)
+        client.releases(repo, per_page: 100) do |releases|
           releases.each do |raw|
             begin
               gh_release = ReleaseFormatter.new(project, raw)
@@ -192,4 +194,5 @@ module Gitlab
         end
       end
     end
+  end
 end

spec/lib/gitlab/github_import/client_spec.rb

@@ -66,6 +66,6 @@ describe Gitlab::GithubImport::Client, lib: true do
     stub_request(:get, /api.github.com/)
     allow(client.api).to receive(:rate_limit!).and_raise(Octokit::NotFound)
 
-    expect { client.issues }.not_to raise_error
+    expect { client.issues {} }.not_to raise_error
   end
 end