Merge branch 'revert-21130' into 'master'

Revert "Merge branch 'ab/pagination-batch-counts' into 'master'"

See merge request gitlab-org/gitlab!21698

lib/api/entities.rb

@@ -188,7 +188,7 @@ module API
     end
 
     class BasicProjectDetails < ProjectIdentity
-      include ::API::ProjectsBatchCounting
+      include ::API::ProjectsRelationBuilder
 
       expose :default_branch, if: -> (project, options) { Ability.allowed?(options[:current_user], :download_code, project) }
       # Avoids an N+1 query: https://github.com/mbleigh/acts-as-taggable-on/issues/91#issuecomment-168273770
@@ -430,7 +430,7 @@ module API
           options: { only_owned: true, limit: projects_limit }
         ).execute
 
-        Entities::Project.preload_and_batch_count!(projects)
+        Entities::Project.prepare_relation(projects)
       end
 
       expose :shared_projects, using: Entities::Project do |group, options|
@@ -440,7 +440,7 @@ module API
           options: { only_shared: true, limit: projects_limit }
         ).execute
 
-        Entities::Project.preload_and_batch_count!(projects)
+        Entities::Project.prepare_relation(projects)
       end
 
       def projects_limit

lib/api/groups.rb

@@ -231,7 +231,7 @@ module API
 
         projects, options = with_custom_attributes(projects, options)
 
-        present options[:with].preload_and_batch_count!(projects), options
+        present options[:with].prepare_relation(projects), options
       end
 
       desc 'Get a list of subgroups in this group.' do

lib/api/projects.rb

@@ -75,17 +75,15 @@ module API
         mutually_exclusive :import_url, :template_name, :template_project_id
       end
 
-      def find_projects
+      def load_projects
         ProjectsFinder.new(current_user: current_user, params: project_finder_params).execute
       end
 
-      # Prepare the full projects query
-      # None of this is supposed to actually execute any database query
-      def prepare_query(projects)
+      def present_projects(projects, options = {})
         projects = reorder_projects(projects)
         projects = apply_filters(projects)
-
-        projects, options = with_custom_attributes(projects)
+        projects = paginate(projects)
+        projects, options = with_custom_attributes(projects, options)
 
         options = options.reverse_merge(
           with: current_user ? Entities::ProjectWithAccess : Entities::BasicProjectDetails,
@@ -93,23 +91,9 @@ module API
           current_user: current_user,
           license: false
         )
-
         options[:with] = Entities::BasicProjectDetails if params[:simple]
 
-        projects = options[:with].preload_relation(projects, options)
-
-        [projects, options]
-      end
-
-      def prepare_and_present(project_relation)
-        projects, options = prepare_query(project_relation)
-
-        projects = paginate_and_retrieve!(projects)
-
-        # Refresh count caches
-        options[:with].execute_batch_counting(projects)
-
-        present projects, options
+        present options[:with].prepare_relation(projects, options), options
       end
 
       def translate_params_for_compatibility(params)
@@ -134,7 +118,7 @@ module API
 
         params[:user] = user
 
-        prepare_and_present find_projects
+        present_projects load_projects
       end
 
       desc 'Get projects starred by a user' do
@@ -150,7 +134,7 @@ module API
         not_found!('User') unless user
 
         starred_projects = StarredProjectsFinder.new(user, params: project_finder_params, current_user: current_user).execute
-        prepare_and_present starred_projects
+        present_projects starred_projects
       end
     end
 
@@ -166,7 +150,7 @@ module API
         use :with_custom_attributes
       end
       get do
-        prepare_and_present find_projects
+        present_projects load_projects
       end
 
       desc 'Create new project' do
@@ -303,7 +287,7 @@ module API
       get ':id/forks' do
         forks = ForkProjectsFinder.new(user_project, params: project_finder_params, current_user: current_user).execute
 
-        prepare_and_present forks
+        present_projects forks
       end
 
       desc 'Check pages access of this project'

lib/api/projects_batch_counting.rb

-# frozen_string_literal: true
-
-module API
-  module ProjectsBatchCounting
-    extend ActiveSupport::Concern
-
-    class_methods do
-      # This adds preloading to the query and executes batch counting
-      # Side-effect: The query will be executed during batch counting
-      def preload_and_batch_count!(projects_relation)
-        preload_relation(projects_relation).tap do |projects|
-          execute_batch_counting(projects)
-        end
-      end
-
-      def execute_batch_counting(projects)
-        ::Projects::BatchForksCountService.new(forks_counting_projects(projects)).refresh_cache
-
-        ::Projects::BatchOpenIssuesCountService.new(projects).refresh_cache
-      end
-
-      def forks_counting_projects(projects)
-        projects
-      end
-    end
-  end
-end

lib/api/projects_relation_builder.rb

+# frozen_string_literal: true
+
+module API
+  module ProjectsRelationBuilder
+    extend ActiveSupport::Concern
+
+    class_methods do
+      def prepare_relation(projects_relation, options = {})
+        projects_relation = preload_relation(projects_relation, options)
+        execute_batch_counting(projects_relation)
+        projects_relation
+      end
+
+      def preload_relation(projects_relation, options = {})
+        projects_relation
+      end
+
+      def forks_counting_projects(projects_relation)
+        projects_relation
+      end
+
+      def batch_forks_counting(projects_relation)
+        ::Projects::BatchForksCountService.new(forks_counting_projects(projects_relation)).refresh_cache
+      end
+
+      def batch_open_issues_counting(projects_relation)
+        ::Projects::BatchOpenIssuesCountService.new(projects_relation).refresh_cache
+      end
+
+      def execute_batch_counting(projects_relation)
+        batch_forks_counting(projects_relation)
+        batch_open_issues_counting(projects_relation)
+      end
+    end
+  end
+end

spec/lib/api/projects_batch_counting_spec.rb

-# frozen_string_literal: true
-
-require 'spec_helper'
-
-describe API::ProjectsBatchCounting do
-  subject do
-    Class.new do
-      include ::API::ProjectsBatchCounting
-    end
-  end
-
-  describe '.preload_and_batch_count!' do
-    let(:projects) { double }
-    let(:preloaded_projects) { double }
-
-    it 'preloads the relation' do
-      allow(subject).to receive(:execute_batch_counting).with(preloaded_projects)
-
-      expect(subject).to receive(:preload_relation).with(projects).and_return(preloaded_projects)
-
-      expect(subject.preload_and_batch_count!(projects)).to eq(preloaded_projects)
-    end
-
-    it 'executes batch counting' do
-      allow(subject).to receive(:preload_relation).with(projects).and_return(preloaded_projects)
-
-      expect(subject).to receive(:execute_batch_counting).with(preloaded_projects)
-
-      subject.preload_and_batch_count!(projects)
-    end
-  end
-
-  describe '.execute_batch_counting' do
-    let(:projects) { create_list(:project, 2) }
-    let(:count_service) { double }
-
-    it 'counts forks' do
-      allow(::Projects::BatchForksCountService).to receive(:new).with(projects).and_return(count_service)
-
-      expect(count_service).to receive(:refresh_cache)
-
-      subject.execute_batch_counting(projects)
-    end
-
-    it 'counts open issues' do
-      allow(::Projects::BatchOpenIssuesCountService).to receive(:new).with(projects).and_return(count_service)
-
-      expect(count_service).to receive(:refresh_cache)
-
-      subject.execute_batch_counting(projects)
-    end
-
-    context 'custom fork counting' do
-      subject do
-        Class.new do
-          include ::API::ProjectsBatchCounting
-          def self.forks_counting_projects(projects)
-            [projects.first]
-          end
-        end
-      end
-
-      it 'counts forks for other projects' do
-        allow(::Projects::BatchForksCountService).to receive(:new).with([projects.first]).and_return(count_service)
-
-        expect(count_service).to receive(:refresh_cache)
-
-        subject.execute_batch_counting(projects)
-      end
-    end
-  end
-end

spec/requests/api/projects_spec.rb

@@ -155,35 +155,6 @@ describe API::Projects do
       project4
     end
 
-    # This is a regression spec for https://gitlab.com/gitlab-org/gitlab/issues/37919
-    context 'batch counting forks and open issues and refreshing count caches' do
-      # We expect to count these projects (only the ones on the first page, not all matching ones)
-      let(:projects) { Project.public_to_user(nil).order(id: :desc).first(per_page) }
-      let(:per_page) { 2 }
-      let(:count_service) { double }
-
-      before do
-        # Create more projects, so we have more than one page
-        create_list(:project, 5, :public)
-      end
-
-      it 'batch counts project forks' do
-        expect(::Projects::BatchForksCountService).to receive(:new).with(projects).and_return(count_service)
-        expect(count_service).to receive(:refresh_cache)
-
-        get api("/projects?per_page=#{per_page}")
-        expect(response.status).to eq 200
-      end
-
-      it 'batch counts open issues' do
-        expect(::Projects::BatchOpenIssuesCountService).to receive(:new).with(projects).and_return(count_service)
-        expect(count_service).to receive(:refresh_cache)
-
-        get api("/projects?per_page=#{per_page}")
-        expect(response.status).to eq 200
-      end
-    end
-
     context 'when unauthenticated' do
       it_behaves_like 'projects response' do
         let(:filter) { { search: project.name } }
@@ -599,87 +570,6 @@ describe API::Projects do
         let(:projects) { Project.all }
       end
     end
-
-    context 'with keyset pagination' do
-      let(:current_user) { user }
-      let(:projects) { [public_project, project, project2, project3] }
-
-      context 'headers and records' do
-        let(:params) { { pagination: 'keyset', order_by: :id, sort: :asc, per_page: 1 } }
-
-        it 'includes a pagination header with link to the next page' do
-          get api('/projects', current_user), params: params
-
-          expect(response.header).to include('Links')
-          expect(response.header['Links']).to include('pagination=keyset')
-          expect(response.header['Links']).to include("id_after=#{public_project.id}")
-        end
-
-        it 'contains only the first project with per_page = 1' do
-          get api('/projects', current_user), params: params
-
-          expect(response).to have_gitlab_http_status(200)
-          expect(json_response).to be_an Array
-          expect(json_response.map { |p| p['id'] }).to contain_exactly(public_project.id)
-        end
-
-        it 'does not include a link if the end has reached and there is no more data' do
-          get api('/projects', current_user), params: params.merge(id_after: project2.id)
-
-          expect(response.header).not_to include('Links')
-        end
-
-        it 'responds with 501 if order_by is different from id' do
-          get api('/projects', current_user), params: params.merge(order_by: :created_at)
-
-          expect(response).to have_gitlab_http_status(405)
-        end
-      end
-
-      context 'with descending sorting' do
-        let(:params) { { pagination: 'keyset', order_by: :id, sort: :desc, per_page: 1 } }
-
-        it 'includes a pagination header with link to the next page' do
-          get api('/projects', current_user), params: params
-
-          expect(response.header).to include('Links')
-          expect(response.header['Links']).to include('pagination=keyset')
-          expect(response.header['Links']).to include("id_before=#{project3.id}")
-        end
-
-        it 'contains only the last project with per_page = 1' do
-          get api('/projects', current_user), params: params
-
-          expect(response).to have_gitlab_http_status(200)
-          expect(json_response).to be_an Array
-          expect(json_response.map { |p| p['id'] }).to contain_exactly(project3.id)
-        end
-      end
-
-      context 'retrieving the full relation' do
-        let(:params) { { pagination: 'keyset', order_by: :id, sort: :desc, per_page: 2 } }
-
-        it 'returns all projects' do
-          url = '/projects'
-          requests = 0
-          ids = []
-
-          while url && requests <= 5 # circuit breaker
-            requests += 1
-            get api(url, current_user), params: params
-
-            links = response.header['Links']
-            url = links&.match(/<[^>]+(\/projects\?[^>]+)>; rel="next"/) do |match|
-              match[1]
-            end
-
-            ids += JSON.parse(response.body).map { |p| p['id'] }
-          end
-
-          expect(ids).to contain_exactly(*projects.map(&:id))
-        end
-      end
-    end
   end
 
   describe 'POST /projects' do