Remove package/ci table cross joins

Updates associations to remove cross join queries
between the package and package_file tables with ci
tables.
Co-authored-by: 10io <dfernandez@gitlab.com>

app/finders/packages/group_packages_finder.rb

@@ -22,7 +22,7 @@ module Packages
 
     def packages_for_group_projects(installable_only: false)
       packages = ::Packages::Package
-        .including_build_info
+        .load_pipelines
         .including_project_route
         .including_tags
         .for_projects(group_projects_visible_to_current_user.select(:id))

app/finders/packages/package_finder.rb

@@ -9,7 +9,7 @@ module Packages
     def execute
       @project
         .packages
-        .including_build_info
+        .load_pipelines
         .including_project_route
         .including_tags
         .displayable

app/finders/packages/packages_finder.rb

@@ -14,9 +14,10 @@ module Packages
 
     def execute
       packages = project.packages
-                        .including_build_info
+                        .load_pipelines
                         .including_project_route
                         .including_tags
+
       packages = filter_with_version(packages)
       packages = filter_by_package_type(packages)
       packages = filter_by_package_name(packages)

app/models/packages/package.rb

@@ -40,7 +40,7 @@ class Packages::Package < ApplicationRecord
   has_one :composer_metadatum, inverse_of: :package, class_name: 'Packages::Composer::Metadatum'
   has_one :rubygems_metadatum, inverse_of: :package, class_name: 'Packages::Rubygems::Metadatum'
   has_many :build_infos, inverse_of: :package
-  has_many :pipelines, through: :build_infos
+  has_many :pipelines, through: :build_infos, disable_joins: -> { disable_cross_joins_to_pipelines? }
   has_one :debian_publication, inverse_of: :package, class_name: 'Packages::Debian::Publication'
   has_one :debian_distribution, through: :debian_publication, source: :distribution, inverse_of: :packages, class_name: 'Packages::Debian::ProjectDistribution'
 
@@ -131,9 +131,11 @@ class Packages::Package < ApplicationRecord
 
   scope :has_version, -> { where.not(version: nil) }
   scope :preload_files, -> { preload(:package_files) }
+  scope :preload_pipelines, -> { preload(pipelines: :user) }
   scope :last_of_each_version, -> { where(id: all.select('MAX(id) AS id').group(:version)) }
   scope :limit_recent, ->(limit) { order_created_desc.limit(limit) }
   scope :select_distinct_name, -> { select(:name).distinct }
+  scope :load_pipelines, -> { disable_cross_joins_to_pipelines? ? preload_pipelines : including_build_info }
 
   # Sorting
   scope :order_created, -> { reorder(created_at: :asc) }
@@ -160,6 +162,10 @@ class Packages::Package < ApplicationRecord
     joins(:project).reorder(keyset_order)
   end
 
+  def self.disable_cross_joins_to_pipelines?
+    ::Feature.enabled?(:packages_remove_cross_joins_to_pipelines, default_enabled: :yaml)
+  end
+
   def self.only_maven_packages_with_path(path, use_cte: false)
     if use_cte
       # This is an optimization fence which assumes that looking up the Metadatum record by path (globally)
@@ -245,7 +251,7 @@ class Packages::Package < ApplicationRecord
 
   def versions
     project.packages
-           .including_build_info
+           .load_pipelines
            .including_tags
            .with_name(name)
            .where.not(version: version)

app/models/packages/package_file.rb

@@ -15,7 +15,7 @@ class Packages::PackageFile < ApplicationRecord
 
   has_one :conan_file_metadatum, inverse_of: :package_file, class_name: 'Packages::Conan::FileMetadatum'
   has_many :package_file_build_infos, inverse_of: :package_file, class_name: 'Packages::PackageFileBuildInfo'
-  has_many :pipelines, through: :package_file_build_infos
+  has_many :pipelines, through: :package_file_build_infos, disable_joins: -> { Packages::Package.disable_cross_joins_to_pipelines? }
   has_one :debian_file_metadatum, inverse_of: :package_file, class_name: 'Packages::Debian::FileMetadatum'
   has_one :helm_file_metadatum, inverse_of: :package_file, class_name: 'Packages::Helm::FileMetadatum'
 
@@ -38,6 +38,7 @@ class Packages::PackageFile < ApplicationRecord
   scope :with_format, ->(format) { where(::Packages::PackageFile.arel_table[:file_name].matches("%.#{format}")) }
 
   scope :preload_package, -> { preload(:package) }
+  scope :preload_pipelines, -> { preload(pipelines: :user) }
   scope :preload_conan_file_metadata, -> { preload(:conan_file_metadatum) }
   scope :preload_debian_file_metadata, -> { preload(:debian_file_metadatum) }
   scope :preload_helm_file_metadata, -> { preload(:helm_file_metadatum) }

config/feature_flags/development/packages_remove_cross_joins_to_pipelines.yml

+---
+name: packages_remove_cross_joins_to_pipelines
+introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/70624
+rollout_issue_url: https://gitlab.com/gitlab-org/gitlab/-/issues/342921
+milestone: '14.5'
+type: development
+group: group::package
+default_enabled: false

lib/api/package_files.rb

@@ -28,7 +28,10 @@ module API
         package = ::Packages::PackageFinder
           .new(user_project, params[:package_id]).execute
 
-        present paginate(package.package_files), with: ::API::Entities::PackageFile
+        files = package.package_files
+        files = files.preload_pipelines if Packages::Package.disable_cross_joins_to_pipelines?
+
+        present paginate(files), with: ::API::Entities::PackageFile
       end
 
       desc 'Remove a package file' do

spec/finders/packages/group_packages_finder_spec.rb

@@ -160,6 +160,30 @@ RSpec.describe Packages::GroupPackagesFinder do
         end
       end
 
+      context 'with pipelines' do
+        let_it_be(:build_info_1) { create(:package_build_info, :with_pipeline, package: package1) }
+        let_it_be(:build_info_2) { create(:package_build_info, :with_pipeline, package: package2) }
+
+        it 'preloads the pipelines' do
+          expect(::Packages::Package).to receive(:preload_pipelines).and_call_original
+          expect(::Packages::Package).not_to receive(:including_build_info)
+
+          expect(subject).to match_array([package1, package2])
+        end
+
+        context 'with packages_remove_cross_joins_to_pipelines disabled' do
+          before do
+            stub_feature_flags(packages_remove_cross_joins_to_pipelines: false)
+          end
+
+          it 'includes the pipelines' do
+            expect(::Packages::Package).to receive(:including_build_info).and_call_original
+            expect(::Packages::Package).not_to receive(:preload_pipelines)
+            expect(subject).to match_array([package1, package2])
+          end
+        end
+      end
+
       it_behaves_like 'concerning versionless param'
       it_behaves_like 'concerning package statuses'
     end

spec/finders/packages/package_finder_spec.rb

@@ -32,5 +32,29 @@ RSpec.describe ::Packages::PackageFinder do
         expect { subject }.to raise_exception(ActiveRecord::RecordNotFound)
       end
     end
+
+    context 'with pipelines' do
+      let_it_be(:build_info) { create(:package_build_info, :with_pipeline, package: maven_package) }
+
+      it 'preloads the pipelines' do
+        expect(::Packages::Package).to receive(:preload_pipelines).and_call_original
+        expect(::Packages::Package).not_to receive(:including_build_info)
+
+        expect(subject).to eq(maven_package)
+      end
+
+      context 'with packages_remove_cross_joins_to_pipelines disabled' do
+        before do
+          stub_feature_flags(packages_remove_cross_joins_to_pipelines: false)
+        end
+
+        it 'includes the pipelines' do
+          expect(::Packages::Package).to receive(:including_build_info).and_call_original
+          expect(::Packages::Package).not_to receive(:preload_pipelines)
+
+          expect(subject).to eq(maven_package)
+        end
+      end
+    end
   end
 end

spec/finders/packages/packages_finder_spec.rb

@@ -81,6 +81,31 @@ RSpec.describe ::Packages::PackagesFinder do
       it { is_expected.to match_array([conan_package, maven_package]) }
     end
 
+    context 'with pipelines' do
+      let_it_be(:build_info1) { create(:package_build_info, :with_pipeline, package: conan_package) }
+      let_it_be(:build_info2) { create(:package_build_info, :with_pipeline, package: maven_package) }
+
+      it 'preloads the pipelines' do
+        expect(::Packages::Package).to receive(:preload_pipelines).and_call_original
+        expect(::Packages::Package).not_to receive(:including_build_info)
+
+        expect(subject).to match_array([conan_package, maven_package])
+      end
+
+      context 'with packages_remove_cross_joins_to_pipelines disabled' do
+        before do
+          stub_feature_flags(packages_remove_cross_joins_to_pipelines: false)
+        end
+
+        it 'includes the pipelines' do
+          expect(::Packages::Package).to receive(:including_build_info).and_call_original
+          expect(::Packages::Package).not_to receive(:preload_pipelines)
+
+          expect(subject).to match_array([conan_package, maven_package])
+        end
+      end
+    end
+
     it_behaves_like 'concerning versionless param'
     it_behaves_like 'concerning package statuses'
   end

spec/models/packages/package_file_spec.rb

@@ -14,7 +14,6 @@ RSpec.describe Packages::PackageFile, type: :model do
     it { is_expected.to belong_to(:package) }
     it { is_expected.to have_one(:conan_file_metadatum) }
     it { is_expected.to have_many(:package_file_build_infos).inverse_of(:package_file) }
-    it { is_expected.to have_many(:pipelines).through(:package_file_build_infos) }
     it { is_expected.to have_one(:debian_file_metadatum).inverse_of(:package_file).class_name('Packages::Debian::FileMetadatum') }
     it { is_expected.to have_one(:helm_file_metadatum).inverse_of(:package_file).class_name('Packages::Helm::FileMetadatum') }
   end
@@ -206,6 +205,28 @@ RSpec.describe Packages::PackageFile, type: :model do
     end
   end
 
+  describe '#pipelines' do
+    let_it_be_with_refind(:package_file) { create(:package_file) }
+
+    subject { package_file.pipelines }
+
+    context 'package_file without pipeline' do
+      it { is_expected.to be_empty }
+    end
+
+    context 'package_file with pipeline' do
+      let_it_be(:pipeline) { create(:ci_pipeline) }
+      let_it_be(:pipeline2) { create(:ci_pipeline) }
+
+      before do
+        package_file.package_file_build_infos.create!(pipeline: pipeline)
+        package_file.package_file_build_infos.create!(pipeline: pipeline2)
+      end
+
+      it { is_expected.to contain_exactly(pipeline, pipeline2) }
+    end
+  end
+
   describe '#update_file_store callback' do
     let_it_be(:package_file) { build(:package_file, :nuget, size: nil) }
 

spec/models/packages/package_spec.rb

@@ -14,7 +14,6 @@ RSpec.describe Packages::Package, type: :model do
     it { is_expected.to have_many(:dependency_links).inverse_of(:package) }
     it { is_expected.to have_many(:tags).inverse_of(:package) }
     it { is_expected.to have_many(:build_infos).inverse_of(:package) }
-    it { is_expected.to have_many(:pipelines).through(:build_infos) }
     it { is_expected.to have_one(:conan_metadatum).inverse_of(:package) }
     it { is_expected.to have_one(:maven_metadatum).inverse_of(:package) }
     it { is_expected.to have_one(:debian_publication).inverse_of(:package).class_name('Packages::Debian::Publication') }
@@ -962,6 +961,33 @@ RSpec.describe Packages::Package, type: :model do
     end
   end
 
+  describe '.load_pipelines' do
+    let_it_be(:package) { create(:maven_package) }
+    let_it_be(:build_info) { create(:package_build_info, :with_pipeline, package: package) }
+
+    subject { described_class.load_pipelines }
+
+    it 'uses preload', :aggregate_failures do
+      expect(described_class).to receive(:preload_pipelines).and_call_original
+      expect(described_class).not_to receive(:including_build_info)
+
+      subject
+    end
+
+    context 'with packages_remove_cross_joins_to_pipelines disabled' do
+      before do
+        stub_feature_flags(packages_remove_cross_joins_to_pipelines: false)
+      end
+
+      it 'uses includes', :aggregate_failures do
+        expect(described_class).to receive(:including_build_info).and_call_original
+        expect(described_class).not_to receive(:preload_pipelines)
+
+        subject
+      end
+    end
+  end
+
   describe '#versions' do
     let_it_be(:project) { create(:project) }
     let_it_be(:package) { create(:maven_package, project: project) }
@@ -975,6 +1001,30 @@ RSpec.describe Packages::Package, type: :model do
     it 'does not return different packages' do
       expect(package.versions).not_to include(package3)
     end
+
+    context 'with pipelines' do
+      let_it_be(:build_info) { create(:package_build_info, :with_pipeline, package: package2) }
+
+      it 'preloads the pipelines' do
+        expect(::Packages::Package).to receive(:preload_pipelines).and_call_original
+        expect(::Packages::Package).not_to receive(:including_build_info)
+
+        expect(package.versions).to contain_exactly(package2)
+      end
+
+      context 'with packages_remove_cross_joins_to_pipelines disabled' do
+        before do
+          stub_feature_flags(packages_remove_cross_joins_to_pipelines: false)
+        end
+
+        it 'includes the pipelines' do
+          expect(::Packages::Package).to receive(:including_build_info).and_call_original
+          expect(::Packages::Package).not_to receive(:preload_pipelines)
+
+          expect(package.versions).to contain_exactly(package2)
+        end
+      end
+    end
   end
 
   describe '#pipeline' do
@@ -999,6 +1049,28 @@ RSpec.describe Packages::Package, type: :model do
     end
   end
 
+  describe '#pipelines' do
+    let_it_be_with_refind(:package) { create(:maven_package) }
+
+    subject { package.pipelines }
+
+    context 'package without pipeline' do
+      it { is_expected.to be_empty }
+    end
+
+    context 'package with pipeline' do
+      let_it_be(:pipeline) { create(:ci_pipeline) }
+      let_it_be(:pipeline2) { create(:ci_pipeline) }
+
+      before do
+        package.build_infos.create!(pipeline: pipeline)
+        package.build_infos.create!(pipeline: pipeline2)
+      end
+
+      it { is_expected.to contain_exactly(pipeline, pipeline2) }
+    end
+  end
+
   describe '#tag_names' do
     let_it_be(:package) { create(:nuget_package) }
 

spec/requests/api/package_files_spec.rb

@@ -27,6 +27,34 @@ RSpec.describe API::PackageFiles do
 
           expect(response).to have_gitlab_http_status(:not_found)
         end
+
+        context 'with pipelines' do
+          let(:package_file_build_info) { create(:package_file_build_info, :with_pipeline, package_file: package.package_files.first) }
+
+          it 'preloads the pipelines' do
+            expect(::Packages::PackageFile).to receive(:preload_pipelines).and_call_original
+
+            get api(url)
+
+            expect(response).to have_gitlab_http_status(:ok)
+          end
+
+          context 'with packages_remove_cross_joins_to_pipelines disabled' do
+            before do
+              stub_feature_flags(packages_remove_cross_joins_to_pipelines: false)
+            end
+
+            it 'does not preload the pipelines' do
+              expect(::Packages::PackageFile).not_to receive(:preload_pipelines)
+
+              ::Gitlab::Database.allow_cross_joins_across_databases(url: 'https://gitlab.com/gitlab-org/gitlab/-/issues/342921') do
+                get api(url)
+              end
+
+              expect(response).to have_gitlab_http_status(:ok)
+            end
+          end
+        end
       end
 
       context 'project is private' do

spec/support/database/cross-join-allowlist.yml

@@ -2,26 +2,10 @@
 - "./ee/spec/services/ci/minutes/additional_packs/change_namespace_service_spec.rb"
 - "./ee/spec/services/ci/minutes/additional_packs/create_service_spec.rb"
 - "./ee/spec/services/ci/minutes/refresh_cached_data_service_spec.rb"
-- "./spec/features/groups/packages_spec.rb"
 - "./spec/features/projects/infrastructure_registry_spec.rb"
-- "./spec/features/projects/packages_spec.rb"
-- "./spec/lib/api/entities/package_spec.rb"
 - "./spec/lib/gitlab/background_migration/copy_ci_builds_columns_to_security_scans_spec.rb"
 - "./spec/lib/gitlab/background_migration/migrate_pages_metadata_spec.rb"
 - "./spec/migrations/20210907211557_finalize_ci_builds_bigint_conversion_spec.rb"
 - "./spec/migrations/associate_existing_dast_builds_with_variables_spec.rb"
 - "./spec/migrations/disable_job_token_scope_when_unused_spec.rb"
 - "./spec/migrations/schedule_copy_ci_builds_columns_to_security_scans2_spec.rb"
-- "./spec/presenters/packages/detail/package_presenter_spec.rb"
-- "./spec/requests/api/graphql/group/packages_spec.rb"
-- "./spec/requests/api/graphql/packages/composer_spec.rb"
-- "./spec/requests/api/graphql/packages/conan_spec.rb"
-- "./spec/requests/api/graphql/packages/maven_spec.rb"
-- "./spec/requests/api/graphql/packages/nuget_spec.rb"
-- "./spec/requests/api/graphql/packages/package_spec.rb"
-- "./spec/requests/api/graphql/packages/pypi_spec.rb"
-- "./spec/requests/api/graphql/project/packages_spec.rb"
-- "./spec/requests/api/package_files_spec.rb"
-- "./spec/services/packages/conan/create_package_file_service_spec.rb"
-- "./spec/services/packages/create_package_file_service_spec.rb"
-- "./spec/services/packages/generic/create_package_file_service_spec.rb"

spec/support/shared_contexts/graphql/requests/packages_shared_context.rb

@@ -10,6 +10,7 @@ RSpec.shared_context 'package details setup' do
   let(:excluded) { %w[metadata apiFuzzingCiConfiguration pipeline packageFiles] }
   let(:package_files) { all_graphql_fields_for('PackageFile') }
   let(:dependency_links) { all_graphql_fields_for('PackageDependencyLink') }
+  let(:pipelines) { all_graphql_fields_for('Pipeline', max_depth: 1) }
   let(:user) { project.owner }
   let(:package_details) { graphql_data_at(:package) }
   let(:metadata_response) { graphql_data_at(:package, :metadata) }
@@ -34,6 +35,11 @@ RSpec.shared_context 'package details setup' do
         #{dependency_links}
       }
     }
+    pipelines {
+      nodes {
+        #{pipelines}
+      }
+    }
     FIELDS
   end
 end

spec/support/shared_examples/requests/api/graphql/packages/package_details_shared_examples.rb

@@ -2,10 +2,43 @@
 
 RSpec.shared_examples 'a package detail' do
   it_behaves_like 'a working graphql query' do
+    it_behaves_like 'matching the package details schema'
+  end
+
+  context 'with pipelines' do
+    let_it_be(:build_info1) { create(:package_build_info, :with_pipeline, package: package) }
+    let_it_be(:build_info2) { create(:package_build_info, :with_pipeline, package: package) }
+    let_it_be(:build_info3) { create(:package_build_info, :with_pipeline, package: package) }
+
+    it_behaves_like 'a working graphql query' do
+      it_behaves_like 'matching the package details schema'
+    end
+
+    context 'with packages_remove_cross_joins_to_pipelines disabled' do
+      # subject is called in a before callback that is executed before the one below
+      # the callback below MUST be executed before the subject
+      # solution: nullify subject and manually call #post_graphql
+      subject {}
+
+      before do
+        stub_feature_flags(packages_remove_cross_joins_to_pipelines: false)
+
+        ::Gitlab::Database.allow_cross_joins_across_databases(url: 'https://gitlab.com/gitlab-org/gitlab/-/issues/342921') do
+          post_graphql(query, current_user: user)
+        end
+      end
+
+      it_behaves_like 'a working graphql query' do
+        it_behaves_like 'matching the package details schema'
+      end
+    end
+  end
+end
+
+RSpec.shared_examples 'matching the package details schema' do
   it 'matches the JSON schema' do
     expect(package_details).to match_schema('graphql/packages/package_details')
   end
-  end
 end
 
 RSpec.shared_examples 'a package with files' do