Commit c21eede7 authored by Steve Abrams's avatar Steve Abrams

Remove package/ci table cross joins

Updates associations to remove cross join queries
between the package and package_file tables with ci
tables.
Co-authored-by: default avatar10io <dfernandez@gitlab.com>
parent a1c814cb
......@@ -22,7 +22,7 @@ module Packages
def packages_for_group_projects(installable_only: false)
packages = ::Packages::Package
.including_build_info
.load_pipelines
.including_project_route
.including_tags
.for_projects(group_projects_visible_to_current_user.select(:id))
......
......@@ -9,7 +9,7 @@ module Packages
def execute
@project
.packages
.including_build_info
.load_pipelines
.including_project_route
.including_tags
.displayable
......
......@@ -14,9 +14,10 @@ module Packages
def execute
packages = project.packages
.including_build_info
.load_pipelines
.including_project_route
.including_tags
packages = filter_with_version(packages)
packages = filter_by_package_type(packages)
packages = filter_by_package_name(packages)
......
......@@ -40,7 +40,7 @@ class Packages::Package < ApplicationRecord
has_one :composer_metadatum, inverse_of: :package, class_name: 'Packages::Composer::Metadatum'
has_one :rubygems_metadatum, inverse_of: :package, class_name: 'Packages::Rubygems::Metadatum'
has_many :build_infos, inverse_of: :package
has_many :pipelines, through: :build_infos
has_many :pipelines, through: :build_infos, disable_joins: -> { disable_cross_joins_to_pipelines? }
has_one :debian_publication, inverse_of: :package, class_name: 'Packages::Debian::Publication'
has_one :debian_distribution, through: :debian_publication, source: :distribution, inverse_of: :packages, class_name: 'Packages::Debian::ProjectDistribution'
......@@ -131,9 +131,11 @@ class Packages::Package < ApplicationRecord
scope :has_version, -> { where.not(version: nil) }
scope :preload_files, -> { preload(:package_files) }
scope :preload_pipelines, -> { preload(pipelines: :user) }
scope :last_of_each_version, -> { where(id: all.select('MAX(id) AS id').group(:version)) }
scope :limit_recent, ->(limit) { order_created_desc.limit(limit) }
scope :select_distinct_name, -> { select(:name).distinct }
scope :load_pipelines, -> { disable_cross_joins_to_pipelines? ? preload_pipelines : including_build_info }
# Sorting
scope :order_created, -> { reorder(created_at: :asc) }
......@@ -160,6 +162,10 @@ class Packages::Package < ApplicationRecord
joins(:project).reorder(keyset_order)
end
def self.disable_cross_joins_to_pipelines?
::Feature.enabled?(:packages_remove_cross_joins_to_pipelines, default_enabled: :yaml)
end
def self.only_maven_packages_with_path(path, use_cte: false)
if use_cte
# This is an optimization fence which assumes that looking up the Metadatum record by path (globally)
......@@ -245,7 +251,7 @@ class Packages::Package < ApplicationRecord
def versions
project.packages
.including_build_info
.load_pipelines
.including_tags
.with_name(name)
.where.not(version: version)
......
......@@ -15,7 +15,7 @@ class Packages::PackageFile < ApplicationRecord
has_one :conan_file_metadatum, inverse_of: :package_file, class_name: 'Packages::Conan::FileMetadatum'
has_many :package_file_build_infos, inverse_of: :package_file, class_name: 'Packages::PackageFileBuildInfo'
has_many :pipelines, through: :package_file_build_infos
has_many :pipelines, through: :package_file_build_infos, disable_joins: -> { Packages::Package.disable_cross_joins_to_pipelines? }
has_one :debian_file_metadatum, inverse_of: :package_file, class_name: 'Packages::Debian::FileMetadatum'
has_one :helm_file_metadatum, inverse_of: :package_file, class_name: 'Packages::Helm::FileMetadatum'
......@@ -38,6 +38,7 @@ class Packages::PackageFile < ApplicationRecord
scope :with_format, ->(format) { where(::Packages::PackageFile.arel_table[:file_name].matches("%.#{format}")) }
scope :preload_package, -> { preload(:package) }
scope :preload_pipelines, -> { preload(pipelines: :user) }
scope :preload_conan_file_metadata, -> { preload(:conan_file_metadatum) }
scope :preload_debian_file_metadata, -> { preload(:debian_file_metadatum) }
scope :preload_helm_file_metadata, -> { preload(:helm_file_metadatum) }
......
---
name: packages_remove_cross_joins_to_pipelines
introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/70624
rollout_issue_url: https://gitlab.com/gitlab-org/gitlab/-/issues/342921
milestone: '14.5'
type: development
group: group::package
default_enabled: false
......@@ -28,7 +28,10 @@ module API
package = ::Packages::PackageFinder
.new(user_project, params[:package_id]).execute
present paginate(package.package_files), with: ::API::Entities::PackageFile
files = package.package_files
files = files.preload_pipelines if Packages::Package.disable_cross_joins_to_pipelines?
present paginate(files), with: ::API::Entities::PackageFile
end
desc 'Remove a package file' do
......
......@@ -160,6 +160,30 @@ RSpec.describe Packages::GroupPackagesFinder do
end
end
context 'with pipelines' do
let_it_be(:build_info_1) { create(:package_build_info, :with_pipeline, package: package1) }
let_it_be(:build_info_2) { create(:package_build_info, :with_pipeline, package: package2) }
it 'preloads the pipelines' do
expect(::Packages::Package).to receive(:preload_pipelines).and_call_original
expect(::Packages::Package).not_to receive(:including_build_info)
expect(subject).to match_array([package1, package2])
end
context 'with packages_remove_cross_joins_to_pipelines disabled' do
before do
stub_feature_flags(packages_remove_cross_joins_to_pipelines: false)
end
it 'includes the pipelines' do
expect(::Packages::Package).to receive(:including_build_info).and_call_original
expect(::Packages::Package).not_to receive(:preload_pipelines)
expect(subject).to match_array([package1, package2])
end
end
end
it_behaves_like 'concerning versionless param'
it_behaves_like 'concerning package statuses'
end
......
......@@ -32,5 +32,29 @@ RSpec.describe ::Packages::PackageFinder do
expect { subject }.to raise_exception(ActiveRecord::RecordNotFound)
end
end
context 'with pipelines' do
let_it_be(:build_info) { create(:package_build_info, :with_pipeline, package: maven_package) }
it 'preloads the pipelines' do
expect(::Packages::Package).to receive(:preload_pipelines).and_call_original
expect(::Packages::Package).not_to receive(:including_build_info)
expect(subject).to eq(maven_package)
end
context 'with packages_remove_cross_joins_to_pipelines disabled' do
before do
stub_feature_flags(packages_remove_cross_joins_to_pipelines: false)
end
it 'includes the pipelines' do
expect(::Packages::Package).to receive(:including_build_info).and_call_original
expect(::Packages::Package).not_to receive(:preload_pipelines)
expect(subject).to eq(maven_package)
end
end
end
end
end
......@@ -81,6 +81,31 @@ RSpec.describe ::Packages::PackagesFinder do
it { is_expected.to match_array([conan_package, maven_package]) }
end
context 'with pipelines' do
let_it_be(:build_info1) { create(:package_build_info, :with_pipeline, package: conan_package) }
let_it_be(:build_info2) { create(:package_build_info, :with_pipeline, package: maven_package) }
it 'preloads the pipelines' do
expect(::Packages::Package).to receive(:preload_pipelines).and_call_original
expect(::Packages::Package).not_to receive(:including_build_info)
expect(subject).to match_array([conan_package, maven_package])
end
context 'with packages_remove_cross_joins_to_pipelines disabled' do
before do
stub_feature_flags(packages_remove_cross_joins_to_pipelines: false)
end
it 'includes the pipelines' do
expect(::Packages::Package).to receive(:including_build_info).and_call_original
expect(::Packages::Package).not_to receive(:preload_pipelines)
expect(subject).to match_array([conan_package, maven_package])
end
end
end
it_behaves_like 'concerning versionless param'
it_behaves_like 'concerning package statuses'
end
......
......@@ -14,7 +14,6 @@ RSpec.describe Packages::PackageFile, type: :model do
it { is_expected.to belong_to(:package) }
it { is_expected.to have_one(:conan_file_metadatum) }
it { is_expected.to have_many(:package_file_build_infos).inverse_of(:package_file) }
it { is_expected.to have_many(:pipelines).through(:package_file_build_infos) }
it { is_expected.to have_one(:debian_file_metadatum).inverse_of(:package_file).class_name('Packages::Debian::FileMetadatum') }
it { is_expected.to have_one(:helm_file_metadatum).inverse_of(:package_file).class_name('Packages::Helm::FileMetadatum') }
end
......@@ -206,6 +205,28 @@ RSpec.describe Packages::PackageFile, type: :model do
end
end
describe '#pipelines' do
let_it_be_with_refind(:package_file) { create(:package_file) }
subject { package_file.pipelines }
context 'package_file without pipeline' do
it { is_expected.to be_empty }
end
context 'package_file with pipeline' do
let_it_be(:pipeline) { create(:ci_pipeline) }
let_it_be(:pipeline2) { create(:ci_pipeline) }
before do
package_file.package_file_build_infos.create!(pipeline: pipeline)
package_file.package_file_build_infos.create!(pipeline: pipeline2)
end
it { is_expected.to contain_exactly(pipeline, pipeline2) }
end
end
describe '#update_file_store callback' do
let_it_be(:package_file) { build(:package_file, :nuget, size: nil) }
......
......@@ -14,7 +14,6 @@ RSpec.describe Packages::Package, type: :model do
it { is_expected.to have_many(:dependency_links).inverse_of(:package) }
it { is_expected.to have_many(:tags).inverse_of(:package) }
it { is_expected.to have_many(:build_infos).inverse_of(:package) }
it { is_expected.to have_many(:pipelines).through(:build_infos) }
it { is_expected.to have_one(:conan_metadatum).inverse_of(:package) }
it { is_expected.to have_one(:maven_metadatum).inverse_of(:package) }
it { is_expected.to have_one(:debian_publication).inverse_of(:package).class_name('Packages::Debian::Publication') }
......@@ -962,6 +961,33 @@ RSpec.describe Packages::Package, type: :model do
end
end
describe '.load_pipelines' do
let_it_be(:package) { create(:maven_package) }
let_it_be(:build_info) { create(:package_build_info, :with_pipeline, package: package) }
subject { described_class.load_pipelines }
it 'uses preload', :aggregate_failures do
expect(described_class).to receive(:preload_pipelines).and_call_original
expect(described_class).not_to receive(:including_build_info)
subject
end
context 'with packages_remove_cross_joins_to_pipelines disabled' do
before do
stub_feature_flags(packages_remove_cross_joins_to_pipelines: false)
end
it 'uses includes', :aggregate_failures do
expect(described_class).to receive(:including_build_info).and_call_original
expect(described_class).not_to receive(:preload_pipelines)
subject
end
end
end
describe '#versions' do
let_it_be(:project) { create(:project) }
let_it_be(:package) { create(:maven_package, project: project) }
......@@ -975,6 +1001,30 @@ RSpec.describe Packages::Package, type: :model do
it 'does not return different packages' do
expect(package.versions).not_to include(package3)
end
context 'with pipelines' do
let_it_be(:build_info) { create(:package_build_info, :with_pipeline, package: package2) }
it 'preloads the pipelines' do
expect(::Packages::Package).to receive(:preload_pipelines).and_call_original
expect(::Packages::Package).not_to receive(:including_build_info)
expect(package.versions).to contain_exactly(package2)
end
context 'with packages_remove_cross_joins_to_pipelines disabled' do
before do
stub_feature_flags(packages_remove_cross_joins_to_pipelines: false)
end
it 'includes the pipelines' do
expect(::Packages::Package).to receive(:including_build_info).and_call_original
expect(::Packages::Package).not_to receive(:preload_pipelines)
expect(package.versions).to contain_exactly(package2)
end
end
end
end
describe '#pipeline' do
......@@ -999,6 +1049,28 @@ RSpec.describe Packages::Package, type: :model do
end
end
describe '#pipelines' do
let_it_be_with_refind(:package) { create(:maven_package) }
subject { package.pipelines }
context 'package without pipeline' do
it { is_expected.to be_empty }
end
context 'package with pipeline' do
let_it_be(:pipeline) { create(:ci_pipeline) }
let_it_be(:pipeline2) { create(:ci_pipeline) }
before do
package.build_infos.create!(pipeline: pipeline)
package.build_infos.create!(pipeline: pipeline2)
end
it { is_expected.to contain_exactly(pipeline, pipeline2) }
end
end
describe '#tag_names' do
let_it_be(:package) { create(:nuget_package) }
......
......@@ -27,6 +27,34 @@ RSpec.describe API::PackageFiles do
expect(response).to have_gitlab_http_status(:not_found)
end
context 'with pipelines' do
let(:package_file_build_info) { create(:package_file_build_info, :with_pipeline, package_file: package.package_files.first) }
it 'preloads the pipelines' do
expect(::Packages::PackageFile).to receive(:preload_pipelines).and_call_original
get api(url)
expect(response).to have_gitlab_http_status(:ok)
end
context 'with packages_remove_cross_joins_to_pipelines disabled' do
before do
stub_feature_flags(packages_remove_cross_joins_to_pipelines: false)
end
it 'does not preload the pipelines' do
expect(::Packages::PackageFile).not_to receive(:preload_pipelines)
::Gitlab::Database.allow_cross_joins_across_databases(url: 'https://gitlab.com/gitlab-org/gitlab/-/issues/342921') do
get api(url)
end
expect(response).to have_gitlab_http_status(:ok)
end
end
end
end
context 'project is private' do
......
......@@ -2,26 +2,10 @@
- "./ee/spec/services/ci/minutes/additional_packs/change_namespace_service_spec.rb"
- "./ee/spec/services/ci/minutes/additional_packs/create_service_spec.rb"
- "./ee/spec/services/ci/minutes/refresh_cached_data_service_spec.rb"
- "./spec/features/groups/packages_spec.rb"
- "./spec/features/projects/infrastructure_registry_spec.rb"
- "./spec/features/projects/packages_spec.rb"
- "./spec/lib/api/entities/package_spec.rb"
- "./spec/lib/gitlab/background_migration/copy_ci_builds_columns_to_security_scans_spec.rb"
- "./spec/lib/gitlab/background_migration/migrate_pages_metadata_spec.rb"
- "./spec/migrations/20210907211557_finalize_ci_builds_bigint_conversion_spec.rb"
- "./spec/migrations/associate_existing_dast_builds_with_variables_spec.rb"
- "./spec/migrations/disable_job_token_scope_when_unused_spec.rb"
- "./spec/migrations/schedule_copy_ci_builds_columns_to_security_scans2_spec.rb"
- "./spec/presenters/packages/detail/package_presenter_spec.rb"
- "./spec/requests/api/graphql/group/packages_spec.rb"
- "./spec/requests/api/graphql/packages/composer_spec.rb"
- "./spec/requests/api/graphql/packages/conan_spec.rb"
- "./spec/requests/api/graphql/packages/maven_spec.rb"
- "./spec/requests/api/graphql/packages/nuget_spec.rb"
- "./spec/requests/api/graphql/packages/package_spec.rb"
- "./spec/requests/api/graphql/packages/pypi_spec.rb"
- "./spec/requests/api/graphql/project/packages_spec.rb"
- "./spec/requests/api/package_files_spec.rb"
- "./spec/services/packages/conan/create_package_file_service_spec.rb"
- "./spec/services/packages/create_package_file_service_spec.rb"
- "./spec/services/packages/generic/create_package_file_service_spec.rb"
......@@ -10,6 +10,7 @@ RSpec.shared_context 'package details setup' do
let(:excluded) { %w[metadata apiFuzzingCiConfiguration pipeline packageFiles] }
let(:package_files) { all_graphql_fields_for('PackageFile') }
let(:dependency_links) { all_graphql_fields_for('PackageDependencyLink') }
let(:pipelines) { all_graphql_fields_for('Pipeline', max_depth: 1) }
let(:user) { project.owner }
let(:package_details) { graphql_data_at(:package) }
let(:metadata_response) { graphql_data_at(:package, :metadata) }
......@@ -34,6 +35,11 @@ RSpec.shared_context 'package details setup' do
#{dependency_links}
}
}
pipelines {
nodes {
#{pipelines}
}
}
FIELDS
end
end
......@@ -2,10 +2,43 @@
RSpec.shared_examples 'a package detail' do
it_behaves_like 'a working graphql query' do
it_behaves_like 'matching the package details schema'
end
context 'with pipelines' do
let_it_be(:build_info1) { create(:package_build_info, :with_pipeline, package: package) }
let_it_be(:build_info2) { create(:package_build_info, :with_pipeline, package: package) }
let_it_be(:build_info3) { create(:package_build_info, :with_pipeline, package: package) }
it_behaves_like 'a working graphql query' do
it_behaves_like 'matching the package details schema'
end
context 'with packages_remove_cross_joins_to_pipelines disabled' do
# subject is called in a before callback that is executed before the one below
# the callback below MUST be executed before the subject
# solution: nullify subject and manually call #post_graphql
subject {}
before do
stub_feature_flags(packages_remove_cross_joins_to_pipelines: false)
::Gitlab::Database.allow_cross_joins_across_databases(url: 'https://gitlab.com/gitlab-org/gitlab/-/issues/342921') do
post_graphql(query, current_user: user)
end
end
it_behaves_like 'a working graphql query' do
it_behaves_like 'matching the package details schema'
end
end
end
end
RSpec.shared_examples 'matching the package details schema' do
it 'matches the JSON schema' do
expect(package_details).to match_schema('graphql/packages/package_details')
end
end
end
RSpec.shared_examples 'a package with files' do
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment