Debian Group and Project Distribution Component Files

See #5835

app/models/concerns/packages/debian/architecture.rb

@@ -7,6 +7,12 @@ module Packages
 
       included do
         belongs_to :distribution, class_name: "Packages::Debian::#{container_type.capitalize}Distribution", inverse_of: :architectures
+        # files must be destroyed by ruby code in order to properly remove carrierwave uploads
+        has_many :files,
+          class_name: "Packages::Debian::#{container_type.capitalize}ComponentFile",
+          foreign_key: :architecture_id,
+          inverse_of: :architecture,
+          dependent: :destroy # rubocop:disable Cop/ActiveRecordDependent
 
         validates :distribution,
           presence: true

app/models/concerns/packages/debian/component.rb

@@ -7,6 +7,12 @@ module Packages
 
       included do
         belongs_to :distribution, class_name: "Packages::Debian::#{container_type.capitalize}Distribution", inverse_of: :components
+        # files must be destroyed by ruby code in order to properly remove carrierwave uploads
+        has_many :files,
+          class_name: "Packages::Debian::#{container_type.capitalize}ComponentFile",
+          foreign_key: :component_id,
+          inverse_of: :component,
+          dependent: :destroy # rubocop:disable Cop/ActiveRecordDependent
 
         validates :distribution,
           presence: true

app/models/concerns/packages/debian/component_file.rb

+# frozen_string_literal: true
+
+module Packages
+  module Debian
+    module ComponentFile
+      extend ActiveSupport::Concern
+
+      included do
+        include Sortable
+        include FileStoreMounter
+
+        def self.container_foreign_key
+          "#{container_type}_id".to_sym
+        end
+
+        def self.distribution_class
+          "::Packages::Debian::#{container_type.capitalize}Distribution".constantize
+        end
+
+        belongs_to :component, class_name: "Packages::Debian::#{container_type.capitalize}Component", inverse_of: :files
+        belongs_to :architecture, class_name: "Packages::Debian::#{container_type.capitalize}Architecture", inverse_of: :files, optional: true
+
+        enum file_type: { packages: 1, source: 2, di_packages: 3 }
+        enum compression_type: { gz: 1, bz2: 2, xz: 3 }
+
+        validates :component, presence: true
+        validates :file_type, presence: true
+        validates :architecture, presence: true, unless: :source?
+        validates :architecture, absence: true, if: :source?
+        validates :file, length: { minimum: 0, allow_nil: false }
+        validates :size, presence: true
+        validates :file_store, presence: true
+        validates :file_md5, presence: true
+        validates :file_sha256, presence: true
+
+        scope :with_container, ->(container) do
+          joins(component: :distribution)
+            .where("packages_debian_#{container_type}_distributions" => { container_foreign_key => container.id })
+        end
+
+        scope :with_codename_or_suite, ->(codename_or_suite) do
+          joins(component: :distribution)
+            .merge(distribution_class.with_codename_or_suite(codename_or_suite))
+        end
+
+        scope :with_component_name, ->(component_name) do
+          joins(:component)
+            .where("packages_debian_#{container_type}_components" => { name: component_name })
+        end
+
+        scope :with_file_type, ->(file_type) { where(file_type: file_type) }
+
+        scope :with_architecture_name, ->(architecture_name) do
+          left_outer_joins(:architecture)
+            .where("packages_debian_#{container_type}_architectures" => { name: architecture_name })
+        end
+
+        scope :with_compression_type, ->(compression_type) { where(compression_type: compression_type) }
+        scope :with_file_sha256, ->(file_sha256) { where(file_sha256: file_sha256) }
+
+        scope :preload_distribution, -> { includes(component: :distribution) }
+
+        mount_file_store_uploader Packages::Debian::ComponentFileUploader
+
+        before_validation :update_size_from_file
+
+        def file_name
+          case file_type
+          when 'di_packages'
+            'Packages'
+          else
+            file_type.capitalize
+          end
+        end
+
+        def relative_path
+          case file_type
+          when 'packages'
+            "#{component.name}/binary-#{architecture.name}/#{file_name}#{extension}"
+          when 'source'
+            "#{component.name}/source/#{file_name}#{extension}"
+          when 'di_packages'
+            "#{component.name}/debian-installer/binary-#{architecture.name}/#{file_name}#{extension}"
+          end
+        end
+
+        private
+
+        def extension
+          return '' unless compression_type
+
+          ".#{compression_type}"
+        end
+
+        def update_size_from_file
+          self.size ||= file.size
+        end
+      end
+    end
+  end
+end

app/models/concerns/packages/debian/distribution.rb

@@ -18,10 +18,16 @@ module Packages
         belongs_to container_type
         belongs_to :creator, class_name: 'User'
 
+        # component_files must be destroyed by ruby code in order to properly remove carrierwave uploads
         has_many :components,
           class_name: "Packages::Debian::#{container_type.capitalize}Component",
           foreign_key: :distribution_id,
-          inverse_of: :distribution
+          inverse_of: :distribution,
+          dependent: :destroy # rubocop:disable Cop/ActiveRecordDependent
+        has_many :component_files,
+          through: :components,
+          source: :files,
+          class_name: "Packages::Debian::#{container_type.capitalize}ComponentFile"
         has_many :architectures,
           class_name: "Packages::Debian::#{container_type.capitalize}Architecture",
           foreign_key: :distribution_id,

app/models/group.rb

@@ -75,7 +75,7 @@ class Group < Namespace
   has_many :dependency_proxy_blobs, class_name: 'DependencyProxy::Blob'
   has_many :dependency_proxy_manifests, class_name: 'DependencyProxy::Manifest'
 
-  # debian_distributions must be destroyed by ruby code in order to properly remove carrierwave uploads
+  # debian_distributions and associated component_files must be destroyed by ruby code in order to properly remove carrierwave uploads
   has_many :debian_distributions, class_name: 'Packages::Debian::GroupDistribution', dependent: :destroy # rubocop:disable Cop/ActiveRecordDependent
 
   accepts_nested_attributes_for :variables, allow_destroy: true

app/models/packages/debian/group_component_file.rb

+# frozen_string_literal: true
+
+class Packages::Debian::GroupComponentFile < ApplicationRecord
+  def self.container_type
+    :group
+  end
+
+  include Packages::Debian::ComponentFile
+end

app/models/packages/debian/project_component_file.rb

+# frozen_string_literal: true
+
+class Packages::Debian::ProjectComponentFile < ApplicationRecord
+  def self.container_type
+    :project
+  end
+
+  include Packages::Debian::ComponentFile
+end

app/models/project.rb

@@ -200,7 +200,7 @@ class Project < ApplicationRecord
   # Packages
   has_many :packages, class_name: 'Packages::Package'
   has_many :package_files, through: :packages, class_name: 'Packages::PackageFile'
-  # debian_distributions must be destroyed by ruby code in order to properly remove carrierwave uploads
+  # debian_distributions and associated component_files must be destroyed by ruby code in order to properly remove carrierwave uploads
   has_many :debian_distributions, class_name: 'Packages::Debian::ProjectDistribution', dependent: :destroy # rubocop:disable Cop/ActiveRecordDependent
 
   has_one :import_state, autosave: true, class_name: 'ProjectImportState', inverse_of: :project

app/uploaders/packages/debian/component_file_uploader.rb

+# frozen_string_literal: true
+class Packages::Debian::ComponentFileUploader < GitlabUploader
+  extend Workhorse::UploadPath
+  include ObjectStorage::Concern
+
+  storage_options Gitlab.config.packages
+
+  after :store, :schedule_background_upload
+
+  alias_method :upload, :model
+
+  def filename
+    model.file_name
+  end
+
+  def store_dir
+    dynamic_segment
+  end
+
+  private
+
+  def dynamic_segment
+    raise ObjectNotReadyError, 'Package model not ready' unless model.id && model.component.distribution.container_id
+
+    Gitlab::HashedPath.new("debian_#{model.class.container_type}_component_file", model.id, root_hash: model.component.distribution.container_id)
+  end
+end

changelogs/unreleased/debian_component_files.yml

+---
+title: Debian Group and Project Distribution Component Files
+merge_request: 52885
+author: Mathieu Parent
+type: added

db/migrate/20201204111400_create_packages_debian_project_component_files.rb

+# frozen_string_literal: true
+
+class CreatePackagesDebianProjectComponentFiles < ActiveRecord::Migration[5.2]
+  include Gitlab::Database::MigrationHelpers
+
+  DOWNTIME = false
+
+  INDEX_ARCHITECTURE = 'idx_packages_debian_project_component_files_on_architecture_id'
+
+  disable_ddl_transaction!
+
+  def up
+    with_lock_retries do
+      unless table_exists?(:packages_debian_project_component_files)
+        create_table :packages_debian_project_component_files do |t|
+          t.timestamps_with_timezone
+          t.references :component,
+            foreign_key: { to_table: :packages_debian_project_components, on_delete: :restrict },
+            null: false,
+            index: true
+          t.references :architecture,
+            foreign_key: { to_table: :packages_debian_project_architectures, on_delete: :restrict },
+            index: { name: INDEX_ARCHITECTURE }
+          t.integer :size, null: false
+          t.integer :file_type, limit: 2, null: false
+          t.integer :compression_type, limit: 2
+          t.integer :file_store, limit: 2, default: 1, null: false
+          t.text :file, null: false
+          t.binary :file_md5, null: false
+          t.binary :file_sha256, null: false
+        end
+      end
+    end
+
+    add_text_limit :packages_debian_project_component_files, :file, 255
+  end
+
+  def down
+    drop_table :packages_debian_project_component_files
+  end
+end

db/migrate/20201204111500_create_packages_debian_group_component_files.rb

+# frozen_string_literal: true
+
+class CreatePackagesDebianGroupComponentFiles < ActiveRecord::Migration[5.2]
+  include Gitlab::Database::MigrationHelpers
+
+  DOWNTIME = false
+
+  INDEX_ARCHITECTURE = 'idx_packages_debian_group_component_files_on_architecture_id'
+
+  disable_ddl_transaction!
+
+  def up
+    with_lock_retries do
+      unless table_exists?(:packages_debian_group_component_files)
+        create_table :packages_debian_group_component_files do |t|
+          t.timestamps_with_timezone
+          t.references :component,
+            foreign_key: { to_table: :packages_debian_group_components, on_delete: :restrict },
+            null: false,
+            index: true
+          t.references :architecture,
+            foreign_key: { to_table: :packages_debian_group_architectures, on_delete: :restrict },
+            index: { name: INDEX_ARCHITECTURE }
+          t.integer :size, null: false
+          t.integer :file_type, limit: 2, null: false
+          t.integer :compression_type, limit: 2
+          t.integer :file_store, limit: 2, default: 1, null: false
+          t.text :file, null: false
+          t.binary :file_md5, null: false
+          t.binary :file_sha256, null: false
+        end
+      end
+    end
+
+    add_text_limit :packages_debian_group_component_files, :file, 255
+  end
+
+  def down
+    drop_table :packages_debian_group_component_files
+  end
+end

db/schema_migrations/20201204111400

+6fcaa4184ae69fabd6f2668cad19c38a8ae7c187053d60cdf4fcbdbc0443aa42
\ No newline at end of file

db/schema_migrations/20201204111500

+3f422a916b50cafd46b4a7486b6c3cc0a9992831a7dbc40c51323c835d845a0a
\ No newline at end of file

db/structure.sql

@@ -14894,6 +14894,31 @@ CREATE SEQUENCE packages_debian_group_architectures_id_seq
 
 ALTER SEQUENCE packages_debian_group_architectures_id_seq OWNED BY packages_debian_group_architectures.id;
 
+CREATE TABLE packages_debian_group_component_files (
+    id bigint NOT NULL,
+    created_at timestamp with time zone NOT NULL,
+    updated_at timestamp with time zone NOT NULL,
+    component_id bigint NOT NULL,
+    architecture_id bigint,
+    size integer NOT NULL,
+    file_type smallint NOT NULL,
+    compression_type smallint,
+    file_store smallint DEFAULT 1 NOT NULL,
+    file text NOT NULL,
+    file_md5 bytea NOT NULL,
+    file_sha256 bytea NOT NULL,
+    CONSTRAINT check_839e1685bc CHECK ((char_length(file) <= 255))
+);
+
+CREATE SEQUENCE packages_debian_group_component_files_id_seq
+    START WITH 1
+    INCREMENT BY 1
+    NO MINVALUE
+    NO MAXVALUE
+    CACHE 1;
+
+ALTER SEQUENCE packages_debian_group_component_files_id_seq OWNED BY packages_debian_group_component_files.id;
+
 CREATE TABLE packages_debian_group_components (
     id bigint NOT NULL,
     created_at timestamp with time zone NOT NULL,
@@ -14971,6 +14996,31 @@ CREATE SEQUENCE packages_debian_project_architectures_id_seq
 
 ALTER SEQUENCE packages_debian_project_architectures_id_seq OWNED BY packages_debian_project_architectures.id;
 
+CREATE TABLE packages_debian_project_component_files (
+    id bigint NOT NULL,
+    created_at timestamp with time zone NOT NULL,
+    updated_at timestamp with time zone NOT NULL,
+    component_id bigint NOT NULL,
+    architecture_id bigint,
+    size integer NOT NULL,
+    file_type smallint NOT NULL,
+    compression_type smallint,
+    file_store smallint DEFAULT 1 NOT NULL,
+    file text NOT NULL,
+    file_md5 bytea NOT NULL,
+    file_sha256 bytea NOT NULL,
+    CONSTRAINT check_e5af03fa2d CHECK ((char_length(file) <= 255))
+);
+
+CREATE SEQUENCE packages_debian_project_component_files_id_seq
+    START WITH 1
+    INCREMENT BY 1
+    NO MINVALUE
+    NO MAXVALUE
+    CACHE 1;
+
+ALTER SEQUENCE packages_debian_project_component_files_id_seq OWNED BY packages_debian_project_component_files.id;
+
 CREATE TABLE packages_debian_project_components (
     id bigint NOT NULL,
     created_at timestamp with time zone NOT NULL,
@@ -19005,12 +19055,16 @@ ALTER TABLE ONLY packages_conan_metadata ALTER COLUMN id SET DEFAULT nextval('pa
 
 ALTER TABLE ONLY packages_debian_group_architectures ALTER COLUMN id SET DEFAULT nextval('packages_debian_group_architectures_id_seq'::regclass);
 
+ALTER TABLE ONLY packages_debian_group_component_files ALTER COLUMN id SET DEFAULT nextval('packages_debian_group_component_files_id_seq'::regclass);
+
 ALTER TABLE ONLY packages_debian_group_components ALTER COLUMN id SET DEFAULT nextval('packages_debian_group_components_id_seq'::regclass);
 
 ALTER TABLE ONLY packages_debian_group_distributions ALTER COLUMN id SET DEFAULT nextval('packages_debian_group_distributions_id_seq'::regclass);
 
 ALTER TABLE ONLY packages_debian_project_architectures ALTER COLUMN id SET DEFAULT nextval('packages_debian_project_architectures_id_seq'::regclass);
 
+ALTER TABLE ONLY packages_debian_project_component_files ALTER COLUMN id SET DEFAULT nextval('packages_debian_project_component_files_id_seq'::regclass);
+
 ALTER TABLE ONLY packages_debian_project_components ALTER COLUMN id SET DEFAULT nextval('packages_debian_project_components_id_seq'::regclass);
 
 ALTER TABLE ONLY packages_debian_project_distributions ALTER COLUMN id SET DEFAULT nextval('packages_debian_project_distributions_id_seq'::regclass);
@@ -20383,6 +20437,9 @@ ALTER TABLE ONLY packages_debian_file_metadata
 ALTER TABLE ONLY packages_debian_group_architectures
     ADD CONSTRAINT packages_debian_group_architectures_pkey PRIMARY KEY (id);
 
+ALTER TABLE ONLY packages_debian_group_component_files
+    ADD CONSTRAINT packages_debian_group_component_files_pkey PRIMARY KEY (id);
+
 ALTER TABLE ONLY packages_debian_group_components
     ADD CONSTRAINT packages_debian_group_components_pkey PRIMARY KEY (id);
 
@@ -20392,6 +20449,9 @@ ALTER TABLE ONLY packages_debian_group_distributions
 ALTER TABLE ONLY packages_debian_project_architectures
     ADD CONSTRAINT packages_debian_project_architectures_pkey PRIMARY KEY (id);
 
+ALTER TABLE ONLY packages_debian_project_component_files
+    ADD CONSTRAINT packages_debian_project_component_files_pkey PRIMARY KEY (id);
+
 ALTER TABLE ONLY packages_debian_project_components
     ADD CONSTRAINT packages_debian_project_components_pkey PRIMARY KEY (id);
 
@@ -21113,6 +21173,10 @@ CREATE UNIQUE INDEX idx_on_compliance_management_frameworks_namespace_id_name ON
 
 CREATE INDEX idx_packages_build_infos_on_package_id ON packages_build_infos USING btree (package_id);
 
+CREATE INDEX idx_packages_debian_group_component_files_on_architecture_id ON packages_debian_group_component_files USING btree (architecture_id);
+
+CREATE INDEX idx_packages_debian_project_component_files_on_architecture_id ON packages_debian_project_component_files USING btree (architecture_id);
+
 CREATE INDEX idx_packages_packages_on_project_id_name_version_package_type ON packages_packages USING btree (project_id, name, version, package_type);
 
 CREATE INDEX idx_pkgs_deb_grp_architectures_on_distribution_id ON packages_debian_group_architectures USING btree (distribution_id);
@@ -22615,10 +22679,14 @@ CREATE UNIQUE INDEX index_packages_conan_file_metadata_on_package_file_id ON pac
 
 CREATE UNIQUE INDEX index_packages_conan_metadata_on_package_id_username_channel ON packages_conan_metadata USING btree (package_id, package_username, package_channel);
 
+CREATE INDEX index_packages_debian_group_component_files_on_component_id ON packages_debian_group_component_files USING btree (component_id);
+
 CREATE INDEX index_packages_debian_group_distributions_on_creator_id ON packages_debian_group_distributions USING btree (creator_id);
 
 CREATE INDEX index_packages_debian_group_distributions_on_group_id ON packages_debian_group_distributions USING btree (group_id);
 
+CREATE INDEX index_packages_debian_project_component_files_on_component_id ON packages_debian_project_component_files USING btree (component_id);
+
 CREATE INDEX index_packages_debian_project_distributions_on_creator_id ON packages_debian_project_distributions USING btree (creator_id);
 
 CREATE INDEX index_packages_debian_project_distributions_on_project_id ON packages_debian_project_distributions USING btree (project_id);
@@ -24825,6 +24893,9 @@ ALTER TABLE ONLY group_group_links
 ALTER TABLE ONLY geo_repository_updated_events
     ADD CONSTRAINT fk_rails_2b70854c08 FOREIGN KEY (project_id) REFERENCES projects(id) ON DELETE CASCADE;
 
+ALTER TABLE ONLY packages_debian_group_component_files
+    ADD CONSTRAINT fk_rails_2b8992dd83 FOREIGN KEY (architecture_id) REFERENCES packages_debian_group_architectures(id) ON DELETE RESTRICT;
+
 ALTER TABLE ONLY boards_epic_board_labels
     ADD CONSTRAINT fk_rails_2bedeb8799 FOREIGN KEY (label_id) REFERENCES labels(id) ON DELETE CASCADE;
 
@@ -25590,6 +25661,9 @@ ALTER TABLE ONLY merge_trains
 ALTER TABLE ONLY application_settings
     ADD CONSTRAINT fk_rails_b53e481273 FOREIGN KEY (custom_project_templates_group_id) REFERENCES namespaces(id) ON DELETE SET NULL;
 
+ALTER TABLE ONLY packages_debian_project_component_files
+    ADD CONSTRAINT fk_rails_b543a9622b FOREIGN KEY (architecture_id) REFERENCES packages_debian_project_architectures(id) ON DELETE RESTRICT;
+
 ALTER TABLE ONLY namespace_aggregation_schedules
     ADD CONSTRAINT fk_rails_b565c8d16c FOREIGN KEY (namespace_id) REFERENCES namespaces(id) ON DELETE CASCADE;
 
@@ -25614,6 +25688,9 @@ ALTER TABLE ONLY lists
 ALTER TABLE ONLY security_findings
     ADD CONSTRAINT fk_rails_bb63863cf1 FOREIGN KEY (scan_id) REFERENCES security_scans(id) ON DELETE CASCADE;
 
+ALTER TABLE ONLY packages_debian_project_component_files
+    ADD CONSTRAINT fk_rails_bbe9ebfbd9 FOREIGN KEY (component_id) REFERENCES packages_debian_project_components(id) ON DELETE RESTRICT;
+
 ALTER TABLE ONLY approval_merge_request_rules_users
     ADD CONSTRAINT fk_rails_bc8972fa55 FOREIGN KEY (user_id) REFERENCES users(id) ON DELETE CASCADE;
 
@@ -25782,6 +25859,9 @@ ALTER TABLE ONLY vulnerability_occurrence_pipelines
 ALTER TABLE ONLY deployment_merge_requests
     ADD CONSTRAINT fk_rails_dcbce9f4df FOREIGN KEY (deployment_id) REFERENCES deployments(id) ON DELETE CASCADE;
 
+ALTER TABLE ONLY packages_debian_group_component_files
+    ADD CONSTRAINT fk_rails_dd262386e9 FOREIGN KEY (component_id) REFERENCES packages_debian_group_components(id) ON DELETE RESTRICT;
+
 ALTER TABLE ONLY user_callouts
     ADD CONSTRAINT fk_rails_ddfdd80f3d FOREIGN KEY (user_id) REFERENCES users(id) ON DELETE CASCADE;
 

spec/factories/packages/debian/component_file.rb

+# frozen_string_literal: true
+
+FactoryBot.define do
+  factory :debian_project_component_file, class: 'Packages::Debian::ProjectComponentFile' do
+    component { association(:debian_project_component) }
+    architecture { association(:debian_project_architecture, distribution: component.distribution) }
+
+    factory :debian_group_component_file, class: 'Packages::Debian::GroupComponentFile' do
+      component { association(:debian_group_component) }
+      architecture { association(:debian_group_architecture, distribution: component.distribution) }
+    end
+
+    file_type { :packages }
+
+    after(:build) do |component_file, evaluator|
+      component_file.file = fixture_file_upload('spec/fixtures/packages/debian/distribution/Packages')
+    end
+
+    file_md5 { '12345abcde' }
+    file_sha256 { 'be93151dc23ac34a82752444556fe79b32c7a1ad' }
+
+    trait(:packages) do
+      file_type { :packages }
+    end
+
+    trait(:source) do
+      file_type { :source }
+      architecture { nil }
+    end
+
+    trait(:di_packages) do
+      file_type { :di_packages }
+    end
+
+    trait(:object_storage) do
+      file_store { Packages::PackageFileUploader::Store::REMOTE }
+    end
+  end
+end

spec/fixtures/packages/debian/distribution/Packages

+Package: example-package
+Description: This is an incomplete Packages file

spec/models/group_spec.rb

@@ -1778,19 +1778,6 @@ RSpec.describe Group do
   describe 'with Debian Distributions' do
     subject { create(:group) }
 
-    let!(:distributions) { create_list(:debian_group_distribution, 2, :with_file, container: subject) }
-
-    it 'removes distribution files on removal' do
-      distribution_file_paths = distributions.map do |distribution|
-        distribution.file.path
-      end
-
-      expect { subject.destroy }
-        .to change {
-          distribution_file_paths.select do |path|
-            File.exist? path
-          end.length
-        }.from(distribution_file_paths.length).to(0)
-    end
+    it_behaves_like 'model with Debian distributions'
   end
 end

spec/models/packages/debian/group_component_file_spec.rb

+# frozen_string_literal: true
+
+require 'spec_helper'
+
+RSpec.describe Packages::Debian::GroupComponentFile do
+  it_behaves_like 'Debian Component File', :group, false
+end

spec/models/packages/debian/project_component_file_spec.rb

+# frozen_string_literal: true
+
+require 'spec_helper'
+
+RSpec.describe Packages::Debian::ProjectComponentFile do
+  it_behaves_like 'Debian Component File', :project, true
+end

spec/models/project_spec.rb

@@ -6388,20 +6388,7 @@ RSpec.describe Project, factory_default: :keep do
   describe 'with Debian Distributions' do
     subject { create(:project) }
 
-    let!(:distributions) { create_list(:debian_project_distribution, 2, :with_file, container: subject) }
-
-    it 'removes distribution files on removal' do
-      distribution_file_paths = distributions.map do |distribution|
-        distribution.file.path
-      end
-
-      expect { subject.destroy }
-        .to change {
-          distribution_file_paths.select do |path|
-            File.exist? path
-          end.length
-        }.from(distribution_file_paths.length).to(0)
-    end
+    it_behaves_like 'model with Debian distributions'
   end
 
   describe '#environments_for_scope' do

spec/services/packages/debian/create_distribution_service_spec.rb

@@ -15,6 +15,8 @@ RSpec.describe Packages::Debian::CreateDistributionService do
           .from(nil).to(expected_components.count)
           .and change { container.debian_distributions.first&.architectures&.count }
           .from(nil).to(expected_architectures.count)
+          .and not_change { Packages::Debian::ProjectComponentFile.count }
+          .and not_change { Packages::Debian::GroupComponentFile.count }
       else
         expect { response }
           .to not_change { container.debian_distributions.klass.all.count }
@@ -23,6 +25,8 @@ RSpec.describe Packages::Debian::CreateDistributionService do
           .and not_change { Packages::Debian::GroupComponent.count }
           .and not_change { Packages::Debian::ProjectArchitecture.count }
           .and not_change { Packages::Debian::GroupArchitecture.count }
+          .and not_change { Packages::Debian::ProjectComponentFile.count }
+          .and not_change { Packages::Debian::GroupComponentFile.count }
       end
 
       expect(response).to be_a(ServiceResponse)

spec/services/packages/debian/destroy_distribution_service_spec.rb

@@ -15,12 +15,15 @@ RSpec.describe Packages::Debian::DestroyDistributionService do
           .from(2).to(0)
           .and change { architecture1.class.all.count }
           .from(3).to(0)
+          .and change { component_file1.class.all.count }
+          .from(4).to(0)
       else
         expect { response }
           .to not_change { container.debian_distributions.klass.all.count }
           .and not_change { container.debian_distributions.count }
           .and not_change { component1.class.all.count }
           .and not_change { architecture1.class.all.count }
+          .and not_change { component_file1.class.all.count }
       end
 
       expect(response).to be_a(ServiceResponse)
@@ -45,6 +48,10 @@ RSpec.describe Packages::Debian::DestroyDistributionService do
       let_it_be(:architecture0, freeze: true) { create("debian_#{container_type}_architecture", distribution: distribution, name: 'all') }
       let_it_be(:architecture1, freeze: can_freeze) { create("debian_#{container_type}_architecture", distribution: distribution, name: 'architecture1') }
       let_it_be(:architecture2, freeze: can_freeze) { create("debian_#{container_type}_architecture", distribution: distribution, name: 'architecture2') }
+      let_it_be(:component_file1, freeze: can_freeze) { create("debian_#{container_type}_component_file", :source, component: component1) }
+      let_it_be(:component_file2, freeze: can_freeze) { create("debian_#{container_type}_component_file", component: component1, architecture: architecture1) }
+      let_it_be(:component_file3, freeze: can_freeze) { create("debian_#{container_type}_component_file", :source, component: component2) }
+      let_it_be(:component_file4, freeze: can_freeze) { create("debian_#{container_type}_component_file", component: component2, architecture: architecture2) }
 
       subject { described_class.new(distribution) }
 

spec/services/packages/debian/update_distribution_service_spec.rb

@@ -3,15 +3,26 @@
 require 'spec_helper'
 
 RSpec.describe Packages::Debian::UpdateDistributionService do
-  RSpec.shared_examples 'Update Debian Distribution' do |expected_message, expected_components, expected_architectures|
+  RSpec.shared_examples 'Update Debian Distribution' do |expected_message, expected_components, expected_architectures, component_file_delta = 0|
     it 'returns ServiceResponse', :aggregate_failures do
       expect(distribution).to receive(:update).with(simple_params).and_call_original if expected_message.nil?
 
-      expect { response }
-        .to not_change { container.debian_distributions.klass.all.count }
-        .and not_change { container.debian_distributions.count }
-        .and not_change { component1.class.all.count }
-        .and not_change { architecture1.class.all.count }
+      if component_file_delta.zero?
+        expect { response }
+          .to not_change { container.debian_distributions.klass.all.count }
+          .and not_change { container.debian_distributions.count }
+          .and not_change { component1.class.all.count }
+          .and not_change { architecture1.class.all.count }
+          .and not_change { component_file1.class.all.count }
+      else
+        expect { response }
+          .to not_change { container.debian_distributions.klass.all.count }
+          .and not_change { container.debian_distributions.count }
+          .and not_change { component1.class.all.count }
+          .and not_change { architecture1.class.all.count }
+          .and change { component_file1.class.all.count }
+          .from(4).to(4 + component_file_delta)
+      end
 
       expect(response).to be_a(ServiceResponse)
       expect(response.success?).to eq(expected_message.nil?)
@@ -48,6 +59,10 @@ RSpec.describe Packages::Debian::UpdateDistributionService do
       let_it_be(:architecture0) { create("debian_#{container_type}_architecture", distribution: distribution, name: 'all') }
       let_it_be(:architecture1) { create("debian_#{container_type}_architecture", distribution: distribution, name: 'architecture1') }
       let_it_be(:architecture2) { create("debian_#{container_type}_architecture", distribution: distribution, name: 'architecture2') }
+      let_it_be(:component_file1) { create("debian_#{container_type}_component_file", :source, component: component1) }
+      let_it_be(:component_file2) { create("debian_#{container_type}_component_file", component: component1, architecture: architecture1) }
+      let_it_be(:component_file3) { create("debian_#{container_type}_component_file", :source, component: component2) }
+      let_it_be(:component_file4) { create("debian_#{container_type}_component_file", component: component2, architecture: architecture2) }
 
       let(:original_params) do
         {
@@ -110,7 +125,7 @@ RSpec.describe Packages::Debian::UpdateDistributionService do
           }
         end
 
-        it_behaves_like 'Update Debian Distribution', nil, %w[component2 component3], %w[all architecture2 architecture3]
+        it_behaves_like 'Update Debian Distribution', nil, %w[component2 component3], %w[all architecture2 architecture3], -2
       end
 
       context 'with invalid components' do

spec/support/shared_examples/models/packages/debian/architecture_shared_examples.rb

@@ -11,6 +11,7 @@ RSpec.shared_examples 'Debian Distribution Architecture' do |factory, container,
 
   describe 'relationships' do
     it { is_expected.to belong_to(:distribution).class_name("Packages::Debian::#{container.capitalize}Distribution").inverse_of(:architectures) }
+    it { is_expected.to have_many(:files).class_name("Packages::Debian::#{container.capitalize}ComponentFile").inverse_of(:architecture) }
   end
 
   describe 'validations' do

spec/support/shared_examples/models/packages/debian/component_file_shared_example.rb

+# frozen_string_literal: true
+
+require 'spec_helper'
+
+RSpec.shared_examples 'Debian Component File' do |container_type, can_freeze|
+  let_it_be(:container1, freeze: can_freeze) { create(container_type) } # rubocop:disable Rails/SaveBang
+  let_it_be(:container2, freeze: can_freeze) { create(container_type) } # rubocop:disable Rails/SaveBang
+  let_it_be(:distribution1, freeze: can_freeze) { create("debian_#{container_type}_distribution", container: container1) }
+  let_it_be(:distribution2, freeze: can_freeze) { create("debian_#{container_type}_distribution", container: container2) }
+  let_it_be(:architecture1_1, freeze: can_freeze) { create("debian_#{container_type}_architecture", distribution: distribution1) }
+  let_it_be(:architecture1_2, freeze: can_freeze) { create("debian_#{container_type}_architecture", distribution: distribution1) }
+  let_it_be(:architecture2_1, freeze: can_freeze) { create("debian_#{container_type}_architecture", distribution: distribution2) }
+  let_it_be(:architecture2_2, freeze: can_freeze) { create("debian_#{container_type}_architecture", distribution: distribution2) }
+  let_it_be(:component1_1, freeze: can_freeze) { create("debian_#{container_type}_component", distribution: distribution1) }
+  let_it_be(:component1_2, freeze: can_freeze) { create("debian_#{container_type}_component", distribution: distribution1) }
+  let_it_be(:component2_1, freeze: can_freeze) { create("debian_#{container_type}_component", distribution: distribution2) }
+  let_it_be(:component2_2, freeze: can_freeze) { create("debian_#{container_type}_component", distribution: distribution2) }
+
+  let_it_be_with_refind(:component_file_with_architecture) { create("debian_#{container_type}_component_file", component: component1_1, architecture: architecture1_1) }
+  let_it_be(:component_file_other_architecture, freeze: can_freeze) { create("debian_#{container_type}_component_file", component: component1_1, architecture: architecture1_2) }
+  let_it_be(:component_file_other_component, freeze: can_freeze) { create("debian_#{container_type}_component_file", component: component1_2, architecture: architecture1_1) }
+  let_it_be(:component_file_other_compression_type, freeze: can_freeze) { create("debian_#{container_type}_component_file", component: component1_1, architecture: architecture1_1, compression_type: :xz) }
+  let_it_be(:component_file_other_file_md5, freeze: can_freeze) { create("debian_#{container_type}_component_file", component: component1_1, architecture: architecture1_1, file_md5: 'other_md5') }
+  let_it_be(:component_file_other_file_sha256, freeze: can_freeze) { create("debian_#{container_type}_component_file", component: component1_1, architecture: architecture1_1, file_sha256: 'other_sha256') }
+  let_it_be(:component_file_other_container, freeze: can_freeze) { create("debian_#{container_type}_component_file", component: component2_1, architecture: architecture2_1) }
+  let_it_be_with_refind(:component_file_with_file_type_source) { create("debian_#{container_type}_component_file", :source, component: component1_1) }
+  let_it_be(:component_file_with_file_type_di_packages, freeze: can_freeze) { create("debian_#{container_type}_component_file", :di_packages, component: component1_1, architecture: architecture1_1) }
+
+  subject { component_file_with_architecture }
+
+  describe 'relationships' do
+    context 'with stubbed uploader' do
+      before do
+        allow_next_instance_of(Packages::Debian::ComponentFileUploader) do |uploader|
+          allow(uploader).to receive(:dynamic_segment).and_return('stubbed')
+        end
+      end
+
+      it { is_expected.to belong_to(:component).class_name("Packages::Debian::#{container_type.capitalize}Component").inverse_of(:files) }
+    end
+
+    context 'with packages file_type' do
+      it { is_expected.to belong_to(:architecture).class_name("Packages::Debian::#{container_type.capitalize}Architecture").inverse_of(:files) }
+    end
+
+    context 'with :source file_type' do
+      subject { component_file_with_file_type_source }
+
+      it { is_expected.to belong_to(:architecture).class_name("Packages::Debian::#{container_type.capitalize}Architecture").inverse_of(:files).optional }
+    end
+  end
+
+  describe 'validations' do
+    describe "#component" do
+      before do
+        allow_next_instance_of(Packages::Debian::ComponentFileUploader) do |uploader|
+          allow(uploader).to receive(:dynamic_segment).and_return('stubbed')
+        end
+      end
+
+      it { is_expected.to validate_presence_of(:component) }
+    end
+
+    describe "#architecture" do
+      context 'with packages file_type' do
+        it { is_expected.to validate_presence_of(:architecture) }
+      end
+
+      context 'with :source file_type' do
+        subject { component_file_with_file_type_source }
+
+        it { is_expected.to validate_absence_of(:architecture) }
+      end
+    end
+
+    describe '#file_type' do
+      it { is_expected.to validate_presence_of(:file_type) }
+
+      it { is_expected.to allow_value(:packages).for(:file_type) }
+    end
+
+    describe '#compression_type' do
+      it { is_expected.not_to validate_presence_of(:compression_type) }
+
+      it { is_expected.to allow_value(nil).for(:compression_type) }
+      it { is_expected.to allow_value(:gz).for(:compression_type) }
+    end
+
+    describe '#file' do
+      subject { component_file_with_architecture.file }
+
+      context 'the uploader api' do
+        it { is_expected.to respond_to(:store_dir) }
+        it { is_expected.to respond_to(:cache_dir) }
+        it { is_expected.to respond_to(:work_dir) }
+      end
+    end
+
+    describe '#file_store' do
+      it { is_expected.to validate_presence_of(:file_store) }
+    end
+
+    describe '#file_md5' do
+      it { is_expected.to validate_presence_of(:file_md5) }
+    end
+
+    describe '#file_sha256' do
+      it { is_expected.to validate_presence_of(:file_sha256) }
+    end
+  end
+
+  describe 'scopes' do
+    describe '.with_container' do
+      subject { described_class.with_container(container2) }
+
+      it do
+        queries = ActiveRecord::QueryRecorder.new do
+          expect(subject.to_a).to contain_exactly(component_file_other_container)
+        end
+
+        expect(queries.count).to eq(1)
+      end
+    end
+
+    describe '.with_codename_or_suite' do
+      subject { described_class.with_codename_or_suite(distribution2.codename) }
+
+      it do
+        queries = ActiveRecord::QueryRecorder.new do
+          expect(subject.to_a).to contain_exactly(component_file_other_container)
+        end
+
+        expect(queries.count).to eq(1)
+      end
+    end
+
+    describe '.with_component_name' do
+      subject { described_class.with_component_name(component1_2.name) }
+
+      it do
+        queries = ActiveRecord::QueryRecorder.new do
+          expect(subject.to_a).to contain_exactly(component_file_other_component)
+        end
+
+        expect(queries.count).to eq(1)
+      end
+    end
+
+    describe '.with_file_type' do
+      subject { described_class.with_file_type(:source) }
+
+      it do
+        # let_it_be_with_refind triggers a query
+        component_file_with_file_type_source
+
+        queries = ActiveRecord::QueryRecorder.new do
+          expect(subject.to_a).to contain_exactly(component_file_with_file_type_source)
+        end
+
+        expect(queries.count).to eq(1)
+      end
+    end
+
+    describe '.with_architecture_name' do
+      subject { described_class.with_architecture_name(architecture1_2.name) }
+
+      it do
+        queries = ActiveRecord::QueryRecorder.new do
+          expect(subject.to_a).to contain_exactly(component_file_other_architecture)
+        end
+
+        expect(queries.count).to eq(1)
+      end
+    end
+
+    describe '.with_compression_type' do
+      subject { described_class.with_compression_type(:xz) }
+
+      it do
+        queries = ActiveRecord::QueryRecorder.new do
+          expect(subject.to_a).to contain_exactly(component_file_other_compression_type)
+        end
+
+        expect(queries.count).to eq(1)
+      end
+    end
+
+    describe '.with_file_sha256' do
+      subject { described_class.with_file_sha256('other_sha256') }
+
+      it do
+        queries = ActiveRecord::QueryRecorder.new do
+          expect(subject.to_a).to contain_exactly(component_file_other_file_sha256)
+        end
+
+        expect(queries.count).to eq(1)
+      end
+    end
+  end
+
+  describe 'callbacks' do
+    let(:component_file) { build("debian_#{container_type}_component_file", component: component1_1, architecture: architecture1_1, size: nil) }
+
+    subject { component_file.save! }
+
+    it 'updates metadata columns' do
+      expect(component_file)
+        .to receive(:update_file_store)
+        .and_call_original
+
+      expect(component_file)
+        .to receive(:update_column)
+        .with(:file_store, ::Packages::PackageFileUploader::Store::LOCAL)
+        .and_call_original
+
+      expect { subject }.to change { component_file.size }.from(nil).to(74)
+    end
+  end
+
+  describe '#relative_path' do
+    context 'with a Packages file_type' do
+      subject { component_file_with_architecture.relative_path }
+
+      it { is_expected.to eq("#{component1_1.name}/binary-#{architecture1_1.name}/Packages") }
+    end
+
+    context 'with a Source file_type' do
+      subject { component_file_with_file_type_source.relative_path }
+
+      it { is_expected.to eq("#{component1_1.name}/source/Source") }
+    end
+
+    context 'with a DI Packages file_type' do
+      subject { component_file_with_file_type_di_packages.relative_path }
+
+      it { is_expected.to eq("#{component1_1.name}/debian-installer/binary-#{architecture1_1.name}/Packages") }
+    end
+
+    context 'with an xz compression_type' do
+      subject { component_file_other_compression_type.relative_path }
+
+      it { is_expected.to eq("#{component1_1.name}/binary-#{architecture1_1.name}/Packages.xz") }
+    end
+  end
+end

spec/support/shared_examples/models/packages/debian/component_shared_examples.rb

@@ -11,6 +11,7 @@ RSpec.shared_examples 'Debian Distribution Component' do |factory, container, ca
 
   describe 'relationships' do
     it { is_expected.to belong_to(:distribution).class_name("Packages::Debian::#{container.capitalize}Distribution").inverse_of(:components) }
+    it { is_expected.to have_many(:files).class_name("Packages::Debian::#{container.capitalize}ComponentFile").inverse_of(:component) }
   end
 
   describe 'validations' do

spec/support/shared_examples/models/with_debian_distributions_shared_examples.rb

+# frozen_string_literal: true
+
+RSpec.shared_examples 'model with Debian distributions' do
+  let(:container_type) { subject.class.name.downcase }
+  let!(:distributions) { create_list("debian_#{container_type}_distribution", 2, :with_file, container: subject) }
+  let!(:components) { create_list("debian_#{container_type}_component", 5, distribution: distributions[0]) }
+  let!(:component_files) { create_list("debian_#{container_type}_component_file", 3, component: components[0]) }
+
+  it 'removes distribution files on removal' do
+    distribution_file_paths = distributions.map do |distribution|
+      [distribution.file.path] +
+      distribution.component_files.map do |component_file|
+        component_file.file.path
+      end
+    end.flatten
+
+    expect { subject.destroy! }
+      .to change {
+        distribution_file_paths.select do |path|
+          File.exist? path
+        end.length
+      }.from(distribution_file_paths.length).to(0)
+  end
+end

spec/uploaders/packages/debian/component_file_uploader_spec.rb

+# frozen_string_literal: true
+require 'spec_helper'
+
+RSpec.describe Packages::Debian::ComponentFileUploader do
+  [:project, :group].each do |container_type|
+    context "Packages::Debian::#{container_type.capitalize}ComponentFile" do
+      let(:factory) { "debian_#{container_type}_component_file" }
+      let(:component_file) { create(factory) } # rubocop:disable Rails/SaveBang
+      let(:uploader) { described_class.new(component_file, :file) }
+      let(:path) { Gitlab.config.packages.storage_path }
+
+      subject { uploader }
+
+      it_behaves_like "builds correct paths",
+                      store_dir: %r[^\h{2}/\h{2}/\h{64}/debian_#{container_type}_component_file/\d+$],
+                      cache_dir: %r[/packages/tmp/cache$],
+                      work_dir: %r[/packages/tmp/work$]
+
+      context 'object store is remote' do
+        before do
+          stub_package_file_object_storage
+        end
+
+        include_context 'with storage', described_class::Store::REMOTE
+
+        it_behaves_like "builds correct paths",
+                        store_dir: %r[^\h{2}/\h{2}/\h{64}/debian_#{container_type}_component_file/\d+$],
+                        cache_dir: %r[/packages/tmp/cache$],
+                        work_dir: %r[/packages/tmp/work$]
+      end
+
+      describe 'remote file' do
+        let(:component_file) { create(factory, :object_storage) }
+
+        context 'with object storage enabled' do
+          before do
+            stub_package_file_object_storage
+          end
+
+          it 'can store file remotely' do
+            allow(ObjectStorage::BackgroundMoveWorker).to receive(:perform_async)
+
+            component_file
+
+            expect(component_file.file_store).to eq(described_class::Store::REMOTE)
+            expect(component_file.file.path).not_to be_blank
+          end
+        end
+      end
+    end
+  end
+end