Merge branch 'debian_plug_sign' into 'master'

Plug Debian SignDistributionService in GenerateDistributionService

See merge request gitlab-org/gitlab!66907

app/services/packages/debian/generate_distribution_service.rb

@@ -161,28 +161,37 @@ module Packages
       end
 
       def generate_release
-        @distribution.file = CarrierWaveStringFile.new(release_header + release_sums)
+        @distribution.key || @distribution.create_key(GenerateDistributionKeyService.new.execute)
+        @distribution.file = CarrierWaveStringFile.new(release_content)
+        @distribution.file_signature = SignDistributionService.new(@distribution, release_content, detach: true).execute
+        @distribution.signed_file = CarrierWaveStringFile.new(
+          SignDistributionService.new(@distribution, release_content).execute
+        )
         @distribution.updated_at = release_date
         @distribution.save!
       end
 
-      def release_header
-        strong_memoize(:release_header) do
-          [
-            %w[origin label suite version codename].map do |attribute|
-              rfc822_field(attribute.capitalize, @distribution.attributes[attribute])
-            end,
-            rfc822_field('Date', release_date.to_formatted_s(:rfc822)),
-            valid_until_field,
-            rfc822_field('NotAutomatic', !@distribution.automatic, !@distribution.automatic),
-            rfc822_field('ButAutomaticUpgrades', @distribution.automatic_upgrades, !@distribution.automatic && @distribution.automatic_upgrades),
-            rfc822_field('Architectures', @distribution.architectures.map { |architecture| architecture.name }.sort.join(' ')),
-            rfc822_field('Components', @distribution.components.map { |component| component.name }.sort.join(' ')),
-            rfc822_field('Description', @distribution.description)
-          ].flatten.compact.join('')
+      def release_content
+        strong_memoize(:release_content) do
+          release_header + release_sums
         end
       end
 
+      def release_header
+        [
+          %w[origin label suite version codename].map do |attribute|
+            rfc822_field(attribute.capitalize, @distribution.attributes[attribute])
+          end,
+          rfc822_field('Date', release_date.to_formatted_s(:rfc822)),
+          valid_until_field,
+          rfc822_field('NotAutomatic', !@distribution.automatic, !@distribution.automatic),
+          rfc822_field('ButAutomaticUpgrades', @distribution.automatic_upgrades, !@distribution.automatic && @distribution.automatic_upgrades),
+          rfc822_field('Architectures', @distribution.architectures.map { |architecture| architecture.name }.sort.join(' ')),
+          rfc822_field('Components', @distribution.components.map { |component| component.name }.sort.join(' ')),
+          rfc822_field('Description', @distribution.description)
+        ].flatten.compact.join('')
+      end
+
       def release_date
         strong_memoize(:release_date) do
           Time.now.utc

app/services/packages/debian/sign_distribution_service.rb

@@ -5,10 +5,10 @@ module Packages
     class SignDistributionService
       include Gitlab::Utils::StrongMemoize
 
-      def initialize(distribution, content, params: {})
+      def initialize(distribution, content, detach: false)
         @distribution = distribution
         @content = content
-        @params = params
+        @detach = detach
       end
 
       def execute
@@ -16,7 +16,7 @@ module Packages
 
         sig_mode = GPGME::GPGME_SIG_MODE_CLEAR
 
-        sig_mode = GPGME::GPGME_SIG_MODE_DETACH if @params[:detach]
+        sig_mode = GPGME::GPGME_SIG_MODE_DETACH if @detach
 
         Gitlab::Gpg.using_tmp_keychain do
           GPGME::Ctx.new(

spec/services/packages/debian/generate_distribution_service_spec.rb

@@ -12,13 +12,7 @@ RSpec.describe Packages::Debian::GenerateDistributionService do
       context "for #{container_type}" do
         include_context 'with Debian distribution', container_type
 
-        context 'with Debian components and architectures' do
-          it_behaves_like 'Generate Debian Distribution and component files'
-        end
-
-        context 'without components and architectures' do
-          it_behaves_like 'Generate minimal Debian Distribution'
-        end
+        it_behaves_like 'Generate Debian Distribution and component files'
       end
     end
   end

spec/services/packages/debian/sign_distribution_service_spec.rb

@@ -6,12 +6,11 @@ RSpec.describe Packages::Debian::SignDistributionService do
   let_it_be(:group) { create(:group, :public) }
 
   let(:content) { FFaker::Lorem.paragraph }
-  let(:params) { {} }
-  let(:service) { described_class.new(distribution, content, params: params) }
+  let(:service) { described_class.new(distribution, content, detach: detach) }
 
   shared_examples 'Sign Distribution' do |container_type, detach: false|
     context "for #{container_type} detach=#{detach}" do
-      let(:params) { { detach: detach } }
+      let(:detach) { detach }
 
       if container_type == :group
         let_it_be(:distribution) { create('debian_group_distribution', container: group) }

spec/support/shared_examples/services/packages/debian/generate_distribution_shared_examples.rb

 # frozen_string_literal: true
 
 RSpec.shared_examples 'Generate Debian Distribution and component files' do
-  let_it_be(:component_main) { create("debian_#{container_type}_component", distribution: distribution, name: 'main') }
-  let_it_be(:component_contrib) { create("debian_#{container_type}_component", distribution: distribution, name: 'contrib') }
-
-  let_it_be(:architecture_all) { create("debian_#{container_type}_architecture", distribution: distribution, name: 'all') }
-  let_it_be(:architecture_amd64) { create("debian_#{container_type}_architecture", distribution: distribution, name: 'amd64') }
-  let_it_be(:architecture_arm64) { create("debian_#{container_type}_architecture", distribution: distribution, name: 'arm64') }
-
-  let_it_be(:component_file1) { create("debian_#{container_type}_component_file", component: component_contrib, architecture: architecture_all,   updated_at: '2020-01-24T08:00:00Z', file_sha256: 'e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855', file_md5: 'd41d8cd98f00b204e9800998ecf8427e', file_fixture: nil, size: 0) } # updated
-  let_it_be(:component_file2) { create("debian_#{container_type}_component_file", component: component_main,    architecture: architecture_all,   updated_at: '2020-01-24T09:00:00Z', file_sha256: 'a') } # destroyed
-  let_it_be(:component_file3) { create("debian_#{container_type}_component_file", component: component_main,    architecture: architecture_amd64, updated_at: '2020-01-24T10:54:59Z', file_sha256: 'b') } # destroyed, 1 second before last generation
-  let_it_be(:component_file4) { create("debian_#{container_type}_component_file", component: component_contrib, architecture: architecture_all,   updated_at: '2020-01-24T10:55:00Z', file_sha256: 'c') } # kept, last generation
-  let_it_be(:component_file5) { create("debian_#{container_type}_component_file", component: component_contrib, architecture: architecture_all,   updated_at: '2020-01-24T10:55:00Z', file_sha256: 'd') } # kept, last generation
-  let_it_be(:component_file6) { create("debian_#{container_type}_component_file", component: component_contrib, architecture: architecture_amd64, updated_at: '2020-01-25T15:17:18Z', file_sha256: 'e') } # kept, less than 1 hour ago
-
-  def check_component_file(release_date, component_name, component_file_type, architecture_name, expected_content)
-    component_file = distribution
-      .component_files
-      .with_component_name(component_name)
-      .with_file_type(component_file_type)
-      .with_architecture_name(architecture_name)
-      .order_updated_asc
-      .last
-
-    expect(component_file).not_to be_nil
-    expect(component_file.updated_at).to eq(release_date)
-
-    unless expected_content.nil?
-      component_file.file.use_file do |file_path|
-        expect(File.read(file_path)).to eq(expected_content)
-      end
+  def check_release_files(expected_release_content)
+    distribution.reload
+
+    distribution.file.use_file do |file_path|
+      expect(File.read(file_path)).to eq(expected_release_content)
+    end
+
+    expect(distribution.file_signature).to start_with("-----BEGIN PGP SIGNATURE-----\n")
+    expect(distribution.file_signature).to end_with("\n-----END PGP SIGNATURE-----\n")
+
+    distribution.signed_file.use_file do |file_path|
+      expect(File.read(file_path)).to start_with("-----BEGIN PGP SIGNED MESSAGE-----\nHash: SHA512\n\n#{expected_release_content}-----BEGIN PGP SIGNATURE-----\n")
+      expect(File.read(file_path)).to end_with("\n-----END PGP SIGNATURE-----\n")
     end
   end
 
-  it 'generates Debian distribution and component files', :aggregate_failures do
-    current_time = Time.utc(2020, 01, 25, 15, 17, 18, 123456)
-
-    travel_to(current_time) do
-      expect(Gitlab::ErrorTracking).not_to receive(:log_exception)
-
-      initial_count = 6
-      destroyed_count = 2
-      # updated_count = 1
-      created_count = 5
-
-      expect { subject }
-        .to not_change { Packages::Package.count }
-        .and not_change { Packages::PackageFile.count }
-        .and change { distribution.reload.updated_at }.to(current_time.round)
-        .and change { distribution.component_files.reset.count }.from(initial_count).to(initial_count - destroyed_count + created_count)
-        .and change { component_file1.reload.updated_at }.to(current_time.round)
-
-      debs = package.package_files.with_debian_file_type(:deb).preload_debian_file_metadata.to_a
-      pool_prefix = 'pool/unstable'
-      pool_prefix += "/#{project.id}" if container_type == :group
-      pool_prefix += "/p/#{package.name}/#{package.version}"
-      expected_main_amd64_content = <<~EOF
-      Package: libsample0
-      Source: #{package.name}
-      Version: #{package.version}
-      Installed-Size: 7
-      Maintainer: #{debs[0].debian_fields['Maintainer']}
-      Architecture: amd64
-      Description: Some mostly empty lib
-       Used in GitLab tests.
-       .
-       Testing another paragraph.
-      Multi-Arch: same
-      Homepage: #{debs[0].debian_fields['Homepage']}
-      Section: libs
-      Priority: optional
-      Filename: #{pool_prefix}/libsample0_1.2.3~alpha2_amd64.deb
-      Size: 409600
-      MD5sum: #{debs[0].file_md5}
-      SHA256: #{debs[0].file_sha256}
-
-      Package: sample-dev
-      Source: #{package.name} (#{package.version})
-      Version: 1.2.3~binary
-      Installed-Size: 7
-      Maintainer: #{debs[1].debian_fields['Maintainer']}
-      Architecture: amd64
-      Depends: libsample0 (= 1.2.3~binary)
-      Description: Some mostly empty development files
-       Used in GitLab tests.
-       .
-       Testing another paragraph.
-      Multi-Arch: same
-      Homepage: #{debs[1].debian_fields['Homepage']}
-      Section: libdevel
-      Priority: optional
-      Filename: #{pool_prefix}/sample-dev_1.2.3~binary_amd64.deb
-      Size: 409600
-      MD5sum: #{debs[1].file_md5}
-      SHA256: #{debs[1].file_sha256}
-      EOF
-
-      check_component_file(current_time.round, 'main', :packages, 'all', nil)
-      check_component_file(current_time.round, 'main', :packages, 'amd64', expected_main_amd64_content)
-      check_component_file(current_time.round, 'main', :packages, 'arm64', nil)
-
-      check_component_file(current_time.round, 'contrib', :packages, 'all', nil)
-      check_component_file(current_time.round, 'contrib', :packages, 'amd64', nil)
-      check_component_file(current_time.round, 'contrib', :packages, 'arm64', nil)
-
-      main_amd64_size = expected_main_amd64_content.length
-      main_amd64_md5sum = Digest::MD5.hexdigest(expected_main_amd64_content)
-      main_amd64_sha256 = Digest::SHA256.hexdigest(expected_main_amd64_content)
-
-      contrib_all_size = component_file1.size
-      contrib_all_md5sum = component_file1.file_md5
-      contrib_all_sha256 = component_file1.file_sha256
-
-      expected_release_content = <<~EOF
-      Codename: unstable
-      Date: Sat, 25 Jan 2020 15:17:18 +0000
-      Valid-Until: Mon, 27 Jan 2020 15:17:18 +0000
-      Architectures: all amd64 arm64
-      Components: contrib main
-      MD5Sum:
-       #{contrib_all_md5sum}        #{contrib_all_size} contrib/binary-all/Packages
-       d41d8cd98f00b204e9800998ecf8427e        0 contrib/binary-amd64/Packages
-       d41d8cd98f00b204e9800998ecf8427e        0 contrib/binary-arm64/Packages
-       d41d8cd98f00b204e9800998ecf8427e        0 main/binary-all/Packages
-       #{main_amd64_md5sum}     #{main_amd64_size} main/binary-amd64/Packages
-       d41d8cd98f00b204e9800998ecf8427e        0 main/binary-arm64/Packages
-      SHA256:
-       #{contrib_all_sha256}        #{contrib_all_size} contrib/binary-all/Packages
-       e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855        0 contrib/binary-amd64/Packages
-       e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855        0 contrib/binary-arm64/Packages
-       e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855        0 main/binary-all/Packages
-       #{main_amd64_sha256}     #{main_amd64_size} main/binary-amd64/Packages
-       e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855        0 main/binary-arm64/Packages
-      EOF
-
-      distribution.file.use_file do |file_path|
-        expect(File.read(file_path)).to eq(expected_release_content)
+  context 'with Debian components and architectures' do
+    let_it_be(:component_main) { create("debian_#{container_type}_component", distribution: distribution, name: 'main') }
+    let_it_be(:component_contrib) { create("debian_#{container_type}_component", distribution: distribution, name: 'contrib') }
+
+    let_it_be(:architecture_all) { create("debian_#{container_type}_architecture", distribution: distribution, name: 'all') }
+    let_it_be(:architecture_amd64) { create("debian_#{container_type}_architecture", distribution: distribution, name: 'amd64') }
+    let_it_be(:architecture_arm64) { create("debian_#{container_type}_architecture", distribution: distribution, name: 'arm64') }
+
+    let_it_be(:component_file1) { create("debian_#{container_type}_component_file", component: component_contrib, architecture: architecture_all,   updated_at: '2020-01-24T08:00:00Z', file_sha256: 'e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855', file_md5: 'd41d8cd98f00b204e9800998ecf8427e', file_fixture: nil, size: 0) } # updated
+    let_it_be(:component_file2) { create("debian_#{container_type}_component_file", component: component_main,    architecture: architecture_all,   updated_at: '2020-01-24T09:00:00Z', file_sha256: 'a') } # destroyed
+    let_it_be(:component_file3) { create("debian_#{container_type}_component_file", component: component_main,    architecture: architecture_amd64, updated_at: '2020-01-24T10:54:59Z', file_sha256: 'b') } # destroyed, 1 second before last generation
+    let_it_be(:component_file4) { create("debian_#{container_type}_component_file", component: component_contrib, architecture: architecture_all,   updated_at: '2020-01-24T10:55:00Z', file_sha256: 'c') } # kept, last generation
+    let_it_be(:component_file5) { create("debian_#{container_type}_component_file", component: component_contrib, architecture: architecture_all,   updated_at: '2020-01-24T10:55:00Z', file_sha256: 'd') } # kept, last generation
+    let_it_be(:component_file6) { create("debian_#{container_type}_component_file", component: component_contrib, architecture: architecture_amd64, updated_at: '2020-01-25T15:17:18Z', file_sha256: 'e') } # kept, less than 1 hour ago
+
+    def check_component_file(release_date, component_name, component_file_type, architecture_name, expected_content)
+      component_file = distribution
+        .component_files
+        .with_component_name(component_name)
+        .with_file_type(component_file_type)
+        .with_architecture_name(architecture_name)
+        .order_updated_asc
+        .last
+
+      expect(component_file).not_to be_nil
+      expect(component_file.updated_at).to eq(release_date)
+
+      unless expected_content.nil?
+        component_file.file.use_file do |file_path|
+          expect(File.read(file_path)).to eq(expected_content)
+        end
+      end
+    end
+
+    it 'generates Debian distribution and component files', :aggregate_failures do
+      current_time = Time.utc(2020, 01, 25, 15, 17, 18, 123456)
+
+      travel_to(current_time) do
+        expect(Gitlab::ErrorTracking).not_to receive(:log_exception)
+
+        initial_count = 6
+        destroyed_count = 2
+        # updated_count = 1
+        created_count = 5
+
+        expect { subject }
+          .to not_change { Packages::Package.count }
+          .and not_change { Packages::PackageFile.count }
+          .and change { distribution.reload.updated_at }.to(current_time.round)
+          .and change { distribution.component_files.reset.count }.from(initial_count).to(initial_count - destroyed_count + created_count)
+          .and change { component_file1.reload.updated_at }.to(current_time.round)
+
+        debs = package.package_files.with_debian_file_type(:deb).preload_debian_file_metadata.to_a
+        pool_prefix = 'pool/unstable'
+        pool_prefix += "/#{project.id}" if container_type == :group
+        pool_prefix += "/p/#{package.name}/#{package.version}"
+        expected_main_amd64_content = <<~EOF
+        Package: libsample0
+        Source: #{package.name}
+        Version: #{package.version}
+        Installed-Size: 7
+        Maintainer: #{debs[0].debian_fields['Maintainer']}
+        Architecture: amd64
+        Description: Some mostly empty lib
+         Used in GitLab tests.
+         .
+         Testing another paragraph.
+        Multi-Arch: same
+        Homepage: #{debs[0].debian_fields['Homepage']}
+        Section: libs
+        Priority: optional
+        Filename: #{pool_prefix}/libsample0_1.2.3~alpha2_amd64.deb
+        Size: 409600
+        MD5sum: #{debs[0].file_md5}
+        SHA256: #{debs[0].file_sha256}
+
+        Package: sample-dev
+        Source: #{package.name} (#{package.version})
+        Version: 1.2.3~binary
+        Installed-Size: 7
+        Maintainer: #{debs[1].debian_fields['Maintainer']}
+        Architecture: amd64
+        Depends: libsample0 (= 1.2.3~binary)
+        Description: Some mostly empty development files
+         Used in GitLab tests.
+         .
+         Testing another paragraph.
+        Multi-Arch: same
+        Homepage: #{debs[1].debian_fields['Homepage']}
+        Section: libdevel
+        Priority: optional
+        Filename: #{pool_prefix}/sample-dev_1.2.3~binary_amd64.deb
+        Size: 409600
+        MD5sum: #{debs[1].file_md5}
+        SHA256: #{debs[1].file_sha256}
+        EOF
+
+        check_component_file(current_time.round, 'main', :packages, 'all', nil)
+        check_component_file(current_time.round, 'main', :packages, 'amd64', expected_main_amd64_content)
+        check_component_file(current_time.round, 'main', :packages, 'arm64', nil)
+
+        check_component_file(current_time.round, 'contrib', :packages, 'all', nil)
+        check_component_file(current_time.round, 'contrib', :packages, 'amd64', nil)
+        check_component_file(current_time.round, 'contrib', :packages, 'arm64', nil)
+
+        main_amd64_size = expected_main_amd64_content.length
+        main_amd64_md5sum = Digest::MD5.hexdigest(expected_main_amd64_content)
+        main_amd64_sha256 = Digest::SHA256.hexdigest(expected_main_amd64_content)
+
+        contrib_all_size = component_file1.size
+        contrib_all_md5sum = component_file1.file_md5
+        contrib_all_sha256 = component_file1.file_sha256
+
+        expected_release_content = <<~EOF
+        Codename: unstable
+        Date: Sat, 25 Jan 2020 15:17:18 +0000
+        Valid-Until: Mon, 27 Jan 2020 15:17:18 +0000
+        Architectures: all amd64 arm64
+        Components: contrib main
+        MD5Sum:
+         #{contrib_all_md5sum}        #{contrib_all_size} contrib/binary-all/Packages
+         d41d8cd98f00b204e9800998ecf8427e        0 contrib/binary-amd64/Packages
+         d41d8cd98f00b204e9800998ecf8427e        0 contrib/binary-arm64/Packages
+         d41d8cd98f00b204e9800998ecf8427e        0 main/binary-all/Packages
+         #{main_amd64_md5sum}     #{main_amd64_size} main/binary-amd64/Packages
+         d41d8cd98f00b204e9800998ecf8427e        0 main/binary-arm64/Packages
+        SHA256:
+         #{contrib_all_sha256}        #{contrib_all_size} contrib/binary-all/Packages
+         e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855        0 contrib/binary-amd64/Packages
+         e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855        0 contrib/binary-arm64/Packages
+         e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855        0 main/binary-all/Packages
+         #{main_amd64_sha256}     #{main_amd64_size} main/binary-amd64/Packages
+         e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855        0 main/binary-arm64/Packages
+        EOF
+
+        check_release_files(expected_release_content)
       end
     end
   end
-end
 
-RSpec.shared_examples 'Generate minimal Debian Distribution' do
-  it 'generates minimal distribution', :aggregate_failures do
-    travel_to(Time.utc(2020, 01, 25, 15, 17, 18, 123456)) do
-      expect(Gitlab::ErrorTracking).not_to receive(:log_exception)
-
-      expect { subject }
-        .to not_change { Packages::Package.count }
-        .and not_change { Packages::PackageFile.count }
-        .and not_change { distribution.component_files.reset.count }
-
-      expected_release_content = <<~EOF
-      Codename: unstable
-      Date: Sat, 25 Jan 2020 15:17:18 +0000
-      Valid-Until: Mon, 27 Jan 2020 15:17:18 +0000
-      MD5Sum:
-      SHA256:
-      EOF
-
-      distribution.file.use_file do |file_path|
-        expect(File.read(file_path)).to eq(expected_release_content)
+  context 'without components and architectures' do
+    it 'generates minimal distribution', :aggregate_failures do
+      travel_to(Time.utc(2020, 01, 25, 15, 17, 18, 123456)) do
+        expect(Gitlab::ErrorTracking).not_to receive(:log_exception)
+
+        expect { subject }
+          .to not_change { Packages::Package.count }
+          .and not_change { Packages::PackageFile.count }
+          .and not_change { distribution.component_files.reset.count }
+
+        expected_release_content = <<~EOF
+        Codename: unstable
+        Date: Sat, 25 Jan 2020 15:17:18 +0000
+        Valid-Until: Mon, 27 Jan 2020 15:17:18 +0000
+        MD5Sum:
+        SHA256:
+        EOF
+
+        check_release_files(expected_release_content)
       end
     end
   end