Merge branch 'update-lf-for-cs' into 'master'

Update location fingerprint for Container Scanning Findings

See merge request gitlab-org/gitlab!41756

db/post_migrate/20200908095446_update_location_fingerprint_column_for_cs.rb

+# frozen_string_literal: true
+
+class UpdateLocationFingerprintColumnForCs < ActiveRecord::Migration[6.0]
+  include Gitlab::Database::MigrationHelpers
+
+  DOWNTIME = false
+
+  disable_ddl_transaction!
+
+  BATCH_SIZE = 1_000
+  INTERVAL = 2.minutes
+
+  # 883_152 records
+  def up
+    return unless Gitlab.ee?
+
+    migration = Gitlab::BackgroundMigration::UpdateLocationFingerprintForContainerScanningFindings
+    migration_name = migration.to_s.demodulize
+    relation = migration::Finding.container_scanning
+    queue_background_migration_jobs_by_range_at_intervals(relation,
+                                                          migration_name,
+                                                          INTERVAL,
+                                                          batch_size: BATCH_SIZE)
+  end
+
+  def down
+    # no-op
+    # intentionally blank
+  end
+end

db/schema_migrations/20200908095446

+3cd8614d1d93340b4607d5270b54ec96b60b04a830c0a15a84b9843048515a12
\ No newline at end of file

ee/changelogs/unreleased/update-lf-for-cs.yml

+---
+title: Update location fingerprint for existing CS vulnerabilities
+merge_request: 41756
+author:
+type: other

ee/lib/ee/gitlab/background_migration/update_location_fingerprint_for_container_scanning_findings.rb

+# frozen_string_literal: true
+
+module EE
+  module Gitlab
+    module BackgroundMigration
+      module UpdateLocationFingerprintForContainerScanningFindings
+        extend ::Gitlab::Utils::Override
+
+        class Finding < ActiveRecord::Base
+          include ::ShaAttribute
+          include ::EachBatch
+
+          self.table_name = 'vulnerability_occurrences'
+
+          REPORT_TYPES = {
+            container_scanning: 2
+          }.with_indifferent_access.freeze
+
+          enum report_type: REPORT_TYPES
+
+          sha_attribute :location_fingerprint
+
+          # Copied from Reports::Security::Locations
+          def calculate_new_fingerprint(image, package_name)
+            return if image.nil? || package_name.nil?
+
+            Digest::SHA1.hexdigest("#{docker_image_name_without_tag(image)}:#{package_name}")
+          end
+
+          private
+
+          def docker_image_name_without_tag(image)
+            base_name, version = image.split(':')
+
+            return image if version_semver_like?(version)
+
+            base_name
+          end
+
+          def version_semver_like?(version)
+            hash_like = /\A[0-9a-f]{32,128}\z/i
+
+            if Gem::Version.correct?(version)
+              !hash_like.match?(version)
+            else
+              false
+            end
+          end
+        end
+
+        override :perform
+        def perform(start_id, stop_id)
+          Finding.container_scanning
+                  .select(:id, "raw_metadata::json->'location' AS loc")
+                  .where(id: start_id..stop_id)
+                  .each do |finding|
+                    next if finding.loc.nil?
+
+                    package = finding.loc.dig('dependency', 'package', 'name')
+                    image = finding.loc.dig('image')
+                    new_fingerprint = finding.calculate_new_fingerprint(image, package)
+
+                    next if new_fingerprint.blank?
+
+                    begin
+                      finding.update_column(:location_fingerprint, new_fingerprint)
+                    rescue ActiveRecord::RecordNotUnique
+                      ::Gitlab::BackgroundMigration::Logger.warn("Duplicate finding found with finding id #{finding.id}")
+                    end
+                  end
+        end
+      end
+    end
+  end
+end

ee/spec/lib/ee/gitlab/background_migration/update_location_fingerprint_for_container_scanning_findings_spec.rb

+# frozen_string_literal: true
+
+require 'spec_helper'
+
+RSpec.describe Gitlab::BackgroundMigration::UpdateLocationFingerprintForContainerScanningFindings, :migration, schema: 20200908095446 do
+  let(:namespaces) { table(:namespaces) }
+  let(:group) { namespaces.create!(name: 'foo', path: 'foo') }
+  let(:projects) { table(:projects) }
+  let(:findings) { table(:vulnerability_occurrences) }
+  let(:scanners) { table(:vulnerability_scanners) }
+  let(:identifiers) { table(:vulnerability_identifiers) }
+
+  let!(:project) { projects.create!(id: 123, namespace_id: group.id, name: 'gitlab', path: 'gitlab') }
+
+  let!(:scanner) do
+    scanners.create!(id: 6, project_id: project.id, external_id: 'clair', name: 'Security Scanner')
+  end
+
+  it 'updates location fingerprint' do
+    raw_metadata = [
+      "{ \"location\":{\"dependency\":{\"package\":{\"name\":\"apparmor\"},\"version\":\"2.10.95-0ubuntu2.11\"},\"operating_system\":\"ubuntu:16.04\",\"image\":\"registry.staging.gitlab.com/gitlab/alpine-ruby2/master:49dda736b6386592f7dd0367bcdd260cb84edfa8\"} }",
+      "{ \"location\":{\"dependency\":{\"package\":{\"name\":\"glibc\"},\"version\":\"2.10.95-0ubuntu2.11\"},\"operating_system\":\"ubuntu:16.04\",\"image\":\"registry.staging.gitlab.com/gitlab/docker/master:2.1\"} }",
+      "{ \"location\":{\"dependency\":{\"package\":{\"name\":\"apt\"},\"version\":\"2.10.95-0ubuntu2.11\"},\"operating_system\":\"ubuntu:16.04\",\"image\":\"registry.staging.gitlab.com/gitlab/gitlab/master:49dda73\"} }"
+    ]
+
+    new_fingerprints = %w(6c871440eb9f7618b9aef25e5246acddff6ed7a1 9d1a47927875f1aee1e2b9f16c25a8ff7586f1a6 d7da2cc109c18d890ab239e833524d451cc45246)
+
+    create_identifier(3)
+
+    vul1 = findings.create!(finding_params(1).merge({ raw_metadata: raw_metadata[0] }))
+    findings.create!(finding_params(2).merge({ raw_metadata: raw_metadata[1] }))
+    vul3 = findings.create!(finding_params(3).merge({ raw_metadata: raw_metadata[2] }))
+
+    expect(findings.where(report_type: 2).count). to eq(3)
+
+    described_class.new.perform(vul1.id, vul3.id)
+
+    location_fingerprints = findings.pluck(:location_fingerprint).flat_map { |x| Gitlab::Database::ShaAttribute.new.deserialize(x) }
+
+    expect(location_fingerprints).to match_array(new_fingerprints)
+  end
+
+  it 'updates the rest when there is a collision' do
+    allow(::Gitlab::BackgroundMigration::Logger).to receive(:warn).with(any_args).and_call_original
+
+    raw_metadata = [
+      "{ \"location\":{\"dependency\":{\"package\":{\"name\":\"apparmor\"},\"version\":\"2.10.95-0ubuntu2.11\"},\"operating_system\":\"ubuntu:16.04\",\"image\":\"registry.staging.gitlab.com/gitlab/alpine-ruby2/master:49dda736b6386592f7dd0367bcdd260cb84edfa8\"} }",
+      "{ \"location\":{\"dependency\":{\"package\":{\"name\":\"glibc\"},\"version\":\"2.10.95-0ubuntu2.11\"},\"operating_system\":\"ubuntu:16.04\",\"image\":\"registry.staging.gitlab.com/gitlab/docker/master:2.1\"} }",
+      "{ \"location\":{\"dependency\":{\"package\":{\"name\":\"apt\"},\"version\":\"2.10.95-0ubuntu2.11\"},\"operating_system\":\"ubuntu:16.04\",\"image\":\"registry.staging.gitlab.com/gitlab/gitlab/master:49dda73\"} }"
+    ]
+
+    new_fingerprints = %w(74657374 6c871440eb9f7618b9aef25e5246acddff6ed7a1 9d1a47927875f1aee1e2b9f16c25a8ff7586f1a6 d7da2cc109c18d890ab239e833524d451cc45246)
+
+    create_identifier(3)
+
+    # exsiting data in db
+    vul1 = findings.create!(finding_params(1).merge({ raw_metadata: raw_metadata[0], location_fingerprint: '36633837313434306562396637363138623961656632356535323436616364646666366564376131' }))
+    findings.create!(finding_params(1).merge({ raw_metadata: raw_metadata[0], location_fingerprint: 'test' }))
+    findings.create!(finding_params(2).merge({ raw_metadata: raw_metadata[1] }))
+    vul3 = findings.create!(finding_params(3).merge({ raw_metadata: raw_metadata[2] }))
+
+    expect(findings.where(report_type: 2).count). to eq(4)
+
+    described_class.new.perform(vul1.id, vul3.id)
+
+    expect(::Gitlab::BackgroundMigration::Logger).to have_received(:warn).with(any_args)
+
+    location_fingerprints = findings.pluck(:location_fingerprint).flat_map { |x| Gitlab::Database::ShaAttribute.new.deserialize(x) }
+
+    expect(location_fingerprints).to match_array(new_fingerprints)
+  end
+
+  def create_identifier(number_of)
+    (1..number_of).each do |identifier_id|
+      identifiers.create!(id: identifier_id,
+                          project_id: 123,
+                          fingerprint: 'd432c2ad2953e8bd587a3a43b3ce309b5b0154c' + identifier_id.to_s,
+                          external_type: 'SECURITY_ID',
+                          external_id: 'SECURITY_0',
+                          name: 'SECURITY_IDENTIFIER 0')
+    end
+  end
+
+  def finding_params(primary_identifier_id)
+    attrs = attributes_for(:vulnerabilities_finding)
+    {
+      severity: 0,
+      confidence: 5,
+      report_type: 2,
+      project_id: 123,
+      scanner_id: 6,
+      primary_identifier_id: primary_identifier_id,
+      project_fingerprint: attrs[:project_fingerprint],
+      location_fingerprint: Digest::SHA1.hexdigest(SecureRandom.hex(10)),
+      uuid: attrs[:uuid],
+      name: attrs[:name],
+      metadata_version: '1.3',
+      raw_metadata: attrs[:raw_metadata]
+    }
+  end
+end

ee/spec/migrations/update_location_fingerprint_column_for_cs_spec.rb

+# frozen_string_literal: true
+
+require 'spec_helper'
+
+require Rails.root.join('db', 'post_migrate', '20200908095446_update_location_fingerprint_column_for_cs.rb')
+
+RSpec.describe UpdateLocationFingerprintColumnForCs, :migration do
+  let(:namespaces) { table(:namespaces) }
+  let(:users) { table(:users) }
+  let(:group) { namespaces.create!(name: 'foo', path: 'foo') }
+  let(:projects) { table(:projects) }
+  let(:findings) { table(:vulnerability_occurrences) }
+  let(:scanners) { table(:vulnerability_scanners) }
+  let(:identifiers) { table(:vulnerability_identifiers) }
+  let!(:project) { projects.create!(id: 123, namespace_id: group.id, name: 'gitlab', path: 'gitlab') }
+
+  let!(:scanner) do
+    scanners.create!(id: 6, project_id: project.id, external_id: 'clair', name: 'Security Scanner')
+  end
+
+  let!(:user) do
+    users.create!(id: 13, email: 'author@example.com', notification_email: 'author@example.com', name: 'author', username: 'author', projects_limit: 10, state: 'active')
+  end
+
+  before do
+    stub_const("#{described_class}::BATCH_SIZE", 2)
+  end
+
+  it 'updates location fingerprint for containter scanning findings', :sidekiq_might_not_need_inline do
+    raw_metadata = [
+      "{ \"location\":{\"dependency\":{\"package\":{\"name\":\"apparmor\"},\"version\":\"2.10.95-0ubuntu2.11\"},\"operating_system\":\"ubuntu:16.04\",\"image\":\"registry.staging.gitlab.com/gitlab/alpine-ruby2/master:49dda736b6386592f7dd0367bcdd260cb84edfa8\"} }",
+      "{ \"location\":{\"dependency\":{\"package\":{\"name\":\"glibc\"},\"version\":\"2.10.95-0ubuntu2.11\"},\"operating_system\":\"ubuntu:16.04\",\"image\":\"registry.staging.gitlab.com/gitlab/docker/master:2.1\"} }"
+    ]
+
+    new_fingerprints = %w(6c871440eb9f7618b9aef25e5246acddff6ed7a1 9d1a47927875f1aee1e2b9f16c25a8ff7586f1a6)
+
+    allow_any_instance_of(Gitlab).to receive(:ee?).and_return(true)
+
+    create_identifier(2)
+
+    findings.create!(finding_params(1).merge({ raw_metadata: raw_metadata[0] }))
+    findings.create!(finding_params(2).merge({ raw_metadata: raw_metadata[1] }))
+
+    migrate!
+
+    location_fingerprints = findings.pluck(:location_fingerprint).flat_map { |x| Gitlab::Database::ShaAttribute.new.deserialize(x) }
+
+    expect(location_fingerprints).to match_array(new_fingerprints)
+  end
+
+  it 'skips migration for ce' do
+    allow_any_instance_of(Gitlab).to receive(:ee?).and_return(false)
+
+    create_identifier(2)
+
+    findings.create!(finding_params(1))
+    findings.create!(finding_params(2))
+
+    before_location_fingerprints = findings.pluck(:location_fingerprint).flat_map { |x| Gitlab::Database::ShaAttribute.new.deserialize(x) }
+
+    migrate!
+
+    after_location_fingerprints = findings.pluck(:location_fingerprint).flat_map { |x| Gitlab::Database::ShaAttribute.new.deserialize(x) }
+
+    expect(after_location_fingerprints).to match_array(before_location_fingerprints)
+  end
+
+  def create_identifier(number_of)
+    (1..number_of).each do |identifier_id|
+      identifiers.create!(id: identifier_id,
+                          project_id: 123,
+                          fingerprint: 'd432c2ad2953e8bd587a3a43b3ce309b5b0154c' + identifier_id.to_s,
+                          external_type: 'SECURITY_ID',
+                          external_id: 'SECURITY_0',
+                          name: 'SECURITY_IDENTIFIER 0')
+    end
+  end
+
+  def finding_params(primary_identifier_id)
+    attrs = attributes_for(:vulnerabilities_finding)
+    {
+      severity: 0,
+      confidence: 5,
+      report_type: 2,
+      project_id: 123,
+      scanner_id: 6,
+      primary_identifier_id: primary_identifier_id,
+      project_fingerprint: attrs[:project_fingerprint],
+      location_fingerprint: Digest::SHA1.hexdigest(SecureRandom.hex(10)),
+      uuid: attrs[:uuid],
+      name: attrs[:name],
+      metadata_version: '1.3',
+      raw_metadata: attrs[:raw_metadata]
+    }
+  end
+end

lib/gitlab/background_migration/update_location_fingerprint_for_container_scanning_findings.rb

+# frozen_string_literal: true
+# rubocop:disable Style/Documentation
+
+module Gitlab
+  module BackgroundMigration
+    class UpdateLocationFingerprintForContainerScanningFindings
+      def perform(start_id, stop_id)
+      end
+    end
+  end
+end
+
+Gitlab::BackgroundMigration::UpdateLocationFingerprintForContainerScanningFindings.prepend_if_ee('EE::Gitlab::BackgroundMigration::UpdateLocationFingerprintForContainerScanningFindings')