Merge branch 'improve_vuln_tracking-db_add_tracking_table' into 'master'

Improve Vulnerability Tracking: Add fingerprints table

See merge request gitlab-org/gitlab!52720

changelogs/unreleased/improve_vuln_tracking-db_add_tracking_table.yml

+---
+title: 'Improve Vulnerability Tracking: Add fingerprints table'
+merge_request: 52720
+author:
+type: added

db/migrate/20201108134919_add_finding_fingerprint_table.rb

+# frozen_string_literal: true
+
+class AddFindingFingerprintTable < ActiveRecord::Migration[6.0]
+  include Gitlab::Database::MigrationHelpers
+
+  DOWNTIME = false
+
+  FINGERPRINT_IDX = :idx_vuln_fingerprints_on_occurrences_id_and_fingerprint
+  UNIQ_IDX = :idx_vuln_fingerprints_uniqueness
+
+  def up
+    with_lock_retries do
+      create_table :vulnerability_finding_fingerprints do |t|
+        t.references :finding,
+          index: true,
+          null: false,
+          foreign_key: { to_table: :vulnerability_occurrences, column: :finding_id, on_delete: :cascade }
+
+        t.timestamps_with_timezone null: false
+
+        t.integer :algorithm_type, null: false
+        t.binary :fingerprint_sha256, null: false
+
+        t.index %i[finding_id fingerprint_sha256],
+          name: FINGERPRINT_IDX,
+          unique: true # only one link should exist between occurrence and the fingerprint
+
+        t.index %i[finding_id algorithm_type fingerprint_sha256],
+          name: UNIQ_IDX,
+          unique: true # these should be unique
+      end
+    end
+  end
+
+  def down
+    with_lock_retries do
+      drop_table :vulnerability_finding_fingerprints
+    end
+  end
+end

db/migrate/20201109080646_create_vulnerability_findings_remediations_join_table.rb

@@ -3,6 +3,7 @@
 class CreateVulnerabilityFindingsRemediationsJoinTable < ActiveRecord::Migration[6.0]
   DOWNTIME = false
 
+  # rubocop:disable Migration/CreateTableWithForeignKeys
   def change
     create_table :vulnerability_findings_remediations do |t|
       t.references :vulnerability_occurrence, index: false, foreign_key: { on_delete: :cascade }
@@ -13,4 +14,5 @@ class CreateVulnerabilityFindingsRemediationsJoinTable < ActiveRecord::Migration
       t.index [:vulnerability_occurrence_id, :vulnerability_remediation_id], unique: true, name: 'index_vulnerability_findings_remediations_on_unique_keys'
     end
   end
+  # rubocop:enable Migration/CreateTableWithForeignKeys
 end

db/schema_migrations/20201108134919

+6643e5b4c5597d92c94115f392bfbd5cfce9884eb0bcb18f9629855f3711eed0
\ No newline at end of file

db/structure.sql

@@ -18141,6 +18141,24 @@ CREATE SEQUENCE vulnerability_feedback_id_seq
 
 ALTER SEQUENCE vulnerability_feedback_id_seq OWNED BY vulnerability_feedback.id;
 
+CREATE TABLE vulnerability_finding_fingerprints (
+    id bigint NOT NULL,
+    finding_id bigint NOT NULL,
+    created_at timestamp with time zone NOT NULL,
+    updated_at timestamp with time zone NOT NULL,
+    algorithm_type integer NOT NULL,
+    fingerprint_sha256 bytea NOT NULL
+);
+
+CREATE SEQUENCE vulnerability_finding_fingerprints_id_seq
+    START WITH 1
+    INCREMENT BY 1
+    NO MINVALUE
+    NO MAXVALUE
+    CACHE 1;
+
+ALTER SEQUENCE vulnerability_finding_fingerprints_id_seq OWNED BY vulnerability_finding_fingerprints.id;
+
 CREATE TABLE vulnerability_finding_links (
     id bigint NOT NULL,
     created_at timestamp with time zone NOT NULL,
@@ -19382,6 +19400,8 @@ ALTER TABLE ONLY vulnerability_external_issue_links ALTER COLUMN id SET DEFAULT
 
 ALTER TABLE ONLY vulnerability_feedback ALTER COLUMN id SET DEFAULT nextval('vulnerability_feedback_id_seq'::regclass);
 
+ALTER TABLE ONLY vulnerability_finding_fingerprints ALTER COLUMN id SET DEFAULT nextval('vulnerability_finding_fingerprints_id_seq'::regclass);
+
 ALTER TABLE ONLY vulnerability_finding_links ALTER COLUMN id SET DEFAULT nextval('vulnerability_finding_links_id_seq'::regclass);
 
 ALTER TABLE ONLY vulnerability_findings_remediations ALTER COLUMN id SET DEFAULT nextval('vulnerability_findings_remediations_id_seq'::regclass);
@@ -20957,6 +20977,9 @@ ALTER TABLE ONLY vulnerability_external_issue_links
 ALTER TABLE ONLY vulnerability_feedback
     ADD CONSTRAINT vulnerability_feedback_pkey PRIMARY KEY (id);
 
+ALTER TABLE ONLY vulnerability_finding_fingerprints
+    ADD CONSTRAINT vulnerability_finding_fingerprints_pkey PRIMARY KEY (id);
+
 ALTER TABLE ONLY vulnerability_finding_links
     ADD CONSTRAINT vulnerability_finding_links_pkey PRIMARY KEY (id);
 
@@ -21294,6 +21317,10 @@ CREATE INDEX idx_security_scans_on_scan_type ON security_scans USING btree (scan
 
 CREATE UNIQUE INDEX idx_serverless_domain_cluster_on_clusters_applications_knative ON serverless_domain_cluster USING btree (clusters_applications_knative_id);
 
+CREATE UNIQUE INDEX idx_vuln_fingerprints_on_occurrences_id_and_fingerprint ON vulnerability_finding_fingerprints USING btree (finding_id, fingerprint_sha256);
+
+CREATE UNIQUE INDEX idx_vuln_fingerprints_uniqueness ON vulnerability_finding_fingerprints USING btree (finding_id, algorithm_type, fingerprint_sha256);
+
 CREATE UNIQUE INDEX idx_vulnerability_ext_issue_links_on_vulne_id_and_ext_issue ON vulnerability_external_issue_links USING btree (vulnerability_id, external_type, external_project_key, external_issue_key);
 
 CREATE UNIQUE INDEX idx_vulnerability_ext_issue_links_on_vulne_id_and_link_type ON vulnerability_external_issue_links USING btree (vulnerability_id, link_type) WHERE (link_type = 1);
@@ -23596,6 +23623,8 @@ CREATE INDEX index_vulnerability_feedback_on_merge_request_id ON vulnerability_f
 
 CREATE INDEX index_vulnerability_feedback_on_pipeline_id ON vulnerability_feedback USING btree (pipeline_id);
 
+CREATE INDEX index_vulnerability_finding_fingerprints_on_finding_id ON vulnerability_finding_fingerprints USING btree (finding_id);
+
 CREATE INDEX index_vulnerability_findings_remediations_on_remediation_id ON vulnerability_findings_remediations USING btree (vulnerability_remediation_id);
 
 CREATE UNIQUE INDEX index_vulnerability_findings_remediations_on_unique_keys ON vulnerability_findings_remediations USING btree (vulnerability_occurrence_id, vulnerability_remediation_id);
@@ -26119,6 +26148,9 @@ ALTER TABLE ONLY merge_trains
 ALTER TABLE ONLY ci_runner_namespaces
     ADD CONSTRAINT fk_rails_f9d9ed3308 FOREIGN KEY (namespace_id) REFERENCES namespaces(id) ON DELETE CASCADE;
 
+ALTER TABLE ONLY vulnerability_finding_fingerprints
+    ADD CONSTRAINT fk_rails_fa411253b2 FOREIGN KEY (finding_id) REFERENCES vulnerability_occurrences(id) ON DELETE CASCADE;
+
 ALTER TABLE ONLY requirements_management_test_reports
     ADD CONSTRAINT fk_rails_fb3308ad55 FOREIGN KEY (requirement_id) REFERENCES requirements(id) ON DELETE CASCADE;
 

rubocop/rubocop-migrations.yml

@@ -38,6 +38,7 @@ Migration/UpdateLargeTable:
     - :users
     - :user_preferences
     - :user_details
+    - :vulnerability_occurrences
     - :web_hook_logs
   DeniedMethods:
     - :change_column_type_concurrently