Commit 637cd4dc authored by GitLab Bot's avatar GitLab Bot

Automatic merge of gitlab-org/gitlab master

parents e78462f1 ed80138c
...@@ -13,7 +13,7 @@ const CLASSES = { ...@@ -13,7 +13,7 @@ const CLASSES = {
const STATUS = { const STATUS = {
opened: [__('Open'), 'issue-open-m'], opened: [__('Open'), 'issue-open-m'],
locked: [__('Open'), 'issue-open-m'], locked: [__('Open'), 'issue-open-m'],
closed: [__('Closed'), 'close'], closed: [__('Closed'), 'issue-close'],
merged: [__('Merged'), 'git-merge'], merged: [__('Merged'), 'git-merge'],
}; };
......
---
title: Fix closed icon for merge requests to match close issue icon
merge_request: 57981
author: jesus beltran
type: fixed
---
title: Rename table/model vulnerability_finding_fingerprints to *_signatures
merge_request: 57840
author:
type: other
---
name: load_balancing_for_bulk_cron_workers
introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/58345
rollout_issue_url: https://gitlab.com/gitlab-org/gitlab/-/issues/326721
milestone: '13.11'
type: development
group: group::global search
default_enabled: false
# frozen_string_literal: true
class AddFindingSignatureTable < ActiveRecord::Migration[6.0]
include Gitlab::Database::MigrationHelpers
DOWNTIME = false
SIGNATURE_IDX = :idx_vuln_signatures_on_occurrences_id_and_signature_sha
UNIQ_IDX = :idx_vuln_signatures_uniqueness_signature_sha
def up
with_lock_retries do
create_table :vulnerability_finding_signatures do |t|
t.references :finding,
index: true,
null: false,
foreign_key: { to_table: :vulnerability_occurrences, column: :finding_id, on_delete: :cascade }
t.timestamps_with_timezone null: false
t.integer :algorithm_type, null: false, limit: 2
t.binary :signature_sha, null: false
t.index %i[finding_id signature_sha],
name: SIGNATURE_IDX,
unique: true # only one link should exist between occurrence and the signature
t.index %i[finding_id algorithm_type signature_sha],
name: UNIQ_IDX,
unique: true # these should be unique
end
end
end
def down
with_lock_retries do
drop_table :vulnerability_finding_signatures
end
end
end
# frozen_string_literal: true
class DropFindingFingerprintTable < ActiveRecord::Migration[6.0]
include Gitlab::Database::MigrationHelpers
DOWNTIME = false
FINGERPRINT_IDX = :idx_vuln_fingerprints_on_occurrences_id_and_fingerprint_sha256
UNIQ_IDX = :idx_vuln_fingerprints_uniqueness_fingerprint_sha256
def up
with_lock_retries do
drop_table :vulnerability_finding_fingerprints
end
end
def down
with_lock_retries do
create_table :vulnerability_finding_fingerprints do |t|
t.references :finding,
index: true,
null: false,
foreign_key: { to_table: :vulnerability_occurrences, column: :finding_id, on_delete: :cascade }
t.timestamps_with_timezone null: false
t.integer :algorithm_type, null: false, limit: 2
t.binary :fingerprint_sha256, null: false
t.index %i[finding_id fingerprint_sha256],
name: FINGERPRINT_IDX,
unique: true # only one link should exist between occurrence and the fingerprint
t.index %i[finding_id algorithm_type fingerprint_sha256],
name: UNIQ_IDX,
unique: true # these should be unique
end
end
end
end
2a49d9f33f7dbcbef3cb5d5537db052c527d5268b37496435fe9918ddbb73095
\ No newline at end of file
de04d010fabd62d9dc995938b69ba178caa5e0a8476af5a78ba68c86698633d6
\ No newline at end of file
...@@ -18538,43 +18538,43 @@ CREATE SEQUENCE vulnerability_finding_evidences_id_seq ...@@ -18538,43 +18538,43 @@ CREATE SEQUENCE vulnerability_finding_evidences_id_seq
ALTER SEQUENCE vulnerability_finding_evidences_id_seq OWNED BY vulnerability_finding_evidences.id; ALTER SEQUENCE vulnerability_finding_evidences_id_seq OWNED BY vulnerability_finding_evidences.id;
CREATE TABLE vulnerability_finding_fingerprints ( CREATE TABLE vulnerability_finding_links (
id bigint NOT NULL, id bigint NOT NULL,
finding_id bigint NOT NULL,
created_at timestamp with time zone NOT NULL, created_at timestamp with time zone NOT NULL,
updated_at timestamp with time zone NOT NULL, updated_at timestamp with time zone NOT NULL,
algorithm_type smallint NOT NULL, vulnerability_occurrence_id bigint NOT NULL,
fingerprint_sha256 bytea NOT NULL name text,
url text NOT NULL,
CONSTRAINT check_55f0a95439 CHECK ((char_length(name) <= 255)),
CONSTRAINT check_b7fe886df6 CHECK ((char_length(url) <= 2048))
); );
CREATE SEQUENCE vulnerability_finding_fingerprints_id_seq CREATE SEQUENCE vulnerability_finding_links_id_seq
START WITH 1 START WITH 1
INCREMENT BY 1 INCREMENT BY 1
NO MINVALUE NO MINVALUE
NO MAXVALUE NO MAXVALUE
CACHE 1; CACHE 1;
ALTER SEQUENCE vulnerability_finding_fingerprints_id_seq OWNED BY vulnerability_finding_fingerprints.id; ALTER SEQUENCE vulnerability_finding_links_id_seq OWNED BY vulnerability_finding_links.id;
CREATE TABLE vulnerability_finding_links ( CREATE TABLE vulnerability_finding_signatures (
id bigint NOT NULL, id bigint NOT NULL,
finding_id bigint NOT NULL,
created_at timestamp with time zone NOT NULL, created_at timestamp with time zone NOT NULL,
updated_at timestamp with time zone NOT NULL, updated_at timestamp with time zone NOT NULL,
vulnerability_occurrence_id bigint NOT NULL, algorithm_type smallint NOT NULL,
name text, signature_sha bytea NOT NULL
url text NOT NULL,
CONSTRAINT check_55f0a95439 CHECK ((char_length(name) <= 255)),
CONSTRAINT check_b7fe886df6 CHECK ((char_length(url) <= 2048))
); );
CREATE SEQUENCE vulnerability_finding_links_id_seq CREATE SEQUENCE vulnerability_finding_signatures_id_seq
START WITH 1 START WITH 1
INCREMENT BY 1 INCREMENT BY 1
NO MINVALUE NO MINVALUE
NO MAXVALUE NO MAXVALUE
CACHE 1; CACHE 1;
ALTER SEQUENCE vulnerability_finding_links_id_seq OWNED BY vulnerability_finding_links.id; ALTER SEQUENCE vulnerability_finding_signatures_id_seq OWNED BY vulnerability_finding_signatures.id;
CREATE TABLE vulnerability_findings_remediations ( CREATE TABLE vulnerability_findings_remediations (
id bigint NOT NULL, id bigint NOT NULL,
...@@ -19827,10 +19827,10 @@ ALTER TABLE ONLY vulnerability_feedback ALTER COLUMN id SET DEFAULT nextval('vul ...@@ -19827,10 +19827,10 @@ ALTER TABLE ONLY vulnerability_feedback ALTER COLUMN id SET DEFAULT nextval('vul
ALTER TABLE ONLY vulnerability_finding_evidences ALTER COLUMN id SET DEFAULT nextval('vulnerability_finding_evidences_id_seq'::regclass); ALTER TABLE ONLY vulnerability_finding_evidences ALTER COLUMN id SET DEFAULT nextval('vulnerability_finding_evidences_id_seq'::regclass);
ALTER TABLE ONLY vulnerability_finding_fingerprints ALTER COLUMN id SET DEFAULT nextval('vulnerability_finding_fingerprints_id_seq'::regclass);
ALTER TABLE ONLY vulnerability_finding_links ALTER COLUMN id SET DEFAULT nextval('vulnerability_finding_links_id_seq'::regclass); ALTER TABLE ONLY vulnerability_finding_links ALTER COLUMN id SET DEFAULT nextval('vulnerability_finding_links_id_seq'::regclass);
ALTER TABLE ONLY vulnerability_finding_signatures ALTER COLUMN id SET DEFAULT nextval('vulnerability_finding_signatures_id_seq'::regclass);
ALTER TABLE ONLY vulnerability_findings_remediations ALTER COLUMN id SET DEFAULT nextval('vulnerability_findings_remediations_id_seq'::regclass); ALTER TABLE ONLY vulnerability_findings_remediations ALTER COLUMN id SET DEFAULT nextval('vulnerability_findings_remediations_id_seq'::regclass);
ALTER TABLE ONLY vulnerability_historical_statistics ALTER COLUMN id SET DEFAULT nextval('vulnerability_historical_statistics_id_seq'::regclass); ALTER TABLE ONLY vulnerability_historical_statistics ALTER COLUMN id SET DEFAULT nextval('vulnerability_historical_statistics_id_seq'::regclass);
...@@ -21455,12 +21455,12 @@ ALTER TABLE ONLY vulnerability_feedback ...@@ -21455,12 +21455,12 @@ ALTER TABLE ONLY vulnerability_feedback
ALTER TABLE ONLY vulnerability_finding_evidences ALTER TABLE ONLY vulnerability_finding_evidences
ADD CONSTRAINT vulnerability_finding_evidences_pkey PRIMARY KEY (id); ADD CONSTRAINT vulnerability_finding_evidences_pkey PRIMARY KEY (id);
ALTER TABLE ONLY vulnerability_finding_fingerprints
ADD CONSTRAINT vulnerability_finding_fingerprints_pkey PRIMARY KEY (id);
ALTER TABLE ONLY vulnerability_finding_links ALTER TABLE ONLY vulnerability_finding_links
ADD CONSTRAINT vulnerability_finding_links_pkey PRIMARY KEY (id); ADD CONSTRAINT vulnerability_finding_links_pkey PRIMARY KEY (id);
ALTER TABLE ONLY vulnerability_finding_signatures
ADD CONSTRAINT vulnerability_finding_signatures_pkey PRIMARY KEY (id);
ALTER TABLE ONLY vulnerability_findings_remediations ALTER TABLE ONLY vulnerability_findings_remediations
ADD CONSTRAINT vulnerability_findings_remediations_pkey PRIMARY KEY (id); ADD CONSTRAINT vulnerability_findings_remediations_pkey PRIMARY KEY (id);
...@@ -21798,9 +21798,9 @@ CREATE INDEX idx_security_scans_on_scan_type ON security_scans USING btree (scan ...@@ -21798,9 +21798,9 @@ CREATE INDEX idx_security_scans_on_scan_type ON security_scans USING btree (scan
CREATE UNIQUE INDEX idx_serverless_domain_cluster_on_clusters_applications_knative ON serverless_domain_cluster USING btree (clusters_applications_knative_id); CREATE UNIQUE INDEX idx_serverless_domain_cluster_on_clusters_applications_knative ON serverless_domain_cluster USING btree (clusters_applications_knative_id);
CREATE UNIQUE INDEX idx_vuln_fingerprints_on_occurrences_id_and_fingerprint_sha256 ON vulnerability_finding_fingerprints USING btree (finding_id, fingerprint_sha256); CREATE UNIQUE INDEX idx_vuln_signatures_on_occurrences_id_and_signature_sha ON vulnerability_finding_signatures USING btree (finding_id, signature_sha);
CREATE UNIQUE INDEX idx_vuln_fingerprints_uniqueness_fingerprint_sha256 ON vulnerability_finding_fingerprints USING btree (finding_id, algorithm_type, fingerprint_sha256); CREATE UNIQUE INDEX idx_vuln_signatures_uniqueness_signature_sha ON vulnerability_finding_signatures USING btree (finding_id, algorithm_type, signature_sha);
CREATE UNIQUE INDEX idx_vulnerability_ext_issue_links_on_vulne_id_and_ext_issue ON vulnerability_external_issue_links USING btree (vulnerability_id, external_type, external_project_key, external_issue_key); CREATE UNIQUE INDEX idx_vulnerability_ext_issue_links_on_vulne_id_and_ext_issue ON vulnerability_external_issue_links USING btree (vulnerability_id, external_type, external_project_key, external_issue_key);
...@@ -24196,7 +24196,7 @@ CREATE INDEX index_vulnerability_feedback_on_merge_request_id ON vulnerability_f ...@@ -24196,7 +24196,7 @@ CREATE INDEX index_vulnerability_feedback_on_merge_request_id ON vulnerability_f
CREATE INDEX index_vulnerability_feedback_on_pipeline_id ON vulnerability_feedback USING btree (pipeline_id); CREATE INDEX index_vulnerability_feedback_on_pipeline_id ON vulnerability_feedback USING btree (pipeline_id);
CREATE INDEX index_vulnerability_finding_fingerprints_on_finding_id ON vulnerability_finding_fingerprints USING btree (finding_id); CREATE INDEX index_vulnerability_finding_signatures_on_finding_id ON vulnerability_finding_signatures USING btree (finding_id);
CREATE INDEX index_vulnerability_findings_remediations_on_remediation_id ON vulnerability_findings_remediations USING btree (vulnerability_remediation_id); CREATE INDEX index_vulnerability_findings_remediations_on_remediation_id ON vulnerability_findings_remediations USING btree (vulnerability_remediation_id);
...@@ -26302,6 +26302,9 @@ ALTER TABLE ONLY analytics_language_trend_repository_languages ...@@ -26302,6 +26302,9 @@ ALTER TABLE ONLY analytics_language_trend_repository_languages
ALTER TABLE ONLY badges ALTER TABLE ONLY badges
ADD CONSTRAINT fk_rails_9df4a56538 FOREIGN KEY (group_id) REFERENCES namespaces(id) ON DELETE CASCADE; ADD CONSTRAINT fk_rails_9df4a56538 FOREIGN KEY (group_id) REFERENCES namespaces(id) ON DELETE CASCADE;
ALTER TABLE ONLY vulnerability_finding_signatures
ADD CONSTRAINT fk_rails_9e0baf9dcd FOREIGN KEY (finding_id) REFERENCES vulnerability_occurrences(id) ON DELETE CASCADE;
ALTER TABLE ONLY clusters_applications_cert_managers ALTER TABLE ONLY clusters_applications_cert_managers
ADD CONSTRAINT fk_rails_9e4f2cb4b2 FOREIGN KEY (cluster_id) REFERENCES clusters(id) ON DELETE CASCADE; ADD CONSTRAINT fk_rails_9e4f2cb4b2 FOREIGN KEY (cluster_id) REFERENCES clusters(id) ON DELETE CASCADE;
...@@ -26797,9 +26800,6 @@ ALTER TABLE ONLY merge_trains ...@@ -26797,9 +26800,6 @@ ALTER TABLE ONLY merge_trains
ALTER TABLE ONLY ci_runner_namespaces ALTER TABLE ONLY ci_runner_namespaces
ADD CONSTRAINT fk_rails_f9d9ed3308 FOREIGN KEY (namespace_id) REFERENCES namespaces(id) ON DELETE CASCADE; ADD CONSTRAINT fk_rails_f9d9ed3308 FOREIGN KEY (namespace_id) REFERENCES namespaces(id) ON DELETE CASCADE;
ALTER TABLE ONLY vulnerability_finding_fingerprints
ADD CONSTRAINT fk_rails_fa411253b2 FOREIGN KEY (finding_id) REFERENCES vulnerability_occurrences(id) ON DELETE CASCADE;
ALTER TABLE ONLY requirements_management_test_reports ALTER TABLE ONLY requirements_management_test_reports
ADD CONSTRAINT fk_rails_fb3308ad55 FOREIGN KEY (requirement_id) REFERENCES requirements(id) ON DELETE CASCADE; ADD CONSTRAINT fk_rails_fb3308ad55 FOREIGN KEY (requirement_id) REFERENCES requirements(id) ON DELETE CASCADE;
...@@ -3,6 +3,8 @@ ...@@ -3,6 +3,8 @@
module Types module Types
# rubocop: disable Graphql/AuthorizeTypes # rubocop: disable Graphql/AuthorizeTypes
class ScanType < BaseObject class ScanType < BaseObject
present_using ::Security::ScanPresenter
graphql_name 'Scan' graphql_name 'Scan'
description 'Represents the security scan information' description 'Represents the security scan information'
...@@ -10,9 +12,5 @@ module Types ...@@ -10,9 +12,5 @@ module Types
field :name, GraphQL::STRING_TYPE, null: false, description: 'Name of the scan.' field :name, GraphQL::STRING_TYPE, null: false, description: 'Name of the scan.'
field :errors, [GraphQL::STRING_TYPE], null: false, description: 'List of errors.' field :errors, [GraphQL::STRING_TYPE], null: false, description: 'List of errors.'
def errors
object.info['errors'].to_a
end
end end
end end
...@@ -34,7 +34,7 @@ module Vulnerabilities ...@@ -34,7 +34,7 @@ module Vulnerabilities
has_many :finding_pipelines, class_name: 'Vulnerabilities::FindingPipeline', inverse_of: :finding, foreign_key: 'occurrence_id' has_many :finding_pipelines, class_name: 'Vulnerabilities::FindingPipeline', inverse_of: :finding, foreign_key: 'occurrence_id'
has_many :pipelines, through: :finding_pipelines, class_name: 'Ci::Pipeline' has_many :pipelines, through: :finding_pipelines, class_name: 'Ci::Pipeline'
has_many :fingerprints, class_name: 'Vulnerabilities::FindingFingerprint', inverse_of: :finding has_many :signatures, class_name: 'Vulnerabilities::FindingSignature', inverse_of: :finding
has_many :finding_evidences, class_name: 'Vulnerabilities::FindingEvidence', inverse_of: :finding, foreign_key: 'vulnerability_occurrence_id' has_many :finding_evidences, class_name: 'Vulnerabilities::FindingEvidence', inverse_of: :finding, foreign_key: 'vulnerability_occurrence_id'
......
# frozen_string_literal: true # frozen_string_literal: true
module Vulnerabilities module Vulnerabilities
class FindingFingerprint < ApplicationRecord class FindingSignature < ApplicationRecord
self.table_name = 'vulnerability_finding_fingerprints' self.table_name = 'vulnerability_finding_signatures'
include BulkInsertSafe include BulkInsertSafe
belongs_to :finding, foreign_key: 'finding_id', inverse_of: :fingerprints, class_name: 'Vulnerabilities::Finding' belongs_to :finding, foreign_key: 'finding_id', inverse_of: :signatures, class_name: 'Vulnerabilities::Finding'
enum algorithm_type: { hash: 1, location: 2, scope_offset: 3 }, _prefix: :algorithm enum algorithm_type: { hash: 1, location: 2, scope_offset: 3 }, _prefix: :algorithm
......
# frozen_string_literal: true
module Security
class ScanPresenter < Gitlab::View::Presenter::Delegated
ERROR_MESSAGE_FORMAT = '[%<type>s] %<message>s'
presents :scan
def errors
info['errors'].to_a.map { |error| format(ERROR_MESSAGE_FORMAT, error.symbolize_keys) }
end
end
end
...@@ -67,7 +67,7 @@ module Security ...@@ -67,7 +67,7 @@ module Security
update_vulnerability_finding(vulnerability_finding, vulnerability_params) update_vulnerability_finding(vulnerability_finding, vulnerability_params)
reset_remediations_for(vulnerability_finding, finding) reset_remediations_for(vulnerability_finding, finding)
update_finding_fingerprints(finding, vulnerability_finding) update_finding_signatures(finding, vulnerability_finding)
# The maximum number of identifiers is not used in validation # The maximum number of identifiers is not used in validation
# we just want to ignore the rest if a finding has more than that. # we just want to ignore the rest if a finding has more than that.
...@@ -175,39 +175,39 @@ module Security ...@@ -175,39 +175,39 @@ module Security
end end
# rubocop: enable CodeReuse/ActiveRecord # rubocop: enable CodeReuse/ActiveRecord
def update_finding_fingerprints(finding, vulnerability_finding) def update_finding_signatures(finding, vulnerability_finding)
to_update = {} to_update = {}
to_create = [] to_create = []
poro_fingerprints = finding.fingerprints.index_by(&:algorithm_type) poro_signatures = finding.signatures.index_by(&:algorithm_type)
vulnerability_finding.fingerprints.each do |fingerprint| vulnerability_finding.signatures.each do |signature|
# NOTE: index_by takes the last entry if there are duplicates of the same algorithm, which should never occur. # NOTE: index_by takes the last entry if there are duplicates of the same algorithm, which should never occur.
poro_fingerprint = poro_fingerprints[fingerprint.algorithm_type] poro_signature = poro_signatures[signature.algorithm_type]
# We're no longer generating these types of fingerprints. Since # We're no longer generating these types of signatures. Since
# we're updating the persisted vulnerability, no need to do anything # we're updating the persisted vulnerability, no need to do anything
# with these fingerprints now. We will track growth with # with these signatures now. We will track growth with
# https://gitlab.com/gitlab-org/gitlab/-/issues/322186 # https://gitlab.com/gitlab-org/gitlab/-/issues/322186
next if poro_fingerprint.nil? next if poro_signature.nil?
poro_fingerprints.delete(fingerprint.algorithm_type) poro_signatures.delete(signature.algorithm_type)
to_update[fingerprint.id] = poro_fingerprint.to_h to_update[signature.id] = poro_signature.to_h
end end
# any remaining poro fingerprints left are new # any remaining poro signatures left are new
poro_fingerprints.values.each do |poro_fingerprint| poro_signatures.values.each do |poro_signature|
attributes = poro_fingerprint.to_h.merge(finding_id: vulnerability_finding.id) attributes = poro_signature.to_h.merge(finding_id: vulnerability_finding.id)
to_create << ::Vulnerabilities::FindingFingerprint.new(attributes: attributes, created_at: Time.zone.now, updated_at: Time.zone.now) to_create << ::Vulnerabilities::FindingSignature.new(attributes: attributes, created_at: Time.zone.now, updated_at: Time.zone.now)
end end
::Vulnerabilities::FindingFingerprint.transaction do ::Vulnerabilities::FindingSignature.transaction do
if to_update.count > 0 if to_update.count > 0
::Vulnerabilities::FindingFingerprint.update(to_update.keys, to_update.values) ::Vulnerabilities::FindingSignature.update(to_update.keys, to_update.values)
end end
if to_create.count > 0 if to_create.count > 0
::Vulnerabilities::FindingFingerprint.bulk_insert!(to_create) ::Vulnerabilities::FindingSignature.bulk_insert!(to_create)
end end
end end
end end
......
...@@ -65,7 +65,7 @@ ...@@ -65,7 +65,7 @@
:urgency: :throttled :urgency: :throttled
:resource_boundary: :unknown :resource_boundary: :unknown
:weight: 1 :weight: 1
:idempotent: true :idempotent:
:tags: [] :tags: []
- :name: cronjob:elastic_index_initial_bulk_cron - :name: cronjob:elastic_index_initial_bulk_cron
:feature_category: :global_search :feature_category: :global_search
...@@ -73,7 +73,7 @@ ...@@ -73,7 +73,7 @@
:urgency: :throttled :urgency: :throttled
:resource_boundary: :unknown :resource_boundary: :unknown
:weight: 1 :weight: 1
:idempotent: true :idempotent:
:tags: [] :tags: []
- :name: cronjob:elastic_migration - :name: cronjob:elastic_migration
:feature_category: :global_search :feature_category: :global_search
......
# frozen_string_literal: true # frozen_string_literal: true
class ElasticIndexBulkCronWorker class ElasticIndexBulkCronWorker # rubocop:disable Scalability/IdempotentWorker
include Elastic::BulkCronWorker include Elastic::BulkCronWorker
feature_category :global_search feature_category :global_search
idempotent!
urgency :throttled urgency :throttled
# Even though this worker is idempotent, until https://gitlab.com/gitlab-org/gitlab/-/issues/325291 is done
# we can't use it with read-only database replicas
data_consistency :delayed, feature_flag: :load_balancing_for_bulk_cron_workers
private private
......
# frozen_string_literal: true # frozen_string_literal: true
class ElasticIndexInitialBulkCronWorker class ElasticIndexInitialBulkCronWorker # rubocop:disable Scalability/IdempotentWorker
include Elastic::BulkCronWorker include Elastic::BulkCronWorker
feature_category :global_search feature_category :global_search
idempotent!
urgency :throttled urgency :throttled
# Even though this worker is idempotent, until https://gitlab.com/gitlab-org/gitlab/-/issues/325291 is done
# we can't use it with read-only database replicas
data_consistency :delayed, feature_flag: :load_balancing_for_bulk_cron_workers
private private
......
---
title: Fix the 'errors' attribute of the 'ScanType' on GraphQL API
merge_request: 57882
author:
type: fixed
...@@ -80,7 +80,7 @@ module Gitlab ...@@ -80,7 +80,7 @@ module Gitlab
links = create_links(data['links']) links = create_links(data['links'])
location = create_location(data['location'] || {}) location = create_location(data['location'] || {})
remediations = create_remediations(data['remediations']) remediations = create_remediations(data['remediations'])
fingerprints = create_fingerprints(tracking_data(data)) signatures = create_signatures(tracking_data(data))
report.add_finding( report.add_finding(
::Gitlab::Ci::Reports::Security::Finding.new( ::Gitlab::Ci::Reports::Security::Finding.new(
...@@ -99,30 +99,30 @@ module Gitlab ...@@ -99,30 +99,30 @@ module Gitlab
raw_metadata: data.to_json, raw_metadata: data.to_json,
metadata_version: report_version, metadata_version: report_version,
details: data['details'] || {}, details: data['details'] || {},
fingerprints: fingerprints)) signatures: signatures))
end end
def create_fingerprints(tracking) def create_signatures(data)
return [] if tracking.nil? || tracking['items'].nil? return [] if data.nil? || data['items'].nil?
fingerprint_algorithms = Hash.new { |hash, key| hash[key] = [] } signature_algorithms = Hash.new { |hash, key| hash[key] = [] }
tracking['items'].each do |item| data['items'].each do |item|
next unless item.key?('fingerprints') next unless item.key?('signatures')
item['fingerprints'].each do |fingerprint| item['signatures'].each do |signature|
alg = fingerprint['algorithm'] alg = signature['algorithm']
fingerprint_algorithms[alg] << fingerprint['value'] signature_algorithms[alg] << signature['value']
end end
end end
fingerprint_algorithms.map do |algorithm, values| signature_algorithms.map do |algorithm, values|
value = values.join('|') value = values.join('|')
begin begin
fingerprint = ::Gitlab::Ci::Reports::Security::FindingFingerprint.new( signature = ::Gitlab::Ci::Reports::Security::FindingSignature.new(
algorithm_type: algorithm, algorithm_type: algorithm,
fingerprint_value: value signature_value: value
) )
fingerprint.valid? ? fingerprint : nil signature.valid? ? signature : nil
rescue ArgumentError => e rescue ArgumentError => e
Gitlab::ErrorTracking.track_and_raise_for_dev_exception(e) Gitlab::ErrorTracking.track_and_raise_for_dev_exception(e)
nil nil
......
...@@ -24,11 +24,11 @@ module Gitlab ...@@ -24,11 +24,11 @@ module Gitlab
attr_reader :uuid attr_reader :uuid
attr_reader :remediations attr_reader :remediations
attr_reader :details attr_reader :details
attr_reader :fingerprints attr_reader :signatures
delegate :file_path, :start_line, :end_line, to: :location delegate :file_path, :start_line, :end_line, to: :location
def initialize(compare_key:, identifiers:, links: [], remediations: [], location:, metadata_version:, name:, raw_metadata:, report_type:, scanner:, scan:, uuid:, confidence: nil, severity: nil, details: {}, fingerprints: []) # rubocop:disable Metrics/ParameterLists def initialize(compare_key:, identifiers:, links: [], remediations: [], location:, metadata_version:, name:, raw_metadata:, report_type:, scanner:, scan:, uuid:, confidence: nil, severity: nil, details: {}, signatures: []) # rubocop:disable Metrics/ParameterLists
@compare_key = compare_key @compare_key = compare_key
@confidence = confidence @confidence = confidence
@identifiers = identifiers @identifiers = identifiers
...@@ -44,7 +44,7 @@ module Gitlab ...@@ -44,7 +44,7 @@ module Gitlab
@uuid = uuid @uuid = uuid
@remediations = remediations @remediations = remediations
@details = details @details = details
@fingerprints = fingerprints @signatures = signatures
@project_fingerprint = generate_project_fingerprint @project_fingerprint = generate_project_fingerprint
end end
......
...@@ -4,27 +4,27 @@ module Gitlab ...@@ -4,27 +4,27 @@ module Gitlab
module Ci module Ci
module Reports module Reports
module Security module Security
class FindingFingerprint class FindingSignature
attr_accessor :algorithm_type, :fingerprint_value attr_accessor :algorithm_type, :signature_value
def initialize(params = {}) def initialize(params = {})
@algorithm_type = params.dig(:algorithm_type) @algorithm_type = params.dig(:algorithm_type)
@fingerprint_value = params.dig(:fingerprint_value) @signature_value = params.dig(:signature_value)
end end
def fingerprint_sha256 def signature_sha
Digest::SHA1.digest(fingerprint_value) Digest::SHA1.digest(signature_value)
end end
def to_h def to_h
{ {
algorithm_type: algorithm_type, algorithm_type: algorithm_type,
fingerprint_sha256: fingerprint_sha256 signature_sha: signature_sha
} }
end end
def valid? def valid?
::Vulnerabilities::FindingFingerprint.algorithm_types.key?(algorithm_type) ::Vulnerabilities::FindingSignature.algorithm_types.key?(algorithm_type)
end end
end end
end end
......
# frozen_string_literal: true # frozen_string_literal: true
FactoryBot.define do FactoryBot.define do
factory :vulnerabilities_finding_fingerprint, class: 'Vulnerabilities::FindingFingerprint' do factory :vulnerabilities_finding_signature, class: 'Vulnerabilities::FindingSignature' do
finding factory: :vulnerabilities_finding finding factory: :vulnerabilities_finding
algorithm_type { ::Vulnerabilities::FindingFingerprint.algorithm_types[:hash] } algorithm_type { ::Vulnerabilities::FindingSignature.algorithm_types[:hash] }
fingerprint_sha256 { ::Digest::SHA1.digest(SecureRandom.hex(50)) } signature_sha { ::Digest::SHA1.digest(SecureRandom.hex(50)) }
end end
end end
...@@ -3,8 +3,40 @@ ...@@ -3,8 +3,40 @@
require 'spec_helper' require 'spec_helper'
RSpec.describe GitlabSchema.types['Scan'] do RSpec.describe GitlabSchema.types['Scan'] do
include GraphqlHelpers
let(:fields) { %i(name errors) } let(:fields) { %i(name errors) }
it { expect(described_class).to have_graphql_fields(fields) } it { expect(described_class).to have_graphql_fields(fields) }
it { expect(described_class).to require_graphql_authorizations(:read_scan) } it { expect(described_class).to require_graphql_authorizations(:read_scan) }
describe 'field values' do
let_it_be(:build) { create(:ee_ci_build, :dast, name: 'foo') }
let_it_be(:security_scan) { build.security_scans.first }
let_it_be(:user) { create(:user) }
subject { resolve_field(field_name, security_scan, current_user: user) }
before do
stub_licensed_features(security_dashboard: true)
build.project.add_developer(user)
end
describe 'name' do
let(:field_name) { :name }
it { is_expected.to eq('foo') }
end
describe 'errors' do
let(:field_name) { :errors }
before do
security_scan.update!(info: { 'errors' => [{ 'type' => 'foo', 'message' => 'bar' }] })
end
it { is_expected.to eq(['[foo] bar']) }
end
end
end end
...@@ -13,7 +13,7 @@ RSpec.describe Gitlab::Ci::Parsers::Security::Common do ...@@ -13,7 +13,7 @@ RSpec.describe Gitlab::Ci::Parsers::Security::Common do
{ {
'type' => 'source', 'type' => 'source',
'items' => [ 'items' => [
'fingerprints' => [ 'signatures' => [
{ 'algorithm' => 'hash', 'value' => 'hash_value' }, { 'algorithm' => 'hash', 'value' => 'hash_value' },
{ 'algorithm' => 'location', 'value' => 'location_value' }, { 'algorithm' => 'location', 'value' => 'location_value' },
{ 'algorithm' => 'scope_offset', 'value' => 'scope_offset_value' } { 'algorithm' => 'scope_offset', 'value' => 'scope_offset_value' }
...@@ -200,21 +200,21 @@ RSpec.describe Gitlab::Ci::Parsers::Security::Common do ...@@ -200,21 +200,21 @@ RSpec.describe Gitlab::Ci::Parsers::Security::Common do
end end
end end
describe 'parsing tracking' do describe 'parsing signature' do
context 'with valid tracking information' do context 'with valid signature information' do
it 'creates fingerprints for each algorithm' do it 'creates signatures for each algorithm' do
finding = report.findings.first finding = report.findings.first
expect(finding.fingerprints.size).to eq(3) expect(finding.signatures.size).to eq(3)
expect(finding.fingerprints.map(&:algorithm_type).to_set).to eq(Set['hash', 'location', 'scope_offset']) expect(finding.signatures.map(&:algorithm_type).to_set).to eq(Set['hash', 'location', 'scope_offset'])
end end
end end
context 'with invalid tracking information' do context 'with invalid signature information' do
let(:tracking_data) do let(:tracking_data) do
{ {
'type' => 'source', 'type' => 'source',
'items' => [ 'items' => [
'fingerprints' => [ 'signatures' => [
{ 'algorithm' => 'hash', 'value' => 'hash_value' }, { 'algorithm' => 'hash', 'value' => 'hash_value' },
{ 'algorithm' => 'location', 'value' => 'location_value' }, { 'algorithm' => 'location', 'value' => 'location_value' },
{ 'algorithm' => 'INVALID', 'value' => 'scope_offset_value' } { 'algorithm' => 'INVALID', 'value' => 'scope_offset_value' }
...@@ -225,8 +225,8 @@ RSpec.describe Gitlab::Ci::Parsers::Security::Common do ...@@ -225,8 +225,8 @@ RSpec.describe Gitlab::Ci::Parsers::Security::Common do
it 'ignores invalid algorithm types' do it 'ignores invalid algorithm types' do
finding = report.findings.first finding = report.findings.first
expect(finding.fingerprints.size).to eq(2) expect(finding.signatures.size).to eq(2)
expect(finding.fingerprints.map(&:algorithm_type).to_set).to eq(Set['hash', 'location']) expect(finding.signatures.map(&:algorithm_type).to_set).to eq(Set['hash', 'location'])
end end
end end
end end
......
...@@ -2,13 +2,13 @@ ...@@ -2,13 +2,13 @@
require 'spec_helper' require 'spec_helper'
RSpec.describe Gitlab::Ci::Reports::Security::FindingFingerprint do RSpec.describe Gitlab::Ci::Reports::Security::FindingSignature do
subject { described_class.new(params.with_indifferent_access) } subject { described_class.new(params.with_indifferent_access) }
let(:params) do let(:params) do
{ {
algorithm_type: 'hash', algorithm_type: 'hash',
fingerprint_value: 'FINGERPRINT' signature_value: 'SIGNATURE'
} }
end end
...@@ -16,16 +16,16 @@ RSpec.describe Gitlab::Ci::Reports::Security::FindingFingerprint do ...@@ -16,16 +16,16 @@ RSpec.describe Gitlab::Ci::Reports::Security::FindingFingerprint do
context 'when a supported algorithm type is given' do context 'when a supported algorithm type is given' do
it 'allows itself to be created' do it 'allows itself to be created' do
expect(subject.algorithm_type).to eq(params[:algorithm_type]) expect(subject.algorithm_type).to eq(params[:algorithm_type])
expect(subject.fingerprint_value).to eq(params[:fingerprint_value]) expect(subject.signature_value).to eq(params[:signature_value])
end end
end end
end end
describe '#to_h' do describe '#to_h' do
it 'returns a hash representation of the fingerprint' do it 'returns a hash representation of the signature' do
expect(subject.to_h).to eq( expect(subject.to_h).to eq(
algorithm_type: params[:algorithm_type], algorithm_type: params[:algorithm_type],
fingerprint_sha256: Digest::SHA1.digest(params[:fingerprint_value]) signature_sha: Digest::SHA1.digest(params[:signature_value])
) )
end end
end end
...@@ -41,7 +41,7 @@ RSpec.describe Gitlab::Ci::Reports::Security::FindingFingerprint do ...@@ -41,7 +41,7 @@ RSpec.describe Gitlab::Ci::Reports::Security::FindingFingerprint do
let(:params) do let(:params) do
{ {
algorithm_type: 'INVALID', algorithm_type: 'INVALID',
fingerprint_value: 'FINGERPRINT' signature_value: 'SIGNATURE'
} }
end end
......
# frozen_string_literal: true
require 'spec_helper'
RSpec.describe Security::ScanPresenter do
let(:presenter) { described_class.new(security_scan) }
let(:security_scan) { build_stubbed(:security_scan, info: { 'errors' => [{ 'type' => 'foo', 'message' => 'bar' }] }) }
describe '#errors' do
subject { presenter.errors }
it { is_expected.to eq(['[foo] bar']) }
end
end
...@@ -30,7 +30,7 @@ RSpec.describe Security::StoreReportService, '#execute' do ...@@ -30,7 +30,7 @@ RSpec.describe Security::StoreReportService, '#execute' do
using RSpec::Parameterized::TableSyntax using RSpec::Parameterized::TableSyntax
where(:case_name, :trait, :scanners, :identifiers, :findings, :finding_identifiers, :finding_pipelines, :remediations, :fingerprints) do where(:case_name, :trait, :scanners, :identifiers, :findings, :finding_identifiers, :finding_pipelines, :remediations, :signatures) do
'with SAST report' | :sast | 1 | 6 | 5 | 7 | 5 | 0 | 2 'with SAST report' | :sast | 1 | 6 | 5 | 7 | 5 | 0 | 2
'with exceeding identifiers' | :with_exceeding_identifiers | 1 | 20 | 1 | 20 | 1 | 0 | 0 'with exceeding identifiers' | :with_exceeding_identifiers | 1 | 20 | 1 | 20 | 1 | 0 | 0
'with Dependency Scanning report' | :dependency_scanning_remediation | 1 | 3 | 2 | 3 | 2 | 1 | 0 'with Dependency Scanning report' | :dependency_scanning_remediation | 1 | 3 | 2 | 3 | 2 | 1 | 0
...@@ -66,8 +66,8 @@ RSpec.describe Security::StoreReportService, '#execute' do ...@@ -66,8 +66,8 @@ RSpec.describe Security::StoreReportService, '#execute' do
expect { subject }.to change { Vulnerability.count }.by(findings) expect { subject }.to change { Vulnerability.count }.by(findings)
end end
it 'inserts all fingerprints' do it 'inserts all signatures' do
expect { subject }.to change { Vulnerabilities::FindingFingerprint.count }.by(fingerprints) expect { subject }.to change { Vulnerabilities::FindingSignature.count }.by(signatures)
end end
end end
...@@ -124,11 +124,11 @@ RSpec.describe Security::StoreReportService, '#execute' do ...@@ -124,11 +124,11 @@ RSpec.describe Security::StoreReportService, '#execute' do
let(:new_build) { create(:ci_build, pipeline: new_pipeline) } let(:new_build) { create(:ci_build, pipeline: new_pipeline) }
let(:new_pipeline) { create(:ci_pipeline, project: project) } let(:new_pipeline) { create(:ci_pipeline, project: project) }
let(:new_report) { new_pipeline.security_reports.get_report(report_type.to_s, artifact) } let(:new_report) { new_pipeline.security_reports.get_report(report_type.to_s, artifact) }
let(:existing_fingerprint) { create(:vulnerabilities_finding_fingerprint, finding: finding) } let(:existing_signature) { create(:vulnerabilities_finding_signature, finding: finding) }
let(:unsupported_fingerprint) do let(:unsupported_signature) do
create(:vulnerabilities_finding_fingerprint, create(:vulnerabilities_finding_signature,
finding: finding, finding: finding,
algorithm_type: ::Vulnerabilities::FindingFingerprint.algorithm_types[:location]) algorithm_type: ::Vulnerabilities::FindingSignature.algorithm_types[:location])
end end
let(:trait) { :sast } let(:trait) { :sast }
...@@ -219,29 +219,29 @@ RSpec.describe Security::StoreReportService, '#execute' do ...@@ -219,29 +219,29 @@ RSpec.describe Security::StoreReportService, '#execute' do
expect(finding.reload).to have_attributes(severity: 'medium', name: 'Cipher with no integrity') expect(finding.reload).to have_attributes(severity: 'medium', name: 'Cipher with no integrity')
end end
it 'updates fingerprints to match new values' do it 'updates signatures to match new values' do
existing_fingerprint existing_signature
unsupported_fingerprint unsupported_signature
expect(finding.fingerprints.count).to eq(2) expect(finding.signatures.count).to eq(2)
fingerprint_algs = finding.fingerprints.map(&:algorithm_type).sort signature_algs = finding.signatures.map(&:algorithm_type).sort
expect(fingerprint_algs).to eq(%w[hash location]) expect(signature_algs).to eq(%w[hash location])
subject subject
finding.reload finding.reload
existing_fingerprint.reload existing_signature.reload
# check that unsupported algorithm is not deleted # check that unsupported algorithm is not deleted
expect(finding.fingerprints.count).to eq(3) expect(finding.signatures.count).to eq(3)
fingerprint_algs = finding.fingerprints.sort.map(&:algorithm_type) signature_algs = finding.signatures.sort.map(&:algorithm_type)
expect(fingerprint_algs).to eq(%w[hash location scope_offset]) expect(signature_algs).to eq(%w[hash location scope_offset])
# check that the existing hash fingerprint was updated/reused # check that the existing hash signature was updated/reused
expect(existing_fingerprint.id).to eq(finding.fingerprints.min.id) expect(existing_signature.id).to eq(finding.signatures.min.id)
# check that the unsupported fingerprint was not deleted # check that the unsupported signature was not deleted
expect(::Vulnerabilities::FindingFingerprint.exists?(unsupported_fingerprint.id)).to eq(true) expect(::Vulnerabilities::FindingSignature.exists?(unsupported_signature.id)).to eq(true)
end end
it 'updates existing vulnerability with new data' do it 'updates existing vulnerability with new data' do
......
...@@ -47,4 +47,9 @@ RSpec.describe ElasticIndexBulkCronWorker do ...@@ -47,4 +47,9 @@ RSpec.describe ElasticIndexBulkCronWorker do
) )
end end
end end
it_behaves_like 'worker with data consistency',
described_class,
feature_flag: :load_balancing_for_bulk_cron_workers,
data_consistency: :delayed
end end
...@@ -161,6 +161,8 @@ module API ...@@ -161,6 +161,8 @@ module API
not_found! unless metadata not_found! unless metadata
track_package_event('pull_package', :composer)
send_git_archive unauthorized_user_project.repository, ref: metadata.target_sha, format: 'zip', append_sha: true send_git_archive unauthorized_user_project.repository, ref: metadata.target_sha, format: 'zip', append_sha: true
end end
end end
......
...@@ -156,7 +156,7 @@ ...@@ -156,7 +156,7 @@
"file": "groovy/src/main/java/com/gitlab/security_products/tests/App.groovy", "file": "groovy/src/main/java/com/gitlab/security_products/tests/App.groovy",
"start_line": 47, "start_line": 47,
"end_line": 47, "end_line": 47,
"fingerprints": [ "signatures": [
{ {
"algorithm": "hash", "algorithm": "hash",
"value": "HASHVALUE" "value": "HASHVALUE"
......
...@@ -27,7 +27,7 @@ const testCases = [ ...@@ -27,7 +27,7 @@ const testCases = [
name: 'Closed', name: 'Closed',
state: 'closed', state: 'closed',
class: 'status-box-mr-closed', class: 'status-box-mr-closed',
icon: 'close', icon: 'issue-close',
}, },
{ {
name: 'Merged', name: 'Merged',
......
...@@ -434,6 +434,7 @@ RSpec.describe API::ComposerPackages do ...@@ -434,6 +434,7 @@ RSpec.describe API::ComposerPackages do
end end
it_behaves_like 'process Composer api request', params[:user_role], params[:expected_status], params[:member] it_behaves_like 'process Composer api request', params[:user_role], params[:expected_status], params[:member]
it_behaves_like 'a package tracking event', described_class.name, 'pull_package'
end end
end end
end end
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment