Add latest changes from gitlab-org/gitlab@master

app/models/concerns/issuable.rb

@@ -4,7 +4,7 @@
 #
 # Contains common functionality shared between Issues and MergeRequests
 #
-# Used by Issue, MergeRequest
+# Used by Issue, MergeRequest, Epic
 #
 module Issuable
   extend ActiveSupport::Concern
@@ -26,6 +26,11 @@ module Issuable
   include IssuableStates
   include ClosedAtFilterable
 
+  TITLE_LENGTH_MAX = 255
+  TITLE_HTML_LENGTH_MAX = 800
+  DESCRIPTION_LENGTH_MAX = 16000
+  DESCRIPTION_HTML_LENGTH_MAX = 48000
+
   # This object is used to gather issuable meta data for displaying
   # upvotes, downvotes, notes and closing merge requests count for issues and merge requests
   # lists avoiding n+1 queries and improving performance.
@@ -72,10 +77,15 @@ module Issuable
              prefix: true
 
     validates :author, presence: true
-    validates :title, presence: true, length: { maximum: 255 }
-    validates :description, length: { maximum: Gitlab::Database::MAX_TEXT_SIZE_LIMIT }, allow_blank: true
+    validates :title, presence: true, length: { maximum: TITLE_LENGTH_MAX }
+    # we validate the description against DESCRIPTION_LENGTH_MAX only for Issuables being created
+    # to avoid breaking the existing Issuables which may have their descriptions longer
+    validates :description, length: { maximum: DESCRIPTION_LENGTH_MAX }, allow_blank: true, on: :create
+    validate :description_max_length_for_new_records_is_valid, on: :update
     validate :milestone_is_valid
 
+    before_validation :truncate_description_on_import!
+
     scope :authored, ->(user) { where(author_id: user) }
     scope :recent, -> { reorder(id: :desc) }
     scope :of_projects, ->(ids) { where(project_id: ids) }
@@ -138,6 +148,16 @@ module Issuable
     def milestone_is_valid
       errors.add(:milestone_id, message: "is invalid") if milestone_id.present? && !milestone_available?
     end
+
+    def description_max_length_for_new_records_is_valid
+      if new_record? && description.length > Issuable::DESCRIPTION_LENGTH_MAX
+        errors.add(:description, :too_long, count: Issuable::DESCRIPTION_LENGTH_MAX)
+      end
+    end
+
+    def truncate_description_on_import!
+      self.description = description&.slice(0, Issuable::DESCRIPTION_LENGTH_MAX) if importing?
+    end
   end
 
   class_methods do

db/migrate/20190929180751_create_vulnerabilities.rb

+# frozen_string_literal: true
+
+# See http://doc.gitlab.com/ce/development/migration_style_guide.html
+# for more information on how to write migrations for GitLab.
+
+class CreateVulnerabilities < ActiveRecord::Migration[5.2]
+  include Gitlab::Database::MigrationHelpers
+
+  DOWNTIME = false
+
+  def change
+    create_table :vulnerabilities do |t|
+      t.bigint "milestone_id"
+      t.bigint "epic_id"
+      t.bigint "project_id", null: false
+      t.bigint "author_id", null: false
+      t.bigint "updated_by_id"
+      t.bigint "last_edited_by_id"
+      t.bigint "start_date_sourcing_milestone_id"
+      t.bigint "due_date_sourcing_milestone_id"
+      t.bigint "closed_by_id"
+      t.datetime_with_timezone "last_edited_at"
+      t.datetime_with_timezone "created_at", null: false
+      t.datetime_with_timezone "updated_at", null: false
+      t.datetime_with_timezone "closed_at"
+      t.date "start_date"
+      t.date "due_date"
+      t.integer "state", limit: 2, default: 1, null: false # initially: open, closed
+      t.integer "severity", limit: 2, null: false # auto-calculated as highest-severity finding, but overrideable
+      t.integer "confidence", limit: 2, null: false # auto-calculated as lowest-confidence finding, but overrideable
+      t.boolean "severity_overridden", default: false
+      t.boolean "confidence_overridden", default: false
+      t.string "title", limit: 255, null: false
+      t.text "title_html", null: false
+      t.text "description"
+      t.text "description_html"
+    end
+  end
+end

db/migrate/20190929180813_add_reference_from_vulnerability_occurrences_to_occurrences.rb

+# frozen_string_literal: true
+
+# See http://doc.gitlab.com/ce/development/migration_style_guide.html
+# for more information on how to write migrations for GitLab.
+
+class AddReferenceFromVulnerabilityOccurrencesToOccurrences < ActiveRecord::Migration[5.2]
+  include Gitlab::Database::MigrationHelpers
+
+  DOWNTIME = false
+
+  def change
+    add_column :vulnerability_occurrences, :vulnerability_id, :bigint
+  end
+end

db/migrate/20190929180827_add_foreign_keys_and_indexes_to_vulnerabilities.rb

+# frozen_string_literal: true
+
+# See http://doc.gitlab.com/ce/development/migration_style_guide.html
+# for more information on how to write migrations for GitLab.
+
+class AddForeignKeysAndIndexesToVulnerabilities < ActiveRecord::Migration[5.2]
+  include Gitlab::Database::MigrationHelpers
+
+  DOWNTIME = false
+
+  disable_ddl_transaction!
+
+  def up
+    add_concurrent_index :vulnerabilities, :milestone_id
+    add_concurrent_foreign_key :vulnerabilities, :milestones, column: :milestone_id, on_delete: :nullify
+
+    add_concurrent_index :vulnerabilities, :epic_id
+    add_concurrent_foreign_key :vulnerabilities, :epics, column: :epic_id, on_delete: :nullify
+
+    add_concurrent_index :vulnerabilities, :project_id
+    add_concurrent_foreign_key :vulnerabilities, :projects, column: :project_id
+
+    add_concurrent_index :vulnerabilities, :author_id
+    add_concurrent_foreign_key :vulnerabilities, :users, column: :author_id, on_delete: :nullify
+
+    add_concurrent_index :vulnerabilities, :updated_by_id
+    add_concurrent_foreign_key :vulnerabilities, :users, column: :updated_by_id, on_delete: :nullify
+
+    add_concurrent_index :vulnerabilities, :last_edited_by_id
+    add_concurrent_foreign_key :vulnerabilities, :users, column: :last_edited_by_id, on_delete: :nullify
+
+    add_concurrent_index :vulnerabilities, :closed_by_id
+    add_concurrent_foreign_key :vulnerabilities, :users, column: :closed_by_id, on_delete: :nullify
+
+    add_concurrent_index :vulnerabilities, :start_date_sourcing_milestone_id
+    add_concurrent_foreign_key :vulnerabilities, :milestones, column: :start_date_sourcing_milestone_id, on_delete: :nullify
+
+    add_concurrent_index :vulnerabilities, :due_date_sourcing_milestone_id
+    add_concurrent_foreign_key :vulnerabilities, :milestones, column: :due_date_sourcing_milestone_id, on_delete: :nullify
+
+    add_concurrent_index :vulnerability_occurrences, :vulnerability_id
+    add_concurrent_foreign_key :vulnerability_occurrences, :vulnerabilities, column: :vulnerability_id, on_delete: :nullify
+  end
+
+  def down
+    remove_foreign_key :vulnerability_occurrences, :vulnerabilities
+    remove_concurrent_index :vulnerability_occurrences, :vulnerability_id
+
+    remove_foreign_key :vulnerabilities, column: :due_date_sourcing_milestone_id
+    remove_concurrent_index :vulnerabilities, :due_date_sourcing_milestone_id
+
+    remove_foreign_key :vulnerabilities, column: :start_date_sourcing_milestone_id
+    remove_concurrent_index :vulnerabilities, :start_date_sourcing_milestone_id
+
+    remove_foreign_key :vulnerabilities, column: :closed_by_id
+    remove_concurrent_index :vulnerabilities, :closed_by_id
+
+    remove_foreign_key :vulnerabilities, column: :last_edited_by_id
+    remove_concurrent_index :vulnerabilities, :last_edited_by_id
+
+    remove_foreign_key :vulnerabilities, column: :updated_by_id
+    remove_concurrent_index :vulnerabilities, :updated_by_id
+
+    remove_foreign_key :vulnerabilities, column: :author_id
+    remove_concurrent_index :vulnerabilities, :author_id
+
+    remove_foreign_key :vulnerabilities, :projects
+    remove_concurrent_index :vulnerabilities, :project_id
+
+    remove_foreign_key :vulnerabilities, :epics
+    remove_concurrent_index :vulnerabilities, :epic_id
+
+    remove_foreign_key :vulnerabilities, :milestones
+    remove_concurrent_index :vulnerabilities, :milestone_id
+  end
+end

db/schema.rb

@@ -10,7 +10,7 @@
 #
 # It's strongly recommended that you check this file into your version control system.
 
-ActiveRecord::Schema.define(version: 2019_09_27_074328) do
+ActiveRecord::Schema.define(version: 2019_09_29_180827) do
 
   # These are extensions that must be enabled in order to support this database
   enable_extension "pg_trgm"
@@ -3705,6 +3705,42 @@ ActiveRecord::Schema.define(version: 2019_09_27_074328) do
     t.index ["user_id", "project_id"], name: "index_users_star_projects_on_user_id_and_project_id", unique: true
   end
 
+  create_table "vulnerabilities", force: :cascade do |t|
+    t.bigint "milestone_id"
+    t.bigint "epic_id"
+    t.bigint "project_id", null: false
+    t.bigint "author_id", null: false
+    t.bigint "updated_by_id"
+    t.bigint "last_edited_by_id"
+    t.date "start_date"
+    t.date "due_date"
+    t.datetime_with_timezone "last_edited_at"
+    t.datetime_with_timezone "created_at", null: false
+    t.datetime_with_timezone "updated_at", null: false
+    t.string "title", limit: 255, null: false
+    t.text "title_html", null: false
+    t.text "description"
+    t.text "description_html"
+    t.bigint "start_date_sourcing_milestone_id"
+    t.bigint "due_date_sourcing_milestone_id"
+    t.bigint "closed_by_id"
+    t.datetime_with_timezone "closed_at"
+    t.integer "state", limit: 2, default: 1, null: false
+    t.integer "severity", limit: 2, null: false
+    t.boolean "severity_overridden", default: false
+    t.integer "confidence", limit: 2, null: false
+    t.boolean "confidence_overridden", default: false
+    t.index ["author_id"], name: "index_vulnerabilities_on_author_id"
+    t.index ["closed_by_id"], name: "index_vulnerabilities_on_closed_by_id"
+    t.index ["due_date_sourcing_milestone_id"], name: "index_vulnerabilities_on_due_date_sourcing_milestone_id"
+    t.index ["epic_id"], name: "index_vulnerabilities_on_epic_id"
+    t.index ["last_edited_by_id"], name: "index_vulnerabilities_on_last_edited_by_id"
+    t.index ["milestone_id"], name: "index_vulnerabilities_on_milestone_id"
+    t.index ["project_id"], name: "index_vulnerabilities_on_project_id"
+    t.index ["start_date_sourcing_milestone_id"], name: "index_vulnerabilities_on_start_date_sourcing_milestone_id"
+    t.index ["updated_by_id"], name: "index_vulnerabilities_on_updated_by_id"
+  end
+
   create_table "vulnerability_feedback", id: :serial, force: :cascade do |t|
     t.datetime_with_timezone "created_at", null: false
     t.datetime_with_timezone "updated_at", null: false
@@ -3772,10 +3808,12 @@ ActiveRecord::Schema.define(version: 2019_09_27_074328) do
     t.string "name", null: false
     t.string "metadata_version", null: false
     t.text "raw_metadata", null: false
+    t.bigint "vulnerability_id"
     t.index ["primary_identifier_id"], name: "index_vulnerability_occurrences_on_primary_identifier_id"
     t.index ["project_id", "primary_identifier_id", "location_fingerprint", "scanner_id"], name: "index_vulnerability_occurrences_on_unique_keys", unique: true
     t.index ["scanner_id"], name: "index_vulnerability_occurrences_on_scanner_id"
     t.index ["uuid"], name: "index_vulnerability_occurrences_on_uuid", unique: true
+    t.index ["vulnerability_id"], name: "index_vulnerability_occurrences_on_vulnerability_id"
   end
 
   create_table "vulnerability_scanners", force: :cascade do |t|
@@ -4202,6 +4240,15 @@ ActiveRecord::Schema.define(version: 2019_09_27_074328) do
   add_foreign_key "users_ops_dashboard_projects", "projects", on_delete: :cascade
   add_foreign_key "users_ops_dashboard_projects", "users", on_delete: :cascade
   add_foreign_key "users_star_projects", "projects", name: "fk_22cd27ddfc", on_delete: :cascade
+  add_foreign_key "vulnerabilities", "epics", name: "fk_1d37cddf91", on_delete: :nullify
+  add_foreign_key "vulnerabilities", "milestones", column: "due_date_sourcing_milestone_id", name: "fk_7c5bb22a22", on_delete: :nullify
+  add_foreign_key "vulnerabilities", "milestones", column: "start_date_sourcing_milestone_id", name: "fk_88b4d546ef", on_delete: :nullify
+  add_foreign_key "vulnerabilities", "milestones", name: "fk_131d289c65", on_delete: :nullify
+  add_foreign_key "vulnerabilities", "projects", name: "fk_efb96ab1e2", on_delete: :cascade
+  add_foreign_key "vulnerabilities", "users", column: "author_id", name: "fk_b1de915a15", on_delete: :nullify
+  add_foreign_key "vulnerabilities", "users", column: "closed_by_id", name: "fk_cf5c60acbf", on_delete: :nullify
+  add_foreign_key "vulnerabilities", "users", column: "last_edited_by_id", name: "fk_1302949740", on_delete: :nullify
+  add_foreign_key "vulnerabilities", "users", column: "updated_by_id", name: "fk_7ac31eacb9", on_delete: :nullify
   add_foreign_key "vulnerability_feedback", "ci_pipelines", column: "pipeline_id", on_delete: :nullify
   add_foreign_key "vulnerability_feedback", "issues", on_delete: :nullify
   add_foreign_key "vulnerability_feedback", "merge_requests", name: "fk_563ff1912e", on_delete: :nullify
@@ -4214,6 +4261,7 @@ ActiveRecord::Schema.define(version: 2019_09_27_074328) do
   add_foreign_key "vulnerability_occurrence_pipelines", "ci_pipelines", column: "pipeline_id", on_delete: :cascade
   add_foreign_key "vulnerability_occurrence_pipelines", "vulnerability_occurrences", column: "occurrence_id", on_delete: :cascade
   add_foreign_key "vulnerability_occurrences", "projects", on_delete: :cascade
+  add_foreign_key "vulnerability_occurrences", "vulnerabilities", name: "fk_97ffe77653", on_delete: :nullify
   add_foreign_key "vulnerability_occurrences", "vulnerability_identifiers", column: "primary_identifier_id", on_delete: :cascade
   add_foreign_key "vulnerability_occurrences", "vulnerability_scanners", column: "scanner_id", on_delete: :cascade
   add_foreign_key "vulnerability_scanners", "projects", on_delete: :cascade

doc/development/README.md

@@ -34,6 +34,7 @@ description: 'Learn how to contribute to GitLab.'
 ## Backend guides
 
 - [GitLab utilities](utilities.md)
+- [Issuable-like Rails models](issuable-like-models.md)
 - [Logging](logging.md)
 - [API styleguide](api_styleguide.md) Use this styleguide if you are
   contributing to the API

doc/development/issuable-like-models.md

+# Issuable-like Rails models utilities
+
+GitLab Rails codebase contains several models that hold common functionality and behave similarly to an [Issue]. Other
+examples of `Issuable`s are [Merge Requests] and [Epics].
+
+This guide accumulates guidelines on working with such Rails models.
+
+## Important text fields
+
+There are max length constraints for the most important text fields for `Issuable`s:
+
+- `title`: 255 chars
+- `title_html`: 800 chars
+- `description`: 16000 chars
+- `description_html`: 48000 chars
+
+[Issue]: https://docs.gitlab.com/ee/user/project/issues
+[Merge Requests]: https://docs.gitlab.com/ee/user/project/merge_requests
+[Epics]: https://docs.gitlab.com/ee/user/group/epics

qa/qa/resource/repository/wiki_push.rb

@@ -25,14 +25,7 @@ module QA
         end
 
         def repository_ssh_uri
-          @repository_ssh_uri ||= begin
-            wiki.visit!
-            Page::Project::Wiki::Show.act do
-              click_clone_repository
-              choose_repository_clone_ssh
-              repository_location.uri
-            end
-          end
+          @repository_ssh_uri ||= wiki.repository_ssh_location.uri
         end
 
         def fabricate!

qa/qa/resource/wiki.rb

@@ -21,6 +21,15 @@ module QA
         end
       end
 
+      attribute :repository_ssh_location do
+        Page::Project::Wiki::Show.perform(&:click_clone_repository)
+
+        Page::Project::Wiki::GitAccess.perform do |git_access|
+          git_access.choose_repository_clone_ssh
+          git_access.repository_location
+        end
+      end
+
       def fabricate!
         project.visit!
 

spec/lib/gitlab/import_export/all_models.yml

@@ -172,7 +172,7 @@ ci_pipelines:
 - downstream_bridges
 - job_artifacts
 - vulnerabilities_occurrence_pipelines
-- vulnerabilities
+- vulnerability_findings
 pipeline_variables:
 - pipeline
 stages:
@@ -389,6 +389,7 @@ project:
 - sourced_pipelines
 - prometheus_metrics
 - vulnerabilities
+- vulnerability_findings
 - vulnerability_feedback
 - vulnerability_identifiers
 - vulnerability_scanners

spec/models/concerns/issuable_spec.rb

@@ -46,7 +46,10 @@ describe Issuable do
       it { is_expected.to validate_presence_of(:author) }
       it { is_expected.to validate_presence_of(:title) }
       it { is_expected.to validate_length_of(:title).is_at_most(255) }
-      it { is_expected.to validate_length_of(:description).is_at_most(1_000_000) }
+      it { is_expected.to validate_length_of(:description).is_at_most(16_000).on(:create) }
+
+      it_behaves_like 'validates description length with custom validation'
+      it_behaves_like 'truncates the description to its allowed maximum length on import'
     end
 
     describe 'milestone' do

spec/support/shared_examples/models/concern/issuable_shared_examples.rb

+# frozen_string_literal: true
+
 shared_examples_for 'matches_cross_reference_regex? fails fast' do
   it 'fails fast for long strings' do
     # took well under 1 second in CI https://dev.gitlab.org/gitlab/gitlabhq/merge_requests/3267#note_172823
@@ -6,3 +8,59 @@ shared_examples_for 'matches_cross_reference_regex? fails fast' do
     end.not_to raise_error
   end
 end
+
+shared_examples_for 'validates description length with custom validation' do
+  let(:issuable) { build(:issue, description: 'x' * 16_001) }
+  let(:context) { :update }
+
+  subject { issuable.validate(context) }
+
+  context 'when Issuable is a new record' do
+    it 'validates the maximum description length' do
+      subject
+      expect(issuable.errors[:description]).to eq(["is too long (maximum is 16000 characters)"])
+    end
+
+    context 'on create' do
+      let(:context) { :create }
+
+      it 'does not validate the maximum description length' do
+        allow(issuable).to receive(:description_max_length_for_new_records_is_valid).and_call_original
+
+        subject
+
+        expect(issuable).not_to have_received(:description_max_length_for_new_records_is_valid)
+      end
+    end
+  end
+
+  context 'when Issuable is an existing record' do
+    before do
+      allow(issuable).to receive(:expire_etag_cache) # to skip the expire_etag_cache callback
+
+      issuable.save!(validate: false)
+    end
+
+    it 'does not validate the maximum description length' do
+      subject
+      expect(issuable.errors).not_to have_key(:description)
+    end
+  end
+end
+
+shared_examples_for 'truncates the description to its allowed maximum length on import' do
+  before do
+    allow(issuable).to receive(:importing?).and_return(true)
+  end
+
+  let(:issuable) { build(:issue, description: 'x' * 16_001) }
+
+  subject { issuable.validate(:create) }
+
+  it 'truncates the description to its allowed maximum length' do
+    subject
+
+    expect(issuable.description).to eq('x' * 16_000)
+    expect(issuable.errors[:description]).to be_empty
+  end
+end