Add many foreign keys to the projects table

This removes the need for relying on Rails' "dependent" option for data
removal, which is _incredibly_ slow (even when using :delete_all) when
deleting large amounts of data. This also ensures data consistency is
enforced on DB level and not on application level (something Rails is
really bad at).

This commit also includes various migrations to add foreign keys to
tables that eventually point to "projects" to ensure no rows get
orphaned upon removing a project.

app/models/ee/project.rb

@@ -449,5 +449,9 @@ module EE
     def destroy_mirror_data
       mirror_data.destroy
     end
+
+    def validate_board_limit(board)
+      # Board limits are disabled in EE, so this method is just a no-op.
+    end
   end
 end

app/models/issue.rb

@@ -33,7 +33,7 @@ class Issue < ActiveRecord::Base
 
   has_many :events, as: :target, dependent: :destroy
 
-  has_many :merge_requests_closing_issues, class_name: 'MergeRequestsClosingIssues', dependent: :delete_all
+  has_many :merge_requests_closing_issues, class_name: 'MergeRequestsClosingIssues'
 
   has_many :issue_assignees
   has_many :assignees, class_name: "User", through: :issue_assignees

app/models/merge_request.rb

@@ -18,7 +18,7 @@ class MergeRequest < ActiveRecord::Base
   has_many :approvals, dependent: :destroy
   has_many :approvers, as: :target, dependent: :destroy
   has_many :approver_groups, as: :target, dependent: :destroy
-  has_many :merge_request_diffs, dependent: :destroy
+  has_many :merge_request_diffs
   has_one :merge_request_diff,
     -> { order('merge_request_diffs.id DESC') }
 
@@ -26,7 +26,7 @@ class MergeRequest < ActiveRecord::Base
 
   has_many :events, as: :target, dependent: :destroy
 
-  has_many :merge_requests_closing_issues, class_name: 'MergeRequestsClosingIssues', dependent: :delete_all
+  has_many :merge_requests_closing_issues, class_name: 'MergeRequestsClosingIssues'
 
   belongs_to :assignee, class_name: "User"
 

app/models/project.rb

@@ -62,6 +62,7 @@ class Project < ActiveRecord::Base
     update_column(:last_repository_updated_at, self.created_at)
   end
 
+  before_destroy :remove_private_deploy_keys
   after_destroy :remove_pages
 
   # update visibility_level of forks
@@ -83,96 +84,108 @@ class Project < ActiveRecord::Base
   belongs_to :namespace
 
   has_one :last_event, -> {order 'events.created_at DESC'}, class_name: 'Event'
-  has_many :boards, dependent: :destroy
+  has_many :boards, before_add: :validate_board_limit
 
   # Project services
-  has_one :campfire_service, dependent: :destroy
-  has_one :drone_ci_service, dependent: :destroy
-  has_one :emails_on_push_service, dependent: :destroy
-  has_one :pipelines_email_service, dependent: :destroy
-  has_one :irker_service, dependent: :destroy
-  has_one :pivotaltracker_service, dependent: :destroy
-  has_one :hipchat_service, dependent: :destroy
-  has_one :flowdock_service, dependent: :destroy
-  has_one :assembla_service, dependent: :destroy
-  has_one :asana_service, dependent: :destroy
-  has_one :gemnasium_service, dependent: :destroy
-  has_one :mattermost_slash_commands_service, dependent: :destroy
-  has_one :mattermost_service, dependent: :destroy
-  has_one :slack_slash_commands_service, dependent: :destroy
-  has_one :slack_service, dependent: :destroy
-  has_one :buildkite_service, dependent: :destroy
-  has_one :bamboo_service, dependent: :destroy
-  has_one :teamcity_service, dependent: :destroy
-  has_one :pushover_service, dependent: :destroy
-  has_one :jira_service, dependent: :destroy
-  has_one :redmine_service, dependent: :destroy
-  has_one :custom_issue_tracker_service, dependent: :destroy
-  has_one :bugzilla_service, dependent: :destroy
-  has_one :gitlab_issue_tracker_service, dependent: :destroy, inverse_of: :project
-  has_one :external_wiki_service, dependent: :destroy
-  has_one :kubernetes_service, dependent: :destroy, inverse_of: :project
-  has_one :prometheus_service, dependent: :destroy, inverse_of: :project
-  has_one :mock_ci_service, dependent: :destroy
-  has_one :mock_deployment_service, dependent: :destroy
-  has_one :mock_monitoring_service, dependent: :destroy
-  has_one :microsoft_teams_service, dependent: :destroy
-
-  has_one  :forked_project_link,  dependent: :destroy, foreign_key: "forked_to_project_id"
+  has_one :campfire_service
+  has_one :drone_ci_service
+  has_one :emails_on_push_service
+  has_one :pipelines_email_service
+  has_one :irker_service
+  has_one :pivotaltracker_service
+  has_one :hipchat_service
+  has_one :flowdock_service
+  has_one :assembla_service
+  has_one :asana_service
+  has_one :gemnasium_service
+  has_one :mattermost_slash_commands_service
+  has_one :mattermost_service
+  has_one :slack_slash_commands_service
+  has_one :slack_service
+  has_one :buildkite_service
+  has_one :bamboo_service
+  has_one :teamcity_service
+  has_one :pushover_service
+  has_one :jira_service
+  has_one :redmine_service
+  has_one :custom_issue_tracker_service
+  has_one :bugzilla_service
+  has_one :gitlab_issue_tracker_service, inverse_of: :project
+  has_one :external_wiki_service
+  has_one :kubernetes_service, inverse_of: :project
+  has_one :prometheus_service, inverse_of: :project
+  has_one :mock_ci_service
+  has_one :mock_deployment_service
+  has_one :mock_monitoring_service
+  has_one :microsoft_teams_service
+
+  has_one  :forked_project_link,  foreign_key: "forked_to_project_id"
   has_one  :forked_from_project,  through:   :forked_project_link
 
   has_many :forked_project_links, foreign_key: "forked_from_project_id"
   has_many :forks,                through:     :forked_project_links, source: :forked_to_project
 
   # Merge Requests for target project should be removed with it
-  has_many :merge_requests,     dependent: :destroy, foreign_key: 'target_project_id'
-  has_many :issues,             dependent: :destroy
-  has_many :labels,             dependent: :destroy, class_name: 'ProjectLabel'
-  has_many :services,           dependent: :destroy
-  has_many :events,             dependent: :destroy
-  has_many :milestones,         dependent: :destroy
-  has_many :notes,              dependent: :destroy
-  has_many :snippets,           dependent: :destroy, class_name: 'ProjectSnippet'
-  has_many :hooks,              dependent: :destroy, class_name: 'ProjectHook'
-  has_many :protected_branches, dependent: :destroy
-  has_many :protected_tags,     dependent: :destroy
+  has_many :merge_requests, foreign_key: 'target_project_id'
+  has_many :issues
+  has_many :labels, class_name: 'ProjectLabel'
+  has_many :services
+  has_many :events
+  has_many :milestones
+  has_many :notes
+  has_many :snippets, class_name: 'ProjectSnippet'
+  has_many :hooks, class_name: 'ProjectHook'
+  has_many :protected_branches
+  has_many :protected_tags
 
   has_many :project_authorizations
   has_many :authorized_users, through: :project_authorizations, source: :user, class_name: 'User'
-  has_many :project_members, -> { where(requested_at: nil) }, dependent: :destroy, as: :source
+  has_many :project_members, -> { where(requested_at: nil) },
+    as: :source, dependent: :delete_all
+
   alias_method :members, :project_members
   has_many :users, through: :project_members
 
-  has_many :requesters, -> { where.not(requested_at: nil) }, dependent: :destroy, as: :source, class_name: 'ProjectMember'
+  has_many :requesters, -> { where.not(requested_at: nil) },
+    as: :source, class_name: 'ProjectMember', dependent: :delete_all
 
-  has_many :deploy_keys_projects, dependent: :destroy
+  has_many :deploy_keys_projects
   has_many :deploy_keys, through: :deploy_keys_projects
-  has_many :users_star_projects, dependent: :destroy
+  has_many :users_star_projects
   has_many :starrers, through: :users_star_projects, source: :user
-  has_many :releases, dependent: :destroy
+  has_many :releases
   has_many :lfs_objects_projects, dependent: :destroy
   has_many :lfs_objects, through: :lfs_objects_projects
-  has_many :project_group_links, dependent: :destroy
+  has_many :project_group_links
   has_many :invited_groups, through: :project_group_links, source: :group
-  has_many :pages_domains, dependent: :destroy
-  has_many :todos, dependent: :destroy
-  has_many :notification_settings, dependent: :destroy, as: :source
+  has_many :pages_domains
+  has_many :todos
+  has_many :notification_settings, as: :source, dependent: :delete_all
+
+  has_one :import_data, class_name: 'ProjectImportData'
+  has_one :project_feature
+  has_one :statistics, class_name: 'ProjectStatistics'
 
-  has_one :import_data, dependent: :delete, class_name: 'ProjectImportData'
-  has_one :project_feature, dependent: :destroy
-  has_one :statistics, class_name: 'ProjectStatistics', dependent: :delete
+  # Container repositories need to remove data from the container registry,
+  # which is not managed by the DB. Hence we're still using dependent: :destroy
+  # here.
   has_many :container_repositories, dependent: :destroy
 
-  has_many :commit_statuses, dependent: :destroy
-  has_many :pipelines, dependent: :destroy, class_name: 'Ci::Pipeline'
-  has_many :builds, class_name: 'Ci::Build' # the builds are created from the commit_statuses
-  has_many :runner_projects, dependent: :destroy, class_name: 'Ci::RunnerProject'
+  has_many :commit_statuses
+  has_many :pipelines, class_name: 'Ci::Pipeline'
+
+  # Ci::Build objects store data on the file system such as artifact files and
+  # build traces. Currently there's no efficient way of removing this data in
+  # bulk that doesn't involve loading the rows into memory. As a result we're
+  # still using `dependent: :destroy` here.
+  has_many :builds, class_name: 'Ci::Build', dependent: :destroy
+  has_many :runner_projects, class_name: 'Ci::RunnerProject'
   has_many :runners, through: :runner_projects, source: :runner, class_name: 'Ci::Runner'
   has_many :variables, class_name: 'Ci::Variable'
-  has_many :triggers, dependent: :destroy, class_name: 'Ci::Trigger'
-  has_many :environments, dependent: :destroy
-  has_many :deployments, dependent: :destroy
-  has_many :pipeline_schedules, dependent: :destroy, class_name: 'Ci::PipelineSchedule'
+  has_many :triggers, class_name: 'Ci::Trigger'
+  has_many :environments
+  has_many :deployments
+  has_many :pipeline_schedules, class_name: 'Ci::PipelineSchedule'
 
   has_many :sourced_pipelines, class_name: Ci::Sources::Pipeline, foreign_key: :source_project_id
 
@@ -1229,7 +1242,13 @@ class Project < ActiveRecord::Base
     File.join(pages_path, 'public')
   end
 
+  def remove_private_deploy_keys
+    deploy_keys.where(public: false).delete_all
+  end
+
   def remove_pages
+    ::Projects::UpdatePagesConfigurationService.new(self).execute
+
     # 1. We rename pages to temporary directory
     # 2. We wait 5 minutes, due to NFS caching
     # 3. We asynchronously remove pages with force

changelogs/unreleased/foreign-keys-for-project-model.yml

+---
+title: Speed up project removals by adding foreign keys with cascading deletes to various tables
+merge_request:
+author:

db/migrate/20170530130129_project_foreign_keys_with_cascading_deletes.rb

+# See http://doc.gitlab.com/ce/development/migration_style_guide.html
+# for more information on how to write migrations for GitLab.
+
+class ProjectForeignKeysWithCascadingDeletes < ActiveRecord::Migration
+  include Gitlab::Database::MigrationHelpers
+
+  DOWNTIME = false
+
+  CONCURRENCY = 4
+
+  disable_ddl_transaction!
+
+  # The tables/columns for which to remove orphans and add foreign keys. Order
+  # matters as some tables/columns should be processed before others.
+  TABLES = [
+    [:boards, :projects, :project_id],
+    [:lists, :labels, :label_id],
+    [:lists, :boards, :board_id],
+    [:services, :projects, :project_id],
+    [:forked_project_links, :projects, :forked_to_project_id],
+    [:merge_requests, :projects, :target_project_id],
+    [:labels, :projects, :project_id],
+    [:issues, :projects, :project_id],
+    [:events, :projects, :project_id],
+    [:milestones, :projects, :project_id],
+    [:notes, :projects, :project_id],
+    [:snippets, :projects, :project_id],
+    [:web_hooks, :projects, :project_id],
+    [:protected_branch_merge_access_levels, :protected_branches, :protected_branch_id],
+    [:protected_branch_push_access_levels, :protected_branches, :protected_branch_id],
+    [:protected_branches, :projects, :project_id],
+    [:protected_tags, :projects, :project_id],
+    [:deploy_keys_projects, :projects, :project_id],
+    [:users_star_projects, :projects, :project_id],
+    [:releases, :projects, :project_id],
+    [:project_group_links, :projects, :project_id],
+    [:pages_domains, :projects, :project_id],
+    [:todos, :projects, :project_id],
+    [:project_import_data, :projects, :project_id],
+    [:project_features, :projects, :project_id],
+    [:ci_builds, :projects, :project_id],
+    [:ci_pipelines, :projects, :project_id],
+    [:ci_runner_projects, :projects, :project_id],
+    [:ci_triggers, :projects, :project_id],
+    [:environments, :projects, :project_id],
+    [:deployments, :projects, :project_id]
+  ]
+
+  def up
+    # These existing foreign keys don't have an "ON DELETE CASCADE" clause.
+    remove_foreign_key_without_error(:boards, :project_id)
+    remove_foreign_key_without_error(:lists, :label_id)
+    remove_foreign_key_without_error(:lists, :board_id)
+    remove_foreign_key_without_error(:protected_branch_merge_access_levels,
+                                     :protected_branch_id)
+
+    remove_foreign_key_without_error(:protected_branch_push_access_levels,
+                                     :protected_branch_id)
+
+    remove_orphaned_rows
+    add_foreign_keys
+
+    # These columns are not indexed yet, meaning a cascading delete would take
+    # forever.
+    add_concurrent_index(:project_group_links, :project_id)
+    add_concurrent_index(:pages_domains, :project_id)
+  end
+
+  def down
+    TABLES.each do |(source, _, column)|
+      remove_foreign_key_without_error(source, column)
+    end
+
+    add_concurrent_foreign_key(:boards, :projects, column: :project_id)
+    add_concurrent_foreign_key(:lists, :labels, column: :label_id)
+    add_concurrent_foreign_key(:lists, :boards, column: :board_id)
+
+    add_concurrent_foreign_key(:protected_branch_merge_access_levels,
+                               :protected_branches,
+                               column: :protected_branch_id)
+
+    add_concurrent_foreign_key(:protected_branch_push_access_levels,
+                               :protected_branches,
+                               column: :protected_branch_id)
+
+    remove_index_without_error(:project_group_links, :project_id)
+    remove_index_without_error(:pages_domains, :project_id)
+  end
+
+  def add_foreign_keys
+    TABLES.each do |(source, target, column)|
+      add_concurrent_foreign_key(source, target, column: column)
+    end
+  end
+
+  # Removes orphans from various tables concurrently.
+  def remove_orphaned_rows
+    Gitlab::Database.with_connection_pool(CONCURRENCY) do |pool|
+      queues = queues_for_rows(TABLES)
+
+      threads = queues.map do |queue|
+        Thread.new do
+          pool.with_connection do |connection|
+            Thread.current[:foreign_key_connection] = connection
+
+            # Disables statement timeouts for the current connection. This is
+            # necessary as removing of orphaned data might otherwise exceed the
+            # statement timeout.
+            disable_statement_timeout
+
+            remove_orphans(*queue.pop) until queue.empty?
+
+            steal_from_queues(queues - [queue])
+          end
+        end
+      end
+
+      threads.each(&:join)
+    end
+  end
+
+  def steal_from_queues(queues)
+    loop do
+      stolen = false
+
+      queues.each do |queue|
+        # Stealing is racy so it's possible a pop might be called on an
+        # already-empty queue.
+        begin
+          remove_orphans(*queue.pop(true))
+          stolen = true
+        rescue ThreadError
+        end
+      end
+
+      break unless stolen
+    end
+  end
+
+  def remove_orphans(source, target, column)
+    quoted_source = quote_table_name(source)
+    quoted_target = quote_table_name(target)
+    quoted_column = quote_column_name(column)
+
+    execute <<-EOF.strip_heredoc
+    DELETE FROM #{quoted_source}
+    WHERE NOT EXISTS (
+      SELECT true
+      FROM #{quoted_target}
+      WHERE #{quoted_target}.id = #{quoted_source}.#{quoted_column}
+    )
+    AND #{quoted_source}.#{quoted_column} IS NOT NULL
+    EOF
+  end
+
+  def remove_foreign_key_without_error(table, column)
+    remove_foreign_key(table, column: column)
+  rescue ArgumentError
+  end
+
+  def remove_index_without_error(table, column)
+    remove_concurrent_index(table, column)
+  rescue ArgumentError
+  end
+
+  def connection
+    # Rails memoizes connection objects, but this causes them to be shared
+    # amongst threads; we don't want that.
+    Thread.current[:foreign_key_connection] || ActiveRecord::Base.connection
+  end
+
+  def queues_for_rows(rows)
+    queues = Array.new(CONCURRENCY) { Queue.new }
+    slice_size = rows.length / CONCURRENCY
+
+    # Divide all the tuples as evenly as possible amongst the queues.
+    rows.each_slice(slice_size).each_with_index do |slice, index|
+      bucket = index % CONCURRENCY
+
+      slice.each do |row|
+        queues[bucket] << row
+      end
+    end
+
+    queues
+  end
+end

db/migrate/20170622130029_correct_protected_branches_foreign_keys.rb

+# See http://doc.gitlab.com/ce/development/migration_style_guide.html
+# for more information on how to write migrations for GitLab.
+
+class CorrectProtectedBranchesForeignKeys < ActiveRecord::Migration
+  include Gitlab::Database::MigrationHelpers
+
+  # Set this constant to true if this migration requires downtime.
+  DOWNTIME = false
+
+  disable_ddl_transaction!
+
+  def up
+    remove_foreign_key_without_error(:protected_branch_push_access_levels,
+                                     column: :protected_branch_id)
+
+    execute <<-EOF
+    DELETE FROM protected_branch_push_access_levels
+    WHERE NOT EXISTS (
+      SELECT true
+      FROM protected_branches
+      WHERE protected_branch_push_access_levels.protected_branch_id = protected_branches.id
+    )
+    AND protected_branch_id IS NOT NULL
+    EOF
+
+    add_concurrent_foreign_key(:protected_branch_push_access_levels,
+                               :protected_branches,
+                               column: :protected_branch_id)
+  end
+
+  def down
+    # Previously there was a foreign key without a CASCADING DELETE, so we'll
+    # just leave the foreign key in place.
+  end
+
+  def remove_foreign_key_without_error(*args)
+    remove_foreign_key(*args)
+  rescue ArgumentError
+  end
+end

db/migrate/20170622132212_add_foreign_key_for_merge_request_diffs.rb

+# See http://doc.gitlab.com/ce/development/migration_style_guide.html
+# for more information on how to write migrations for GitLab.
+
+class AddForeignKeyForMergeRequestDiffs < ActiveRecord::Migration
+  include Gitlab::Database::MigrationHelpers
+
+  # Set this constant to true if this migration requires downtime.
+  DOWNTIME = false
+
+  disable_ddl_transaction!
+
+  def up
+    execute <<-EOF
+    DELETE FROM merge_request_diffs
+    WHERE NOT EXISTS (
+      SELECT true
+      FROM merge_requests
+      WHERE merge_requests.id = merge_request_diffs.merge_request_id
+    )
+    EOF
+
+    add_concurrent_foreign_key(:merge_request_diffs,
+                               :merge_requests,
+                               column: :merge_request_id)
+  end
+
+  def down
+    remove_foreign_key(:merge_request_diffs, column: :merge_request_id)
+  end
+end

db/schema.rb

@@ -1165,6 +1165,7 @@ ActiveRecord::Schema.define(version: 20170627211700) do
   end
 
   add_index "pages_domains", ["domain"], name: "index_pages_domains_on_domain", unique: true, using: :btree
+  add_index "pages_domains", ["project_id"], name: "index_pages_domains_on_project_id", using: :btree
 
   create_table "path_locks", force: :cascade do |t|
     t.string "path", null: false
@@ -1226,6 +1227,7 @@ ActiveRecord::Schema.define(version: 20170627211700) do
   end
 
   add_index "project_group_links", ["group_id"], name: "index_project_group_links_on_group_id", using: :btree
+  add_index "project_group_links", ["project_id"], name: "index_project_group_links_on_project_id", using: :btree
 
   create_table "project_import_data", force: :cascade do |t|
     t.integer "project_id"
@@ -1813,14 +1815,17 @@ ActiveRecord::Schema.define(version: 20170627211700) do
   add_index "web_hooks", ["type"], name: "index_web_hooks_on_type", using: :btree
 
   add_foreign_key "approver_groups", "namespaces", column: "group_id", on_delete: :cascade
-  add_foreign_key "boards", "projects"
+  add_foreign_key "boards", "projects", name: "fk_f15266b5f9", on_delete: :cascade
   add_foreign_key "chat_teams", "namespaces", on_delete: :cascade
   add_foreign_key "ci_builds", "ci_pipelines", column: "auto_canceled_by_id", name: "fk_a2141b1522", on_delete: :nullify
   add_foreign_key "ci_builds", "ci_stages", column: "stage_id", name: "fk_3a9eaa254d", on_delete: :cascade
+  add_foreign_key "ci_builds", "projects", name: "fk_befce0568a", on_delete: :cascade
   add_foreign_key "ci_pipeline_schedules", "projects", name: "fk_8ead60fcc4", on_delete: :cascade
   add_foreign_key "ci_pipeline_schedules", "users", column: "owner_id", name: "fk_9ea99f58d2", on_delete: :nullify
   add_foreign_key "ci_pipelines", "ci_pipeline_schedules", column: "pipeline_schedule_id", name: "fk_3d34ab2e06", on_delete: :nullify
   add_foreign_key "ci_pipelines", "ci_pipelines", column: "auto_canceled_by_id", name: "fk_262d4c2d19", on_delete: :nullify
+  add_foreign_key "ci_pipelines", "projects", name: "fk_86635dbd80", on_delete: :cascade
+  add_foreign_key "ci_runner_projects", "projects", name: "fk_4478a6f1e4", on_delete: :cascade
   add_foreign_key "ci_sources_pipelines", "ci_builds", column: "source_job_id", name: "fk_3f0c88d7dc", on_delete: :cascade
   add_foreign_key "ci_sources_pipelines", "ci_pipelines", column: "pipeline_id", name: "fk_b8c0fac459", on_delete: :cascade
   add_foreign_key "ci_sources_pipelines", "ci_pipelines", column: "source_pipeline_id", name: "fk_3a3e3cb83a", on_delete: :cascade
@@ -1829,9 +1834,15 @@ ActiveRecord::Schema.define(version: 20170627211700) do
   add_foreign_key "ci_stages", "ci_pipelines", column: "pipeline_id", name: "fk_fb57e6cc56", on_delete: :cascade
   add_foreign_key "ci_stages", "projects", name: "fk_2360681d1d", on_delete: :cascade
   add_foreign_key "ci_trigger_requests", "ci_triggers", column: "trigger_id", name: "fk_b8ec8b7245", on_delete: :cascade
+  add_foreign_key "ci_triggers", "projects", name: "fk_e3e63f966e", on_delete: :cascade
   add_foreign_key "ci_triggers", "users", column: "owner_id", name: "fk_e8e10d1964", on_delete: :cascade
   add_foreign_key "ci_variables", "projects", name: "fk_ada5eb64b3", on_delete: :cascade
   add_foreign_key "container_repositories", "projects"
+  add_foreign_key "deploy_keys_projects", "projects", name: "fk_58a901ca7e", on_delete: :cascade
+  add_foreign_key "deployments", "projects", name: "fk_b9a3851b82", on_delete: :cascade
+  add_foreign_key "environments", "projects", name: "fk_d1c8c1da6a", on_delete: :cascade
+  add_foreign_key "events", "projects", name: "fk_0434b48643", on_delete: :cascade
+  add_foreign_key "forked_project_links", "projects", column: "forked_to_project_id", name: "fk_434510edb0", on_delete: :cascade
   add_foreign_key "geo_event_log", "geo_repository_deleted_events", column: "repository_deleted_event_id", name: "fk_c4b1c1f66e", on_delete: :cascade
   add_foreign_key "geo_event_log", "geo_repository_updated_events", column: "repository_updated_event_id", on_delete: :cascade
   add_foreign_key "geo_repository_updated_events", "projects", on_delete: :cascade
@@ -1840,40 +1851,58 @@ ActiveRecord::Schema.define(version: 20170627211700) do
   add_foreign_key "issue_links", "issues", column: "source_id", name: "fk_c900194ff2", on_delete: :cascade
   add_foreign_key "issue_links", "issues", column: "target_id", name: "fk_e71bb44f1f", on_delete: :cascade
   add_foreign_key "issue_metrics", "issues", on_delete: :cascade
+  add_foreign_key "issues", "projects", name: "fk_899c8f3231", on_delete: :cascade
   add_foreign_key "label_priorities", "labels", on_delete: :cascade
   add_foreign_key "label_priorities", "projects", on_delete: :cascade
   add_foreign_key "labels", "namespaces", column: "group_id", on_delete: :cascade
-  add_foreign_key "lists", "boards"
-  add_foreign_key "lists", "labels"
+  add_foreign_key "labels", "projects", name: "fk_7de4989a69", on_delete: :cascade
+  add_foreign_key "lists", "boards", name: "fk_0d3f677137", on_delete: :cascade
+  add_foreign_key "lists", "labels", name: "fk_7a5553d60f", on_delete: :cascade
   add_foreign_key "merge_request_diff_files", "merge_request_diffs", on_delete: :cascade
+  add_foreign_key "merge_request_diffs", "merge_requests", name: "fk_8483f3258f", on_delete: :cascade
   add_foreign_key "merge_request_metrics", "ci_pipelines", column: "pipeline_id", on_delete: :cascade
   add_foreign_key "merge_request_metrics", "merge_requests", on_delete: :cascade
+  add_foreign_key "merge_requests", "projects", column: "target_project_id", name: "fk_a6963e8447", on_delete: :cascade
   add_foreign_key "merge_requests_closing_issues", "issues", on_delete: :cascade
   add_foreign_key "merge_requests_closing_issues", "merge_requests", on_delete: :cascade
+  add_foreign_key "milestones", "projects", name: "fk_9bd0a0c791", on_delete: :cascade
   add_foreign_key "namespace_statistics", "namespaces", on_delete: :cascade
+  add_foreign_key "notes", "projects", name: "fk_99e097b079", on_delete: :cascade
   add_foreign_key "oauth_openid_requests", "oauth_access_grants", column: "access_grant_id", name: "fk_oauth_openid_requests_oauth_access_grants_access_grant_id"
+  add_foreign_key "pages_domains", "projects", name: "fk_ea2f6dfc6f", on_delete: :cascade
   add_foreign_key "path_locks", "projects"
   add_foreign_key "path_locks", "users"
   add_foreign_key "personal_access_tokens", "users"
   add_foreign_key "project_authorizations", "projects", on_delete: :cascade
   add_foreign_key "project_authorizations", "users", on_delete: :cascade
+  add_foreign_key "project_features", "projects", name: "fk_18513d9b92", on_delete: :cascade
+  add_foreign_key "project_group_links", "projects", name: "fk_daa8cee94c", on_delete: :cascade
+  add_foreign_key "project_import_data", "projects", name: "fk_ffb9ee3a10", on_delete: :cascade
   add_foreign_key "project_mirror_data", "projects", name: "fk_d1aad367d7", on_delete: :cascade
   add_foreign_key "project_statistics", "projects", on_delete: :cascade
   add_foreign_key "protected_branch_merge_access_levels", "namespaces", column: "group_id"
-  add_foreign_key "protected_branch_merge_access_levels", "protected_branches"
+  add_foreign_key "protected_branch_merge_access_levels", "protected_branches", name: "fk_8a3072ccb3", on_delete: :cascade
   add_foreign_key "protected_branch_merge_access_levels", "users"
   add_foreign_key "protected_branch_push_access_levels", "namespaces", column: "group_id"
-  add_foreign_key "protected_branch_push_access_levels", "protected_branches"
+  add_foreign_key "protected_branch_push_access_levels", "protected_branches", name: "fk_9ffc86a3d9", on_delete: :cascade
   add_foreign_key "protected_branch_push_access_levels", "users"
+  add_foreign_key "protected_branches", "projects", name: "fk_7a9c6d93e7", on_delete: :cascade
   add_foreign_key "protected_tag_create_access_levels", "namespaces", column: "group_id"
   add_foreign_key "protected_tag_create_access_levels", "protected_tags"
   add_foreign_key "protected_tag_create_access_levels", "users"
+  add_foreign_key "protected_tags", "projects", name: "fk_8e4af87648", on_delete: :cascade
+  add_foreign_key "releases", "projects", name: "fk_47fe2a0596", on_delete: :cascade
   add_foreign_key "remote_mirrors", "projects"
+  add_foreign_key "services", "projects", name: "fk_71cce407f9", on_delete: :cascade
+  add_foreign_key "snippets", "projects", name: "fk_be41fd4bb7", on_delete: :cascade
   add_foreign_key "subscriptions", "projects", on_delete: :cascade
   add_foreign_key "system_note_metadata", "notes", name: "fk_d83a918cb1", on_delete: :cascade
   add_foreign_key "timelogs", "issues", name: "fk_timelogs_issues_issue_id", on_delete: :cascade
   add_foreign_key "timelogs", "merge_requests", name: "fk_timelogs_merge_requests_merge_request_id", on_delete: :cascade
+  add_foreign_key "todos", "projects", name: "fk_45054f9c45", on_delete: :cascade
   add_foreign_key "trending_projects", "projects", on_delete: :cascade
   add_foreign_key "u2f_registrations", "users"
+  add_foreign_key "users_star_projects", "projects", name: "fk_22cd27ddfc", on_delete: :cascade
   add_foreign_key "web_hook_logs", "web_hooks", on_delete: :cascade
+  add_foreign_key "web_hooks", "projects", name: "fk_0c8ca6d9d1", on_delete: :cascade
 end

spec/models/concerns/issuable_spec.rb

@@ -155,7 +155,7 @@ describe Issuable do
   end
 
   describe "#sort" do
-    let(:project) { build_stubbed(:empty_project) }
+    let(:project) { create(:empty_project) }
 
     context "by weight" do
       let!(:issue)  { create(:issue, project: project) }

spec/models/forked_project_link_spec.rb

@@ -5,6 +5,15 @@ describe ForkedProjectLink, "add link on fork" do
   let(:user) { create(:user) }
   let(:namespace) { user.namespace }
 
+  def fork_project(from_project, user)
+    shell = double('gitlab_shell', fork_repository: true)
+
+    service = Projects::ForkService.new(from_project, user)
+    allow(service).to receive(:gitlab_shell).and_return(shell)
+
+    service.execute
+  end
+
   before do
     create(:project_member, :reporter, user: user, project: project_from)
     @project_to = fork_project(project_from, user)
@@ -20,7 +29,7 @@ describe ForkedProjectLink, "add link on fork" do
 end
 
 describe '#forked?' do
-  let(:forked_project_link) { build(:forked_project_link) }
+  let(:forked_project_link) { create(:forked_project_link) }
   let(:project_from) { create(:project, :repository) }
   let(:project_to) { create(:project, forked_project_link: forked_project_link) }
 
@@ -39,16 +48,8 @@ describe '#forked?' do
   end
 
   it "project_to.destroy destroys fork_link" do
-    expect(forked_project_link).to receive(:destroy)
     project_to.destroy
-  end
-end
-
-def fork_project(from_project, user)
-  shell = double('gitlab_shell', fork_repository: true)
 
-  service = Projects::ForkService.new(from_project, user)
-  allow(service).to receive(:gitlab_shell).and_return(shell)
-
-  service.execute
+    expect(ForkedProjectLink.exists?(id: forked_project_link.id)).to eq(false)
+  end
 end

spec/models/merge_request_spec.rb

@@ -10,8 +10,8 @@ describe MergeRequest, models: true do
     it { is_expected.to belong_to(:source_project).class_name('Project') }
     it { is_expected.to belong_to(:merge_user).class_name("User") }
     it { is_expected.to belong_to(:assignee) }
-    it { is_expected.to have_many(:merge_request_diffs).dependent(:destroy) }
-    it { is_expected.to have_many(:approver_groups).dependent(:destroy) }
+    it { is_expected.to have_many(:merge_request_diffs) }
+    it { is_expected.to have_many(:approver_groups).dependent(:delete_all) }
   end
 
   describe 'modules' do

spec/models/project_spec.rb

@@ -7,50 +7,50 @@ describe Project, models: true do
     it { is_expected.to belong_to(:creator).class_name('User') }
     it { is_expected.to have_many(:users) }
     it { is_expected.to have_many(:services) }
-    it { is_expected.to have_many(:events).dependent(:destroy) }
-    it { is_expected.to have_many(:merge_requests).dependent(:destroy) }
-    it { is_expected.to have_many(:issues).dependent(:destroy) }
-    it { is_expected.to have_many(:milestones).dependent(:destroy) }
-    it { is_expected.to have_many(:project_members).dependent(:destroy) }
+    it { is_expected.to have_many(:events) }
+    it { is_expected.to have_many(:merge_requests) }
+    it { is_expected.to have_many(:issues) }
+    it { is_expected.to have_many(:milestones) }
+    it { is_expected.to have_many(:project_members).dependent(:delete_all) }
     it { is_expected.to have_many(:users).through(:project_members) }
-    it { is_expected.to have_many(:requesters).dependent(:destroy) }
-    it { is_expected.to have_many(:notes).dependent(:destroy) }
-    it { is_expected.to have_many(:snippets).class_name('ProjectSnippet').dependent(:destroy) }
-    it { is_expected.to have_many(:deploy_keys_projects).dependent(:destroy) }
+    it { is_expected.to have_many(:requesters).dependent(:delete_all) }
+    it { is_expected.to have_many(:notes) }
+    it { is_expected.to have_many(:snippets).class_name('ProjectSnippet') }
+    it { is_expected.to have_many(:deploy_keys_projects) }
     it { is_expected.to have_many(:deploy_keys) }
-    it { is_expected.to have_many(:hooks).dependent(:destroy) }
-    it { is_expected.to have_many(:protected_branches).dependent(:destroy) }
-    it { is_expected.to have_one(:forked_project_link).dependent(:destroy) }
-    it { is_expected.to have_one(:slack_service).dependent(:destroy) }
-    it { is_expected.to have_one(:microsoft_teams_service).dependent(:destroy) }
-    it { is_expected.to have_one(:mattermost_service).dependent(:destroy) }
-    it { is_expected.to have_one(:pushover_service).dependent(:destroy) }
-    it { is_expected.to have_one(:asana_service).dependent(:destroy) }
-    it { is_expected.to have_many(:boards).dependent(:destroy) }
-    it { is_expected.to have_one(:campfire_service).dependent(:destroy) }
-    it { is_expected.to have_one(:drone_ci_service).dependent(:destroy) }
-    it { is_expected.to have_one(:emails_on_push_service).dependent(:destroy) }
-    it { is_expected.to have_one(:pipelines_email_service).dependent(:destroy) }
-    it { is_expected.to have_one(:irker_service).dependent(:destroy) }
-    it { is_expected.to have_one(:pivotaltracker_service).dependent(:destroy) }
-    it { is_expected.to have_one(:hipchat_service).dependent(:destroy) }
-    it { is_expected.to have_one(:flowdock_service).dependent(:destroy) }
-    it { is_expected.to have_one(:assembla_service).dependent(:destroy) }
-    it { is_expected.to have_one(:slack_slash_commands_service).dependent(:destroy) }
-    it { is_expected.to have_one(:mattermost_slash_commands_service).dependent(:destroy) }
-    it { is_expected.to have_one(:gemnasium_service).dependent(:destroy) }
-    it { is_expected.to have_one(:buildkite_service).dependent(:destroy) }
-    it { is_expected.to have_one(:bamboo_service).dependent(:destroy) }
-    it { is_expected.to have_one(:teamcity_service).dependent(:destroy) }
-    it { is_expected.to have_one(:jira_service).dependent(:destroy) }
-    it { is_expected.to have_one(:redmine_service).dependent(:destroy) }
-    it { is_expected.to have_one(:custom_issue_tracker_service).dependent(:destroy) }
-    it { is_expected.to have_one(:bugzilla_service).dependent(:destroy) }
-    it { is_expected.to have_one(:gitlab_issue_tracker_service).dependent(:destroy) }
-    it { is_expected.to have_one(:external_wiki_service).dependent(:destroy) }
-    it { is_expected.to have_one(:project_feature).dependent(:destroy) }
-    it { is_expected.to have_one(:statistics).class_name('ProjectStatistics').dependent(:delete) }
-    it { is_expected.to have_one(:import_data).class_name('ProjectImportData').dependent(:delete) }
+    it { is_expected.to have_many(:hooks) }
+    it { is_expected.to have_many(:protected_branches) }
+    it { is_expected.to have_one(:forked_project_link) }
+    it { is_expected.to have_one(:slack_service) }
+    it { is_expected.to have_one(:microsoft_teams_service) }
+    it { is_expected.to have_one(:mattermost_service) }
+    it { is_expected.to have_one(:pushover_service) }
+    it { is_expected.to have_one(:asana_service) }
+    it { is_expected.to have_many(:boards) }
+    it { is_expected.to have_one(:campfire_service) }
+    it { is_expected.to have_one(:drone_ci_service) }
+    it { is_expected.to have_one(:emails_on_push_service) }
+    it { is_expected.to have_one(:pipelines_email_service) }
+    it { is_expected.to have_one(:irker_service) }
+    it { is_expected.to have_one(:pivotaltracker_service) }
+    it { is_expected.to have_one(:hipchat_service) }
+    it { is_expected.to have_one(:flowdock_service) }
+    it { is_expected.to have_one(:assembla_service) }
+    it { is_expected.to have_one(:slack_slash_commands_service) }
+    it { is_expected.to have_one(:mattermost_slash_commands_service) }
+    it { is_expected.to have_one(:gemnasium_service) }
+    it { is_expected.to have_one(:buildkite_service) }
+    it { is_expected.to have_one(:bamboo_service) }
+    it { is_expected.to have_one(:teamcity_service) }
+    it { is_expected.to have_one(:jira_service) }
+    it { is_expected.to have_one(:redmine_service) }
+    it { is_expected.to have_one(:custom_issue_tracker_service) }
+    it { is_expected.to have_one(:bugzilla_service) }
+    it { is_expected.to have_one(:gitlab_issue_tracker_service) }
+    it { is_expected.to have_one(:external_wiki_service) }
+    it { is_expected.to have_one(:project_feature) }
+    it { is_expected.to have_one(:statistics).class_name('ProjectStatistics') }
+    it { is_expected.to have_one(:import_data).class_name('ProjectImportData') }
     it { is_expected.to have_one(:mirror_data).class_name('ProjectMirrorData').dependent(:delete) }
     it { is_expected.to have_one(:last_event).class_name('Event') }
     it { is_expected.to have_one(:forked_from_project).through(:forked_project_link) }
@@ -64,19 +64,19 @@ describe Project, models: true do
     it { is_expected.to have_many(:triggers) }
     it { is_expected.to have_many(:pages_domains) }
     it { is_expected.to have_many(:path_locks).dependent(:destroy) }
-    it { is_expected.to have_many(:labels).class_name('ProjectLabel').dependent(:destroy) }
-    it { is_expected.to have_many(:users_star_projects).dependent(:destroy) }
-    it { is_expected.to have_many(:environments).dependent(:destroy) }
-    it { is_expected.to have_many(:deployments).dependent(:destroy) }
-    it { is_expected.to have_many(:todos).dependent(:destroy) }
-    it { is_expected.to have_many(:releases).dependent(:destroy) }
-    it { is_expected.to have_many(:lfs_objects_projects).dependent(:destroy) }
-    it { is_expected.to have_many(:project_group_links).dependent(:destroy) }
-    it { is_expected.to have_many(:notification_settings).dependent(:destroy) }
+    it { is_expected.to have_many(:labels).class_name('ProjectLabel') }
+    it { is_expected.to have_many(:users_star_projects) }
+    it { is_expected.to have_many(:environments) }
+    it { is_expected.to have_many(:deployments) }
+    it { is_expected.to have_many(:todos) }
+    it { is_expected.to have_many(:releases) }
+    it { is_expected.to have_many(:lfs_objects_projects) }
+    it { is_expected.to have_many(:project_group_links) }
+    it { is_expected.to have_many(:notification_settings).dependent(:delete_all) }
     it { is_expected.to have_many(:forks).through(:forked_project_links) }
     it { is_expected.to have_many(:approver_groups).dependent(:destroy) }
     it { is_expected.to have_many(:uploads).dependent(:destroy) }
-    it { is_expected.to have_many(:pipeline_schedules).dependent(:destroy) }
+    it { is_expected.to have_many(:pipeline_schedules) }
     it { is_expected.to have_many(:sourced_pipelines) }
     it { is_expected.to have_many(:source_pipelines) }
 
@@ -2660,4 +2660,21 @@ describe Project, models: true do
       end
     end
   end
+
+  describe '#remove_private_deploy_keys' do
+    it 'removes the private deploy keys of a project' do
+      project = create(:empty_project)
+
+      private_key = create(:deploy_key, public: false)
+      public_key = create(:deploy_key, public: true)
+
+      create(:deploy_keys_project, deploy_key: private_key, project: project)
+      create(:deploy_keys_project, deploy_key: public_key, project: project)
+
+      project.remove_private_deploy_keys
+
+      expect(project.deploy_keys.where(public: false).any?).to eq(false)
+      expect(project.deploy_keys.where(public: true).any?).to eq(true)
+    end
+  end
 end

spec/presenters/ci/build_presenter_spec.rb

@@ -85,7 +85,7 @@ describe Ci::BuildPresenter do
 
   describe 'quack like a Ci::Build permission-wise' do
     context 'user is not allowed' do
-      let(:project) { build_stubbed(:empty_project, public_builds: false) }
+      let(:project) { create(:empty_project, public_builds: false) }
 
       it 'returns false' do
         expect(presenter.can?(nil, :read_build)).to be_falsy
@@ -93,7 +93,7 @@ describe Ci::BuildPresenter do
     end
 
     context 'user is allowed' do
-      let(:project) { build_stubbed(:empty_project, :public) }
+      let(:project) { create(:empty_project, :public) }
 
       it 'returns true' do
         expect(presenter.can?(nil, :read_build)).to be_truthy

spec/workers/expire_build_instance_artifacts_worker_spec.rb

@@ -30,20 +30,6 @@ describe ExpireBuildInstanceArtifactsWorker do
           expect(build.reload.artifacts_file_identifier).to be_nil
         end
       end
-
-      context 'when associated project was removed' do
-        let(:build) do
-          create(:ci_build, :artifacts, artifacts_expiry) do |build|
-            build.project.pending_delete = true
-          end
-        end
-
-        it 'does not remove artifacts' do
-          expect do
-            build.reload.artifacts_file
-          end.not_to raise_error
-        end
-      end
     end
 
     context 'with not yet expired artifacts' do