Merge branch 'bvl-fork-networks-for-deleted-projects' into 'master'

Create fork networks for deleted source projects

Closes #40072

See merge request gitlab-org/gitlab-ce!15595

changelogs/unreleased/bvl-fork-networks-for-deleted-projects.yml

+---
+title: Create a fork network for forks with a deleted source
+merge_request: 15595
+author:
+type: fixed

db/post_migrate/20171124150326_reschedule_fork_network_creation.rb

+class RescheduleForkNetworkCreation < ActiveRecord::Migration
+  include Gitlab::Database::MigrationHelpers
+
+  DOWNTIME = false
+
+  MIGRATION = 'PopulateForkNetworksRange'.freeze
+  BATCH_SIZE = 100
+  DELAY_INTERVAL = 15.seconds
+
+  disable_ddl_transaction!
+
+  class ForkedProjectLink < ActiveRecord::Base
+    include EachBatch
+
+    self.table_name = 'forked_project_links'
+  end
+
+  def up
+    say 'Populating the `fork_networks` based on existing `forked_project_links`'
+
+    queue_background_migration_jobs_by_range_at_intervals(ForkedProjectLink, MIGRATION, DELAY_INTERVAL, batch_size: BATCH_SIZE)
+  end
+
+  def down
+    # nothing
+  end
+end

db/schema.rb

@@ -11,7 +11,7 @@
 #
 # It's strongly recommended that you check this file into your version control system.
 
-ActiveRecord::Schema.define(version: 20171124132536) do
+ActiveRecord::Schema.define(version: 20171124150326) do
 
   # These are extensions that must be enabled in order to support this database
   enable_extension "plpgsql"

lib/gitlab/background_migration/populate_fork_networks_range.rb

 # frozen_string_literal: true
-# rubocop:disable Metrics/MethodLength
-# rubocop:disable Metrics/LineLength
-# rubocop:disable Style/Documentation
 
 module Gitlab
   module BackgroundMigration
+    # This background migration is going to create all `fork_networks` and
+    # the `fork_network_members` for the roots of fork networks based on the
+    # existing `forked_project_links`.
+    #
+    # When the source of a fork is deleted, we will create the fork with the
+    # target project as the root. This way, when there are forks of the target
+    # project, they will be joined into the same fork network.
+    #
+    # When the `fork_networks` and memberships for the root projects are created
+    # the `CreateForkNetworkMembershipsRange` migration is scheduled. This
+    # migration will create the memberships for all remaining forks-of-forks
     class PopulateForkNetworksRange
       def perform(start_id, end_id)
-        log("Creating fork networks for forked project links: #{start_id} - #{end_id}")
+        create_fork_networks_for_existing_projects(start_id, end_id)
+        create_fork_networks_for_missing_projects(start_id, end_id)
+        create_fork_networks_memberships_for_root_projects(start_id, end_id)
 
+        delay = BackgroundMigration::CreateForkNetworkMembershipsRange::RESCHEDULE_DELAY # rubocop:disable Metrics/LineLength
+        BackgroundMigrationWorker.perform_in(
+          delay, "CreateForkNetworkMembershipsRange", [start_id, end_id]
+        )
+      end
+
+      def create_fork_networks_for_existing_projects(start_id, end_id)
+        log("Creating fork networks: #{start_id} - #{end_id}")
         ActiveRecord::Base.connection.execute <<~INSERT_NETWORKS
         INSERT INTO fork_networks (root_project_id)
           SELECT DISTINCT forked_project_links.forked_from_project_id
 
           FROM forked_project_links
 
+          -- Exclude the forks that are not the first level fork of a project
           WHERE NOT EXISTS (
             SELECT true
             FROM forked_project_links inner_links
             WHERE inner_links.forked_to_project_id = forked_project_links.forked_from_project_id
           )
+
+          /* Exclude the ones that are already created, in case the fork network
+             was already created for another fork of the project.
+          */
           AND NOT EXISTS (
             SELECT true
             FROM fork_networks
             WHERE forked_project_links.forked_from_project_id = fork_networks.root_project_id
           )
+
+          -- Only create a fork network for a root project that still exists
           AND EXISTS (
             SELECT true
             FROM projects
@@ -32,7 +57,45 @@ module Gitlab
           )
           AND forked_project_links.id BETWEEN #{start_id} AND #{end_id}
         INSERT_NETWORKS
+      end
+
+      def create_fork_networks_for_missing_projects(start_id, end_id)
+        log("Creating fork networks with missing root: #{start_id} - #{end_id}")
+        ActiveRecord::Base.connection.execute <<~INSERT_NETWORKS
+        INSERT INTO fork_networks (root_project_id)
+          SELECT DISTINCT forked_project_links.forked_to_project_id
+
+          FROM forked_project_links
+
+          -- Exclude forks that are not the root forks
+          WHERE NOT EXISTS (
+            SELECT true
+            FROM forked_project_links inner_links
+            WHERE inner_links.forked_to_project_id = forked_project_links.forked_from_project_id
+          )
+
+          /* Exclude the ones that are already created, in case this migration is
+             re-run
+          */
+          AND NOT EXISTS (
+            SELECT true
+            FROM fork_networks
+            WHERE forked_project_links.forked_to_project_id = fork_networks.root_project_id
+          )
+
+          /* Exclude projects for which the project still exists, those are
+             Processed in the previous step of this migration
+          */
+          AND NOT EXISTS (
+            SELECT true
+            FROM projects
+            WHERE projects.id = forked_project_links.forked_from_project_id
+          )
+          AND forked_project_links.id BETWEEN #{start_id} AND #{end_id}
+        INSERT_NETWORKS
+      end
 
+      def create_fork_networks_memberships_for_root_projects(start_id, end_id)
         log("Creating memberships for root projects: #{start_id} - #{end_id}")
 
         ActiveRecord::Base.connection.execute <<~INSERT_ROOT
@@ -41,8 +104,12 @@ module Gitlab
 
           FROM fork_networks
 
+          /* Joining both on forked_from- and forked_to- so we could create the
+             memberships for forks for which the source was deleted
+          */
           INNER JOIN forked_project_links
               ON forked_project_links.forked_from_project_id = fork_networks.root_project_id
+              OR forked_project_links.forked_to_project_id = fork_networks.root_project_id
 
           WHERE NOT EXISTS (
             SELECT true
@@ -51,9 +118,6 @@ module Gitlab
           )
           AND forked_project_links.id BETWEEN #{start_id} AND #{end_id}
         INSERT_ROOT
-
-        delay = BackgroundMigration::CreateForkNetworkMembershipsRange::RESCHEDULE_DELAY
-        BackgroundMigrationWorker.perform_in(delay, "CreateForkNetworkMembershipsRange", [start_id, end_id])
       end
 
       def log(message)

spec/lib/gitlab/background_migration/populate_fork_networks_range_spec.rb

@@ -62,12 +62,15 @@ describe Gitlab::BackgroundMigration::PopulateForkNetworksRange, :migration, sch
     expect(base2_membership).not_to be_nil
   end
 
-  it 'skips links that had their source project deleted' do
-    forked_project_links.create(id: 6, forked_from_project_id: 99999, forked_to_project_id: create(:project).id)
+  it 'creates a fork network for the fork of which the source was deleted' do
+    fork = create(:project)
+    forked_project_links.create(id: 6, forked_from_project_id: 99999, forked_to_project_id: fork.id)
 
     migration.perform(5, 8)
 
     expect(fork_networks.find_by(root_project_id: 99999)).to be_nil
+    expect(fork_networks.find_by(root_project_id: fork.id)).not_to be_nil
+    expect(fork_network_members.find_by(project_id: fork.id)).not_to be_nil
   end
 
   it 'schedules a job for inserting memberships for forks-of-forks' do