Create BulkImport::Entity model for import data

We want to start importing Group and Project data directly from
another GitLab instance, via the API. To do this, we need somewhere
to store a mapping between the source entity and the destination
entity.

Create the Entity model which is associated with a BulkImport,
and used to store the data required to link the entity (project or
group) on the source instance and the destination instance.

This model introduces:

1. `bulk_import_id` -> the bulk import that this data belongs to.
2. `parent_id` -> the parent ImportEntity that this one should be
   imported into
3. `source_type` -> what kind of entity this is (group or project)
4. `source_full_path` -> path to access the entity on the source
5. `destination_name` -> what to call the entity on the when it's
   created
6. `destination_namespace` -> where to store the entity on the
   destination.
7. `status` -> for use with the state machine to determine the import
   status of this entity
8. `jid` -> the job ID of the sidekiq job that will/has processed this
   import.

This is a component of the Group Migration MVC epic:
https://gitlab.com/groups/gitlab-org/-/epics/4374

MR: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/42978
Issue: https://gitlab.com/gitlab-org/gitlab/-/issues/250280

app/models/bulk_import.rb

@@ -4,6 +4,7 @@ class BulkImport < ApplicationRecord
   belongs_to :user, optional: false
 
   has_one :configuration, class_name: 'BulkImports::Configuration'
+  has_many :entities, class_name: 'BulkImports::Entity'
 
   validates :source_type, :status, presence: true
 

app/models/bulk_imports/entity.rb

+# frozen_string_literal: true
+
+class BulkImports::Entity < ApplicationRecord
+  self.table_name = 'bulk_import_entities'
+
+  belongs_to :bulk_import, optional: false
+  belongs_to :parent, class_name: 'BulkImports::Entity', optional: true
+
+  belongs_to :project, optional: true
+  belongs_to :group, foreign_key: :namespace_id, optional: true
+
+  validates :project, absence: true, if: :group
+  validates :group, absence: true, if: :project
+  validates :source_type, :source_full_path, :destination_name,
+            :destination_namespace, presence: true
+
+  validate :validate_parent_is_a_group, if: :parent
+  validate :validate_imported_entity_type
+
+  enum source_type: { group_entity: 0, project_entity: 1 }
+
+  state_machine :status, initial: :created do
+    state :created, value: 0
+  end
+
+  private
+
+  def validate_parent_is_a_group
+    unless parent.group_entity?
+      errors.add(:parent, s_('BulkImport|must be a group'))
+    end
+  end
+
+  def validate_imported_entity_type
+    if group.present? && project_entity?
+      errors.add(:group, s_('BulkImport|expected an associated Project but has an associated Group'))
+    end
+
+    if project.present? && group_entity?
+      errors.add(:project, s_('BulkImport|expected an associated Group but has an associated Project'))
+    end
+  end
+end

db/migrate/20200925114522_create_bulk_import_entities.rb

+# frozen_string_literal: true
+
+class CreateBulkImportEntities < ActiveRecord::Migration[6.0]
+  include Gitlab::Database::MigrationHelpers
+
+  DOWNTIME = false
+
+  disable_ddl_transaction!
+
+  def up
+    create_table :bulk_import_entities, if_not_exists: true do |t|
+      t.bigint :bulk_import_id, index: true, null: false
+      t.bigint :parent_id, index: true
+      t.bigint :namespace_id, index: true
+      t.bigint :project_id, index: true
+
+      t.integer :source_type, null: false, limit: 2
+      t.text :source_full_path, null: false
+
+      t.text :destination_name, null: false
+      t.text :destination_namespace, null: false
+
+      t.integer :status, null: false, limit: 2
+      t.text :jid
+
+      t.timestamps_with_timezone
+    end
+
+    add_text_limit(:bulk_import_entities, :source_full_path, 255)
+    add_text_limit(:bulk_import_entities, :destination_name, 255)
+    add_text_limit(:bulk_import_entities, :destination_namespace, 255)
+    add_text_limit(:bulk_import_entities, :jid, 255)
+  end
+
+  def down
+    drop_table :bulk_import_entities
+  end
+end

db/migrate/20200925153423_add_bulk_import_foreign_key_to_bulk_import_entities.rb

+# frozen_string_literal: true
+
+class AddBulkImportForeignKeyToBulkImportEntities < ActiveRecord::Migration[6.0]
+  include Gitlab::Database::MigrationHelpers
+
+  DOWNTIME = false
+
+  disable_ddl_transaction!
+
+  def up
+    add_concurrent_foreign_key :bulk_import_entities, :bulk_imports, column: :bulk_import_id, on_delete: :cascade
+  end
+
+  def down
+    remove_foreign_key :bulk_import_entities, column: :bulk_import_id
+  end
+end

db/migrate/20200925193815_add_parent_foreign_key_to_bulk_import_entities.rb

+# frozen_string_literal: true
+
+class AddParentForeignKeyToBulkImportEntities < ActiveRecord::Migration[6.0]
+  include Gitlab::Database::MigrationHelpers
+
+  DOWNTIME = false
+
+  disable_ddl_transaction!
+
+  def up
+    add_concurrent_foreign_key :bulk_import_entities, :bulk_import_entities, column: :parent_id, on_delete: :cascade
+  end
+
+  def down
+    remove_foreign_key :bulk_import_entities, column: :parent_id
+  end
+end

db/migrate/20200925193906_add_namespace_foreign_key_to_bulk_import_entities.rb

+# frozen_string_literal: true
+
+class AddNamespaceForeignKeyToBulkImportEntities < ActiveRecord::Migration[6.0]
+  include Gitlab::Database::MigrationHelpers
+
+  DOWNTIME = false
+
+  disable_ddl_transaction!
+
+  def up
+    add_concurrent_foreign_key :bulk_import_entities, :namespaces, column: :namespace_id
+  end
+
+  def down
+    with_lock_retries do
+      remove_foreign_key :bulk_import_entities, column: :namespace_id
+    end
+  end
+end

db/migrate/20200925194006_add_project_foreign_key_to_bulk_import_entities.rb

+# frozen_string_literal: true
+
+class AddProjectForeignKeyToBulkImportEntities < ActiveRecord::Migration[6.0]
+  include Gitlab::Database::MigrationHelpers
+
+  DOWNTIME = false
+
+  disable_ddl_transaction!
+
+  def up
+    add_concurrent_foreign_key :bulk_import_entities, :projects, column: :project_id
+  end
+
+  def down
+    with_lock_retries do
+      remove_foreign_key :bulk_import_entities, column: :project_id
+    end
+  end
+end

db/schema_migrations/20200925114522

+7d43d2fa91e27eaf9399cf0ce9e4375e849deb71b12d4891455bc51392bce14a
\ No newline at end of file

db/schema_migrations/20200925153423

+f445704e51dad2369719d8c0931c3314793fa90ba6b5a383df503ea4f6dafd20
\ No newline at end of file

db/schema_migrations/20200925193815

+a814b745b4911fc6c80971e6c0c19e6d64ca30cb94fa87a94bc1adf8c07b1c87
\ No newline at end of file

db/schema_migrations/20200925193906

+a915ccf5df0ec803286205916ffcd34b1410d1cc4da84f8299b63b3665d69e09
\ No newline at end of file

db/schema_migrations/20200925194006

+28a71a380be0ef08389defac604c351f0a7f31b6c03a7c40aabe47bf09e6a485
\ No newline at end of file

db/structure.sql

@@ -9836,6 +9836,35 @@ CREATE SEQUENCE bulk_import_configurations_id_seq
 
 ALTER SEQUENCE bulk_import_configurations_id_seq OWNED BY bulk_import_configurations.id;
 
+CREATE TABLE bulk_import_entities (
+    id bigint NOT NULL,
+    bulk_import_id bigint NOT NULL,
+    parent_id bigint,
+    namespace_id bigint,
+    project_id bigint,
+    source_type smallint NOT NULL,
+    source_full_path text NOT NULL,
+    destination_name text NOT NULL,
+    destination_namespace text NOT NULL,
+    status smallint NOT NULL,
+    jid text,
+    created_at timestamp with time zone NOT NULL,
+    updated_at timestamp with time zone NOT NULL,
+    CONSTRAINT check_13f279f7da CHECK ((char_length(source_full_path) <= 255)),
+    CONSTRAINT check_715d725ea2 CHECK ((char_length(destination_name) <= 255)),
+    CONSTRAINT check_796a4d9cc6 CHECK ((char_length(jid) <= 255)),
+    CONSTRAINT check_b834fff4d9 CHECK ((char_length(destination_namespace) <= 255))
+);
+
+CREATE SEQUENCE bulk_import_entities_id_seq
+    START WITH 1
+    INCREMENT BY 1
+    NO MINVALUE
+    NO MAXVALUE
+    CACHE 1;
+
+ALTER SEQUENCE bulk_import_entities_id_seq OWNED BY bulk_import_entities.id;
+
 CREATE TABLE bulk_imports (
     id bigint NOT NULL,
     user_id integer NOT NULL,
@@ -17344,6 +17373,8 @@ ALTER TABLE ONLY broadcast_messages ALTER COLUMN id SET DEFAULT nextval('broadca
 
 ALTER TABLE ONLY bulk_import_configurations ALTER COLUMN id SET DEFAULT nextval('bulk_import_configurations_id_seq'::regclass);
 
+ALTER TABLE ONLY bulk_import_entities ALTER COLUMN id SET DEFAULT nextval('bulk_import_entities_id_seq'::regclass);
+
 ALTER TABLE ONLY bulk_imports ALTER COLUMN id SET DEFAULT nextval('bulk_imports_id_seq'::regclass);
 
 ALTER TABLE ONLY chat_names ALTER COLUMN id SET DEFAULT nextval('chat_names_id_seq'::regclass);
@@ -18325,6 +18356,9 @@ ALTER TABLE ONLY broadcast_messages
 ALTER TABLE ONLY bulk_import_configurations
     ADD CONSTRAINT bulk_import_configurations_pkey PRIMARY KEY (id);
 
+ALTER TABLE ONLY bulk_import_entities
+    ADD CONSTRAINT bulk_import_entities_pkey PRIMARY KEY (id);
+
 ALTER TABLE ONLY bulk_imports
     ADD CONSTRAINT bulk_imports_pkey PRIMARY KEY (id);
 
@@ -19835,6 +19869,14 @@ CREATE INDEX index_broadcast_message_on_ends_at_and_broadcast_type_and_id ON bro
 
 CREATE INDEX index_bulk_import_configurations_on_bulk_import_id ON bulk_import_configurations USING btree (bulk_import_id);
 
+CREATE INDEX index_bulk_import_entities_on_bulk_import_id ON bulk_import_entities USING btree (bulk_import_id);
+
+CREATE INDEX index_bulk_import_entities_on_namespace_id ON bulk_import_entities USING btree (namespace_id);
+
+CREATE INDEX index_bulk_import_entities_on_parent_id ON bulk_import_entities USING btree (parent_id);
+
+CREATE INDEX index_bulk_import_entities_on_project_id ON bulk_import_entities USING btree (project_id);
+
 CREATE INDEX index_bulk_imports_on_user_id ON bulk_imports USING btree (user_id);
 
 CREATE UNIQUE INDEX index_chat_names_on_service_id_and_team_id_and_chat_id ON chat_names USING btree (service_id, team_id, chat_id);
@@ -22467,6 +22509,9 @@ ALTER TABLE ONLY ci_builds
 ALTER TABLE ONLY vulnerabilities
     ADD CONSTRAINT fk_88b4d546ef FOREIGN KEY (start_date_sourcing_milestone_id) REFERENCES milestones(id) ON DELETE SET NULL;
 
+ALTER TABLE ONLY bulk_import_entities
+    ADD CONSTRAINT fk_88c725229f FOREIGN KEY (namespace_id) REFERENCES namespaces(id) ON DELETE CASCADE;
+
 ALTER TABLE ONLY issues
     ADD CONSTRAINT fk_899c8f3231 FOREIGN KEY (project_id) REFERENCES projects(id) ON DELETE CASCADE;
 
@@ -22545,6 +22590,9 @@ ALTER TABLE ONLY ci_builds
 ALTER TABLE ONLY ci_pipelines
     ADD CONSTRAINT fk_a23be95014 FOREIGN KEY (merge_request_id) REFERENCES merge_requests(id) ON DELETE CASCADE;
 
+ALTER TABLE ONLY bulk_import_entities
+    ADD CONSTRAINT fk_a44ff95be5 FOREIGN KEY (parent_id) REFERENCES bulk_import_entities(id) ON DELETE CASCADE;
+
 ALTER TABLE ONLY users
     ADD CONSTRAINT fk_a4b8fefe3e FOREIGN KEY (managing_group_id) REFERENCES namespaces(id) ON DELETE SET NULL;
 
@@ -22587,6 +22635,9 @@ ALTER TABLE ONLY project_access_tokens
 ALTER TABLE ONLY protected_tag_create_access_levels
     ADD CONSTRAINT fk_b4eb82fe3c FOREIGN KEY (group_id) REFERENCES namespaces(id) ON DELETE CASCADE;
 
+ALTER TABLE ONLY bulk_import_entities
+    ADD CONSTRAINT fk_b69fa2b2df FOREIGN KEY (bulk_import_id) REFERENCES bulk_imports(id) ON DELETE CASCADE;
+
 ALTER TABLE ONLY compliance_management_frameworks
     ADD CONSTRAINT fk_b74c45b71f FOREIGN KEY (namespace_id) REFERENCES namespaces(id) ON DELETE CASCADE;
 
@@ -22647,6 +22698,9 @@ ALTER TABLE ONLY todos
 ALTER TABLE ONLY geo_event_log
     ADD CONSTRAINT fk_cff7185ad2 FOREIGN KEY (reset_checksum_event_id) REFERENCES geo_reset_checksum_events(id) ON DELETE CASCADE;
 
+ALTER TABLE ONLY bulk_import_entities
+    ADD CONSTRAINT fk_d06d023c30 FOREIGN KEY (project_id) REFERENCES projects(id) ON DELETE CASCADE;
+
 ALTER TABLE ONLY project_mirror_data
     ADD CONSTRAINT fk_d1aad367d7 FOREIGN KEY (project_id) REFERENCES projects(id) ON DELETE CASCADE;
 

locale/gitlab.pot

@@ -4419,6 +4419,15 @@ msgstr ""
 msgid "Bulk request concurrency"
 msgstr ""
 
+msgid "BulkImport|expected an associated Group but has an associated Project"
+msgstr ""
+
+msgid "BulkImport|expected an associated Project but has an associated Group"
+msgstr ""
+
+msgid "BulkImport|must be a group"
+msgstr ""
+
 msgid "Burndown chart"
 msgstr ""
 

spec/factories/bulk_import/entities.rb

+# frozen_string_literal: true
+
+FactoryBot.define do
+  factory :bulk_import_entity, class: 'BulkImports::Entity' do
+    bulk_import
+
+    source_type { :group_entity }
+    sequence(:source_full_path) { |n| "source-path-#{n}" }
+
+    sequence(:destination_namespace) { |n| "destination-path-#{n}" }
+    destination_name { 'Imported Entity' }
+
+    trait(:group_entity) do
+      source_type { :group_entity }
+    end
+
+    trait(:project_entity) do
+      source_type { :project_entity }
+    end
+  end
+end

spec/models/bulk_import_spec.rb

@@ -6,6 +6,7 @@ RSpec.describe BulkImport, type: :model do
   describe 'associations' do
     it { is_expected.to belong_to(:user).required }
     it { is_expected.to have_one(:configuration) }
+    it { is_expected.to have_many(:entities) }
   end
 
   describe 'validations' do

spec/models/bulk_imports/entity_spec.rb

+# frozen_string_literal: true
+
+require 'spec_helper'
+
+RSpec.describe BulkImports::Entity, type: :model do
+  describe 'associations' do
+    it { is_expected.to belong_to(:bulk_import).required }
+    it { is_expected.to belong_to(:parent) }
+    it { is_expected.to belong_to(:group) }
+    it { is_expected.to belong_to(:project) }
+  end
+
+  describe 'validations' do
+    it { is_expected.to validate_presence_of(:source_type) }
+    it { is_expected.to validate_presence_of(:source_full_path) }
+    it { is_expected.to validate_presence_of(:destination_name) }
+    it { is_expected.to validate_presence_of(:destination_namespace) }
+
+    it { is_expected.to define_enum_for(:source_type).with_values(%i[group_entity project_entity]) }
+
+    context 'when associated with a group and project' do
+      it 'is invalid' do
+        entity = build(:bulk_import_entity, group: build(:group), project: build(:project))
+
+        expect(entity).not_to be_valid
+        expect(entity.errors).to include(:project, :group)
+      end
+    end
+
+    context 'when not associated with a group or project' do
+      it 'is valid' do
+        entity = build(:bulk_import_entity, group: nil, project: nil)
+
+        expect(entity).to be_valid
+      end
+    end
+
+    context 'when associated with a group and no project' do
+      it 'is valid as a group_entity' do
+        entity = build(:bulk_import_entity, :group_entity, group: build(:group), project: nil)
+
+        expect(entity).to be_valid
+      end
+
+      it 'is invalid as a project_entity' do
+        entity = build(:bulk_import_entity, :project_entity, group: build(:group), project: nil)
+
+        expect(entity).not_to be_valid
+        expect(entity.errors).to include(:group)
+      end
+    end
+
+    context 'when associated with a project and no group' do
+      it 'is valid' do
+        entity = build(:bulk_import_entity, :project_entity, group: nil, project: build(:project))
+
+        expect(entity).to be_valid
+      end
+
+      it 'is invalid as a project_entity' do
+        entity = build(:bulk_import_entity, :group_entity, group: nil, project: build(:project))
+
+        expect(entity).not_to be_valid
+        expect(entity.errors).to include(:project)
+      end
+    end
+
+    context 'when the parent is a group import' do
+      it 'is valid' do
+        entity = build(:bulk_import_entity, parent: build(:bulk_import_entity, :group_entity))
+
+        expect(entity).to be_valid
+      end
+    end
+
+    context 'when the parent is a project import' do
+      it 'is invalid' do
+        entity = build(:bulk_import_entity, parent: build(:bulk_import_entity, :project_entity))
+
+        expect(entity).not_to be_valid
+        expect(entity.errors).to include(:parent)
+      end
+    end
+  end
+end