Merge branch 'sh-geo-add-project-deletion-events' into 'master'

Add Geo delete event log

See merge request !2126

app/models/geo/event_log.rb

@@ -5,5 +5,9 @@ module Geo
     belongs_to :repository_updated_event,
       class_name: 'Geo::RepositoryUpdatedEvent',
       foreign_key: :repository_updated_event_id
+
+    belongs_to :repository_deleted_event,
+      class_name: 'Geo::RepositoryDeletedEvent',
+      foreign_key: :repository_deleted_event_id
   end
 end

app/models/geo/repository_deleted_event.rb

+module Geo
+  class RepositoryDeletedEvent < ActiveRecord::Base
+    include Geo::Model
+
+    belongs_to :project
+
+    validates :project, presence: true
+  end
+end

app/services/ee/projects/destroy_service.rb

+module EE
+  module Projects
+    module DestroyService
+      def execute
+        raise NotImplementedError unless defined?(super)
+
+        super
+
+        log_geo_event(project)
+      end
+
+      def log_geo_event(project)
+        ::Geo::RepositoryDeletedEventStore.new(project,
+          repo_path: repo_path,
+          wiki_path: wiki_path).create
+      end
+
+      # Removes physical repository in a Geo replicated secondary node
+      # There is no need to do any database operation as it will be
+      # replicated by itself.
+      def geo_replicate
+        # Flush the cache for both repositories. This has to be done _before_
+        # removing the physical repositories as some expiration code depends on
+        # Git data (e.g. a list of branch names).
+        flush_caches(project, wiki_path)
+
+        trash_repositories!
+        remove_tracking_entries!
+        log_info("Project \"#{project.name}\" was removed")
+      end
+    end
+  end
+end

app/services/geo/repository_deleted_event_store.rb

+module Geo
+  class RepositoryDeletedEventStore
+    attr_reader :project, :repo_path, :wiki_path
+
+    def initialize(project, repo_path:, wiki_path:)
+      @project = project
+      @repo_path = repo_path
+      @wiki_path = wiki_path
+    end
+
+    def create
+      return unless Gitlab::Geo.primary?
+
+      Geo::EventLog.transaction do
+        event_log = Geo::EventLog.new
+        deleted_event = Geo::RepositoryDeletedEvent.new(
+          project: project,
+          repository_storage_name: project.repository.storage,
+          repository_storage_path: project.repository_storage_path,
+          deleted_path: repo_path,
+          deleted_wiki_path: wiki_path,
+          deleted_project_name: project.name)
+        event_log.repository_deleted_event = deleted_event
+        event_log.save
+      end
+    end
+  end
+end

app/services/projects/destroy_service.rb

 module Projects
   class DestroyService < BaseService
     include Gitlab::ShellAdapter
+    prepend ::EE::Projects::DestroyService
 
     DestroyError = Class.new(StandardError)
 
@@ -45,21 +46,8 @@ module Projects
 
       log_info("Project \"#{project.path_with_namespace}\" was removed")
       system_hook_service.execute_hooks_for(project, :destroy)
-      true
-    end
 
-    # Removes physical repository in a Geo replicated secondary node
-    # There is no need to do any database operation as it will be
-    # replicated by itself.
-    def geo_replicate
-      # Flush the cache for both repositories. This has to be done _before_
-      # removing the physical repositories as some expiration code depends on
-      # Git data (e.g. a list of branch names).
-      flush_caches(project, wiki_path)
-
-      trash_repositories!
-      remove_tracking_entries!
-      log_info("Project \"#{project.name}\" was removed")
+      true
     end
 
     private

db/migrate/20170614053336_create_geo_repository_deleted_events.rb

+class CreateGeoRepositoryDeletedEvents < ActiveRecord::Migration
+  include Gitlab::Database::MigrationHelpers
+
+  DOWNTIME = false
+
+  def change
+    create_table :geo_repository_deleted_events, id: :bigserial do |t|
+      # If a project is deleted, we need to retain this entry
+      t.references :project, index: true, foreign_key: false, null: false
+      t.text :repository_storage_name, null: false
+      t.text :repository_storage_path, null: false
+      t.text :deleted_path, null: false
+      t.text :deleted_wiki_path
+      t.text :deleted_project_name, null: false
+    end
+
+    add_timestamps_with_timezone :geo_repository_deleted_events
+    add_column :geo_event_log, :repository_deleted_event_id, :integer, limit: 8
+  end
+end

db/migrate/20170614061551_add_geo_repository_deleted_events_foreign_key.rb

+class AddGeoRepositoryDeletedEventsForeignKey < ActiveRecord::Migration
+  include Gitlab::Database::MigrationHelpers
+
+  DOWNTIME = false
+
+  disable_ddl_transaction!
+
+  def up
+    add_concurrent_foreign_key :geo_event_log, :geo_repository_deleted_events,
+                               column: :repository_deleted_event_id, on_delete: :cascade
+  end
+
+  def down
+    remove_foreign_key :geo_event_log, column: :repository_deleted_event_id
+  end
+end

db/schema.rb

@@ -582,6 +582,7 @@ ActiveRecord::Schema.define(version: 20170621102400) do
   create_table "geo_event_log", id: :bigserial, force: :cascade do |t|
     t.datetime "created_at", null: false
     t.integer "repository_updated_event_id", limit: 8
+    t.integer "repository_deleted_event_id", limit: 8
   end
 
   add_index "geo_event_log", ["repository_updated_event_id"], name: "index_geo_event_log_on_repository_updated_event_id", using: :btree
@@ -606,6 +607,19 @@ ActiveRecord::Schema.define(version: 20170621102400) do
   add_index "geo_nodes", ["host"], name: "index_geo_nodes_on_host", using: :btree
   add_index "geo_nodes", ["primary"], name: "index_geo_nodes_on_primary", using: :btree
 
+  create_table "geo_repository_deleted_events", id: :bigserial, force: :cascade do |t|
+    t.integer "project_id", null: false
+    t.text "repository_storage_name", null: false
+    t.text "repository_storage_path", null: false
+    t.text "deleted_path", null: false
+    t.text "deleted_wiki_path"
+    t.text "deleted_project_name", null: false
+    t.datetime "created_at", null: false
+    t.datetime "updated_at", null: false
+  end
+
+  add_index "geo_repository_deleted_events", ["project_id"], name: "index_geo_repository_deleted_events_on_project_id", using: :btree
+
   create_table "geo_repository_updated_events", id: :bigserial, force: :cascade do |t|
     t.datetime "created_at", null: false
     t.integer "branches_affected", null: false
@@ -1816,6 +1830,7 @@ ActiveRecord::Schema.define(version: 20170621102400) do
   add_foreign_key "ci_triggers", "users", column: "owner_id", name: "fk_e8e10d1964", on_delete: :cascade
   add_foreign_key "ci_variables", "projects", name: "fk_ada5eb64b3", on_delete: :cascade
   add_foreign_key "container_repositories", "projects"
+  add_foreign_key "geo_event_log", "geo_repository_deleted_events", column: "repository_deleted_event_id", name: "fk_c4b1c1f66e", on_delete: :cascade
   add_foreign_key "geo_event_log", "geo_repository_updated_events", column: "repository_updated_event_id", on_delete: :cascade
   add_foreign_key "geo_repository_updated_events", "projects", on_delete: :cascade
   add_foreign_key "issue_assignees", "issues", name: "fk_b7d881734a", on_delete: :cascade

spec/services/ee/projects/destroy_service_spec.rb

+# rubocop:disable RSpec/FilePath
+require 'spec_helper'
+
+describe Projects::DestroyService, services: true do
+  let!(:user) { create(:user) }
+  let!(:project) { create(:project, :repository, namespace: user.namespace) }
+  let!(:project_id) { project.id }
+  let!(:project_name) { project.name }
+  let!(:project_path) { project.path_with_namespace }
+  let!(:wiki_path) { project.path_with_namespace + '.wiki' }
+  let!(:storage_name) { project.repository_storage }
+  let!(:storage_path) { project.repository_storage_path }
+  let!(:geo_node) { create(:geo_node, :primary, :current) }
+
+  before do
+    stub_container_registry_config(enabled: true)
+    stub_container_registry_tags(repository: :any, tags: [])
+  end
+
+  context 'Geo primary' do
+    it 'logs the event' do
+      # Run sidekiq immediatly to check that renamed repository will be removed
+      Sidekiq::Testing.inline! { destroy_project(project, user, {}) }
+
+      event = Geo::RepositoryDeletedEvent.first
+
+      expect(Geo::EventLog.count).to eq(1)
+      expect(Geo::RepositoryDeletedEvent.count).to eq(1)
+      expect(event.project_id).to eq(project_id)
+      expect(event.deleted_path).to eq(project_path)
+      expect(event.deleted_wiki_path).to eq(wiki_path)
+      expect(event.deleted_project_name).to eq(project_name)
+      expect(event.repository_storage_name).to eq(storage_name)
+      expect(event.repository_storage_path).to eq(storage_path)
+    end
+  end
+
+  def destroy_project(project, user, params = {})
+    described_class.new(project, user, params).execute
+  end
+end

spec/services/users/destroy_service_spec.rb

@@ -18,7 +18,7 @@ describe Users::DestroyService, services: true do
       end
 
       it 'will delete the project' do
-        expect_any_instance_of(Projects::DestroyService).to receive(:execute).once
+        expect_any_instance_of(EE::Projects::DestroyService).to receive(:execute).once
 
         service.execute(user)
       end
@@ -31,7 +31,7 @@ describe Users::DestroyService, services: true do
       end
 
       it 'destroys a project in pending_delete' do
-        expect_any_instance_of(Projects::DestroyService).to receive(:execute).once
+        expect_any_instance_of(EE::Projects::DestroyService).to receive(:execute).once
 
         service.execute(user)
 

spec/workers/geo_repository_destroy_worker_spec.rb

@@ -7,7 +7,7 @@ describe GeoRepositoryDestroyWorker do
   let(:perform!) { subject.perform(project.id, project.name, path) }
 
   it 'delegates project removal to Projects::DestroyService' do
-    expect_any_instance_of(::Projects::DestroyService).to receive(:geo_replicate)
+    expect_any_instance_of(EE::Projects::DestroyService).to receive(:geo_replicate)
 
     perform!
   end