Update namespace statistics when project is destroyed

When a project is destroyed, Projects::ProjectDeletedEvent is published.

A Namespaces::UpdateRootStatisticsWorker is subscribed to
Projects::ProjectDeletedEvent. The worker will
enqueue Namespaces::ScheduleAggregationWorker to
update the namespace's root statistics.

This change is behind a feature flag for publishing ProjectDeletedEvent.

Changelog: fixed

app/events/projects/project_deleted_event.rb

+# frozen_string_literal: true
+
+module Projects
+  class ProjectDeletedEvent < ::Gitlab::EventStore::Event
+    def schema
+      {
+        'type' => 'object',
+        'properties' => {
+          'project_id' => { 'type' => 'integer' },
+          'namespace_id' => { 'type' => 'integer' }
+        },
+        'required' => %w[project_id namespace_id]
+      }
+    end
+  end
+end

app/services/projects/destroy_service.rb

@@ -37,6 +37,8 @@ module Projects
       system_hook_service.execute_hooks_for(project, :destroy)
       log_info("Project \"#{project.full_path}\" was deleted")
 
+      publish_project_deleted_event_for(project) if Feature.enabled?(:publish_project_deleted_event, default_enabled: :yaml)
+
       current_user.invalidate_personal_projects_count
 
       true
@@ -260,6 +262,12 @@ module Projects
     def flush_caches(project)
       Projects::ForksCountService.new(project).delete_cache
     end
+
+    def publish_project_deleted_event_for(project)
+      data = { project_id: project.id, namespace_id: project.namespace_id }
+      event = Projects::ProjectDeletedEvent.new(data: data)
+      Gitlab::EventStore.publish(event)
+    end
   end
 end
 

app/workers/all_queues.yml

@@ -2569,6 +2569,15 @@
   :weight: 1
   :idempotent: true
   :tags: []
+- :name: namespaces_update_root_statistics
+  :worker_name: Namespaces::UpdateRootStatisticsWorker
+  :feature_category: :source_code_management
+  :has_external_dependencies:
+  :urgency: :low
+  :resource_boundary: :unknown
+  :weight: 1
+  :idempotent: true
+  :tags: []
 - :name: new_issue
   :worker_name: NewIssueWorker
   :feature_category: :team_planning

app/workers/namespaces/update_root_statistics_worker.rb

+# frozen_string_literal: true
+
+module Namespaces
+  class UpdateRootStatisticsWorker
+    include Gitlab::EventStore::Subscriber
+
+    data_consistency :always
+
+    idempotent!
+
+    feature_category :source_code_management
+
+    def handle_event(event)
+      ScheduleAggregationWorker.perform_async(event.data[:namespace_id])
+    end
+  end
+end

config/feature_flags/development/publish_project_deleted_event.yml

+---
+name: publish_project_deleted_event
+introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/78862
+rollout_issue_url: https://gitlab.com/gitlab-org/gitlab/-/issues/351073
+milestone: '14.8'
+type: development
+group: group::pipeline insights
+default_enabled: false

config/sidekiq_queues.yml

@@ -295,6 +295,8 @@
   - 1
 - - namespaces_sync_namespace_name
   - 1
+- - namespaces_update_root_statistics
+  - 1
 - - new_epic
   - 2
 - - new_issue

lib/gitlab/event_store.rb

@@ -34,6 +34,7 @@ module Gitlab
       # Add subscriptions here:
 
       store.subscribe ::MergeRequests::UpdateHeadPipelineWorker, to: ::Ci::PipelineCreatedEvent
+      store.subscribe ::Namespaces::UpdateRootStatisticsWorker, to: ::Projects::ProjectDeletedEvent
     end
     private_class_method :configure!
   end

spec/events/projects/project_deleted_event_spec.rb

+# frozen_string_literal: true
+
+require 'spec_helper'
+
+RSpec.describe Projects::ProjectDeletedEvent do
+  where(:data, :valid) do
+    [
+      [{ project_id: 1, namespace_id: 2 }, true],
+      [{ project_id: 1 }, false],
+      [{ namespace_id: 1 }, false],
+      [{ project_id: 'foo', namespace_id: 2 }, false],
+      [{ project_id: 1, namespace_id: 'foo' }, false],
+      [{ project_id: [], namespace_id: 2 }, false],
+      [{ project_id: 1, namespace_id: [] }, false],
+      [{ project_id: {}, namespace_id: 2 }, false],
+      [{ project_id: 1, namespace_id: {} }, false],
+      ['foo', false],
+      [123, false],
+      [[], false]
+    ]
+  end
+
+  with_them do
+    it 'validates data' do
+      constructor = -> { described_class.new(data: data) }
+
+      if valid
+        expect { constructor.call }.not_to raise_error
+      else
+        expect { constructor.call }.to raise_error(Gitlab::EventStore::InvalidEvent)
+      end
+    end
+  end
+end

spec/services/projects/destroy_service_spec.rb

@@ -18,17 +18,35 @@ RSpec.describe Projects::DestroyService, :aggregate_failures do
   end
 
   shared_examples 'deleting the project' do
-    before do
-      # Run sidekiq immediately to check that renamed repository will be removed
+    it 'deletes the project', :sidekiq_inline do
       destroy_project(project, user, {})
-    end
 
-    it 'deletes the project', :sidekiq_inline do
       expect(Project.unscoped.all).not_to include(project)
 
       expect(project.gitlab_shell.repository_exists?(project.repository_storage, path + '.git')).to be_falsey
       expect(project.gitlab_shell.repository_exists?(project.repository_storage, remove_path + '.git')).to be_falsey
     end
+
+    it 'publishes a ProjectDeleted event with project id and namespace id' do
+      expected_data = { project_id: project.id, namespace_id: project.namespace_id }
+      expect(Gitlab::EventStore)
+        .to receive(:publish)
+        .with(event_type(Projects::ProjectDeletedEvent).containing(expected_data))
+
+      destroy_project(project, user, {})
+    end
+
+    context 'when feature flag publish_project_deleted_event is disabled' do
+      before do
+        stub_feature_flags(publish_project_deleted_event: false)
+      end
+
+      it 'does not publish an event' do
+        expect(Gitlab::EventStore).not_to receive(:publish)
+
+        destroy_project(project, user, {})
+      end
+    end
   end
 
   shared_examples 'deleting the project with pipeline and build' do

spec/support/matchers/event_store.rb

+# frozen_string_literal: true
+
+RSpec::Matchers.define :event_type do |event_class|
+  match do |actual|
+    actual.instance_of?(event_class) &&
+      actual.data == @expected_data
+  end
+
+  chain :containing do |expected_data|
+    @expected_data = expected_data
+  end
+end

spec/tooling/quality/test_level_spec.rb

@@ -28,7 +28,7 @@ RSpec.describe Quality::TestLevel do
     context 'when level is unit' do
       it 'returns a pattern' do
         expect(subject.pattern(:unit))
-          .to eq("spec/{bin,channels,config,db,dependencies,elastic,elastic_integration,experiments,factories,finders,frontend,graphql,haml_lint,helpers,initializers,javascripts,lib,metrics_server,models,policies,presenters,rack_servers,replicators,routing,rubocop,scripts,serializers,services,sidekiq,sidekiq_cluster,spam,support_specs,tasks,uploaders,validators,views,workers,tooling}{,/**/}*_spec.rb")
+          .to eq("spec/{bin,channels,config,db,dependencies,elastic,elastic_integration,experiments,events,factories,finders,frontend,graphql,haml_lint,helpers,initializers,javascripts,lib,metrics_server,models,policies,presenters,rack_servers,replicators,routing,rubocop,scripts,serializers,services,sidekiq,sidekiq_cluster,spam,support_specs,tasks,uploaders,validators,views,workers,tooling}{,/**/}*_spec.rb")
       end
     end
 
@@ -110,7 +110,7 @@ RSpec.describe Quality::TestLevel do
     context 'when level is unit' do
       it 'returns a regexp' do
         expect(subject.regexp(:unit))
-          .to eq(%r{spec/(bin|channels|config|db|dependencies|elastic|elastic_integration|experiments|factories|finders|frontend|graphql|haml_lint|helpers|initializers|javascripts|lib|metrics_server|models|policies|presenters|rack_servers|replicators|routing|rubocop|scripts|serializers|services|sidekiq|sidekiq_cluster|spam|support_specs|tasks|uploaders|validators|views|workers|tooling)})
+          .to eq(%r{spec/(bin|channels|config|db|dependencies|elastic|elastic_integration|experiments|events|factories|finders|frontend|graphql|haml_lint|helpers|initializers|javascripts|lib|metrics_server|models|policies|presenters|rack_servers|replicators|routing|rubocop|scripts|serializers|services|sidekiq|sidekiq_cluster|spam|support_specs|tasks|uploaders|validators|views|workers|tooling)})
       end
     end
 

spec/workers/every_sidekiq_worker_spec.rb

@@ -349,6 +349,7 @@ RSpec.describe 'Every Sidekiq worker' do
         'Namespaces::OnboardingPipelineCreatedWorker' => 3,
         'Namespaces::OnboardingProgressWorker' => 3,
         'Namespaces::OnboardingUserAddedWorker' => 3,
+        'Namespaces::RefreshRootStatisticsWorker' => 3,
         'Namespaces::RootStatisticsWorker' => 3,
         'Namespaces::ScheduleAggregationWorker' => 3,
         'NetworkPolicyMetricsWorker' => 3,

spec/workers/namespaces/update_root_statistics_worker_spec.rb

+# frozen_string_literal: true
+
+require 'spec_helper'
+
+RSpec.describe Namespaces::UpdateRootStatisticsWorker do
+  let(:namespace_id) { 123 }
+
+  let(:event) do
+    Projects::ProjectDeletedEvent.new(data: { project_id: 1, namespace_id: namespace_id })
+  end
+
+  subject { consume_event(event) }
+
+  def consume_event(event)
+    described_class.new.perform(event.class.name, event.data)
+  end
+
+  it 'enqueues ScheduleAggregationWorker' do
+    expect(Namespaces::ScheduleAggregationWorker).to receive(:perform_async).with(namespace_id)
+
+    subject
+  end
+end

tooling/quality/test_level.rb

@@ -24,6 +24,7 @@ module Quality
         elastic
         elastic_integration
         experiments
+        events
         factories
         finders
         frontend