Merge branch 'mk/fix-design-update-failures' into 'master'

Geo: Fix update event bugs

Closes #238137 and #241668

See merge request gitlab-org/gitlab!40643

ee/app/models/geo_node.rb

@@ -244,10 +244,23 @@ class GeoNode < ApplicationRecord
     ContainerRepository.project_id_in(projects)
   end
 
+  def container_repositories_include?(container_repository_id)
+    return false unless Geo::ContainerRepositoryRegistry.replication_enabled?
+    return true unless selective_sync?
+
+    container_repositories.where(id: container_repository_id).exists?
+  end
+
   def designs
     projects.with_designs
   end
 
+  def designs_include?(project_id)
+    return true unless selective_sync?
+
+    designs.where(id: project_id).exists?
+  end
+
   def lfs_objects
     return LfsObject.all unless selective_sync?
 

ee/changelogs/unreleased/mk-fix-design-update-failures.yml

+---
+title: 'Geo: Fix design repository failures with selective sync, and make container repository updates more robust'
+merge_request: 40643
+author:
+type: fixed

ee/lib/gitlab/geo/log_cursor/events/base_event.rb

@@ -37,8 +37,14 @@ module Gitlab
             yield if healthy_shard_for?(event)
           end
 
-          def replicable_project?(project_id)
-            strong_memoize(:"replicable_project_#{project_id}") do
+          def replicable_project?
+            memoize_and_short_circuit_if_registry_is_persisted(:"replicable_project_#{event.project_id}", registry) do
+              Gitlab::Geo.current_node.projects_include?(event.project_id)
+            end
+          end
+
+          def memoize_and_short_circuit_if_registry_is_persisted(memoize_key, registry, &block)
+            strong_memoize(memoize_key) do
               # If a registry exists, then it *should* be replicated. The
               # registry will be removed by the delete event or
               # RegistryConsistencyWorker if it should no longer be replicated.
@@ -47,7 +53,7 @@ module Gitlab
               # for repository updates which are a large proportion of events.
               next true if registry.persisted?
 
-              Gitlab::Geo.current_node.projects_include?(project_id)
+              yield
             end
           end
 

ee/lib/gitlab/geo/log_cursor/events/container_repository_updated_event.rb

@@ -8,8 +8,9 @@ module Gitlab
           include BaseEvent
 
           def process
-            if should_sync?
+            if replicable_container_repository?
               registry.repository_updated!
+              registry.save
 
               job_id = ::Geo::ContainerRepositorySyncWorker.perform_async(event.container_repository_id)
             end
@@ -19,11 +20,21 @@ module Gitlab
 
           private
 
-          def should_sync?
-            strong_memoize(:should_sync) do
-              ::Geo::ContainerRepositoryRegistry.replication_enabled? &&
-                registry.container_repository &&
-                replicable_project?(registry.container_repository.project_id)
+          def replicable_container_repository?
+            id = event.container_repository_id
+
+            strong_memoize(:"replicable_container_repository_#{id}") do
+              next false unless ::Geo::ContainerRepositoryRegistry.replication_enabled?
+
+              # If a registry exists, then it *should* be replicated. The
+              # registry will be removed by the delete event or
+              # RegistryConsistencyWorker if it should no longer be replicated.
+              #
+              # This early exit helps keep processing of update events
+              # efficient.
+              next true if registry.persisted?
+
+              Gitlab::Geo.current_node.container_repositories_include?(id)
             end
           end
 
@@ -38,10 +49,9 @@ module Gitlab
           def log_event(job_id)
             super(
               'Docker Repository update',
-              container_repository_id: registry.container_repository_id,
-              should_sync: should_sync?,
+              container_repository_id: event.container_repository_id,
               replication_enabled: ::Geo::ContainerRepositoryRegistry.replication_enabled?,
-              replicable_project: replicable_project?(registry.container_repository.project_id),
+              replicable_container_repository: replicable_container_repository?,
               project_id: registry.container_repository.project_id,
               job_id: job_id)
           end

ee/lib/gitlab/geo/log_cursor/events/design_repository_updated_event.rb

@@ -9,7 +9,7 @@ module Gitlab
 
           def process
             job_id =
-              if replicable_project?(event.project_id)
+              if replicable_design?
                 registry.repository_updated!
                 registry.save
 
@@ -25,6 +25,12 @@ module Gitlab
             @registry ||= ::Geo::DesignRegistry.find_or_initialize_by(project_id: event.project_id) # rubocop: disable CodeReuse/ActiveRecord
           end
 
+          def replicable_design?
+            memoize_and_short_circuit_if_registry_is_persisted(:"replicable_design_#{event.project_id}", registry) do
+              Gitlab::Geo.current_node.designs_include?(event.project_id)
+            end
+          end
+
           def schedule_job(event)
             enqueue_job_if_shard_healthy(event) do
               ::Geo::DesignRepositorySyncWorker.perform_async(event.project_id)
@@ -36,7 +42,7 @@ module Gitlab
               'Design repository update',
               project_id: event.project_id,
               scheduled_at: Time.now,
-              replicable_project: replicable_project?(event.project_id),
+              replicable_design: replicable_design?,
               job_id: job_id)
           end
         end

ee/lib/gitlab/geo/log_cursor/events/repository_created_event.rb

@@ -8,7 +8,7 @@ module Gitlab
           include BaseEvent
 
           def process
-            if replicable_project?(event.project_id)
+            if replicable_project?
               registry.repository_created!(event)
 
               job_id = nil
@@ -31,7 +31,7 @@ module Gitlab
               wiki_path: event.wiki_path,
               resync_repository: registry.resync_repository,
               resync_wiki: registry.resync_wiki,
-              replicable_project: replicable_project?(event.project_id),
+              replicable_project: replicable_project?,
               job_id: job_id)
           end
         end

ee/lib/gitlab/geo/log_cursor/events/repository_updated_event.rb

@@ -8,7 +8,7 @@ module Gitlab
           include BaseEvent
 
           def process
-            if replicable_project?(event.project_id)
+            if replicable_project?
               registry.repository_updated!(event.source, scheduled_at)
 
               job_id = enqueue_job_if_shard_healthy(event) do
@@ -33,7 +33,7 @@ module Gitlab
               resync_repository: registry.resync_repository,
               resync_wiki: registry.resync_wiki,
               scheduled_at: scheduled_at,
-              replicable_project: replicable_project?(event.project_id),
+              replicable_project: replicable_project?,
               job_id: job_id)
           end
 

ee/spec/lib/gitlab/geo/log_cursor/events/container_repository_updated_event_spec.rb

@@ -30,25 +30,11 @@ RSpec.describe Gitlab::Geo::LogCursor::Events::ContainerRepositoryUpdatedEvent,
         stub_config(geo: { registry_replication: { enabled: true } })
       end
 
-      context "when the container repository's project is not excluded by selective sync" do
-        # TODO: Fix https://gitlab.com/gitlab-org/gitlab/-/issues/233514 and
-        # use this test, and remove the other tests in this context.
-        # it_behaves_like 'event should trigger a sync'
-
-        context 'when a registry does not yet exist' do
-          it_behaves_like 'event schedules a sync worker'
-          it_behaves_like 'logs event source info'
-        end
-
-        context 'when a registry exists' do
-          let!(:registry) { create(registry_factory, *registry_factory_args) }
-
-          it_behaves_like 'event schedules a sync worker'
-          it_behaves_like 'logs event source info'
-        end
+      context "when the container repository is not excluded by selective sync" do
+        it_behaves_like 'event should trigger a sync'
       end
 
-      context "when the container repository's project is excluded by selective sync" do
+      context "when the container repository is excluded by selective sync" do
         before do
           stub_current_geo_node(secondary_excludes_all_projects)
         end

ee/spec/lib/gitlab/geo/log_cursor/events/design_repository_updated_event_spec.rb

@@ -30,11 +30,24 @@ RSpec.describe Gitlab::Geo::LogCursor::Events::DesignRepositoryUpdatedEvent, :cl
         allow(Gitlab::ShardHealthCache).to receive(:healthy_shard?).with('default').and_return(true)
       end
 
-      context "when the container repository's project is not excluded by selective sync" do
+      context 'when the design repository is not excluded by selective sync' do
         it_behaves_like 'event should trigger a sync'
+
+        context 'when the project is included in selective sync but there is no design' do
+          before do
+            node = create(:geo_node, selective_sync_type: 'shards', selective_sync_shards: [project.repository_storage])
+            stub_current_geo_node(node)
+          end
+
+          context 'when a registry does not yet exist' do
+            it_behaves_like 'event does not create a registry'
+            it_behaves_like 'event does not schedule a sync worker'
+            it_behaves_like 'logs event source info'
+          end
+        end
       end
 
-      context "when the container repository's project is excluded by selective sync" do
+      context "when the design repository is excluded by selective sync" do
         before do
           stub_current_geo_node(secondary_excludes_all_projects)
         end