Geo: Inform users about current replication lag in the UI on secondaries

On a secondary, every page now contains the replication lag statistic
see https://gitlab.com/gitlab-org/gitlab-ee/issues/9214

ee/app/helpers/ee/application_helper.rb

@@ -3,6 +3,10 @@
 module EE
   module ApplicationHelper
     extend ::Gitlab::Utils::Override
+    DB_LAG_SHOW_THRESHOLD = 60 # seconds
+    LOG_CURSOR_CHECK_TIME = ::Gitlab::Geo::LogCursor::Daemon::SECONDARY_CHECK_INTERVAL
+    EVENT_PROCESSING_TIME = 60.seconds
+    EVENT_LAG_SHOW_THRESHOLD = DB_LAG_SHOW_THRESHOLD.seconds + LOG_CURSOR_CHECK_TIME + EVENT_PROCESSING_TIME
 
     override :read_only_message
     def read_only_message
@@ -11,8 +15,25 @@ module EE
       if @limited_actions_message
         s_('Geo|You are on a secondary, <b>read-only</b> Geo node. You may be able to make a limited amount of changes or perform a limited amount of actions on this page.').html_safe
       else
-        (s_('Geo|You are on a secondary, <b>read-only</b> Geo node. If you want to make changes, you must visit this page on the %{primary_node}.') %
+        message = (s_('Geo|You are on a secondary, <b>read-only</b> Geo node. If you want to make changes, you must visit this page on the %{primary_node}.') %
           { primary_node: link_to('primary node', ::Gitlab::Geo.primary_node&.url || '#') }).html_safe
+
+        return "#{message} #{lag_message}".html_safe if lag_message
+
+        message
+      end
+    end
+
+    def lag_message
+      if db_lag > DB_LAG_SHOW_THRESHOLD
+        return (s_('Geo|The database is currently %{db_lag} behind the primary node.') %
+          { db_lag: time_ago_in_words(db_lag.seconds.ago) }).html_safe
+      end
+
+      if unprocessed_too_old?
+        minutes_behind = time_ago_in_words(next_unprocessed_event.created_at)
+        return (s_('Geo|The node is currently %{minutes_behind} behind the primary node.') %
+          { minutes_behind: minutes_behind }).html_safe
       end
     end
 
@@ -101,5 +122,23 @@ module EE
     def appearance
       ::Appearance.current
     end
+
+    def db_lag
+      @db_lag ||= Rails.cache.fetch('geo:db_lag', expires_in: 20.seconds) do
+        ::Gitlab::Geo::HealthCheck.new.db_replication_lag_seconds
+      end
+    end
+
+    def next_unprocessed_event
+      @next_unprocessed_event ||= Geo::EventLog.next_unprocessed_event
+    end
+
+    def unprocessed_too_old?
+      Rails.cache.fetch('geo:unprocessed_too_old', expires_in: 20.seconds) do
+        break false unless next_unprocessed_event
+
+        next_unprocessed_event.created_at < EVENT_LAG_SHOW_THRESHOLD.ago
+      end
+    end
   end
 end

ee/app/models/geo/event_log.rb

@@ -70,6 +70,13 @@ module Geo
       order(id: :desc).first
     end
 
+    def self.next_unprocessed_event
+      last_processed = Geo::EventLogState.last_processed
+      return first unless last_processed
+
+      where('id > ?', last_processed.event_id).first
+    end
+
     def self.event_classes
       EVENT_CLASSES.map(&:constantize)
     end

ee/changelogs/unreleased/9214-geo-inform-users-about-current-replication-lag-in-the-ui-on-secondaries.yml

+---
+title: 'Geo: Inform users about current replication lag in the UI on secondaries'
+merge_request: 10807
+author:
+type: added

ee/spec/helpers/application_helper_spec.rb

@@ -44,6 +44,51 @@ describe ApplicationHelper do
           expect(helper.read_only_message).to match(/You may be able to make a limited amount of changes or perform a limited amount of actions on this page/)
           expect(helper.read_only_message).not_to include('http://')
         end
+
+        it 'includes a warning about database lag' do
+          allow_any_instance_of(::Gitlab::Geo::HealthCheck).to receive(:db_replication_lag_seconds).and_return(120)
+
+          expect(helper.read_only_message).to match(/If you want to make changes, you must visit this page on the .*primary node/)
+          expect(helper.read_only_message).to match(/The database is currently 2 minutes behind the primary node/)
+          expect(helper.read_only_message).to include(geo_primary.url)
+        end
+
+        context 'event lag' do
+          it 'includes a lag warning about a node lag' do
+            event_log = create(:geo_event_log, created_at: 4.minutes.ago)
+            create(:geo_event_log, created_at: 3.minutes.ago)
+            create(:geo_event_log_state, event_id: event_log.id)
+
+            expect(helper.read_only_message).to match(/If you want to make changes, you must visit this page on the .*primary node/)
+            expect(helper.read_only_message).to match(/The node is currently 3 minutes behind the primary/)
+            expect(helper.read_only_message).to include(geo_primary.url)
+          end
+
+          it 'does not include a lag warning because the last event is too fresh' do
+            event_log = create(:geo_event_log, created_at: 3.minutes.ago)
+            create(:geo_event_log)
+            create(:geo_event_log_state, event_id: event_log.id)
+
+            expect(helper.read_only_message).to match(/If you want to make changes, you must visit this page on the .*primary node/)
+            expect(helper.read_only_message).not_to match(/The node is currently 3 minutes behind the primary/)
+            expect(helper.read_only_message).to include(geo_primary.url)
+          end
+
+          it 'does not include a lag warning because the last event is processed' do
+            event_log = create(:geo_event_log, created_at: 3.minutes.ago)
+            create(:geo_event_log_state, event_id: event_log.id)
+
+            expect(helper.read_only_message).to match(/If you want to make changes, you must visit this page on the .*primary node/)
+            expect(helper.read_only_message).not_to match(/The node is currently 3 minutes behind the primary/)
+            expect(helper.read_only_message).to include(geo_primary.url)
+          end
+
+          it 'does not include a lag warning because there are no events yet' do
+            expect(helper.read_only_message).to match(/If you want to make changes, you must visit this page on the .*primary node/)
+            expect(helper.read_only_message).not_to match(/minutes behind the primary/)
+            expect(helper.read_only_message).to include(geo_primary.url)
+          end
+        end
       end
     end
   end

ee/spec/models/geo/event_log_spec.rb

@@ -15,6 +15,27 @@ RSpec.describe Geo::EventLog, type: :model do
     it { is_expected.to belong_to(:job_artifact_deleted_event).class_name('Geo::JobArtifactDeletedEvent').with_foreign_key('job_artifact_deleted_event_id') }
   end
 
+  describe '.next_unprocessed_event' do
+    it 'returns next unprocessed event' do
+      processed_event = create(:geo_event_log)
+      unprocessed_event = create(:geo_event_log)
+      create(:geo_event_log_state, event_id: processed_event.id)
+
+      expect(described_class.next_unprocessed_event).to eq unprocessed_event
+    end
+
+    it 'returns the oldest event when there are no processed events yet' do
+      oldest_event = create(:geo_event_log)
+      create(:geo_event_log)
+
+      expect(described_class.next_unprocessed_event).to eq oldest_event
+    end
+
+    it 'returns nil when there are no events yet' do
+      expect(described_class.next_unprocessed_event).to be_nil
+    end
+  end
+
   describe '.event_classes' do
     it 'returns all event class reflections' do
       reflections = described_class.reflections.map { |_k, v| v.class_name.constantize }

locale/gitlab.pot

@@ -5526,6 +5526,12 @@ msgstr ""
 msgid "Geo|The URL defined on the primary node that secondary nodes should use to contact it. Returns `url` if not set"
 msgstr ""
 
+msgid "Geo|The database is currently %{db_lag} behind the primary node."
+msgstr ""
+
+msgid "Geo|The node is currently %{minutes_behind} behind the primary node."
+msgstr ""
+
 msgid "Geo|This is a primary node"
 msgstr ""