Decrease the push mirror backoff delay and add rate limiting to the push...

Decrease the push mirror backoff delay and add rate limiting to the push mirror scheduling algorithm.

changelogs/unreleased-ee/3832-decrease-push-mirror-delay.yml

+---
+title: Decrease scheduling delay and add rate limiting to push mirror.
+merge_request: 3575
+author:
+type: added

doc/workflow/repository_mirroring.md

@@ -219,7 +219,8 @@ mirror to the repository in GitLab, you are advised not to push commits directly
 to the mirrored repository. Instead, all changes will end up in the mirrored repository
 whenever commits are pushed to GitLab, or when a [forced update](#forcing-an-update) is initiated.
 
-Pushes into GitLab are automatically pushed to the remote mirror 5 minutes after they come in.
+Pushes into GitLab are automatically pushed to the remote mirror at least once every 5 minutes 
+after they come in or 1 minute if **push only protected branches** is enabled.
 
 In case of a diverged branch, you will see an error indicated at the
 **Mirror repository** settings.

ee/app/models/remote_mirror.rb

 class RemoteMirror < ActiveRecord::Base
   include AfterCommitQueue
 
-  BACKOFF_DELAY = 5.minutes
+  PROTECTED_BACKOFF_DELAY   = 1.minute
+  UNPROTECTED_BACKOFF_DELAY = 5.minutes
 
   attr_encrypted :credentials,
                  key: Gitlab::Application.secrets.db_key_base,
@@ -84,7 +85,11 @@ class RemoteMirror < ActiveRecord::Base
     return unless enabled?
     return if Gitlab::Geo.secondary?
 
-    RepositoryUpdateRemoteMirrorWorker.perform_in(BACKOFF_DELAY, self.id, Time.now)
+    if recently_scheduled?
+      RepositoryUpdateRemoteMirrorWorker.perform_in(backoff_delay, self.id, Time.now)
+    else
+      RepositoryUpdateRemoteMirrorWorker.perform_async(self.id, Time.now)
+    end
   end
 
   def enabled
@@ -139,6 +144,20 @@ class RemoteMirror < ActiveRecord::Base
 
   private
 
+  def recently_scheduled?
+    return false unless self.last_update_started_at
+
+    self.last_update_started_at >= Time.now - backoff_delay
+  end
+
+  def backoff_delay
+    if self.only_protected_branches
+      PROTECTED_BACKOFF_DELAY
+    else
+      UNPROTECTED_BACKOFF_DELAY
+    end
+  end
+
   def url_availability
     return unless project
 

spec/models/remote_mirror_spec.rb

@@ -121,14 +121,6 @@ describe RemoteMirror do
       end
     end
 
-    context 'with remote mirroring enabled' do
-      it 'schedules a RepositoryUpdateRemoteMirrorWorker to run within a certain backoff delay' do
-        expect(RepositoryUpdateRemoteMirrorWorker).to receive(:perform_in).with(RemoteMirror::BACKOFF_DELAY, remote_mirror.id, Time.now)
-
-        remote_mirror.sync
-      end
-    end
-
     context 'with remote mirroring disabled' do
       it 'returns nil' do
         remote_mirror.update_attributes(enabled: false)
@@ -144,6 +136,52 @@ describe RemoteMirror do
         expect(remote_mirror.sync).to be_nil
       end
     end
+
+    context 'with remote mirroring enabled' do
+      context 'with only protected branches enabled' do
+        context 'when it did not update in the last minute' do
+          it 'schedules a RepositoryUpdateRemoteMirrorWorker to run now' do
+            expect(RepositoryUpdateRemoteMirrorWorker).to receive(:perform_async).with(remote_mirror.id, Time.now)
+
+            remote_mirror.sync
+          end
+        end
+
+        context 'when it did update in the last minute' do
+          it 'schedules a RepositoryUpdateRemoteMirrorWorker to run in the next minute' do
+            remote_mirror.last_update_started_at = Time.now - 30.seconds
+
+            expect(RepositoryUpdateRemoteMirrorWorker).to receive(:perform_in).with(RemoteMirror::PROTECTED_BACKOFF_DELAY, remote_mirror.id, Time.now)
+
+            remote_mirror.sync
+          end
+        end
+      end
+
+      context 'with only protected branches disabled' do
+        before do
+          remote_mirror.only_protected_branches = false
+        end
+
+        context 'when it did not update in the last 5 minutes' do
+          it 'schedules a RepositoryUpdateRemoteMirrorWorker to run now' do
+            expect(RepositoryUpdateRemoteMirrorWorker).to receive(:perform_async).with(remote_mirror.id, Time.now)
+
+            remote_mirror.sync
+          end
+        end
+
+        context 'when it did update within the last 5 minutes' do
+          it 'schedules a RepositoryUpdateRemoteMirrorWorker to run in the next 5 minutes' do
+            remote_mirror.last_update_started_at = Time.now - 30.seconds
+
+            expect(RepositoryUpdateRemoteMirrorWorker).to receive(:perform_in).with(RemoteMirror::UNPROTECTED_BACKOFF_DELAY, remote_mirror.id, Time.now)
+
+            remote_mirror.sync
+          end
+        end
+      end
+    end
   end
 
   context '#updated_since?' do