Allow configuring new circuitbreaker settings from the UI and API

app/helpers/application_settings_helper.rb

@@ -121,6 +121,15 @@ module ApplicationSettingsHelper
     message.html_safe
   end
 
+  def circuitbreaker_access_retries_help_text
+    _('The number of attempts GitLab will make to access a storage.')
+  end
+
+  def circuitbreaker_backoff_threshold_help_text
+    _("The number of failures after which GitLab will start temporarily "\
+      "disabling access to a storage shard on a host")
+  end
+
   def circuitbreaker_failure_wait_time_help_text
     _("When access to a storage fails. GitLab will prevent access to the "\
       "storage for the time specified here. This allows the filesystem to "\
@@ -145,6 +154,8 @@ module ApplicationSettingsHelper
       :akismet_api_key,
       :akismet_enabled,
       :auto_devops_enabled,
+      :circuitbreaker_access_retries,
+      :circuitbreaker_backoff_threshold,
       :circuitbreaker_failure_count_threshold,
       :circuitbreaker_failure_reset_time,
       :circuitbreaker_failure_wait_time,

app/models/application_setting.rb

@@ -166,13 +166,25 @@ class ApplicationSetting < ActiveRecord::Base
             presence: true,
             numericality: { greater_than_or_equal_to: 0 }
 
-  validates :circuitbreaker_failure_count_threshold,
+  validates :circuitbreaker_backoff_threshold,
+            :circuitbreaker_failure_count_threshold,
             :circuitbreaker_failure_wait_time,
             :circuitbreaker_failure_reset_time,
             :circuitbreaker_storage_timeout,
             presence: true,
             numericality: { only_integer: true, greater_than_or_equal_to: 0 }
 
+  validates :circuitbreaker_access_retries,
+            presence: true,
+            numericality: { only_integer: true, greater_than_or_equal_to: 1 }
+
+  validates_each :circuitbreaker_backoff_threshold do |record, attr, value|
+    if value.to_i >= record.circuitbreaker_failure_count_threshold
+      record.errors.add(attr, _("The circuitbreaker backoff threshold should be "\
+                                "lower than the failure count threshold"))
+    end
+  end
+
   SUPPORTED_KEY_TYPES.each do |type|
     validates :"#{type}_key_restriction", presence: true, key_restriction: { type: type }
   end

app/views/admin/application_settings/_form.html.haml

@@ -571,11 +571,23 @@
   %fieldset
     %legend Git Storage Circuitbreaker settings
     .form-group
-      = f.label :circuitbreaker_failure_count_threshold, _('Maximum git storage failures'), class: 'control-label col-sm-2'
+      = f.label :circuitbreaker_access_retries, _('Number of access attempts'), class: 'control-label col-sm-2'
       .col-sm-10
-        = f.number_field :circuitbreaker_failure_count_threshold, class: 'form-control'
+        = f.number_field :circuitbreaker_access_retries, class: 'form-control'
         .help-block
-          = circuitbreaker_failure_count_help_text
+          = circuitbreaker_access_retries_help_text
+    .form-group
+      = f.label :circuitbreaker_storage_timeout, _('Seconds to wait for a storage access attempt'), class: 'control-label col-sm-2'
+      .col-sm-10
+        = f.number_field :circuitbreaker_storage_timeout, class: 'form-control'
+        .help-block
+          = circuitbreaker_storage_timeout_help_text
+    .form-group
+      = f.label :circuitbreaker_backoff_threshold, _('Number of failures before backing off'), class: 'control-label col-sm-2'
+      .col-sm-10
+        = f.number_field :circuitbreaker_backoff_threshold, class: 'form-control'
+        .help-block
+          = circuitbreaker_backoff_threshold_help_text
     .form-group
       = f.label :circuitbreaker_failure_wait_time, _('Seconds to wait after a storage failure'), class: 'control-label col-sm-2'
       .col-sm-10
@@ -583,17 +595,17 @@
         .help-block
           = circuitbreaker_failure_wait_time_help_text
     .form-group
-      = f.label :circuitbreaker_failure_reset_time, _('Seconds before reseting failure information'), class: 'control-label col-sm-2'
+      = f.label :circuitbreaker_failure_count_threshold, _('Maximum git storage failures'), class: 'control-label col-sm-2'
       .col-sm-10
-        = f.number_field :circuitbreaker_failure_reset_time, class: 'form-control'
+        = f.number_field :circuitbreaker_failure_count_threshold, class: 'form-control'
         .help-block
-          = circuitbreaker_failure_reset_time_help_text
+          = circuitbreaker_failure_count_help_text
     .form-group
-      = f.label :circuitbreaker_storage_timeout, _('Seconds to wait for a storage access attempt'), class: 'control-label col-sm-2'
+      = f.label :circuitbreaker_failure_reset_time, _('Seconds before reseting failure information'), class: 'control-label col-sm-2'
       .col-sm-10
-        = f.number_field :circuitbreaker_storage_timeout, class: 'form-control'
+        = f.number_field :circuitbreaker_failure_reset_time, class: 'form-control'
         .help-block
-          = circuitbreaker_storage_timeout_help_text
+          = circuitbreaker_failure_reset_time_help_text
 
   %fieldset
     %legend Repository Checks

doc/administration/repository_storage_paths.md

@@ -109,6 +109,11 @@ This can be configured from the admin interface:
 
 ![circuitbreaker configuration](img/circuitbreaker_config.png)
 
+**Number of access attempts**: The number of attempts GitLab will make to access a
+storage when probing a shard.
+
+**Number of failures before backing off**: The number of failures after which
+GitLab will start temporarily disabling access to a storage shard on a host.
 
 **Maximum git storage failures:** The number of failures of after which GitLab will
 completely prevent access to the storage. The number of failures can be reset in

doc/api/settings.md

@@ -72,6 +72,8 @@ PUT /application/settings
 | `akismet_enabled`                        | boolean          | no                                            | Enable or disable akismet spam protection                                                                                                                                                                                                                                                                                                                                                                                                                 |
 | `allow_group_owners_to_manage_ldap`      | boolean          | no                                            | Set to `true` to allow group owners to manage LDAP                                                                                                                                                                                                                                                                                                                                                                                                        |
 | `authorized_keys_enabled`                | boolean          | no                                            | By default, we write to the "authorized_keys" file to support Git over SSH without additional configuration. GitLab can be optimized to authenticate SSH keys via the database file. Only disable this if you have configured your OpenSSH server to use the AuthorizedKeysCommand.                                                                                                                                                                       |
+| `circuitbreaker_access_retries           | integer          | no                                            | The number of attempts GitLab will make to access a storage.                                                                                                                                                                                                                                                                                                                                                                                              |
+| `circuitbreaker_backoff_threshold        | integer          | no                                            | The number of failures after which GitLab will start temporarily disabling access to a storage shard on a host.                                                                                                                                                                                                                                                                                                                                                                                                                                                        |
 | `circuitbreaker_failure_count_threshold` | integer          | no                                            | The number of failures of after which GitLab will completely prevent access to the storage.                                                                                                                                                                                                                                                                                                                                                               |
 | `circuitbreaker_failure_reset_time`      | integer          | no                                            | Time in seconds GitLab will keep storage failure information. When no failures occur during this time, the failure information is reset.                                                                                                                                                                                                                                                                                                                  |
 | `circuitbreaker_failure_wait_time`       | integer          | no                                            | Time in seconds GitLab will block access to a failing storage to allow it to recover.                                                                                                                                                                                                                                                                                                                                                                     |

spec/models/application_setting_spec.rb

@@ -115,7 +115,8 @@ describe ApplicationSetting do
     end
 
     context 'circuitbreaker settings' do
-      [:circuitbreaker_failure_count_threshold,
+      [:circuitbreaker_backoff_threshold,
+       :circuitbreaker_failure_count_threshold,
        :circuitbreaker_failure_wait_time,
        :circuitbreaker_failure_reset_time,
        :circuitbreaker_storage_timeout].each do |field|
@@ -125,6 +126,16 @@ describe ApplicationSetting do
                            .is_greater_than_or_equal_to(0)
         end
       end
+
+      it 'requires the `backoff_threshold` to be lower than the `failure_count_threshold`' do
+        setting.circuitbreaker_failure_count_threshold = 10
+        setting.circuitbreaker_backoff_threshold = 15
+        failure_message = "The circuitbreaker backoff threshold should be lower "\
+                          "than the failure count threshold"
+
+        expect(setting).not_to be_valid
+        expect(setting.errors[:circuitbreaker_backoff_threshold]).to include(failure_message)
+      end
     end
 
     context 'repository storages' do