[ci skip]Merge branch 'master' of gitlab.com:gitlab-org/gitlab-ee into ce_upstream

app/assets/javascripts/autosave.js

-/* eslint-disable func-names, space-before-function-paren, wrap-iife, no-param-reassign, quotes, prefer-template, no-var, one-var, no-unused-vars, one-var-declaration-per-line, no-void, consistent-return, no-empty, max-len */
+/* eslint-disable no-param-reassign, prefer-template, no-var, no-void, consistent-return */
+
 import AccessorUtilities from './lib/utils/accessor';
 
-window.Autosave = (function() {
-  function Autosave(field, key, resource) {
+export default class Autosave {
+  constructor(field, key, resource) {
     this.field = field;
     this.isLocalStorageAvailable = AccessorUtilities.isLocalStorageAccessSafe();
     this.resource = resource;
@@ -12,14 +13,10 @@ window.Autosave = (function() {
     this.key = 'autosave/' + key;
     this.field.data('autosave', this);
     this.restore();
-    this.field.on('input', (function(_this) {
-      return function() {
-        return _this.save();
-      };
-    })(this));
+    this.field.on('input', () => this.save());
   }
 
-  Autosave.prototype.restore = function() {
+  restore() {
     var text;
 
     if (!this.isLocalStorageAvailable) return;
@@ -40,9 +37,9 @@ window.Autosave = (function() {
         field.dispatchEvent(event);
       }
     }
-  };
+  }
 
-  Autosave.prototype.save = function() {
+  save() {
     var text;
     text = this.field.val();
 
@@ -51,15 +48,11 @@ window.Autosave = (function() {
     }
 
     return this.reset();
-  };
+  }
 
-  Autosave.prototype.reset = function() {
+  reset() {
     if (!this.isLocalStorageAvailable) return;
 
     return window.localStorage.removeItem(this.key);
-  };
-
-  return Autosave;
-})();
-
-export default window.Autosave;
+  }
+}

app/assets/javascripts/new_sidebar.js → app/assets/javascripts/contextual_sidebar.js

@@ -2,7 +2,7 @@ import Cookies from 'js-cookie';
 import _ from 'underscore';
 import bp from './breakpoints';
 
-export default class NewNavSidebar {
+export default class ContextualSidebar {
   constructor() {
     this.initDomElements();
     this.render();
@@ -55,7 +55,7 @@ export default class NewNavSidebar {
       this.$sidebar.toggleClass('sidebar-icons-only', collapsed);
       this.$page.toggleClass('page-with-icon-sidebar', breakpoint === 'sm' ? true : collapsed);
     }
-    NewNavSidebar.setCollapsedCookie(collapsed);
+    ContextualSidebar.setCollapsedCookie(collapsed);
 
     this.toggleSidebarOverflow();
   }

app/assets/javascripts/issuable_form.js

 /* eslint-disable func-names, space-before-function-paren, no-var, prefer-rest-params, wrap-iife, no-use-before-define, no-useless-escape, no-new, quotes, object-shorthand, no-unused-vars, comma-dangle, no-alert, consistent-return, no-else-return, prefer-template, one-var, one-var-declaration-per-line, curly, max-len */
 /* global GitLab */
-/* global Autosave */
 /* global GroupsSelect */
 
 import Pikaday from 'pikaday';
+import Autosave from './autosave';
 import UsersSelect from './users_select';
 import GfmAutoComplete from './gfm_auto_complete';
 import ZenMode from './zen_mode';

app/assets/javascripts/layout_nav.js

 /* eslint-disable func-names, space-before-function-paren, no-var, prefer-arrow-callback, no-unused-vars, one-var, one-var-declaration-per-line, vars-on-top, max-len */
 import _ from 'underscore';
 import Cookies from 'js-cookie';
-import NewNavSidebar from './new_sidebar';
+import ContextualSidebar from './contextual_sidebar';
 import initFlyOutNav from './fly_out_nav';
 
 (function() {
@@ -51,8 +51,8 @@ import initFlyOutNav from './fly_out_nav';
   });
 
   $(() => {
-    const newNavSidebar = new NewNavSidebar();
-    newNavSidebar.bindEvents();
+    const contextualSidebar = new ContextualSidebar();
+    contextualSidebar.bindEvents();
 
     initFlyOutNav();
   });

app/assets/javascripts/main.js

@@ -41,7 +41,6 @@ import './behaviors/';
 import './activities';
 import './admin';
 import './aside';
-import './autosave';
 import loadAwardsHandler from './awards_handler';
 import bp from './breakpoints';
 import './commits';

app/assets/javascripts/notes.js

@@ -5,7 +5,7 @@ default-case, prefer-template, consistent-return, no-alert, no-return-assign,
 no-param-reassign, prefer-arrow-callback, no-else-return, comma-dangle, no-new,
 brace-style, no-lonely-if, vars-on-top, no-unused-vars, no-sequences, no-shadow,
 newline-per-chained-call, no-useless-escape, class-methods-use-this */
-/* global Autosave */
+
 /* global ResolveService */
 /* global mrRefreshWidgetUrl */
 
@@ -20,7 +20,12 @@ import Flash from './flash';
 import CommentTypeToggle from './comment_type_toggle';
 import GLForm from './gl_form';
 import loadAwardsHandler from './awards_handler';
+<<<<<<< HEAD
 import './autosave';
+=======
+import Autosave from './autosave';
+import './dropzone_input';
+>>>>>>> 1c317af47772ed2862a9c1ed22ab952bb1a84874
 import TaskList from './task_list';
 import { ajaxPost, isInViewport, getPagePath, scrollToElement, isMetaKey } from './lib/utils/common_utils';
 import imageDiffHelper from './image_diff/helpers/index';

app/assets/javascripts/notes/components/issue_comment_form.vue

 <script>
-  /* global Autosave */
   import { mapActions, mapGetters } from 'vuex';
   import _ from 'underscore';
   import autosize from 'vendor/autosize';
   import Flash from '../../flash';
-  import '../../autosave';
+  import Autosave from '../../autosave';
   import TaskList from '../../task_list';
   import * as constants from '../constants';
   import eventHub from '../event_hub';

app/assets/javascripts/notes/mixins/autosave.js

-/* globals Autosave */
-import '../../autosave';
+import Autosave from '../../autosave';
 
 export default {
   methods: {

app/assets/stylesheets/framework.scss

@@ -36,7 +36,7 @@
 @import "framework/secondary-navigation-elements";
 @import "framework/selects";
 @import "framework/sidebar";
-@import "framework/new-sidebar";
+@import "framework/contextual-sidebar";
 @import "framework/tables";
 @import "framework/notes";
 @import "framework/tabs";

app/assets/stylesheets/framework/new-sidebar.scss → app/assets/stylesheets/framework/contextual-sidebar.scss

+<<<<<<< HEAD:app/assets/stylesheets/framework/new-sidebar.scss
 @import "framework/variables";
 @import 'framework/tw_bootstrap_variables';
 @import "bootstrap/variables";
@@ -13,12 +14,15 @@ $new-sidebar-width: 220px;
 $new-sidebar-collapsed-width: 50px;
 
 .page-with-new-sidebar {
+=======
+.page-with-contextual-sidebar {
+>>>>>>> 1c317af47772ed2862a9c1ed22ab952bb1a84874:app/assets/stylesheets/framework/contextual-sidebar.scss
   @media (min-width: $screen-md-min) {
-    padding-left: $new-sidebar-collapsed-width;
+    padding-left: $contextual-sidebar-collapsed-width;
   }
 
   @media (min-width: $screen-lg-min) {
-    padding-left: $new-sidebar-width;
+    padding-left: $contextual-sidebar-width;
   }
 
   // Override position: absolute
@@ -34,7 +38,7 @@ $new-sidebar-collapsed-width: 50px;
 
 .page-with-icon-sidebar {
   @media (min-width: $screen-sm-min) {
-    padding-left: $new-sidebar-collapsed-width;
+    padding-left: $contextual-sidebar-collapsed-width;
   }
 }
 
@@ -52,12 +56,12 @@ $new-sidebar-collapsed-width: 50px;
 
   &:hover,
   a:hover {
-    background-color: $hover-background;
-    color: $hover-color;
+    background-color: $link-hover-background;
+    color: $gl-text-color;
 
     .settings-avatar {
       svg {
-        fill: $hover-color;
+        fill: $gl-text-color;
       }
     }
   }
@@ -85,7 +89,7 @@ $new-sidebar-collapsed-width: 50px;
 .nav-sidebar {
   position: fixed;
   z-index: 400;
-  width: $new-sidebar-width;
+  width: $contextual-sidebar-width;
   transition: left $sidebar-transition-duration;
   top: $header-height;
   bottom: 0;
@@ -103,7 +107,7 @@ $new-sidebar-collapsed-width: 50px;
 
   &.sidebar-icons-only {
     width: auto;
-    min-width: $new-sidebar-collapsed-width;
+    min-width: $contextual-sidebar-collapsed-width;
 
     .nav-sidebar-inner-scroll {
       overflow-x: hidden;
@@ -149,11 +153,11 @@ $new-sidebar-collapsed-width: 50px;
       display: flex;
       align-items: center;
       padding: 12px 16px;
-      color: $inactive-color;
+      color: $gl-text-color-secondary;
     }
 
     svg {
-      fill: $inactive-color;
+      fill: $gl-text-color-secondary;
     }
   }
 
@@ -168,7 +172,7 @@ $new-sidebar-collapsed-width: 50px;
   }
 
   @media (max-width: $screen-xs-max) {
-    left: (-$new-sidebar-width);
+    left: (-$contextual-sidebar-width);
   }
 
   .nav-icon-container {
@@ -210,8 +214,8 @@ $new-sidebar-collapsed-width: 50px;
 
       &:hover,
       &:focus {
-        background: $active-hover-background;
-        color: $active-hover-color;
+        background: $link-active-background;
+        color: $gl-text-color;
       }
     }
 
@@ -220,7 +224,7 @@ $new-sidebar-collapsed-width: 50px;
         &,
         &:hover,
         &:focus {
-          background: $active-background;
+          background: $link-active-background;
         }
       }
     }
@@ -308,11 +312,11 @@ $new-sidebar-collapsed-width: 50px;
 
     .badge {
       background-color: $inactive-badge-background;
-      color: $inactive-color;
+      color: $gl-text-color-secondary;
     }
 
     &.active {
-      background: $active-background;
+      background: $link-active-background;
 
       > a {
         margin-left: 4px;
@@ -330,7 +334,11 @@ $new-sidebar-collapsed-width: 50px;
 
     &.active > a:hover,
     &.is-over > a {
+<<<<<<< HEAD:app/assets/stylesheets/framework/new-sidebar.scss
       background-color: $hover-background;
+=======
+      background-color: $link-hover-background;
+>>>>>>> 1c317af47772ed2862a9c1ed22ab952bb1a84874:app/assets/stylesheets/framework/contextual-sidebar.scss
     }
   }
 }
@@ -340,7 +348,7 @@ $new-sidebar-collapsed-width: 50px;
 
 .toggle-sidebar-button,
 .close-nav-button {
-  width: $new-sidebar-width - 2px;
+  width: $contextual-sidebar-width - 2px;
   position: fixed;
   bottom: 0;
   padding: 16px;
@@ -407,7 +415,7 @@ $new-sidebar-collapsed-width: 50px;
   }
 
   .toggle-sidebar-button {
-    width: $new-sidebar-collapsed-width - 2px;
+    width: $contextual-sidebar-collapsed-width - 2px;
     padding: 16px;
 
     .collapse-text,

app/assets/stylesheets/framework/variables.scss

@@ -9,6 +9,8 @@ $sidebar-transition-duration: .15s;
 $sidebar-breakpoint: 1024px;
 $default-transition-duration: .15s;
 $right-sidebar-transition-duration: .3s;
+$contextual-sidebar-width: 220px;
+$contextual-sidebar-collapsed-width: 50px;
 
 /*
  * Color schema
@@ -365,6 +367,13 @@ $dropdown-item-hover-bg: $gray-darker;
 $filtered-search-term-shadow-color: rgba(0, 0, 0, 0.09);
 $dropdown-hover-color: $blue-400;
 
+/*
+* Contextual Sidebar
+*/
+$link-active-background: rgba(0, 0, 0, .04);
+$link-hover-background: rgba(0, 0, 0, .06);
+$inactive-badge-background: rgba(0, 0, 0, .08);
+
 /*
 * Buttons
 */
@@ -411,7 +420,6 @@ $note-targe3-inside: #ffffd3;
 $note-line2-border: #ddd;
 $note-icon-gutter-width: 55px;
 
-
 /*
 * Zen
 */

app/assets/stylesheets/pages/boards.scss

@@ -476,7 +476,7 @@
   border-top: 1px solid $border-color;
 }
 
-.page-with-new-sidebar.page-with-sidebar .issue-boards-sidebar {
+.page-with-contextual-sidebar.page-with-sidebar .issue-boards-sidebar {
   .issuable-sidebar-header {
     position: relative;
   }

app/helpers/application_settings_helper.rb

@@ -121,6 +121,15 @@ module ApplicationSettingsHelper
     message.html_safe
   end
 
+  def circuitbreaker_access_retries_help_text
+    _('The number of attempts GitLab will make to access a storage.')
+  end
+
+  def circuitbreaker_backoff_threshold_help_text
+    _("The number of failures after which GitLab will start temporarily "\
+      "disabling access to a storage shard on a host")
+  end
+
   def circuitbreaker_failure_wait_time_help_text
     _("When access to a storage fails. GitLab will prevent access to the "\
       "storage for the time specified here. This allows the filesystem to "\
@@ -145,6 +154,8 @@ module ApplicationSettingsHelper
       :akismet_api_key,
       :akismet_enabled,
       :auto_devops_enabled,
+      :circuitbreaker_access_retries,
+      :circuitbreaker_backoff_threshold,
       :circuitbreaker_failure_count_threshold,
       :circuitbreaker_failure_reset_time,
       :circuitbreaker_failure_wait_time,

app/helpers/nav_helper.rb

 module NavHelper
   def page_with_sidebar_class
     class_name = page_gutter_class
-    class_name << 'page-with-new-sidebar' if defined?(@left_sidebar) && @left_sidebar
+    class_name << 'page-with-contextual-sidebar' if defined?(@left_sidebar) && @left_sidebar
     class_name << 'page-with-icon-sidebar' if collapsed_sidebar? && @left_sidebar
 
     class_name

app/helpers/storage_health_helper.rb

@@ -16,17 +16,16 @@ module StorageHealthHelper
   def message_for_circuit_breaker(circuit_breaker)
     maximum_failures = circuit_breaker.failure_count_threshold
     current_failures = circuit_breaker.failure_count
-    permanently_broken = circuit_breaker.circuit_broken? && current_failures >= maximum_failures
 
     translation_params = { number_of_failures: current_failures,
                            maximum_failures: maximum_failures,
                            number_of_seconds: circuit_breaker.failure_wait_time }
 
-    if permanently_broken
+    if circuit_breaker.circuit_broken?
       s_("%{number_of_failures} of %{maximum_failures} failures. GitLab will not "\
          "retry automatically. Reset storage information when the problem is "\
          "resolved.") % translation_params
-    elsif circuit_breaker.circuit_broken?
+    elsif circuit_breaker.backing_off?
       _("%{number_of_failures} of %{maximum_failures} failures. GitLab will "\
         "block access for %{number_of_seconds} seconds.") % translation_params
     else

app/models/application_setting.rb

@@ -166,13 +166,25 @@ class ApplicationSetting < ActiveRecord::Base
             presence: true,
             numericality: { greater_than_or_equal_to: 0 }
 
-  validates :circuitbreaker_failure_count_threshold,
+  validates :circuitbreaker_backoff_threshold,
+            :circuitbreaker_failure_count_threshold,
             :circuitbreaker_failure_wait_time,
             :circuitbreaker_failure_reset_time,
             :circuitbreaker_storage_timeout,
             presence: true,
             numericality: { only_integer: true, greater_than_or_equal_to: 0 }
 
+  validates :circuitbreaker_access_retries,
+            presence: true,
+            numericality: { only_integer: true, greater_than_or_equal_to: 1 }
+
+  validates_each :circuitbreaker_backoff_threshold do |record, attr, value|
+    if value.to_i >= record.circuitbreaker_failure_count_threshold
+      record.errors.add(attr, _("The circuitbreaker backoff threshold should be "\
+                                "lower than the failure count threshold"))
+    end
+  end
+
   SUPPORTED_KEY_TYPES.each do |type|
     validates :"#{type}_key_restriction", presence: true, key_restriction: { type: type }
   end

app/views/admin/application_settings/_form.html.haml

@@ -571,11 +571,23 @@
   %fieldset
     %legend Git Storage Circuitbreaker settings
     .form-group
-      = f.label :circuitbreaker_failure_count_threshold, _('Maximum git storage failures'), class: 'control-label col-sm-2'
+      = f.label :circuitbreaker_access_retries, _('Number of access attempts'), class: 'control-label col-sm-2'
       .col-sm-10
-        = f.number_field :circuitbreaker_failure_count_threshold, class: 'form-control'
+        = f.number_field :circuitbreaker_access_retries, class: 'form-control'
         .help-block
-          = circuitbreaker_failure_count_help_text
+          = circuitbreaker_access_retries_help_text
+    .form-group
+      = f.label :circuitbreaker_storage_timeout, _('Seconds to wait for a storage access attempt'), class: 'control-label col-sm-2'
+      .col-sm-10
+        = f.number_field :circuitbreaker_storage_timeout, class: 'form-control'
+        .help-block
+          = circuitbreaker_storage_timeout_help_text
+    .form-group
+      = f.label :circuitbreaker_backoff_threshold, _('Number of failures before backing off'), class: 'control-label col-sm-2'
+      .col-sm-10
+        = f.number_field :circuitbreaker_backoff_threshold, class: 'form-control'
+        .help-block
+          = circuitbreaker_backoff_threshold_help_text
     .form-group
       = f.label :circuitbreaker_failure_wait_time, _('Seconds to wait after a storage failure'), class: 'control-label col-sm-2'
       .col-sm-10
@@ -583,17 +595,17 @@
         .help-block
           = circuitbreaker_failure_wait_time_help_text
     .form-group
-      = f.label :circuitbreaker_failure_reset_time, _('Seconds before reseting failure information'), class: 'control-label col-sm-2'
+      = f.label :circuitbreaker_failure_count_threshold, _('Maximum git storage failures'), class: 'control-label col-sm-2'
       .col-sm-10
-        = f.number_field :circuitbreaker_failure_reset_time, class: 'form-control'
+        = f.number_field :circuitbreaker_failure_count_threshold, class: 'form-control'
         .help-block
-          = circuitbreaker_failure_reset_time_help_text
+          = circuitbreaker_failure_count_help_text
     .form-group
-      = f.label :circuitbreaker_storage_timeout, _('Seconds to wait for a storage access attempt'), class: 'control-label col-sm-2'
+      = f.label :circuitbreaker_failure_reset_time, _('Seconds before reseting failure information'), class: 'control-label col-sm-2'
       .col-sm-10
-        = f.number_field :circuitbreaker_storage_timeout, class: 'form-control'
+        = f.number_field :circuitbreaker_failure_reset_time, class: 'form-control'
         .help-block
-          = circuitbreaker_storage_timeout_help_text
+          = circuitbreaker_failure_reset_time_help_text
 
   %fieldset
     %legend Repository Checks

changelogs/unreleased/bvl-circuitbreaker-backoff.yml

+---
+title: Make the circuitbreaker more robust by adding higher thresholds, and multiple
+  access attempts.
+merge_request: 14933
+author:
+type: fixed

db/migrate/20171017145932_add_new_circuitbreaker_settings_to_application_settings.rb

+class AddNewCircuitbreakerSettingsToApplicationSettings < ActiveRecord::Migration
+  include Gitlab::Database::MigrationHelpers
+
+  DOWNTIME = false
+
+  def change
+    add_column :application_settings,
+               :circuitbreaker_access_retries,
+               :integer,
+               default: 3
+    add_column :application_settings,
+               :circuitbreaker_backoff_threshold,
+               :integer,
+               default: 80
+  end
+end

db/schema.rb

@@ -11,7 +11,7 @@
 #
 # It's strongly recommended that you check this file into your version control system.
 
-ActiveRecord::Schema.define(version: 20171017130239) do
+ActiveRecord::Schema.define(version: 20171017145932) do
 
   # These are extensions that must be enabled in order to support this database
   enable_extension "plpgsql"
@@ -162,6 +162,8 @@ ActiveRecord::Schema.define(version: 20171017130239) do
     t.integer "circuitbreaker_failure_reset_time", default: 1800
     t.integer "circuitbreaker_storage_timeout", default: 30
     t.boolean "remote_mirror_available", default: true, null: false
+    t.integer "circuitbreaker_access_retries", default: 3
+    t.integer "circuitbreaker_backoff_threshold", default: 80
   end
 
   create_table "approvals", force: :cascade do |t|

doc/administration/high_availability/alpha_database.md

@@ -376,10 +376,6 @@ Select one node as a primary node.
     CREATE EXTENSION pg_trgm;
     ```
 
-    # Output:
-
-    CREATE EXTENSION
-
 1. Exit the database prompt by typing `\q` and Enter.
 1. Verify the cluster is initialized with one node:
 

doc/administration/repository_storage_paths.md

@@ -109,6 +109,11 @@ This can be configured from the admin interface:
 
 ![circuitbreaker configuration](img/circuitbreaker_config.png)
 
+**Number of access attempts**: The number of attempts GitLab will make to access a
+storage when probing a shard.
+
+**Number of failures before backing off**: The number of failures after which
+GitLab will start temporarily disabling access to a storage shard on a host.
 
 **Maximum git storage failures:** The number of failures of after which GitLab will
 completely prevent access to the storage. The number of failures can be reset in
@@ -126,6 +131,15 @@ mount is reset.
 **Seconds to wait for a storage access attempt:** The time in seconds GitLab will
 try to access storage. After this time a timeout error will be raised.
 
+To enable the circuitbreaker for repository storage you can flip the feature flag from a rails console:
+
+```
+Feature.enable('git_storage_circuit_breaker')
+```
+
+Alternatively it can be enabled by setting `true` in the `GIT_STORAGE_CIRCUIT_BREAKER` environment variable.
+This approach would be used when enabling the circuit breaker on a single host.
+
 When storage failures occur, this will be visible in the admin interface like this:
 
 ![failing storage](img/failing_storage.png)

doc/api/settings.md

@@ -72,6 +72,8 @@ PUT /application/settings
 | `akismet_enabled`                        | boolean          | no                                            | Enable or disable akismet spam protection                                                                                                                                                                                                                                                                                                                                                                                                                 |
 | `allow_group_owners_to_manage_ldap`      | boolean          | no                                            | Set to `true` to allow group owners to manage LDAP                                                                                                                                                                                                                                                                                                                                                                                                        |
 | `authorized_keys_enabled`                | boolean          | no                                            | By default, we write to the "authorized_keys" file to support Git over SSH without additional configuration. GitLab can be optimized to authenticate SSH keys via the database file. Only disable this if you have configured your OpenSSH server to use the AuthorizedKeysCommand.                                                                                                                                                                       |
+| `circuitbreaker_access_retries           | integer          | no                                            | The number of attempts GitLab will make to access a storage.                                                                                                                                                                                                                                                                                                                                                                                              |
+| `circuitbreaker_backoff_threshold        | integer          | no                                            | The number of failures after which GitLab will start temporarily disabling access to a storage shard on a host.                                                                                                                                                                                                                                                                                                                                                                                                                                                        |
 | `circuitbreaker_failure_count_threshold` | integer          | no                                            | The number of failures of after which GitLab will completely prevent access to the storage.                                                                                                                                                                                                                                                                                                                                                               |
 | `circuitbreaker_failure_reset_time`      | integer          | no                                            | Time in seconds GitLab will keep storage failure information. When no failures occur during this time, the failure information is reset.                                                                                                                                                                                                                                                                                                                  |
 | `circuitbreaker_failure_wait_time`       | integer          | no                                            | Time in seconds GitLab will block access to a failing storage to allow it to recover.                                                                                                                                                                                                                                                                                                                                                                     |

doc/gitlab-geo/configuration.md

@@ -190,7 +190,11 @@ used for all GitLab Geo installations.
    you created previously. **Do NOT** check the box 'This is a primary node'.
 1. Added in GitLab 9.5: Choose which namespaces should be replicated by the secondary node. Leave blank to replicate all. Read more in [selective replication](#selective-replication).
 1. Click the **Add node** button.
+1. Restart GitLab on the secondary:
 
+    ```
+    gitlab-ctl restart
+    ```
 ---
 
 After the **Add Node** button is pressed, the primary node will start to notify

doc/gitlab-geo/database.md

@@ -186,41 +186,16 @@ The following guide assumes that:
     geo_secondary_role['enable'] = true
     ```
 
-1. Optional since GitLab 9.1, and required for GitLab 10.0 or higher:
-   [Enable tracking database on the secondary server](#enable-tracking-database-on-the-secondary-server)
-
-1. Otherwise, continue to [initiate the replication process](#step-3-initiate-the-replication-process).
-
-#### Enable tracking database on the secondary server
-
-Geo secondary nodes use a tracking database to keep track of replication status and recover
-automatically from some replication issues.
-
-It is added in GitLab 9.1, and since GitLab 10.0 it is required.
-
-> **IMPORTANT:** For this feature to work correctly, all nodes must be
-with their clocks synchronized. It is not required for all nodes to be set to
-the same time zone, but when the respective times are converted to UTC time,
-the clocks must be synchronized to within 60 seconds of each other.
+1. [Reconfigure GitLab][] for the changes to take effect.
 
 1. Setup clock synchronization service in your Linux distro.
    This can easily be done via any NTP-compatible daemon. For example,
    here are [instructions for setting up NTP with Ubuntu](https://help.ubuntu.com/lts/serverguide/NTP.html).
 
-1. Edit `/etc/gitlab/gitlab.rb` and add the following:
-
-    ```ruby
-    geo_postgresql['enable'] = true
-    ```
-
-1. Set up the Geo tracking database:
-
-    ```
-    sudo gitlab-rake geo:db:migrate
-    ```
-
-1. [Reconfigure GitLab][] for the changes to take effect.
-1. Continue to [initiate the replication process](#step-3-initiate-the-replication-process).
+    **IMPORTANT:** For Geo to work correctly, all nodes must be with their
+    clocks synchronized. It is not required for all nodes to be set to the
+    same time zone, but when the respective times are converted to UTC time,
+    the clocks must be synchronized to within 60 seconds of each other.
 
 ### Step 3. Initiate the replication process
 

doc/user/project/milestones/burndown_charts.md

@@ -40,6 +40,8 @@ it was taken care of closely throughout the whole quarter
 
 ## How it works
 
+>**Note:** Burndown charts are only available for project milestones. They will be available for group milestones [in the future](https://gitlab.com/gitlab-org/gitlab-ee/issues/3064).
+
 A Burndown Chart is available for every project milestone that has been attributed a **start
 date** and a **due date**.
 

lib/gitlab/git/storage.rb

@@ -12,6 +12,7 @@ module Gitlab
 
       CircuitOpen = Class.new(Inaccessible)
       Misconfiguration = Class.new(Inaccessible)
+      Failing = Class.new(Inaccessible)
 
       REDIS_KEY_PREFIX = 'storage_accessible:'.freeze
 

lib/gitlab/git/storage/circuit_breaker.rb

@@ -54,7 +54,7 @@ module Gitlab
         end
 
         def perform
-          return yield unless Feature.enabled?('git_storage_circuit_breaker')
+          return yield unless enabled?
 
           check_storage_accessible!
 
@@ -64,10 +64,27 @@ module Gitlab
         def circuit_broken?
           return false if no_failures?
 
+          failure_count > failure_count_threshold
+        end
+
+        def backing_off?
+          return false if no_failures?
+
           recent_failure = last_failure > failure_wait_time.seconds.ago
-          too_many_failures = failure_count > failure_count_threshold
+          too_many_failures = failure_count > backoff_threshold
 
-          recent_failure || too_many_failures
+          recent_failure && too_many_failures
+        end
+
+        private
+
+        # The circuitbreaker can be enabled for the entire fleet using a Feature
+        # flag.
+        #
+        # Enabling it for a single host can be done setting the
+        # `GIT_STORAGE_CIRCUIT_BREAKER` environment variable.
+        def enabled?
+          ENV['GIT_STORAGE_CIRCUIT_BREAKER'].present? || Feature.enabled?('git_storage_circuit_breaker')
         end
 
         def failure_info
@@ -83,7 +100,7 @@ module Gitlab
           return @storage_available if @storage_available
 
           if @storage_available = Gitlab::Git::Storage::ForkedStorageCheck
-                                    .storage_available?(storage_path, storage_timeout)
+                                    .storage_available?(storage_path, storage_timeout, access_retries)
             track_storage_accessible
           else
             track_storage_inaccessible
@@ -94,7 +111,11 @@ module Gitlab
 
         def check_storage_accessible!
           if circuit_broken?
-            raise Gitlab::Git::Storage::CircuitOpen.new("Circuit for #{storage} is broken", failure_wait_time)
+            raise Gitlab::Git::Storage::CircuitOpen.new("Circuit for #{storage} is broken", failure_reset_time)
+          end
+
+          if backing_off?
+            raise Gitlab::Git::Storage::Failing.new("Backing off access to #{storage}", failure_wait_time)
           end
 
           unless storage_available?
@@ -131,12 +152,6 @@ module Gitlab
           end
         end
 
-        def cache_key
-          @cache_key ||= "#{Gitlab::Git::Storage::REDIS_KEY_PREFIX}#{storage}:#{hostname}"
-        end
-
-        private
-
         def get_failure_info
           last_failure, failure_count = Gitlab::Git::Storage.redis.with do |redis|
             redis.hmget(cache_key, :last_failure, :failure_count)
@@ -146,6 +161,10 @@ module Gitlab
 
           FailureInfo.new(last_failure, failure_count.to_i)
         end
+
+        def cache_key
+          @cache_key ||= "#{Gitlab::Git::Storage::REDIS_KEY_PREFIX}#{storage}:#{hostname}"
+        end
       end
     end
   end

lib/gitlab/git/storage/circuit_breaker_settings.rb

@@ -18,6 +18,14 @@ module Gitlab
           application_settings.circuitbreaker_storage_timeout
         end
 
+        def access_retries
+          application_settings.circuitbreaker_access_retries
+        end
+
+        def backoff_threshold
+          application_settings.circuitbreaker_backoff_threshold
+        end
+
         private
 
         def application_settings

lib/gitlab/git/storage/forked_storage_check.rb

@@ -4,8 +4,17 @@ module Gitlab
       module ForkedStorageCheck
         extend self
 
-        def storage_available?(path, timeout_seconds = 5)
-          status = timeout_check(path, timeout_seconds)
+        def storage_available?(path, timeout_seconds = 5, retries = 1)
+          partial_timeout = timeout_seconds / retries
+          status = timeout_check(path, partial_timeout)
+
+          # If the status check did not succeed the first time, we retry a few
+          # more times to avoid one-off failures
+          current_attempts = 1
+          while current_attempts < retries && !status.success?
+            status = timeout_check(path, partial_timeout)
+            current_attempts += 1
+          end
 
           status.success?
         end

lib/gitlab/git/storage/null_circuit_breaker.rb

@@ -25,6 +25,10 @@ module Gitlab
           !!@error
         end
 
+        def backing_off?
+          false
+        end
+
         def last_failure
           circuit_broken? ? Time.now : nil
         end

spec/lib/gitlab/git/storage/forked_storage_check_spec.rb

@@ -33,6 +33,21 @@ describe Gitlab::Git::Storage::ForkedStorageCheck, broken_storage: true, skip_da
       expect(runtime).to be < 1.0
     end
 
+    it 'will try the specified amount of times before failing'  do
+      allow(described_class).to receive(:check_filesystem_in_process) do
+        Process.spawn("sleep 10")
+      end
+
+      expect(Process).to receive(:spawn).with('sleep 10').twice
+                           .and_call_original
+
+      runtime = Benchmark.realtime do
+        described_class.storage_available?(existing_path, 0.5, 2)
+      end
+
+      expect(runtime).to be < 1.0
+    end
+
     describe 'when using paths with spaces' do
       let(:test_dir) { Rails.root.join('tmp', 'tests', 'storage_check') }
       let(:path_with_spaces) { File.join(test_dir, 'path with spaces') }

spec/lib/gitlab/git/storage/null_circuit_breaker_spec.rb

@@ -65,17 +65,6 @@ describe Gitlab::Git::Storage::NullCircuitBreaker do
     ours = described_class.public_instance_methods
     theirs = Gitlab::Git::Storage::CircuitBreaker.public_instance_methods
 
-    # These methods are not part of the public API, but are public to allow the
-    # CircuitBreaker specs to operate. They should be made private over time.
-    exceptions = %i[
-      cache_key
-      check_storage_accessible!
-      no_failures?
-      storage_available?
-      track_storage_accessible
-      track_storage_inaccessible
-    ]
-
-    expect(theirs - ours).to contain_exactly(*exceptions)
+    expect(theirs - ours).to be_empty
   end
 end

spec/models/application_setting_spec.rb

@@ -115,7 +115,8 @@ describe ApplicationSetting do
     end
 
     context 'circuitbreaker settings' do
-      [:circuitbreaker_failure_count_threshold,
+      [:circuitbreaker_backoff_threshold,
+       :circuitbreaker_failure_count_threshold,
        :circuitbreaker_failure_wait_time,
        :circuitbreaker_failure_reset_time,
        :circuitbreaker_storage_timeout].each do |field|
@@ -125,6 +126,16 @@ describe ApplicationSetting do
                            .is_greater_than_or_equal_to(0)
         end
       end
+
+      it 'requires the `backoff_threshold` to be lower than the `failure_count_threshold`' do
+        setting.circuitbreaker_failure_count_threshold = 10
+        setting.circuitbreaker_backoff_threshold = 15
+        failure_message = "The circuitbreaker backoff threshold should be lower "\
+                          "than the failure count threshold"
+
+        expect(setting).not_to be_valid
+        expect(setting.errors[:circuitbreaker_backoff_threshold]).to include(failure_message)
+      end
     end
 
     context 'repository storages' do