Restart Action Cable server when Redis disconnects

Prevent Puma from crashing when there are connection errors to Redis.
This change catches the error and restarts the Action Cable server when
there are connection errors.

We need to restart Action Cable and disconnect all clients because once
we reconnect to Redis, all the previous SUBSCRIBE commands we made are
no longer applicable.

Changelog: fixed

config/initializers/action_cable.rb

@@ -10,6 +10,7 @@ Rails.application.configure do
 end
 
 ActionCable::SubscriptionAdapter::Base.prepend(Gitlab::Patch::ActionCableSubscriptionAdapterIdentifier)
+ActionCable::SubscriptionAdapter::Redis::Listener.prepend(Gitlab::Patch::ActionCableRedisListener)
 
 # https://github.com/rails/rails/blob/bb5ac1623e8de08c1b7b62b1368758f0d3bb6379/actioncable/lib/action_cable/subscription_adapter/redis.rb#L18
 ActionCable::SubscriptionAdapter::Redis.redis_connector = lambda do |config|

lib/gitlab/patch/action_cable_redis_listener.rb

+# frozen_string_literal: true
+
+# Modifies https://github.com/rails/rails/blob/v6.1.4.6/actioncable/lib/action_cable/subscription_adapter/redis.rb
+# so that it is resilient to Redis connection errors.
+# See https://github.com/rails/rails/issues/27659.
+
+# rubocop:disable Gitlab/ModuleWithInstanceVariables
+module Gitlab
+  module Patch
+    module ActionCableRedisListener
+      private
+
+      def ensure_listener_running
+        @thread ||= Thread.new do
+          Thread.current.abort_on_exception = true
+
+          conn = @adapter.redis_connection_for_subscriptions
+          listen conn
+        rescue ::Redis::BaseConnectionError
+          @thread = @raw_client = nil
+          ::ActionCable.server.restart
+        end
+      end
+    end
+  end
+end

spec/lib/gitlab/patch/action_cable_redis_listener_spec.rb

+# frozen_string_literal: true
+
+require 'spec_helper'
+
+RSpec.describe Gitlab::Patch::ActionCableRedisListener do
+  let(:adapter) { instance_double('ActionCable::SubscriptionAdapter::Redis') }
+  let(:connection) { instance_double('Redis') }
+  let(:listener) { ActionCable::SubscriptionAdapter::Redis::Listener.new(adapter, nil) }
+
+  before do
+    allow(Thread).to receive(:new).and_yield
+    allow(adapter).to receive(:redis_connection_for_subscriptions).and_return(connection)
+  end
+
+  it 'catches Redis connection errors and restarts Action Cable' do
+    expect(connection).to receive(:without_reconnect).and_raise Redis::ConnectionError
+    expect(ActionCable).to receive_message_chain(:server, :restart)
+
+    expect { listener.add_channel('test_channel', nil) }.not_to raise_error
+  end
+
+  it 're-raises other exceptions' do
+    expect(connection).to receive(:without_reconnect).and_raise StandardError
+    expect(ActionCable).not_to receive(:server)
+
+    expect { listener.add_channel('test_channel', nil) }.to raise_error(StandardError)
+  end
+end