Fix authorized_keys file if needed

GITLAB_SHELL_VERSION

-5.0.6
+5.1.0

app/workers/gitlab_shell_worker.rb

@@ -4,6 +4,19 @@ class GitlabShellWorker
   include DedicatedSidekiqQueue
 
   def perform(action, *arg)
-    gitlab_shell.send(action, *arg)
+    if action.to_s == 'batch_add_keys_in_db_starting_from'
+      batch_add_keys_in_db_starting_from(arg.first)
+    else
+      gitlab_shell.send(action, *arg)
+    end
+  end
+
+  # Not added to Gitlab::Shell because I don't expect this to be used again
+  def batch_add_keys_in_db_starting_from(start_id)
+    gitlab_shell.batch_add_keys do |adder|
+      Key.find_each(start: start_id, batch_size: 1000) do |key|
+        adder.add_key(key.shell_id, key.key)
+      end
+    end
   end
 end

db/migrate/20170626202753_update_authorized_keys_file.rb

+# See http://doc.gitlab.com/ce/development/migration_style_guide.html
+# for more information on how to write migrations for GitLab.
+
+class UpdateAuthorizedKeysFile < ActiveRecord::Migration
+  include Gitlab::Database::MigrationHelpers
+
+  class ApplicationSetting < ActiveRecord::Base; end
+  class Key < ActiveRecord::Base; end
+
+  # Set this constant to true if this migration requires downtime.
+  DOWNTIME = false
+
+  # We started deploying 9.3.0 to GitLab.com at 2017-06-22T17:24:00+00:00.
+  # TODO look at the time we pushed to the package server
+  DATETIME_9_3_0_RELEASED = DateTime.parse('2017-06-22T17:24:00+00:00')
+
+  def up
+    if authorized_keys_file_in_use_and_stale?
+      # Update nil authorized_keys_enabled to true to ensure that Gitlab::Shell
+      # key methods work properly for workers running 9.3.0 during the
+      # migration. If the setting remained nil, the workers would not edit the
+      # file.
+      update_nil_setting_to_true
+
+      update_authorized_keys_file_since(DATETIME_9_3_0_RELEASED)
+    end
+  end
+
+  def down
+    # Do nothing
+  end
+
+  def authorized_keys_file_in_use_and_stale?
+    return false unless ran_broken_migration?
+
+    uncached_setting = ApplicationSetting.last
+
+    # If there is no ApplicationSetting record in the DB, then the instance was
+    # never in a state where `authorized_keys_enabled` field was `nil`. So the
+    # file is not stale.
+    return false unless uncached_setting
+
+    if uncached_setting.authorized_keys_enabled == false # not falsey!
+      # If authorized_keys_enabled is explicitly false, then the file is not in
+      # use, so it doesn't need to be fixed. I.e. GitLab.com.
+      #
+      # Unfortunately it is possible some users may have saved Application
+      # Settings without realizing the new option existed, and since it
+      # mistakenly defaulted to unchecked, now it is explicitly false. These
+      # users need this warning.
+      say false_negative_warning
+      return false
+    end
+
+    # If authorized_keys_enabled is true or nil, then we need to rebuild the
+    # file in case it is stale.
+    true
+  end
+
+  def ran_broken_migration?
+    # If the column is already fixed, then the migration wasn't run before now.
+    if Gitlab::Database.postgresql?
+      !column_exists?(:application_settings, :authorized_keys_enabled, :boolean, default: 'true', null: false)
+    else
+      !column_exists?(:application_settings, :authorized_keys_enabled, :boolean, default: '1', null: false)
+    end
+  end
+
+  def false_negative_warning
+    <<-MSG.strip_heredoc
+      WARNING
+
+      If you did not intentionally disable the "Write to authorized_keys file"
+      option in Application Settings as outlined in the Speed up SSH
+      documentation,
+
+      https://docs.gitlab.com/ee/administration/operations/speed_up_ssh.html
+
+      then the authorized_keys file may be out-of-date, affecting SSH
+      operations.
+
+      If you are affected, please check the "Write to authorized_keys file"
+      checkbox, and Save. Then rebuild the authorized_keys file as shown here:
+
+      https://docs.gitlab.com/ee/administration/raketasks/maintenance.html#rebuild-authorized_keys-file
+
+      For more information, see the issue:
+
+      https://gitlab.com/gitlab-org/gitlab-ee/issues/2738
+    MSG
+  end
+
+  def update_nil_setting_to_true
+    setting = ApplicationSetting.last # guaranteed to be found since authorized_keys_file_in_use_and_stale? is true
+    setting.update!(authorized_keys_enabled: true) if setting.authorized_keys_enabled.nil?
+  end
+
+  def update_authorized_keys_file_since(cutoff_datetime)
+    add_keys_since(cutoff_datetime)
+
+    remove_keys_not_found_in_db
+  end
+
+  def add_keys_since(cutoff_datetime)
+    start_key = Key.where("created_at >= ?", cutoff_datetime).first
+    if start_key
+      GitlabShellWorker.perform_async(:batch_add_keys_in_db_starting_from, start_key.id)
+    end
+  end
+
+  def remove_keys_not_found_in_db
+    GitlabShellWorker.perform_async(:remove_keys_not_found_in_db)
+  end
+end

db/schema.rb

@@ -11,7 +11,7 @@
 #
 # It's strongly recommended that you check this file into your version control system.
 
-ActiveRecord::Schema.define(version: 20170622162730) do
+ActiveRecord::Schema.define(version: 20170626202753) do
 
   # These are extensions that must be enabled in order to support this database
   enable_extension "plpgsql"

lib/gitlab/shell.rb

@@ -220,11 +220,12 @@ module Gitlab
     # Ex.
     #   remove_key("key-342", "sha-rsa ...")
     #
-    def remove_key(key_id, key_content)
+    def remove_key(key_id, key_content = nil)
       return unless self.authorized_keys_enabled?
 
-      Gitlab::Utils.system_silent([gitlab_shell_keys_path,
-                                   'rm-key', key_id, key_content])
+      args = [gitlab_shell_keys_path, 'rm-key', key_id]
+      args << key_content if key_content
+      Gitlab::Utils.system_silent(args)
     end
 
     # Remove all ssh keys from gitlab shell
@@ -238,6 +239,54 @@ module Gitlab
       Gitlab::Utils.system_silent([gitlab_shell_keys_path, 'clear'])
     end
 
+    # Remove ssh keys from gitlab shell that are not in the DB
+    #
+    # Ex.
+    #   remove_keys_not_found_in_db
+    #
+    def remove_keys_not_found_in_db
+      return unless self.authorized_keys_enabled?
+
+      batch_read_key_ids do |ids_in_file|
+        keys_in_db = Key.where(id: ids_in_file)
+        if ids_in_file.size > keys_in_db.count
+          ids_to_remove = ids_in_file - keys_in_db.pluck(:id)
+          ids_to_remove.each do |id|
+            remove_key("key-#{id}")
+          end
+        end
+      end
+    end
+
+    # Iterate over all ssh key IDs from gitlab shell, in batches
+    #
+    # Ex.
+    #   batch_read_key_ids { |batch| keys = Key.where(id: batch) }
+    #
+    def batch_read_key_ids(batch_size: 100, &block)
+      return unless self.authorized_keys_enabled?
+
+      list_key_ids do |key_id_stream|
+        key_id_stream.lazy.each_slice(batch_size) do |lines|
+          key_ids = lines.map { |l| l.chomp.to_i }
+          yield(key_ids)
+        end
+      end
+    end
+
+    # Stream all ssh key IDs from gitlab shell, separated by newlines
+    #
+    # Ex.
+    #   list_key_ids
+    #
+    def list_key_ids(&block)
+      return unless self.authorized_keys_enabled?
+
+      IO.popen(%W(#{gitlab_shell_path}/bin/gitlab-keys list-key-ids)) do |key_id_stream|
+        yield(key_id_stream)
+      end
+    end
+
     # Add empty directory for storing repositories
     #
     # Ex.

spec/lib/gitlab/shell_spec.rb

@@ -177,6 +177,17 @@ describe Gitlab::Shell, lib: true do
         gitlab_shell.remove_key('key-123', 'ssh-rsa foobar')
       end
     end
+
+    context 'when key content is not given' do
+      it 'calls rm-key with only one argument' do
+        allow(gitlab_shell).to receive(:gitlab_shell_keys_path).and_return(:gitlab_shell_keys_path)
+        expect(Gitlab::Utils).to receive(:system_silent).with(
+          [:gitlab_shell_keys_path, 'rm-key', 'key-123']
+        )
+
+        gitlab_shell.remove_key('key-123')
+      end
+    end
   end
 
   describe '#remove_all_keys' do
@@ -202,6 +213,89 @@ describe Gitlab::Shell, lib: true do
     end
   end
 
+  describe '#remove_keys_not_found_in_db' do
+    context 'when keys are in the file that are not in the DB' do
+      before do
+        gitlab_shell.remove_all_keys
+        gitlab_shell.add_key('key-1234', 'ssh-rsa ASDFASDF')
+        gitlab_shell.add_key('key-9876', 'ssh-rsa ASDFASDF')
+        @another_key = create(:key) # this one IS in the DB
+      end
+
+      it 'removes the keys' do
+        expect(find_in_authorized_keys_file(1234)).to be_truthy
+        expect(find_in_authorized_keys_file(9876)).to be_truthy
+        expect(find_in_authorized_keys_file(@another_key.id)).to be_truthy
+        gitlab_shell.remove_keys_not_found_in_db
+        expect(find_in_authorized_keys_file(1234)).to be_falsey
+        expect(find_in_authorized_keys_file(9876)).to be_falsey
+        expect(find_in_authorized_keys_file(@another_key.id)).to be_truthy
+      end
+    end
+  end
+
+  describe '#batch_read_key_ids' do
+    context 'when there are keys in the authorized_keys file' do
+      before do
+        gitlab_shell.remove_all_keys
+        (1..4).each do |i|
+          gitlab_shell.add_key("key-#{i}", "ssh-rsa ASDFASDF#{i}")
+        end
+      end
+
+      it 'iterates over the key IDs in the file, in batches' do
+        loop_count = 0
+        first_batch = [1, 2]
+        second_batch = [3, 4]
+
+        gitlab_shell.batch_read_key_ids(batch_size: 2) do |batch|
+          expected = (loop_count == 0 ? first_batch : second_batch)
+          expect(batch).to eq(expected)
+          loop_count += 1
+        end
+      end
+    end
+  end
+
+  describe '#list_key_ids' do
+    context 'when there are keys in the authorized_keys file' do
+      before do
+        gitlab_shell.remove_all_keys
+        (1..4).each do |i|
+          gitlab_shell.add_key("key-#{i}", "ssh-rsa ASDFASDF#{i}")
+        end
+      end
+
+      it 'outputs the key IDs in the file, separated by newlines' do
+        ids = []
+        gitlab_shell.list_key_ids do |io|
+          io.each do |line|
+            ids << line
+          end
+        end
+
+        expect(ids).to eq(["1\n", "2\n", "3\n", "4\n"])
+      end
+    end
+
+    context 'when there are no keys in the authorized_keys file' do
+      before do
+        gitlab_shell.remove_all_keys
+      end
+
+      it 'outputs nothing, not even an empty string' do
+        ids = []
+        gitlab_shell.list_key_ids do |io|
+          io.each do |line|
+            ids << line
+          end
+        end
+
+        expect(ids).to eq([])
+      end
+    end
+  end
+
   describe Gitlab::Shell::KeyAdder, lib: true do
     describe '#add_key' do
       it 'removes trailing garbage' do
@@ -276,4 +370,12 @@ describe Gitlab::Shell, lib: true do
       end
     end
   end
+
+  def find_in_authorized_keys_file(key_id)
+    gitlab_shell.batch_read_key_ids do |ids|
+      return true if ids.include?(key_id)
+    end
+
+    false
+  end
 end

spec/migrations/update_authorized_keys_file_spec.rb

+require 'spec_helper'
+require Rails.root.join('db', 'migrate', '20170626202753_update_authorized_keys_file.rb')
+
+describe UpdateAuthorizedKeysFile do
+  let(:migration) { described_class.new }
+
+  describe '#up' do
+    context 'when authorized_keys_enabled is nil' do
+      before do
+        # Ensure the column can be null for the test
+        ActiveRecord::Base.connection.change_column_null :application_settings, :authorized_keys_enabled, true
+        ActiveRecord::Base.connection.change_column :application_settings, :authorized_keys_enabled, :boolean, default: nil
+
+        ApplicationSetting.create!(authorized_keys_enabled: nil)
+      end
+
+      it 'sets authorized_keys_enabled to true' do
+        migration.up
+
+        expect(ApplicationSetting.last.authorized_keys_enabled).to be_truthy
+      end
+
+      context 'there are keys created before and after the cutoff datetime' do
+        before do
+          Timecop.freeze
+        end
+        after do
+          Timecop.return
+        end
+
+        before do
+          @cutoff_datetime = UpdateAuthorizedKeysFile::DATETIME_9_3_0_RELEASED
+          @keys = []
+          Timecop.travel(@cutoff_datetime - 1.day)
+          2.times { @keys << create(:key) } # 2 keys before cutoff
+          Timecop.travel(@cutoff_datetime + 1.day)
+          2.times { @keys << create(:key) } # 2 keys after cutoff
+        end
+
+        it 'adds the keys created after the cutoff datetime to the authorized_keys file' do
+          Gitlab::Shell.new.remove_all_keys
+
+          migration.up
+
+          file = File.read(Rails.root.join('tmp/tests/.ssh/authorized_keys'))
+          expect(file.scan(/ssh-rsa/).count).to eq(2)
+
+          expect(file).not_to include(Gitlab::Shell.strip_key(@keys[0].key))
+          expect(file).not_to include(Gitlab::Shell.strip_key(@keys[1].key))
+          expect(file).to include(Gitlab::Shell.strip_key(@keys[2].key))
+          expect(file).to include(Gitlab::Shell.strip_key(@keys[3].key))
+        end
+      end
+
+      context 'when an SSH key exists in authorized_keys but not in the DB' do
+        before do
+          @key_to_stay = create(:key)
+          @key_to_delete = create(:key)
+          @key_to_delete.delete
+        end
+
+        it 'deletes the SSH key from authorized_keys' do
+          file = File.read(Rails.root.join('tmp/tests/.ssh/authorized_keys'))
+          expect(file).to include(Gitlab::Shell.strip_key(@key_to_stay.key))
+          expect(file).to include(Gitlab::Shell.strip_key(@key_to_delete.key))
+
+          migration.up
+
+          file = File.read(Rails.root.join('tmp/tests/.ssh/authorized_keys'))
+          expect(file).to include(Gitlab::Shell.strip_key(@key_to_stay.key))
+          expect(file).not_to include(Gitlab::Shell.strip_key(@key_to_delete.key))
+        end
+      end
+    end
+  end
+
+  describe '#authorized_keys_file_in_use_and_stale?' do
+    subject { migration.authorized_keys_file_in_use_and_stale? }
+
+    context 'when the customer ran the broken migration' do
+      before do
+        allow(migration).to receive(:ran_broken_migration?).and_return(true)
+      end
+
+      context 'when is a record in application_settings table' do
+        before do
+          ApplicationSetting.create!(authorized_keys_enabled: true)
+        end
+
+        context 'when authorized_keys_enabled is true' do
+          it { is_expected.to be_truthy }
+        end
+
+        context 'when authorized_keys_enabled is nil' do
+          before do
+            # Ensure the column can be null for the test
+            ActiveRecord::Base.connection.change_column_null :application_settings, :authorized_keys_enabled, true
+            ActiveRecord::Base.connection.change_column :application_settings, :authorized_keys_enabled, :boolean, default: nil
+
+            ApplicationSetting.first.update(authorized_keys_enabled: nil)
+          end
+
+          it { is_expected.to be_truthy }
+        end
+
+        context 'when authorized_keys_enabled is explicitly false' do
+          before do
+            ApplicationSetting.first.update(authorized_keys_enabled: false)
+          end
+
+          it { is_expected.to be_falsey }
+
+          it 'outputs a warning message for users who unintentionally Saved the setting unchecked' do
+            expect{ subject }.to output(/warning.*intentionally/mi).to_stdout
+          end
+        end
+      end
+
+      context 'when there is no record in application_settings table' do
+        before do
+          expect(ApplicationSetting.count).to eq(0)
+        end
+
+        it { is_expected.to be_falsey }
+      end
+    end
+
+    context 'when the customer did not run the broken migration' do
+      before do
+        allow(migration).to receive(:ran_broken_migration?).and_return(false)
+      end
+
+      it { is_expected.to be_falsey }
+    end
+  end
+
+  describe '#ran_broken_migration?' do
+    subject { migration.ran_broken_migration? }
+
+    context 'for unaffected customers: the authorized_keys_enabled column has a default (so the fixed migration ran)' do
+      before do
+        ActiveRecord::Base.connection.change_column :application_settings, :authorized_keys_enabled, :boolean, default: true
+        ActiveRecord::Base.connection.change_column_null :application_settings, :authorized_keys_enabled, false, true
+      end
+
+      it 'returns false' do
+        expect(subject).to be_falsey
+      end
+    end
+
+    context 'for affected customers: the authorized_keys_enabled column does not have a default (so the broken migration ran)' do
+      before do
+        ActiveRecord::Base.connection.change_column_null :application_settings, :authorized_keys_enabled, true
+        ActiveRecord::Base.connection.change_column :application_settings, :authorized_keys_enabled, :boolean, default: nil
+      end
+
+      it 'returns true' do
+        expect(subject).to be_truthy
+      end
+    end
+  end
+
+  describe '#update_authorized_keys_file_since' do
+    let!(:cutoff_datetime) { DateTime.now }
+
+    subject { migration.update_authorized_keys_file_since(cutoff_datetime) }
+
+    context 'when an SSH key was created after the cutoff datetime' do
+      before do
+        Timecop.freeze
+      end
+      after do
+        Timecop.return
+      end
+
+      before do
+        Timecop.travel 1.day.from_now
+        @key = create(:key)
+      end
+
+      it 'queues a batch_add_keys_from call to GitlabShellWorker, including the start key ID' do
+        expect(GitlabShellWorker).to receive(:perform_async).with(:batch_add_keys_in_db_starting_from, @key.id)
+        allow(GitlabShellWorker).to receive(:perform_async).with(:remove_keys_not_found_in_db)
+        subject
+      end
+    end
+
+    it 'queues a remove_keys_not_found_in_db call to GitlabShellWorker' do
+      expect(GitlabShellWorker).to receive(:perform_async).with(:remove_keys_not_found_in_db)
+      subject
+    end
+  end
+
+  describe '#add_keys_since' do
+    let!(:cutoff_datetime) { DateTime.now }
+
+    subject { migration.add_keys_since(cutoff_datetime) }
+
+    before do
+      Timecop.freeze
+    end
+    after do
+      Timecop.return
+    end
+
+    context 'when an SSH key was created after the cutoff datetime' do
+      before do
+        Timecop.travel 1.day.from_now
+        @key = create(:key)
+      end
+
+      it 'queues a batch_add_keys_from call to GitlabShellWorker, including the start key ID' do
+        expect(GitlabShellWorker).to receive(:perform_async).with(:batch_add_keys_in_db_starting_from, @key.id)
+        subject
+      end
+    end
+
+    context 'when an SSH key was not created after the cutoff datetime' do
+      it 'does not use GitlabShellWorker' do
+        expect(GitlabShellWorker).not_to receive(:perform_async)
+        subject
+      end
+    end
+  end
+
+  describe '#remove_keys_not_found_in_db' do
+    it 'queues a rm_keys_not_in_db call to GitlabShellWorker' do
+      expect(GitlabShellWorker).to receive(:perform_async).with(:remove_keys_not_found_in_db)
+      migration.remove_keys_not_found_in_db
+    end
+  end
+end

spec/workers/gitlab_shell_worker_spec.rb

+require 'spec_helper'
+
+describe GitlabShellWorker do
+  let(:worker) { described_class.new }
+
+  describe '#perform with add_key' do
+    it 'calls add_key on Gitlab::Shell' do
+      expect_any_instance_of(Gitlab::Shell).to receive(:add_key).with('foo', 'bar')
+      worker.perform(:add_key, 'foo', 'bar')
+    end
+  end
+
+  describe '#perform with batch_add_keys_in_db_starting_from' do
+    context 'when there are many keys in the DB' do
+      before do
+        @keys = []
+        10.times do
+          @keys << create(:key)
+        end
+      end
+
+      it 'adds all the keys in the DB, starting from the given ID, to the authorized_keys file' do
+        Gitlab::Shell.new.remove_all_keys
+
+        worker.perform(:batch_add_keys_in_db_starting_from, @keys[3].id)
+
+        file = File.read(Rails.root.join('tmp/tests/.ssh/authorized_keys'))
+        expect(file.scan(/ssh-rsa/).count).to eq(7)
+
+        expect(file).to_not include(Gitlab::Shell.strip_key(@keys[0].key))
+        expect(file).to_not include(Gitlab::Shell.strip_key(@keys[2].key))
+        expect(file).to include(Gitlab::Shell.strip_key(@keys[3].key))
+        expect(file).to include(Gitlab::Shell.strip_key(@keys[9].key))
+      end
+    end
+  end
+end