Add support for replicating terraform states

- Add migration that creates terraform state registry
- Add model for terraform state registry
- Add associations to terraform state
- Add replicator for terraform state
- Add verification columns to terraform_states
- Fix issue in registry consistency worker where it did
  not pick up disabled replicables

Relates to https://gitlab.com/gitlab-org/gitlab/issues/220953

app/models/terraform/state.rb

@@ -33,8 +33,14 @@ module Terraform
       super || StateUploader.default_store
     end
 
+    def local?
+      file_store == ObjectStorage::Store::LOCAL
+    end
+
     def locked?
       self.lock_xid.present?
     end
   end
 end
+
+Terraform::State.prepend_if_ee('EE::Terraform::State')

config/initializers_before_autoloader/000_inflections.rb

@@ -25,6 +25,7 @@ ActiveSupport::Inflector.inflections do |inflect|
     project_registry
     project_statistics
     system_note_metadata
+    terraform_state_registry
     vulnerabilities_feedback
     vulnerability_feedback
   )

db/migrate/20200710152642_add_verification_state_to_terraform_states.rb

+# frozen_string_literal: true
+
+class AddVerificationStateToTerraformStates < ActiveRecord::Migration[6.0]
+  DOWNTIME = false
+
+  def change
+    change_table(:terraform_states) do |t|
+      t.column :verification_retry_at, :datetime_with_timezone
+      t.column :verified_at, :datetime_with_timezone
+      t.integer :verification_retry_count, limit: 2
+      t.binary :verification_checksum, using: 'verification_checksum::bytea'
+
+      # rubocop:disable Migration/AddLimitToTextColumns
+      # limit is added in 20200710153009_add_verification_failure_limit_and_index_to_terraform_states
+      t.text :verification_failure
+      # rubocop:enable Migration/AddLimitToTextColumns
+    end
+  end
+end

db/migrate/20200710153009_add_verification_failure_limit_and_index_to_terraform_states.rb

+# frozen_string_literal: true
+
+class AddVerificationFailureLimitAndIndexToTerraformStates < ActiveRecord::Migration[6.0]
+  include Gitlab::Database::MigrationHelpers
+
+  DOWNTIME = false
+
+  disable_ddl_transaction!
+
+  def up
+    add_concurrent_index :terraform_states, :verification_failure, where: "(verification_failure IS NOT NULL)", name: "terraform_states_verification_failure_partial"
+    add_concurrent_index :terraform_states, :verification_checksum, where: "(verification_checksum IS NOT NULL)", name: "terraform_states_verification_checksum_partial"
+    add_text_limit :terraform_states, :verification_failure, 255
+  end
+
+  def down
+    remove_concurrent_index :terraform_states, :verification_failure
+    remove_concurrent_index :terraform_states, :verification_checksum
+    remove_text_limit :terraform_states, :verification_failure
+  end
+end

db/schema_migrations/20200710152642

+0e4151d0fa03777383015a9efa6ce7ff6b2ef548978853da313500bf29448530
\ No newline at end of file

db/schema_migrations/20200710153009

+338199b853aa81cd1cc961eb6d8be313d794f885b182be8e7e3b227a3eca5be5
\ No newline at end of file

db/structure.sql

@@ -15672,7 +15672,13 @@ CREATE TABLE public.terraform_states (
     locked_at timestamp with time zone,
     locked_by_user_id bigint,
     uuid character varying(32) NOT NULL,
-    name character varying(255)
+    name character varying(255),
+    verification_retry_at timestamp with time zone,
+    verified_at timestamp with time zone,
+    verification_retry_count smallint,
+    verification_checksum bytea,
+    verification_failure text,
+    CONSTRAINT check_21a47163ea CHECK ((char_length(verification_failure) <= 255))
 );
 
 CREATE SEQUENCE public.terraform_states_id_seq
@@ -20925,6 +20931,10 @@ CREATE UNIQUE INDEX taggings_idx ON public.taggings USING btree (tag_id, taggabl
 
 CREATE UNIQUE INDEX term_agreements_unique_index ON public.term_agreements USING btree (user_id, term_id);
 
+CREATE INDEX terraform_states_verification_checksum_partial ON public.terraform_states USING btree (verification_checksum) WHERE (verification_checksum IS NOT NULL);
+
+CREATE INDEX terraform_states_verification_failure_partial ON public.terraform_states USING btree (verification_failure) WHERE (verification_failure IS NOT NULL);
+
 CREATE INDEX tmp_build_stage_position_index ON public.ci_builds USING btree (stage_id, stage_idx) WHERE (stage_idx IS NOT NULL);
 
 CREATE INDEX tmp_idx_on_user_id_where_bio_is_filled ON public.users USING btree (id) WHERE ((COALESCE(bio, ''::character varying))::text IS DISTINCT FROM ''::text);

ee/app/models/concerns/geo/replicable_registry.rb

@@ -22,10 +22,10 @@ module Geo::ReplicableRegistry
     def declarative_policy_class
       'Geo::RegistryPolicy'
     end
-  end
 
-  def registry_consistency_worker_enabled?
-    replicator_class.enabled?
+    def registry_consistency_worker_enabled?
+      replicator_class.enabled?
+    end
   end
 
   def replicator_class

ee/app/models/ee/terraform/state.rb

+# frozen_string_literal: true
+
+module EE
+  module Terraform
+    module State
+      extend ActiveSupport::Concern
+
+      prepended do
+        include ::Gitlab::Geo::ReplicableModel
+        with_replicator Geo::TerraformStateReplicator
+
+        scope :with_files_stored_locally, -> { where(file_store: ::ObjectStorage::Store::LOCAL) }
+        scope :project_id_in, ->(ids) { where(project_id: ids) }
+      end
+
+      class_methods do
+        def replicables_for_geo_node(node = ::Gitlab::Geo.current_node)
+          selective_sync_scope(node).merge(object_storage_scope(node))
+        end
+
+        private
+
+        def object_storage_scope(node)
+          return all if node.sync_object_storage?
+
+          with_files_stored_locally
+        end
+
+        def selective_sync_scope(node)
+          return all unless node.selective_sync?
+
+          project_id_in(node.projects)
+        end
+      end
+
+      def log_geo_deleted_event
+        # Keep empty for now. Should be addressed in future
+        # by https://gitlab.com/gitlab-org/gitlab/-/issues/232917
+      end
+    end
+  end
+end

ee/app/models/geo/terraform_state_registry.rb

+# frozen_string_literal: true
+
+class Geo::TerraformStateRegistry < Geo::BaseRegistry
+  include Geo::ReplicableRegistry
+
+  MODEL_CLASS = ::Terraform::State
+  MODEL_FOREIGN_KEY = :terraform_state_id
+
+  belongs_to :terraform_state, class_name: 'Terraform::State'
+end

ee/app/replicators/geo/terraform_state_replicator.rb

+# frozen_string_literal: true
+
+module Geo
+  class TerraformStateReplicator < Gitlab::Geo::Replicator
+    include ::Geo::BlobReplicatorStrategy
+
+    def carrierwave_uploader
+      model_record.file
+    end
+
+    def self.model
+      ::Terraform::State
+    end
+
+    def self.replication_enabled_by_default?
+      false
+    end
+  end
+end

ee/app/workers/geo/secondary/registry_consistency_worker.rb

@@ -22,6 +22,7 @@ module Geo
         Geo::LfsObjectRegistry,
         Geo::PackageFileRegistry,
         Geo::ProjectRegistry,
+        Geo::TerraformStateRegistry,
         Geo::UploadRegistry
       ].freeze
 

ee/changelogs/unreleased/alexives-220953-replicate_terraform_state.yml

+---
+title: Add Geo replication columns and tables for terraform states
+merge_request: 36594
+author:
+type: added

ee/db/geo/migrate/20200707210300_create_terraform_state_registry.rb

+# frozen_string_literal: true
+
+class CreateTerraformStateRegistry < ActiveRecord::Migration[6.0]
+  include Gitlab::Database::MigrationHelpers
+
+  DOWNTIME = false
+
+  disable_ddl_transaction!
+
+  def up
+    unless table_exists?(:terraform_state_registry)
+      ActiveRecord::Base.transaction do
+        create_table :terraform_state_registry, id: :bigserial, force: :cascade do |t|
+          t.datetime_with_timezone :retry_at
+          t.datetime_with_timezone :last_synced_at
+          t.datetime_with_timezone :created_at, null: false
+          t.bigint :terraform_state_id, null: false
+          t.integer :state, default: 0, null: false, limit: 2
+          t.integer :retry_count, default: 0, limit: 2
+          t.text :last_sync_failure
+
+          t.index :terraform_state_id
+          t.index :retry_at
+          t.index :state
+        end
+      end
+    end
+
+    add_text_limit :terraform_state_registry, :last_sync_failure, 255
+  end
+
+  def down
+    drop_table :terraform_state_registry
+  end
+end

ee/db/geo/schema.rb

@@ -168,6 +168,19 @@ ActiveRecord::Schema.define(version: 2020_07_10_194046) do
     t.index ["wiki_verification_checksum_sha"], name: "idx_project_registry_on_wiki_checksum_sha_partial", where: "(wiki_verification_checksum_sha IS NULL)"
   end
 
+  create_table "terraform_state_registry", force: :cascade do |t|
+    t.datetime_with_timezone "retry_at"
+    t.datetime_with_timezone "last_synced_at"
+    t.datetime_with_timezone "created_at", null: false
+    t.bigint "terraform_state_id", null: false
+    t.integer "state", limit: 2, default: 0, null: false
+    t.integer "retry_count", limit: 2, default: 0
+    t.text "last_sync_failure"
+    t.index ["retry_at"], name: "index_terraform_state_registry_on_retry_at"
+    t.index ["state"], name: "index_terraform_state_registry_on_state"
+    t.index ["terraform_state_id"], name: "index_terraform_state_registry_on_terraform_state_id"
+  end
+
   create_table "vulnerability_export_registry", force: :cascade do |t|
     t.datetime_with_timezone "retry_at"
     t.datetime_with_timezone "last_synced_at"

ee/lib/gitlab/geo.rb

@@ -162,7 +162,10 @@ module Gitlab
     # solutions can be found at
     # https://gitlab.com/gitlab-org/gitlab/-/issues/227693
     def self.replicator_classes
-      classes = [::Geo::PackageFileReplicator]
+      classes = [
+          ::Geo::PackageFileReplicator,
+          ::Geo::TerraformStateReplicator
+      ]
 
       classes.select(&:enabled?)
     end

ee/spec/factories/geo/terraform_state_registry.rb

+# frozen_string_literal: true
+
+FactoryBot.define do
+  factory :geo_terraform_state_registry, class: 'Geo::TerraformStateRegistry' do
+    association :terraform_state, factory: :terraform_state
+    state { Geo::TerraformStateRegistry.state_value(:pending) }
+
+    trait :synced do
+      state { Geo::TerraformStateRegistry.state_value(:synced) }
+      last_synced_at { 5.days.ago }
+    end
+
+    trait :failed do
+      state { Geo::TerraformStateRegistry.state_value(:failed) }
+      last_synced_at { 1.day.ago }
+      retry_count { 2 }
+      last_sync_failure { 'Random error' }
+    end
+
+    trait :started do
+      state { Geo::TerraformStateRegistry.state_value(:started) }
+      last_synced_at { 1.day.ago }
+      retry_count { 0 }
+    end
+  end
+end

ee/spec/lib/gitlab/geo/geo_node_status_check_spec.rb

@@ -13,6 +13,8 @@ RSpec.describe Gitlab::Geo::GeoNodeStatusCheck do
   describe '#replication_verification_complete?' do
     before do
       allow(Gitlab.config.geo.registry_replication).to receive(:enabled).and_return(true)
+
+      stub_feature_flags(geo_terraform_state_replication: false)
     end
 
     it 'prints messages for all verification checks' do

ee/spec/models/ee/terraform/state_spec.rb

+# frozen_string_literal: true
+
+require 'spec_helper'
+
+RSpec.describe Terraform::State do
+  using RSpec::Parameterized::TableSyntax
+  include EE::GeoHelpers
+
+  let_it_be(:group) { create(:group) }
+  let_it_be(:project) { create(:project, group: group) }
+  subject { create(:terraform_state, :with_file) }
+
+  describe '.with_files_stored_locally' do
+    it 'includes states with local storage' do
+      create_list(:terraform_state, 5, :with_file)
+
+      expect(described_class.with_files_stored_locally).to have_attributes(count: 5)
+    end
+
+    it 'excludes states with local storage' do
+      stub_terraform_state_object_storage(Terraform::StateUploader)
+
+      create_list(:terraform_state, 5, :with_file)
+
+      expect(described_class.with_files_stored_locally).to have_attributes(count: 0)
+    end
+  end
+
+  describe '.replicables_for_geo_node' do
+    where(:selective_sync_enabled, :object_storage_sync_enabled, :terraform_object_storage_enabled, :synced_states) do
+      true  | true  | true  | 5
+      true  | true  | false | 5
+      true  | false | true  | 0
+      true  | false | false | 5
+      false | false | false | 10
+      false | false | true  | 0
+      false | true  | true  | 10
+      false | true  | false | 10
+      true  | true  | false | 5
+    end
+
+    with_them do
+      let(:secondary) do
+        node = build(:geo_node, sync_object_storage: object_storage_sync_enabled)
+
+        if selective_sync_enabled
+          node.selective_sync_type = 'namespaces'
+          node.namespaces = [group]
+        end
+
+        node.save!
+        node
+      end
+
+      before do
+        stub_current_geo_node(secondary)
+        stub_terraform_state_object_storage(Terraform::StateUploader) if terraform_object_storage_enabled
+
+        create_list(:terraform_state, 5, project: project)
+        create_list(:terraform_state, 5, project: create(:project))
+      end
+
+      it 'returns the proper number of terraform states' do
+        expect(Terraform::State.replicables_for_geo_node.count).to eq(synced_states)
+      end
+    end
+  end
+end

ee/spec/models/geo/terraform_state_registry_spec.rb

+# frozen_string_literal: true
+
+require 'spec_helper'
+
+RSpec.describe Geo::TerraformStateRegistry, :geo, type: :model do
+  let_it_be(:registry) { create(:geo_terraform_state_registry) }
+
+  specify 'factory is valid' do
+    expect(registry).to be_valid
+  end
+
+  include_examples 'a Geo framework registry'
+end

ee/spec/replicators/geo/terraform_state_replicator_spec.rb

+# frozen_string_literal: true
+
+require 'spec_helper'
+
+RSpec.describe Geo::TerraformStateReplicator do
+  let(:model_record) { build(:terraform_state) }
+
+  it_behaves_like 'a blob replicator'
+end

ee/spec/services/geo/registry_consistency_service_spec.rb

@@ -15,6 +15,7 @@ RSpec.describe Geo::RegistryConsistencyService, :geo, :use_clean_rails_memory_st
   def model_class_factory_name(registry_class)
     return :project_with_design if registry_class == ::Geo::DesignRegistry
     return :package_file_with_file if registry_class == ::Geo::PackageFileRegistry
+    return :terraform_state if registry_class == ::Geo::TerraformStateRegistry
 
     registry_class::MODEL_CLASS.underscore.tr('/', '_').to_sym
   end

ee/spec/tasks/geo_rake_spec.rb

@@ -8,6 +8,7 @@ RSpec.describe 'geo rake tasks', :geo do
   before do
     Rake.application.rake_require 'tasks/geo'
     stub_licensed_features(geo: true)
+    stub_feature_flags(geo_terraform_state_replication: false)
   end
 
   it 'Gitlab:Geo::DatabaseTasks responds to all methods used in Geo rake tasks' do

ee/spec/tasks/gitlab/geo_rake_spec.rb

@@ -8,6 +8,7 @@ RSpec.describe 'gitlab:geo rake tasks', :geo do
   before do
     Rake.application.rake_require 'tasks/gitlab/geo'
     stub_licensed_features(geo: true)
+    stub_feature_flags(geo_terraform_state_replication: false)
   end
 
   describe 'gitlab:geo:check_replication_verification_status' do

ee/spec/workers/geo/secondary/registry_consistency_worker_spec.rb

@@ -85,6 +85,7 @@ RSpec.describe Geo::Secondary::RegistryConsistencyWorker, :geo do
       upload = create(:upload)
       package_file = create(:conan_package_file, :conan_package)
       container_repository = create(:container_repository, project: project)
+      terraform_state = create(:terraform_state, project: project)
 
       expect(Geo::LfsObjectRegistry.where(lfs_object_id: lfs_object.id).count).to eq(0)
       expect(Geo::JobArtifactRegistry.where(artifact_id: job_artifact.id).count).to eq(0)
@@ -93,6 +94,7 @@ RSpec.describe Geo::Secondary::RegistryConsistencyWorker, :geo do
       expect(Geo::UploadRegistry.where(file_id: upload.id).count).to eq(0)
       expect(Geo::PackageFileRegistry.where(package_file_id: package_file.id).count).to eq(0)
       expect(Geo::ContainerRepositoryRegistry.where(container_repository_id: container_repository.id).count).to eq(0)
+      expect(Geo::TerraformStateRegistry.where(terraform_state_id: terraform_state.id).count).to eq(0)
 
       subject.perform
 
@@ -103,6 +105,24 @@ RSpec.describe Geo::Secondary::RegistryConsistencyWorker, :geo do
       expect(Geo::UploadRegistry.where(file_id: upload.id).count).to eq(1)
       expect(Geo::PackageFileRegistry.where(package_file_id: package_file.id).count).to eq(1)
       expect(Geo::ContainerRepositoryRegistry.where(container_repository_id: container_repository.id).count).to eq(1)
+      expect(Geo::TerraformStateRegistry.where(terraform_state_id: terraform_state.id).count).to eq(1)
+    end
+
+    context 'when geo_terraform_state_replication is disabled' do
+      before do
+        stub_feature_flags(geo_terraform_state_replication: false)
+      end
+
+      it 'returns false' do
+        expect(subject.perform).to be_falsey
+      end
+
+      it 'does not execute RegistryConsistencyService for terraform states' do
+        allow(Geo::RegistryConsistencyService).to receive(:new).and_call_original
+        expect(Geo::RegistryConsistencyService).not_to receive(:new).with(Geo::TerraformStateRegistry, batch_size: batch_size)
+
+        subject.perform
+      end
     end
 
     context 'when the current Geo node is disabled or primary' do