Support multiple DBs for schema_migrations/ directory

db/ci_schema_migrations/20210617101848

+1b74312f59f6f8937cd0dd754d22dc72e9bdc7302e6254a2fda5762afebe303c
\ No newline at end of file

db/ci_structure.sql

@@ -115,7 +115,5 @@ CREATE UNIQUE INDEX index_ci_instance_variables_on_key ON public.ci_instance_var
 
 SET search_path TO "$user", public;
 
-INSERT INTO "schema_migrations" (version) VALUES
-('20210617101848');
 
 

lib/gitlab/database/postgresql_adapter/dump_schema_versions_mixin.rb

@@ -7,10 +7,9 @@ module Gitlab
         extend ActiveSupport::Concern
 
         def dump_schema_information # :nodoc:
-          return super unless ActiveRecord::Base.configurations.primary?(pool.db_config.name)
-
           versions = schema_migration.all_versions
-          Gitlab::Database::SchemaVersionFiles.touch_all(versions) if versions.any?
+          file_versions = migration_context.migrations.map { |m| m.version.to_s }
+          Gitlab::Database::SchemaVersionFiles.touch_all(pool.db_config.name, versions, file_versions) if versions.any?
 
           nil
         end

lib/gitlab/database/postgresql_database_tasks/load_schema_versions_mixin.rb

@@ -7,13 +7,9 @@ module Gitlab
         extend ActiveSupport::Concern
 
         def structure_load(...)
-          result = super(...)
+          super(...)
 
-          if ActiveRecord::Base.configurations.primary?(connection.pool.db_config.name)
-            Gitlab::Database::SchemaVersionFiles.load_all
-          else
-            result
-          end
+          Gitlab::Database::SchemaVersionFiles.load_all(connection.pool.db_config.name)
         end
       end
     end

lib/gitlab/database/schema_version_files.rb

@@ -3,19 +3,18 @@
 module Gitlab
   module Database
     class SchemaVersionFiles
-      SCHEMA_DIRECTORY = 'db/schema_migrations'
-      MIGRATION_DIRECTORIES = %w[db/migrate db/post_migrate].freeze
+      SCHEMA_DIRECTORY = 'schema_migrations'
       MIGRATION_VERSION_GLOB = '20[0-9][0-9]*'
 
-      def self.touch_all(versions_from_database)
-        versions_from_migration_files = find_versions_from_migration_files
+      def self.touch_all(database_name, versions_from_database, versions_from_migration_files)
+        schema_directory = schema_directory_for(database_name)
 
-        version_filepaths = find_version_filenames.map { |f| schema_directory.join(f) }
+        version_filepaths = find_version_filenames(schema_directory).map { |f| File.join(schema_directory, f) }
         FileUtils.rm(version_filepaths)
 
         versions_to_create = versions_from_database & versions_from_migration_files
         versions_to_create.each do |version|
-          version_filepath = schema_directory.join(version)
+          version_filepath = File.join(schema_directory, version)
 
           File.open(version_filepath, 'w') do |file|
             file << Digest::SHA256.hexdigest(version)
@@ -23,8 +22,9 @@ module Gitlab
         end
       end
 
-      def self.load_all
-        version_filenames = find_version_filenames
+      def self.load_all(database_name)
+        schema_directory = schema_directory_for(database_name)
+        version_filenames = find_version_filenames(schema_directory)
         return if version_filenames.empty?
 
         values = version_filenames.map { |vf| "('#{connection.quote_string(vf)}')" }
@@ -35,27 +35,22 @@ module Gitlab
         SQL
       end
 
-      def self.schema_directory
-        @schema_directory ||= Rails.root.join(SCHEMA_DIRECTORY)
+      def self.db_dir
+        @db_dir ||= Rails.application.config.paths["db"].first
       end
 
-      def self.migration_directories
-        @migration_directories ||= MIGRATION_DIRECTORIES.map { |dir| Rails.root.join(dir) }
+      def self.schema_directory_for(database_name)
+        if ActiveRecord::Base.configurations.primary?(database_name)
+          File.join(db_dir, SCHEMA_DIRECTORY)
+        else
+          File.join(db_dir, "#{database_name}_#{SCHEMA_DIRECTORY}")
+        end
       end
 
-      def self.find_version_filenames
+      def self.find_version_filenames(schema_directory)
         Dir.glob(MIGRATION_VERSION_GLOB, base: schema_directory)
       end
 
-      def self.find_versions_from_migration_files
-        migration_directories.each_with_object([]) do |directory, migration_versions|
-          directory_migrations = Dir.glob(MIGRATION_VERSION_GLOB, base: directory)
-          directory_versions = directory_migrations.map! { |m| m.split('_').first }
-
-          migration_versions.concat(directory_versions)
-        end
-      end
-
       def self.connection
         ActiveRecord::Base.connection
       end

spec/lib/gitlab/database/postgresql_adapter/dump_schema_versions_mixin_spec.rb

@@ -4,9 +4,15 @@ require 'spec_helper'
 
 RSpec.describe Gitlab::Database::PostgresqlAdapter::DumpSchemaVersionsMixin do
   let(:schema_migration) { double('schema_migration', all_versions: versions) }
+  let(:migration_context) { double('migration_context', migrations: versions.map {|v| migration_struct.new(v) }) }
+
   let(:db_name) { 'primary' }
   let(:versions) { %w(5 2 1000 200 4 93 2) }
 
+  let(:migration_struct) do
+    Struct.new(:version)
+  end
+
   let(:instance_class) do
     klass = Class.new do
       def dump_schema_information
@@ -26,6 +32,7 @@ RSpec.describe Gitlab::Database::PostgresqlAdapter::DumpSchemaVersionsMixin do
 
   before do
     allow(instance).to receive(:schema_migration).and_return(schema_migration)
+    allow(instance).to receive(:migration_context).and_return(migration_context)
 
     # pool is from ActiveRecord::ConnectionAdapters::PostgreSQLAdapter
     allow(instance).to receive_message_chain(:pool, :db_config, :name).and_return(db_name)
@@ -34,7 +41,7 @@ RSpec.describe Gitlab::Database::PostgresqlAdapter::DumpSchemaVersionsMixin do
   context 'when database name is primary' do
     context 'when version files exist' do
       it 'touches version files' do
-        expect(Gitlab::Database::SchemaVersionFiles).to receive(:touch_all).with(versions)
+        expect(Gitlab::Database::SchemaVersionFiles).to receive(:touch_all).with(db_name, versions, versions)
         expect(instance).not_to receive(:original_dump_schema_information)
 
         instance.dump_schema_information
@@ -56,9 +63,9 @@ RSpec.describe Gitlab::Database::PostgresqlAdapter::DumpSchemaVersionsMixin do
   context 'when database name is ci' do
     let(:db_name) { 'ci' }
 
-    it 'does not touch version files' do
-      expect(Gitlab::Database::SchemaVersionFiles).not_to receive(:touch_all)
-      expect(instance).to receive(:original_dump_schema_information)
+    it 'touches version files' do
+      expect(Gitlab::Database::SchemaVersionFiles).to receive(:touch_all).with(db_name, versions, versions)
+      expect(instance).not_to receive(:original_dump_schema_information)
 
       instance.dump_schema_information
     end

spec/lib/gitlab/database/postgresql_database_tasks/load_schema_versions_mixin_spec.rb

@@ -28,8 +28,8 @@ RSpec.describe Gitlab::Database::PostgresqlDatabaseTasks::LoadSchemaVersionsMixi
   end
 
   context 'when database is primary' do
-    it 'loads version files' do
-      expect(Gitlab::Database::SchemaVersionFiles).to receive(:load_all)
+    it 'loads version files for primary database' do
+      expect(Gitlab::Database::SchemaVersionFiles).to receive(:load_all).with(db_name)
       expect(instance).to receive(:original_structure_load)
 
       instance.structure_load
@@ -39,8 +39,8 @@ RSpec.describe Gitlab::Database::PostgresqlDatabaseTasks::LoadSchemaVersionsMixi
   context 'when the database is ci' do
     let(:db_name) { 'ci' }
 
-    it 'does not load version files' do
-      expect(Gitlab::Database::SchemaVersionFiles).not_to receive(:load_all)
+    it 'loads version files for ci database' do
+      expect(Gitlab::Database::SchemaVersionFiles).to receive(:load_all).with(db_name)
       expect(instance).to receive(:original_structure_load)
 
       instance.structure_load

spec/lib/gitlab/database/schema_version_files_spec.rb

@@ -8,6 +8,8 @@ RSpec.describe Gitlab::Database::SchemaVersionFiles do
     let(:version2) { '20200410' }
     let(:version3) { '20200602' }
     let(:version4) { '20200809' }
+
+    let(:database_name) { 'primary' }
     let(:relative_schema_directory) { 'db/schema_migrations' }
     let(:relative_migrate_directory) { 'db/migrate' }
     let(:relative_post_migrate_directory) { 'db/post_migrate' }
@@ -33,10 +35,9 @@ RSpec.describe Gitlab::Database::SchemaVersionFiles do
 
         expect(File.exist?(old_version_filepath)).to be(true)
 
-        allow(described_class).to receive(:schema_directory).and_return(schema_directory)
-        allow(described_class).to receive(:migration_directories).and_return([migrate_directory, post_migrate_directory])
+        allow(described_class).to receive(:schema_directory_for).with(database_name).and_return(schema_directory)
 
-        described_class.touch_all([version1, version2, version3, version4])
+        described_class.touch_all(database_name, [version1, version2, version3, version4], [version1, version2])
 
         expect(File.exist?(old_version_filepath)).to be(false)
         [version1, version2].each do |version|
@@ -56,6 +57,7 @@ RSpec.describe Gitlab::Database::SchemaVersionFiles do
   end
 
   describe '.load_all' do
+    let(:database_name) { 'primary' }
     let(:connection) { double('connection') }
 
     before do
@@ -70,7 +72,7 @@ RSpec.describe Gitlab::Database::SchemaVersionFiles do
         expect(connection).not_to receive(:quote_string)
         expect(connection).not_to receive(:execute)
 
-        described_class.load_all
+        described_class.load_all(database_name)
       end
     end
 
@@ -88,7 +90,7 @@ RSpec.describe Gitlab::Database::SchemaVersionFiles do
           ON CONFLICT DO NOTHING
         SQL
 
-        described_class.load_all
+        described_class.load_all(database_name)
       end
     end
   end