Commit fff0e82e authored by Thong Kuah's avatar Thong Kuah Committed by Mayra Cabrera

Setup new CI DB (decomposition)

parent 0b27fd04
# frozen_string_literal: true
module Ci
# TODO: https://gitlab.com/groups/gitlab-org/-/epics/6168
#
# Do not use this yet outside of `ci_instance_variables`.
# This class is part of a migration to move all CI classes to a new separate database.
# Initially we are only going to be moving the `Ci::InstanceVariable` model and it will be duplicated in the main and CI tables
# Do not extend this class in any other models.
class ApplicationRecord < ::ApplicationRecord
self.abstract_class = true
if Gitlab::Database.has_config?(:ci)
connects_to database: { writing: :ci, reading: :ci }
end
end
end
# frozen_string_literal: true
module Ci
class InstanceVariable < ApplicationRecord
class InstanceVariable < ::Ci::ApplicationRecord
extend Gitlab::Ci::Model
extend Gitlab::ProcessMemoryCache::Helper
include Ci::NewHasVariable
......
# frozen_string_literal: true
# Because we use Gitlab::Database, which in turn uses prepend_mod_with,
# we need this intializer to be after config/initializers/0_inject_enterprise_edition_module.rb.
# Post deployment migrations are included by default. This file must be loaded
# before other initializers as Rails may otherwise memoize a list of migrations
# excluding the post deployment migrations.
......
# frozen_string_literal: true
ci_db_config = Gitlab::Application.config.database_configuration[Rails.env]["ci"]
if ci_db_config.present?
raise "migrations_paths setting for ci database must be `db/ci_migrate`" unless ci_db_config["migrations_paths"] == 'db/ci_migrate'
end
# frozen_string_literal: true
class CreateCiInstanceVariablesOnCi < ActiveRecord::Migration[6.1]
include Gitlab::Database::MigrationHelpers
disable_ddl_transaction!
def up
unless table_exists?(:ci_instance_variables)
create_table :ci_instance_variables do |t|
t.integer :variable_type, null: false, limit: 2, default: 1
t.boolean :masked, default: false, allow_null: false
t.boolean :protected, default: false, allow_null: false
t.text :key, null: false
t.text :encrypted_value
t.text :encrypted_value_iv
t.index [:key], name: 'index_ci_instance_variables_on_key', unique: true, using: :btree
end
end
add_text_limit(:ci_instance_variables, :key, 255)
# Use constraint_name generated from db/migrate/20200625193358_increase_size_on_instance_level_variable_values.rb
add_text_limit(:ci_instance_variables, :encrypted_value, 13_579, constraint_name: 'check_956afd70f1')
add_text_limit(:ci_instance_variables, :encrypted_value_iv, 255)
end
def down
drop_table :ci_instance_variables
end
end
SET statement_timeout = 0;
SET lock_timeout = 0;
SET idle_in_transaction_session_timeout = 0;
SET client_encoding = 'UTF8';
SET standard_conforming_strings = on;
SELECT pg_catalog.set_config('search_path', '', false);
SET check_function_bodies = false;
SET xmloption = content;
SET client_min_messages = warning;
SET row_security = off;
SET default_tablespace = '';
SET default_table_access_method = heap;
--
-- Name: ar_internal_metadata; Type: TABLE; Schema: public; Owner: -
--
CREATE TABLE public.ar_internal_metadata (
key character varying NOT NULL,
value character varying,
created_at timestamp(6) without time zone NOT NULL,
updated_at timestamp(6) without time zone NOT NULL
);
--
-- Name: ci_instance_variables; Type: TABLE; Schema: public; Owner: -
--
CREATE TABLE public.ci_instance_variables (
id bigint NOT NULL,
variable_type smallint DEFAULT 1 NOT NULL,
masked boolean DEFAULT false,
protected boolean DEFAULT false,
key text NOT NULL,
encrypted_value text,
encrypted_value_iv text,
CONSTRAINT check_07a45a5bcb CHECK ((char_length(encrypted_value_iv) <= 255)),
CONSTRAINT check_5aede12208 CHECK ((char_length(key) <= 255)),
CONSTRAINT check_956afd70f1 CHECK ((char_length(encrypted_value) <= 13579))
);
--
-- Name: ci_instance_variables_id_seq; Type: SEQUENCE; Schema: public; Owner: -
--
CREATE SEQUENCE public.ci_instance_variables_id_seq
START WITH 1
INCREMENT BY 1
NO MINVALUE
NO MAXVALUE
CACHE 1;
--
-- Name: ci_instance_variables_id_seq; Type: SEQUENCE OWNED BY; Schema: public; Owner: -
--
ALTER SEQUENCE public.ci_instance_variables_id_seq OWNED BY public.ci_instance_variables.id;
--
-- Name: schema_migrations; Type: TABLE; Schema: public; Owner: -
--
CREATE TABLE public.schema_migrations (
version character varying NOT NULL
);
--
-- Name: ci_instance_variables id; Type: DEFAULT; Schema: public; Owner: -
--
ALTER TABLE ONLY public.ci_instance_variables ALTER COLUMN id SET DEFAULT nextval('public.ci_instance_variables_id_seq'::regclass);
--
-- Name: ar_internal_metadata ar_internal_metadata_pkey; Type: CONSTRAINT; Schema: public; Owner: -
--
ALTER TABLE ONLY public.ar_internal_metadata
ADD CONSTRAINT ar_internal_metadata_pkey PRIMARY KEY (key);
--
-- Name: ci_instance_variables ci_instance_variables_pkey; Type: CONSTRAINT; Schema: public; Owner: -
--
ALTER TABLE ONLY public.ci_instance_variables
ADD CONSTRAINT ci_instance_variables_pkey PRIMARY KEY (id);
--
-- Name: schema_migrations schema_migrations_pkey; Type: CONSTRAINT; Schema: public; Owner: -
--
ALTER TABLE ONLY public.schema_migrations
ADD CONSTRAINT schema_migrations_pkey PRIMARY KEY (version);
--
-- Name: index_ci_instance_variables_on_key; Type: INDEX; Schema: public; Owner: -
--
CREATE UNIQUE INDEX index_ci_instance_variables_on_key ON public.ci_instance_variables USING btree (key);
--
-- PostgreSQL database dump complete
--
SET search_path TO "$user", public;
INSERT INTO "schema_migrations" (version) VALUES
('20210617101848');
---
stage: Enablement
group: Sharding
info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments
---
# Multiple Databases
In order to scale GitLab, the GitLab application database
will be [decomposed into multiple
databases](https://gitlab.com/groups/gitlab-org/-/epics/6168).
## CI Database
Support for configuring the GitLab Rails application to use a distinct
database for CI tables was added in [GitLab
14.1](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/64289). This
feature is still under development, and is not ready for production use.
By default, GitLab is configured to use only one main database. To
opt-in to use a main database, and CI database, modify the
`config/database.yml` file to have a `primary` and a `ci` database
configurations. For example, given a `config/database.yml` like below:
```yaml
development:
adapter: postgresql
encoding: unicode
database: gitlabhq_development
host: /path/to/gdk/postgresql
pool: 10
prepared_statements: false
variables:
statement_timeout: 120s
test: &test
adapter: postgresql
encoding: unicode
database: gitlabhq_test
host: /path/to/gdk/postgresql
pool: 10
prepared_statements: false
variables:
statement_timeout: 120s
```
Edit the `config/database.yml` to look like this:
```yaml
development:
primary:
adapter: postgresql
encoding: unicode
database: gitlabhq_development
host: /path/to/gdk/postgresql
pool: 10
prepared_statements: false
variables:
statement_timeout: 120s
ci:
adapter: postgresql
encoding: unicode
database: gitlabhq_development_ci
migrations_paths: db/ci_migrate
host: /path/to/gdk/postgresql
pool: 10
prepared_statements: false
variables:
statement_timeout: 120s
test: &test
primary:
adapter: postgresql
encoding: unicode
database: gitlabhq_test
host: /path/to/gdk/postgresql
pool: 10
prepared_statements: false
variables:
statement_timeout: 120s
ci:
adapter: postgresql
encoding: unicode
database: gitlabhq_test_ci
migrations_paths: db/ci_migrate
host: /path/to/gdk/postgresql
pool: 10
prepared_statements: false
variables:
statement_timeout: 120s
```
Note that we use `primary` in the `config/database.yml` to refer to the main
database. This is to match the default name Rails has.
### Migrations
Any migrations that affect `Ci::ApplicationRecord` models
and their tables must be placed in two directories for now:
- `db/migrate`
- `db/ci_migrate`
We aim to keep the schema for both tables the same across both databases.
......@@ -2,6 +2,8 @@
module Gitlab
module Database
CI_DATABASE_NAME = 'ci'
# This constant is used when renaming tables concurrently.
# If you plan to rename a table using the `rename_table_safely` method, add your table here one milestone before the rename.
# Example:
......@@ -68,6 +70,14 @@ module Gitlab
end
end
def self.has_config?(database_name)
Gitlab::Application.config.database_configuration[Rails.env].include?(database_name.to_s)
end
def self.ci_database?(name)
name == CI_DATABASE_NAME
end
def self.username
config['username'] || ENV['USER']
end
......
......@@ -7,6 +7,8 @@ module Gitlab
extend ActiveSupport::Concern
def dump_schema_information # :nodoc:
return super unless ActiveRecord::Base.configurations.primary?(pool.db_config.name)
versions = schema_migration.all_versions
Gitlab::Database::SchemaVersionFiles.touch_all(versions) if versions.any?
......
......@@ -6,9 +6,14 @@ module Gitlab
module LoadSchemaVersionsMixin
extend ActiveSupport::Concern
def structure_load(*args)
super(*args)
def structure_load(...)
result = super(...)
if ActiveRecord::Base.configurations.primary?(connection.pool.db_config.name)
Gitlab::Database::SchemaVersionFiles.load_all
else
result
end
end
end
end
......
......@@ -4,20 +4,38 @@ require 'spec_helper'
RSpec.describe Gitlab::Database::PostgresqlAdapter::DumpSchemaVersionsMixin do
let(:schema_migration) { double('schema_migration', all_versions: versions) }
let(:db_name) { 'primary' }
let(:versions) { %w(5 2 1000 200 4 93 2) }
let(:instance_class) do
klass = Class.new do
def dump_schema_information
original_dump_schema_information
end
def original_dump_schema_information
end
end
klass.prepend(described_class)
let(:instance) do
Object.new.extend(described_class)
klass
end
let(:instance) { instance_class.new }
before do
allow(instance).to receive(:schema_migration).and_return(schema_migration)
# pool is from ActiveRecord::ConnectionAdapters::PostgreSQLAdapter
allow(instance).to receive_message_chain(:pool, :db_config, :name).and_return(db_name)
end
context 'when database name is primary' do
context 'when version files exist' do
let(:versions) { %w(5 2 1000 200 4 93 2) }
it 'touches version files' do
expect(Gitlab::Database::SchemaVersionFiles).to receive(:touch_all).with(versions)
expect(instance).not_to receive(:original_dump_schema_information)
instance.dump_schema_information
end
......@@ -28,6 +46,19 @@ RSpec.describe Gitlab::Database::PostgresqlAdapter::DumpSchemaVersionsMixin do
it 'does not touch version files' do
expect(Gitlab::Database::SchemaVersionFiles).not_to receive(:touch_all)
expect(instance).not_to receive(:original_dump_schema_information)
instance.dump_schema_information
end
end
end
context 'when database name is ci' do
let(:db_name) { 'ci' }
it 'does not touch version files' do
expect(Gitlab::Database::SchemaVersionFiles).not_to receive(:touch_all)
expect(instance).to receive(:original_dump_schema_information)
instance.dump_schema_information
end
......
# frozen_string_literal: true
require 'spec_helper'
RSpec.describe Gitlab::Database::PostgresqlDatabaseTasks::LoadSchemaVersionsMixin do
let(:db_name) { 'primary' }
let(:instance_class) do
klass = Class.new do
def structure_load
original_structure_load
end
def original_structure_load
end
end
klass.prepend(described_class)
klass
end
let(:instance) { instance_class.new }
before do
# connection is available in ActiveRecord::Tasks::PostgreSQLDatabaseTasks
allow(instance).to receive_message_chain(:connection, :pool, :db_config, :name).and_return(db_name)
end
context 'when database is primary' do
it 'loads version files' do
expect(Gitlab::Database::SchemaVersionFiles).to receive(:load_all)
expect(instance).to receive(:original_structure_load)
instance.structure_load
end
end
context 'when the database is ci' do
let(:db_name) { 'ci' }
it 'does not load version files' do
expect(Gitlab::Database::SchemaVersionFiles).not_to receive(:load_all)
expect(instance).to receive(:original_structure_load)
instance.structure_load
end
end
end
......@@ -41,6 +41,45 @@ RSpec.describe Gitlab::Database do
end
end
describe '.has_config?' do
context 'two tier database config' do
before do
allow(Gitlab::Application).to receive_message_chain(:config, :database_configuration, :[]).with(Rails.env)
.and_return({ "adapter" => "postgresql", "database" => "gitlabhq_test" })
end
it 'returns false for primary' do
expect(described_class.has_config?(:primary)).to eq(false)
end
it 'returns false for ci' do
expect(described_class.has_config?(:ci)).to eq(false)
end
end
context 'three tier database config' do
before do
allow(Gitlab::Application).to receive_message_chain(:config, :database_configuration, :[]).with(Rails.env)
.and_return({
"primary" => { "adapter" => "postgresql", "database" => "gitlabhq_test" },
"ci" => { "adapter" => "postgresql", "database" => "gitlabhq_test_ci" }
})
end
it 'returns true for primary' do
expect(described_class.has_config?(:primary)).to eq(true)
end
it 'returns true for ci' do
expect(described_class.has_config?(:ci)).to eq(true)
end
it 'returns false for non-existent' do
expect(described_class.has_config?(:nonexistent)).to eq(false)
end
end
end
describe '.adapter_name' do
it 'returns the name of the adapter' do
expect(described_class.adapter_name).to be_an_instance_of(String)
......
......@@ -7,10 +7,10 @@ require 'spec_helper'
RSpec.describe ActiveRecord::Schema, schema: :latest do
let(:all_migrations) do
migrations_paths = %w[db/migrate db/post_migrate]
.map { |path| Rails.root.join(*path, '*') }
migrations_directories = %w[db/migrate db/post_migrate].map { |path| Rails.root.join(path).to_s }
migrations_paths = migrations_directories.map { |path| File.join(path, '*') }
migrations = Dir[*migrations_paths]
migrations = Dir[*migrations_paths] - migrations_directories
migrations.map { |migration| File.basename(migration).split('_').first.to_i }.sort
end
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment