Merge branch '323677-swap-base-and-partitioned-web-hook-logs' into 'master'

Add migration to swap partitioned web_hook_logs

See merge request gitlab-org/gitlab!60184

app/models/hooks/web_hook_log.rb

@@ -5,9 +5,12 @@ class WebHookLog < ApplicationRecord
   include Presentable
   include DeleteWithLimit
   include CreatedAtFilterable
+  include PartitionedTable
 
   self.primary_key = :id
 
+  partitioned_by :created_at, strategy: :monthly
+
   belongs_to :web_hook
 
   serialize :request_headers, Hash # rubocop:disable Cop/ActiveRecordSerialize

app/models/hooks/web_hook_log_partitioned.rb → app/models/hooks/web_hook_log_archived.rb

 # frozen_string_literal: true
 
-# This model is not yet intended to be used.
-# It is in a transitioning phase while we are partitioning
-# the web_hook_logs table on the database-side.
+# This model is not intended to be used.
+# It is a temporary reference to the old non-partitioned
+# web_hook_logs table.
 # Please refer to https://gitlab.com/groups/gitlab-org/-/epics/5558
 # for details.
 # rubocop:disable Gitlab/NamespacedClass: This is a temporary class with no relevant namespace
 #  WebHook, WebHookLog and all hooks are defined outside of a namespace
-class WebHookLogPartitioned < ApplicationRecord
-  include PartitionedTable
-
-  self.table_name = 'web_hook_logs_part_0c5294f417'
-  self.primary_key = :id
-
-  partitioned_by :created_at, strategy: :monthly
+class WebHookLogArchived < ApplicationRecord
+  self.table_name = 'web_hook_logs_archived'
 end

changelogs/unreleased/323677-swap-base-and-partitioned-web-hook-logs.yml

+---
+title: Add migration to swap partitioned web_hook_logs
+merge_request: 60184
+author:
+type: other

config/initializers/postgres_partitioning.rb

@@ -4,7 +4,7 @@
 # (even with eager loading disabled).
 
 Gitlab::Database::Partitioning::PartitionCreator.register(AuditEvent)
-Gitlab::Database::Partitioning::PartitionCreator.register(WebHookLogPartitioned)
+Gitlab::Database::Partitioning::PartitionCreator.register(WebHookLog)
 
 begin
   Gitlab::Database::Partitioning::PartitionCreator.new.create_partitions unless ENV['DISABLE_POSTGRES_PARTITION_CREATION_ON_STARTUP']

db/migrate/20210424115725_swap_partitioned_web_hook_logs.rb

+# frozen_string_literal: true
+
+class SwapPartitionedWebHookLogs < ActiveRecord::Migration[6.0]
+  include Gitlab::Database::PartitioningMigrationHelpers
+
+  DOWNTIME = false
+
+  def up
+    replace_with_partitioned_table :web_hook_logs
+  end
+
+  def down
+    rollback_replace_with_partitioned_table :web_hook_logs
+  end
+end

db/schema_migrations/20210424115725

+66c4c48c53131a156c0bf45982691e9017373aefae1f7b6d7053467481b3f5cf
\ No newline at end of file

db/structure.sql

@@ -46,9 +46,9 @@ CREATE FUNCTION table_sync_function_29bc99d6db() RETURNS trigger
     AS $$
 BEGIN
 IF (TG_OP = 'DELETE') THEN
-  DELETE FROM web_hook_logs_part_0c5294f417 where id = OLD.id;
+  DELETE FROM web_hook_logs_archived where id = OLD.id;
 ELSIF (TG_OP = 'UPDATE') THEN
-  UPDATE web_hook_logs_part_0c5294f417
+  UPDATE web_hook_logs_archived
   SET web_hook_id = NEW.web_hook_id,
     trigger = NEW.trigger,
     url = NEW.url,
@@ -59,11 +59,11 @@ ELSIF (TG_OP = 'UPDATE') THEN
     response_status = NEW.response_status,
     execution_duration = NEW.execution_duration,
     internal_error_message = NEW.internal_error_message,
-    updated_at = NEW.updated_at,
-    created_at = NEW.created_at
-  WHERE web_hook_logs_part_0c5294f417.id = NEW.id;
+    created_at = NEW.created_at,
+    updated_at = NEW.updated_at
+  WHERE web_hook_logs_archived.id = NEW.id;
 ELSIF (TG_OP = 'INSERT') THEN
-  INSERT INTO web_hook_logs_part_0c5294f417 (id,
+  INSERT INTO web_hook_logs_archived (id,
     web_hook_id,
     trigger,
     url,
@@ -74,8 +74,8 @@ ELSIF (TG_OP = 'INSERT') THEN
     response_status,
     execution_duration,
     internal_error_message,
-    updated_at,
-    created_at)
+    created_at,
+    updated_at)
   VALUES (NEW.id,
     NEW.web_hook_id,
     NEW.trigger,
@@ -87,8 +87,8 @@ ELSIF (TG_OP = 'INSERT') THEN
     NEW.response_status,
     NEW.execution_duration,
     NEW.internal_error_message,
-    NEW.updated_at,
-    NEW.created_at);
+    NEW.created_at,
+    NEW.updated_at);
 END IF;
 RETURN NULL;
 
@@ -163,7 +163,7 @@ CREATE TABLE audit_events (
 )
 PARTITION BY RANGE (created_at);
 
-CREATE TABLE web_hook_logs_part_0c5294f417 (
+CREATE TABLE web_hook_logs (
     id bigint NOT NULL,
     web_hook_id integer NOT NULL,
     trigger character varying,
@@ -18915,7 +18915,7 @@ CREATE SEQUENCE vulnerability_user_mentions_id_seq
 
 ALTER SEQUENCE vulnerability_user_mentions_id_seq OWNED BY vulnerability_user_mentions.id;
 
-CREATE TABLE web_hook_logs (
+CREATE TABLE web_hook_logs_archived (
     id integer NOT NULL,
     web_hook_id integer NOT NULL,
     trigger character varying,
@@ -21608,11 +21608,11 @@ ALTER TABLE ONLY vulnerability_statistics
 ALTER TABLE ONLY vulnerability_user_mentions
     ADD CONSTRAINT vulnerability_user_mentions_pkey PRIMARY KEY (id);
 
-ALTER TABLE ONLY web_hook_logs_part_0c5294f417
-    ADD CONSTRAINT web_hook_logs_part_0c5294f417_pkey PRIMARY KEY (id, created_at);
+ALTER TABLE ONLY web_hook_logs_archived
+    ADD CONSTRAINT web_hook_logs_archived_pkey PRIMARY KEY (id);
 
 ALTER TABLE ONLY web_hook_logs
-    ADD CONSTRAINT web_hook_logs_pkey PRIMARY KEY (id);
+    ADD CONSTRAINT web_hook_logs_pkey PRIMARY KEY (id, created_at);
 
 ALTER TABLE ONLY web_hooks
     ADD CONSTRAINT web_hooks_pkey PRIMARY KEY (id);
@@ -24368,13 +24368,13 @@ CREATE UNIQUE INDEX index_vulns_user_mentions_on_vulnerability_id ON vulnerabili
 
 CREATE UNIQUE INDEX index_vulns_user_mentions_on_vulnerability_id_and_note_id ON vulnerability_user_mentions USING btree (vulnerability_id, note_id);
 
-CREATE INDEX index_web_hook_logs_on_created_at_and_web_hook_id ON web_hook_logs USING btree (created_at, web_hook_id);
+CREATE INDEX index_web_hook_logs_on_created_at_and_web_hook_id ON web_hook_logs_archived USING btree (created_at, web_hook_id);
 
-CREATE INDEX index_web_hook_logs_on_web_hook_id ON web_hook_logs USING btree (web_hook_id);
+CREATE INDEX index_web_hook_logs_on_web_hook_id ON web_hook_logs_archived USING btree (web_hook_id);
 
-CREATE INDEX index_web_hook_logs_part_on_created_at_and_web_hook_id ON ONLY web_hook_logs_part_0c5294f417 USING btree (created_at, web_hook_id);
+CREATE INDEX index_web_hook_logs_part_on_created_at_and_web_hook_id ON ONLY web_hook_logs USING btree (created_at, web_hook_id);
 
-CREATE INDEX index_web_hook_logs_part_on_web_hook_id ON ONLY web_hook_logs_part_0c5294f417 USING btree (web_hook_id);
+CREATE INDEX index_web_hook_logs_part_on_web_hook_id ON ONLY web_hook_logs USING btree (web_hook_id);
 
 CREATE INDEX index_web_hooks_on_group_id ON web_hooks USING btree (group_id) WHERE ((type)::text = 'GroupHook'::text);
 
@@ -26149,7 +26149,7 @@ ALTER TABLE ONLY operations_feature_flags_clients
 ALTER TABLE ONLY namespace_admin_notes
     ADD CONSTRAINT fk_rails_666166ea7b FOREIGN KEY (namespace_id) REFERENCES namespaces(id) ON DELETE CASCADE;
 
-ALTER TABLE ONLY web_hook_logs
+ALTER TABLE ONLY web_hook_logs_archived
     ADD CONSTRAINT fk_rails_666826e111 FOREIGN KEY (web_hook_id) REFERENCES web_hooks(id) ON DELETE CASCADE;
 
 ALTER TABLE ONLY jira_imports
@@ -26587,7 +26587,7 @@ ALTER TABLE ONLY approval_project_rules_users
 ALTER TABLE ONLY lists
     ADD CONSTRAINT fk_rails_baed5f39b7 FOREIGN KEY (milestone_id) REFERENCES milestones(id) ON DELETE CASCADE;
 
-ALTER TABLE web_hook_logs_part_0c5294f417
+ALTER TABLE web_hook_logs
     ADD CONSTRAINT fk_rails_bb3355782d FOREIGN KEY (web_hook_id) REFERENCES web_hooks(id) ON DELETE CASCADE;
 
 ALTER TABLE ONLY security_findings

spec/models/hooks/web_hook_log_archived_spec.rb

+# frozen_string_literal: true
+
+require 'spec_helper'
+
+RSpec.describe WebHookLogArchived do
+  let(:source_table) { WebHookLog }
+  let(:destination_table) { described_class }
+
+  it 'has the same columns as the source table' do
+    column_names_from_source_table = column_names(source_table)
+    column_names_from_destination_table = column_names(destination_table)
+
+    expect(column_names_from_destination_table).to match_array(column_names_from_source_table)
+  end
+
+  it 'has the same null constraints as the source table' do
+    constraints_from_source_table = null_constraints(source_table)
+    constraints_from_destination_table = null_constraints(destination_table)
+
+    expect(constraints_from_destination_table.to_a).to match_array(constraints_from_source_table.to_a)
+  end
+
+  it 'inserts the same record as the one in the source table', :aggregate_failures do
+    expect { create(:web_hook_log) }.to change { destination_table.count }.by(1)
+
+    event_from_source_table = source_table.connection.select_one(
+      "SELECT * FROM #{source_table.table_name} ORDER BY created_at desc LIMIT 1"
+    )
+    event_from_destination_table = destination_table.connection.select_one(
+      "SELECT * FROM #{destination_table.table_name} ORDER BY created_at desc LIMIT 1"
+    )
+
+    expect(event_from_destination_table).to eq(event_from_source_table)
+  end
+
+  def column_names(table)
+    table.connection.select_all(<<~SQL)
+      SELECT c.column_name
+      FROM information_schema.columns c
+      WHERE c.table_name = '#{table.table_name}'
+    SQL
+  end
+
+  def null_constraints(table)
+    table.connection.select_all(<<~SQL)
+      SELECT c.column_name, c.is_nullable
+      FROM information_schema.columns c
+      WHERE c.table_name = '#{table.table_name}'
+      AND c.column_name != 'created_at'
+    SQL
+  end
+end