Debug gitlab transactions stack

On CI, debug only in system tests

.gitlab/ci/rails.gitlab-ci.yml

@@ -312,6 +312,8 @@ rspec system pg12:
     - .rspec-base-pg12
     - .rails:rules:ee-and-foss-system
     - .rspec-system-parallel
+  variables:
+    DEBUG_GITLAB_TRANSACTION_STACK: "true"
 
 rspec system pg12 minimal:
   extends:

app/models/concerns/cross_database_modification.rb

@@ -4,15 +4,45 @@ module CrossDatabaseModification
   extend ActiveSupport::Concern
 
   class TransactionStackTrackRecord
+    DEBUG_STACK = Rails.env.test? && ENV['DEBUG_GITLAB_TRANSACTION_STACK']
+    LOG_FILENAME = Rails.root.join("log", "gitlab_transaction_stack.log")
+
+    def self.logger
+      @logger ||= Logger.new(LOG_FILENAME, formatter: ->(_, _, _, msg) { Gitlab::Json.dump(msg) + "\n" })
+    end
+
+    def self.log_gitlab_transactions_stack(action: nil, example: nil)
+      return unless DEBUG_STACK
+
+      message = "gitlab_transactions_stack performing #{action}"
+      message += " in example #{example}" if example
+
+      cleaned_backtrace = Gitlab::BacktraceCleaner.clean_backtrace(caller)
+        .reject { |line| line.include?('lib/gitlab/database/query_analyzer') }
+        .first(5)
+
+      logger.warn({
+        message: message,
+        action: action,
+        gitlab_transactions_stack: ::ApplicationRecord.gitlab_transactions_stack,
+        caller: cleaned_backtrace,
+        thread: Thread.current.object_id
+      })
+    end
+
     def initialize(subject, gitlab_schema)
       @subject = subject
       @gitlab_schema = gitlab_schema
       @subject.gitlab_transactions_stack.push(gitlab_schema)
+
+      self.class.log_gitlab_transactions_stack(action: :after_push)
     end
 
     def done!
       unless @done
         @done = true
+
+        self.class.log_gitlab_transactions_stack(action: :before_pop)
         @subject.gitlab_transactions_stack.pop
       end
 

lib/gitlab/database/query_analyzers/prevent_cross_database_modification.rb

@@ -56,6 +56,9 @@ module Gitlab
             context[:transaction_depth_by_db][database] -= 1
             if context[:transaction_depth_by_db][database] == 0
               context[:modified_tables_by_db][database].clear
+
+              # Attempt to troubleshoot  https://gitlab.com/gitlab-org/gitlab/-/issues/351531
+              ::CrossDatabaseModification::TransactionStackTrackRecord.log_gitlab_transactions_stack(action: :end_of_transaction)
             elsif context[:transaction_depth_by_db][database] < 0
               context[:transaction_depth_by_db][database] = 0
               raise CrossDatabaseModificationAcrossUnsupportedTablesError, "Misaligned cross-DB transactions discovered at query #{sql}. This could be a bug in #{self.class} or a valid issue to investigate. Read more at https://docs.gitlab.com/ee/development/database/multiple_databases.html#removing-cross-database-transactions ."

spec/spec_helper.rb

@@ -114,6 +114,15 @@ RSpec.configure do |config|
     config.run_all_when_everything_filtered = true
   end
 
+  # Attempt to troubleshoot  https://gitlab.com/gitlab-org/gitlab/-/issues/351531
+  config.after do |example|
+    if example.exception.is_a?(Gitlab::Database::QueryAnalyzers::PreventCrossDatabaseModification::CrossDatabaseModificationAcrossUnsupportedTablesError)
+      ::CrossDatabaseModification::TransactionStackTrackRecord.log_gitlab_transactions_stack(action: :after_failure, example: example.description)
+    else
+      ::CrossDatabaseModification::TransactionStackTrackRecord.log_gitlab_transactions_stack(action: :after_example, example: example.description)
+    end
+  end
+
   # Re-run failures locally with `--only-failures`
   config.example_status_persistence_file_path = ENV.fetch('RSPEC_LAST_RUN_RESULTS_FILE', './spec/examples.txt')