Merge branch 'ee-62747-gather-ci-tests-memory-usage-data' into 'master'

Gather memory usage data during tests

See merge request gitlab-org/gitlab-ee!14588

.gitlab/ci/rails.gitlab-ci.yml

@@ -66,6 +66,8 @@
     - scripts/gitaly-test-spawn
     - date
     - 'export KNAPSACK_TEST_FILE_PATTERN=$(ruby -r./lib/quality/test_level.rb -e "puts Quality::TestLevel.new.pattern(:${TEST_LEVEL})")'
+    - mkdir -p tmp/memory_test
+    - export MEMORY_TEST_PATH="tmp/memory_test/${TEST_TOOL}_${TEST_LEVEL}_${DATABASE}_node_${CI_NODE_INDEX}_${CI_NODE_TOTAL}_memory.csv"
     - knapsack rspec "--color --format documentation --format RspecJunitFormatter --out junit_rspec.xml --tag level:${TEST_LEVEL} --tag ~geo"
     - date
   artifacts:
@@ -77,6 +79,7 @@
       - rspec_flaky/
       - rspec_profiling/
       - tmp/capybara/
+      - tmp/memory_test/
 #    reports:
 #      junit: junit_rspec.xml
 
@@ -273,6 +276,7 @@ coverage:
   stage: post-test
   script:
     - bundle exec scripts/merge-simplecov
+    - bundle exec scripts/gather-test-memory-data
   coverage: '/LOC \((\d+\.\d+%)\) covered.$/'
   artifacts:
     name: coverage
@@ -280,6 +284,7 @@ coverage:
     paths:
       - coverage/index.html
       - coverage/assets/
+      - tmp/memory_test/
   except:
     - /(^docs[\/-].*|.*-docs$)/
     - /(^qa[\/-].*|.*-qa$)/
@@ -307,6 +312,8 @@ coverage:
     - scripts/gitaly-test-spawn
     - date
     - 'export KNAPSACK_TEST_FILE_PATTERN=$(ruby -r./lib/quality/test_level.rb -e "puts Quality::TestLevel.new(%(ee/)).pattern(:${TEST_LEVEL})")'
+    - mkdir -p tmp/memory_test
+    - export MEMORY_TEST_PATH="tmp/memory_test/ee_${TEST_TOOL}_${TEST_LEVEL}_${DATABASE}_node_${CI_NODE_INDEX}_${CI_NODE_TOTAL}_memory.csv"
     - knapsack rspec "--color --format documentation --format RspecJunitFormatter --out junit_rspec.xml --tag level:${TEST_LEVEL} --tag ~geo"
     - date
 
@@ -332,11 +339,15 @@ rspec system pg ee:
   script:
     - JOB_NAME=( $CI_JOB_NAME )
     - TEST_TOOL=${JOB_NAME[0]}
+    - TEST_LEVEL=${JOB_NAME[1]}
+    - DATABASE=${JOB_NAME[2]}
     - export KNAPSACK_TEST_FILE_PATTERN="ee/spec/**{,/*/**}/*_spec.rb" KNAPSACK_GENERATE_REPORT=true CACHE_CLASSES=true
     - export KNAPSACK_REPORT_PATH=knapsack/${CI_PROJECT_NAME}/${TEST_TOOL}_node_${CI_NODE_INDEX}_${CI_NODE_TOTAL}_report.json
     - cp ${EE_KNAPSACK_RSPEC_SUITE_REPORT_PATH} ${KNAPSACK_REPORT_PATH}
     - source scripts/prepare_postgres_fdw.sh
     - scripts/gitaly-test-spawn
+    - mkdir -p tmp/memory_test
+    - export MEMORY_TEST_PATH="tmp/memory_test/ee_${TEST_TOOL}_${TEST_LEVEL}_${DATABASE}_node_${CI_NODE_INDEX}_${CI_NODE_TOTAL}_memory.csv"
     - knapsack rspec "-Ispec --color --format documentation --format RspecJunitFormatter --out junit_rspec.xml --tag geo"
 
 rspec geo pg ee:

scripts/gather-test-memory-data

+#!/usr/bin/env ruby
+
+require 'csv'
+
+def join_csv_files(output_path, input_paths)
+  return if input_paths.empty?
+
+  input_csvs = input_paths.map do |input_path|
+    CSV.read(input_path, headers: true)
+  end
+
+  CSV.open(output_path, "w", headers: input_csvs.first.headers, write_headers: true) do |output_csv|
+    input_csvs.each do |input_csv|
+      input_csv.each do |line|
+        output_csv << line
+      end
+    end
+  end
+end
+
+join_csv_files('tmp/memory_test/report.csv', Dir['tmp/memory_test/*.csv'].sort)

spec/spec_helper.rb

@@ -105,6 +105,7 @@ RSpec.configure do |config|
   config.include RedisHelpers
   config.include Rails.application.routes.url_helpers, type: :routing
   config.include PolicyHelpers, type: :policy
+  config.include MemoryUsageHelper
 
   if ENV['CI']
     # This includes the first try, i.e. tests will be run 4 times before failing.

spec/support/helpers/memory_usage_helper.rb

+# frozen_string_literal: true
+
+module MemoryUsageHelper
+  extend ActiveSupport::Concern
+
+  def gather_memory_data(csv_path)
+    write_csv_entry(csv_path,
+      {
+        example_group_path: TestEnv.topmost_example_group[:location],
+        example_group_description: TestEnv.topmost_example_group[:description],
+        time: Time.current,
+        job_name: ENV['CI_JOB_NAME']
+      }.merge(get_memory_usage))
+  end
+
+  def write_csv_entry(path, entry)
+    CSV.open(path, "a", headers: entry.keys, write_headers: !File.exist?(path)) do |file|
+      file << entry.values
+    end
+  end
+
+  def get_memory_usage
+    output, status = Gitlab::Popen.popen(%w(free -m))
+    abort "`free -m` return code is #{status}: #{output}" unless status.zero?
+
+    result = output.split("\n")[1].split(" ")[1..-1]
+    attrs = %i(m_total m_used m_free m_shared m_buffers_cache m_available).freeze
+
+    attrs.zip(result).to_h
+  end
+
+  included do |config|
+    config.after(:all) do
+      gather_memory_data(ENV['MEMORY_TEST_PATH']) if ENV['MEMORY_TEST_PATH']
+    end
+  end
+end

spec/support/helpers/test_env.rb

@@ -2,6 +2,7 @@ require 'rspec/mocks'
 require 'toml-rb'
 
 module TestEnv
+  extend ActiveSupport::Concern
   extend self
 
   ComponentFailedToInstallError = Class.new(StandardError)
@@ -108,6 +109,12 @@ module TestEnv
     setup_forked_repo
   end
 
+  included do |config|
+    config.append_before do
+      set_current_example_group
+    end
+  end
+
   def disable_mailer
     allow_any_instance_of(NotificationService).to receive(:mailer)
       .and_return(double.as_null_object)
@@ -297,8 +304,23 @@ module TestEnv
     FileUtils.rm_rf(path)
   end
 
+  def current_example_group
+    Thread.current[:current_example_group]
+  end
+
+  # looking for a top-level `describe`
+  def topmost_example_group
+    example_group = current_example_group
+    example_group = example_group[:parent_example_group] until example_group[:parent_example_group].nil?
+    example_group
+  end
+
   private
 
+  def set_current_example_group
+    Thread.current[:current_example_group] = ::RSpec.current_example.metadata[:example_group]
+  end
+
   # These are directories that should be preserved at cleanup time
   def test_dirs
     @test_dirs ||= %w[