Use median over multiple runs for mem-on-boot

.gitlab/ci/memory.gitlab-ci.yml

@@ -9,7 +9,7 @@
   artifacts:
     reports:
       metrics: "${METRICS_FILE}"
-    expire_in: 31d
+    expire_in: 62d
 
 
 # Show memory usage caused by invoking require per gem.
@@ -26,11 +26,17 @@ memory-on-boot:
     NODE_ENV: "production"
     RAILS_ENV: "production"
     SETUP_DB: "true"
-    MEMORY_ON_BOOT_FILE: "tmp/memory_on_boot.txt"
+    MEMORY_ON_BOOT_FILE_PREFIX: "tmp/memory_on_boot_"
+    TEST_COUNT: 5
   script:
-    - PATH_TO_HIT="/users/sign_in" CUT_OFF=0.3 bundle exec derailed exec perf:mem >> "${MEMORY_ON_BOOT_FILE}"
-    - scripts/generate-memory-metrics-on-boot "${MEMORY_ON_BOOT_FILE}" >> "${METRICS_FILE}"
+    - |
+      for i in $(seq 1 $TEST_COUNT)
+      do
+        echo "Starting run $i out of $TEST_COUNT"
+        PATH_TO_HIT="/users/sign_in" CUT_OFF=0.3 bundle exec derailed exec perf:mem >> "${MEMORY_ON_BOOT_FILE_PREFIX}$i.txt"
+      done
+    - scripts/generate-memory-metrics-on-boot "${MEMORY_ON_BOOT_FILE_PREFIX}" "$TEST_COUNT" >> "${METRICS_FILE}"
   artifacts:
     paths:
       - "${METRICS_FILE}"
-      - "${MEMORY_ON_BOOT_FILE}"
+      - "${MEMORY_ON_BOOT_FILE_PREFIX}*.txt"

scripts/generate-memory-metrics-on-boot

 #!/usr/bin/env ruby
 # frozen_string_literal: true
 
-abort "usage: #{__FILE__} <memory_bundle_mem_file_name>" unless ARGV.length == 1
-memory_bundle_mem_file_name = ARGV.first
+abort "usage: #{__FILE__} <memory_bundle_mem_file_name_prefix> <test_count>" unless ARGV.length == 2
+memory_bundle_mem_file_name_prefix = ARGV.first
+test_count = ARGV.last.to_i
 
-full_report = File.open(memory_bundle_mem_file_name).read
+results = []
+(1..test_count).each do |i|
+  report_filename = "#{memory_bundle_mem_file_name_prefix}#{i}.txt"
 
-stats = /TOP: (?<total_mibs_str>.*) MiB/.match(full_report)
-abort 'failed to process the benchmark output' unless stats
+  stats = nil
+  File.foreach(report_filename).detect do |line|
+    stats = /TOP: (?<total_mibs_str>.*) MiB/.match(line)
+  end
+  abort 'failed to process the benchmark output' unless stats
 
-puts "total_memory_used_by_dependencies_on_boot_prod_env_mb #{stats[:total_mibs_str].to_f.round(1)}"
+  total_mibs = stats[:total_mibs_str].to_f
+  results << total_mibs
+end
+
+res = results.sort
+median = (res[(test_count - 1) / 2] + res[test_count / 2]) / 2.0
+
+METRIC_NAME = "total_memory_used_by_dependencies_on_boot_prod_env_mb"
+
+puts "# TYPE #{METRIC_NAME} gauge"
+puts "# UNIT #{METRIC_NAME} mebibytes"
+puts "#{METRIC_NAME} #{median.round(1)}"