Update docs and calculate process start time via proc table

This updates monitor docs to reflect the new ruby and unicorn metrics as
well as making it so we fetch process start time via the proc table
instead of via CLOCK_BOOTTIME

doc/administration/monitoring/prometheus/gitlab_metrics.md

@@ -43,10 +43,11 @@ The following metrics are available:
 | redis_ping_latency_seconds        | Gauge     | 9.4   | Round trip time of the redis ping |
 | user_session_logins_total         | Counter   | 9.4   | Counter of how many users have logged in |
 | upload_file_does_not_exist        | Counter   | 10.7 in EE, 11.5 in CE  | Number of times an upload record could not find its file |
-| failed_login_captcha_total        | Gauge | 11.0 | Counter of failed CAPTCHA attempts during login |
-| successful_login_captcha_total    | Gauge | 11.0 | Counter of successful CAPTCHA attempts during login |
-| unicorn_active_connections        | Gauge | 11.0 | The number of active Unicorn connections (workers) |
-| unicorn_queued_connections        | Gauge | 11.0 | The number of queued Unicorn connections |
+| failed_login_captcha_total        | Gauge | 11.0  | Counter of failed CAPTCHA attempts during login |
+| successful_login_captcha_total    | Gauge | 11.0  | Counter of successful CAPTCHA attempts during login |
+| unicorn_active_connections        | Gauge | 11.0  | The number of active Unicorn connections (workers) |
+| unicorn_queued_connections        | Gauge | 11.0  | The number of queued Unicorn connections |
+| unicorn_workers                   | Gauge | 11.11 | The number of Unicorn workers |
 
 ### Ruby metrics
 
@@ -57,8 +58,11 @@ Some basic Ruby runtime metrics are available:
 | ruby_gc_duration_seconds_total         | Counter   | 11.1  | Time spent by Ruby in GC |
 | ruby_gc_stat_...                       | Gauge     | 11.1  | Various metrics from [GC.stat] |
 | ruby_file_descriptors                  | Gauge     | 11.1  | File descriptors per process |
-| ruby_memory_bytes                      | Gauge     | 11.1  | Memory usage by process |
+| ruby_process_resident_memory_bytes     | Gauge     | 11.1  | Memory usage by process |
 | ruby_sampler_duration_seconds_total    | Counter   | 11.1  | Time spent collecting stats |
+| ruby_process_cpu_seconds_total         | Gauge     | 11.11 | Total amount of cpu time per process |
+| ruby_process_max_fds                   | Gauge     | 11.11 | Maximum number of open file descriptors per process |
+| ruby_process_start_time_seconds        | Gauge     | 11.11 | The time the process started after system boot in seconds |
 
 [GC.stat]: https://ruby-doc.org/core-2.3.0/GC.html#method-c-stat
 

lib/gitlab/metrics/samplers/ruby_sampler.rb

@@ -24,13 +24,13 @@ module Gitlab
 
         def init_metrics
           metrics = {
-            file_descriptors:           ::Gitlab::Metrics.gauge(with_prefix(:file, :descriptors), 'File descriptors used', labels, :livesum),
-            memory_usage:               ::Gitlab::Metrics.gauge(with_prefix(:memory, :bytes), 'Memory used', labels, :livesum),
-            process_cpu_seconds_total:  ::Gitlab::Metrics.gauge(:process_cpu_seconds_total, 'Process CPU seconds total'),
-            process_max_fds:            ::Gitlab::Metrics.gauge(:process_max_fds, 'Process max fds'),
-            process_start_time_seconds: ::Gitlab::Metrics.gauge(:process_start_time_seconds, 'Process start time seconds'),
-            sampler_duration:           ::Gitlab::Metrics.counter(with_prefix(:sampler, :duration_seconds_total), 'Sampler time', labels),
-            total_time:                 ::Gitlab::Metrics.counter(with_prefix(:gc, :duration_seconds_total), 'Total GC time', labels)
+            file_descriptors:               ::Gitlab::Metrics.gauge(with_prefix(:file, :descriptors), 'File descriptors used', labels, :livesum),
+            process_cpu_seconds_total:      ::Gitlab::Metrics.gauge(with_prefix(:process, :cpu_seconds_total), 'Process CPU seconds total'),
+            process_max_fds:                ::Gitlab::Metrics.gauge(with_prefix(:process, :max_fds), 'Process max fds'),
+            process_resident_memory_bytes:  ::Gitlab::Metrics.gauge(with_prefix(:process, :resident_memory_bytes), 'Memory used', labels, :livesum),
+            process_start_time_seconds:     ::Gitlab::Metrics.gauge(with_prefix(:process, :start_time_seconds), 'Process start time seconds'),
+            sampler_duration:               ::Gitlab::Metrics.counter(with_prefix(:sampler, :duration_seconds_total), 'Sampler time', labels),
+            total_time:                     ::Gitlab::Metrics.counter(with_prefix(:gc, :duration_seconds_total), 'Total GC time', labels)
           }
 
           GC.stat.keys.each do |key|
@@ -44,10 +44,10 @@ module Gitlab
           start_time = System.monotonic_time
 
           metrics[:file_descriptors].set(labels.merge(worker_label), System.file_descriptor_count)
-          metrics[:memory_usage].set(labels.merge(worker_label), System.memory_usage)
           metrics[:process_cpu_seconds_total].set(labels.merge(worker_label), ::Gitlab::Metrics::System.cpu_time)
-          metrics[:process_start_time_seconds].set(labels.merge(worker_label), ::Gitlab::Metrics::System.process_start_time)
           metrics[:process_max_fds].set(labels.merge(worker_label), ::Gitlab::Metrics::System.max_open_file_descriptors)
+          metrics[:process_resident_memory_bytes].set(labels.merge(worker_label), System.memory_usage)
+          metrics[:process_start_time_seconds].set(labels.merge(worker_label), ::Gitlab::Metrics::System.process_start_time)
           sample_gc
 
           metrics[:sampler_duration].increment(labels, System.monotonic_time - start_time)

lib/gitlab/metrics/system.rb

@@ -33,6 +33,13 @@ module Gitlab
 
           max_fds
         end
+
+        def self.process_start_time
+          start_time_in_jiffies = Sys::ProcTable.ps(pid: Process.pid).starttime
+          return 0 unless start_time_in_jiffies
+
+          start_time_in_jiffies / 100
+        end
       else
         def self.memory_usage
           0.0
@@ -45,6 +52,10 @@ module Gitlab
         def self.max_open_file_descriptors
           0
         end
+
+        def self.process_start_time
+          0
+        end
       end
 
       # THREAD_CPUTIME is not supported on OS X
@@ -60,17 +71,6 @@ module Gitlab
         end
       end
 
-      # CLOCK_BOOTTIME is not supported on OS X
-      if Process.const_defined?(:CLOCK_BOOTTIME)
-        def self.process_start_time
-          Process
-            .clock_gettime(Process::CLOCK_BOOTTIME, :float_second)
-        end
-      else
-        def self.process_start_time
-          0.0
-        end
-      end
       # Returns the current real time in a given precision.
       #
       # Returns the time as a Float for precision = :float_second.

spec/lib/gitlab/metrics/samplers/ruby_sampler_spec.rb

@@ -20,10 +20,10 @@ describe Gitlab::Metrics::Samplers::RubySampler do
       sampler.sample
     end
 
-    it 'adds a metric containing the memory usage' do
+    it 'adds a metric containing the process resident memory bytes' do
       expect(Gitlab::Metrics::System).to receive(:memory_usage).and_return(9000)
 
-      expect(sampler.metrics[:memory_usage]).to receive(:set).with({}, 9000)
+      expect(sampler.metrics[:process_resident_memory_bytes]).to receive(:set).with({}, 9000)
 
       sampler.sample
     end
@@ -37,7 +37,7 @@ describe Gitlab::Metrics::Samplers::RubySampler do
       sampler.sample
     end
 
-    it 'adds a metric containing the processes total cpu time' do
+    it 'adds a metric containing the process total cpu time' do
       expect(Gitlab::Metrics::System).to receive(:cpu_time).and_return(0.51)
       expect(sampler.metrics[:process_cpu_seconds_total]).to receive(:set).with({}, 0.51)
 

spec/lib/gitlab/metrics/system_spec.rb

@@ -19,6 +19,12 @@ describe Gitlab::Metrics::System do
         expect(described_class.max_open_file_descriptors).to be > 0
       end
     end
+
+    describe '.process_start_time' do
+      it 'returns the process start time' do
+        expect(described_class.process_start_time).to be > 0
+      end
+    end
   else
     describe '.memory_usage' do
       it 'returns 0.0' do
@@ -37,6 +43,12 @@ describe Gitlab::Metrics::System do
         expect(described_class.max_open_file_descriptors).to eq(0)
       end
     end
+
+    describe 'process_start_time' do
+      it 'returns 0' do
+        expect(described_class.process_start_time).to eq(0)
+      end
+    end
   end
 
   describe '.cpu_time' do
@@ -56,10 +68,4 @@ describe Gitlab::Metrics::System do
       expect(described_class.monotonic_time).to be_an(Float)
     end
   end
-
-  describe '.process_start_time' do
-    it 'returns a Float' do
-      expect(described_class.process_start_time).to be_an(Float)
-    end
-  end
 end