Check if build trace is corrupted by comparing size

This commit adds support for incrementing a Prometheus counter in case
of detecting a malformed / corrupted build log. We store a bytesize of a
build log in the ci_build_pending_states table and compare it to the
size calculated using stored build trace chunks.

If CRC32 checksum of a build log is invalid and a bytesize differs too,
we increment `corrupted` build logs counter and send an appropritate log
to Sentry.

app/services/ci/update_build_state_service.rb

@@ -82,6 +82,10 @@ module Ci
         unless checksum.valid?
           metrics.increment_trace_operation(operation: :invalid)
 
+          if checksum.corrupted?
+            metrics.increment_trace_operation(operation: :corrupted)
+          end
+
           next unless log_invalid_chunks?
 
           ::Gitlab::ErrorTracking.log_exception(InvalidTraceError.new,
@@ -89,7 +93,8 @@ module Ci
             build_id: build.id,
             state_crc32: checksum.state_crc32,
             chunks_crc32: checksum.chunks_crc32,
-            chunks_count: checksum.chunks_count
+            chunks_count: checksum.chunks_count,
+            chunks_corrupted: checksum.corrupted?
           )
         end
       end
@@ -151,13 +156,21 @@ module Ci
     end
 
     def has_checksum?
-      params.dig(:checksum).present?
+      trace_checksum.present?
     end
 
     def build_running?
       build_state == 'running'
     end
 
+    def trace_checksum
+      params.dig(:output, :checksum) || params.dig(:checksum)
+    end
+
+    def trace_bytesize
+      params.dig(:output, :bytesize)
+    end
+
     def pending_state
       strong_memoize(:pending_state) { ensure_pending_state }
     end
@@ -166,7 +179,8 @@ module Ci
       build_state = Ci::BuildPendingState.safe_find_or_create_by(
         build_id: build.id,
         state: params.fetch(:state),
-        trace_checksum: params.fetch(:checksum),
+        trace_checksum: trace_checksum,
+        trace_bytesize: trace_bytesize,
         failure_reason: params.dig(:failure_reason)
       )
 

changelogs/unreleased/feature-gb-build-trace-malformed-size-metric.yml

+---
+title: Detect corrupted build logs and report them by incrementing Prometheus counter
+merge_request: 49004
+author:
+type: added

db/migrate/20201204141038_add_trace_bytesize_to_ci_build_pending_states.rb

+# frozen_string_literal: true
+
+class AddTraceBytesizeToCiBuildPendingStates < ActiveRecord::Migration[6.0]
+  DOWNTIME = false
+
+  def change
+    add_column :ci_build_pending_states, :trace_bytesize, :bigint
+  end
+end

db/schema_migrations/20201204141038

+9dc8d6b557198a60def4690ea06ec3dc9a29deca9082b7b03666aaed483a42f1
\ No newline at end of file

db/structure.sql

@@ -10051,7 +10051,8 @@ CREATE TABLE ci_build_pending_states (
     build_id bigint NOT NULL,
     state smallint,
     failure_reason smallint,
-    trace_checksum bytea
+    trace_checksum bytea,
+    trace_bytesize bigint
 );
 
 CREATE SEQUENCE ci_build_pending_states_id_seq

lib/api/ci/runner.rb

@@ -176,6 +176,10 @@ module API
           optional :state, type: String, desc: %q(Job's status: success, failed)
           optional :checksum, type: String, desc: %q(Job's trace CRC32 checksum)
           optional :failure_reason, type: String, desc: %q(Job's failure_reason)
+          optional :output, type: Hash, desc: %q(Build log state) do
+            optional :checksum, type: String, desc: %q(Job's trace CRC32 checksum)
+            optional :bytesize, type: Integer, desc: %q(Job's trace size in bytes)
+          end
         end
         put '/:id' do
           job = authenticate_job!

lib/gitlab/ci/trace/checksum.rb

@@ -64,12 +64,24 @@ module Gitlab
           end
         end
 
+        def state_bytesize
+          strong_memoize(:state_crc32) { build.pending_state&.trace_bytesize }
+        end
+
         def trace_size
-          trace_chunks.reduce(0) do |total, chunk|
-            total + chunk_size(chunk)
+          strong_memoize(:trace_size) do
+            trace_chunks.reduce(0) do |total, chunk|
+              total + chunk_size(chunk)
+            end
           end
         end
 
+        def corrupted?
+          return false if valid? || state_bytesize.nil?
+
+          state_bytesize.to_i != trace_size.to_i
+        end
+
         def chunks_count
           trace_chunks.to_a.size
         end