Merge branch 'dz-et-add-gzip-support' into 'master'

Add gzip support to error tracking collector

See merge request gitlab-org/gitlab!71949

config/application.rb

@@ -28,6 +28,7 @@ module Gitlab
     require_dependency Rails.root.join('lib/gitlab/redis/sessions')
     require_dependency Rails.root.join('lib/gitlab/current_settings')
     require_dependency Rails.root.join('lib/gitlab/middleware/read_only')
+    require_dependency Rails.root.join('lib/gitlab/middleware/compressed_json')
     require_dependency Rails.root.join('lib/gitlab/middleware/basic_health_check')
     require_dependency Rails.root.join('lib/gitlab/middleware/same_site_cookies')
     require_dependency Rails.root.join('lib/gitlab/middleware/handle_ip_spoof_attack_error')
@@ -319,6 +320,8 @@ module Gitlab
 
     config.middleware.insert_after Rack::Sendfile, ::Gitlab::Middleware::RackMultipartTempfileFactory
 
+    config.middleware.insert_before Rack::Runtime, ::Gitlab::Middleware::CompressedJson
+
     # Allow access to GitLab API from other domains
     config.middleware.insert_before Warden::Manager, Rack::Cors do
       headers_to_expose = %w[Link X-Total X-Total-Pages X-Per-Page X-Page X-Next-Page X-Prev-Page X-Gitlab-Blob-Id X-Gitlab-Commit-Id X-Gitlab-Content-Sha256 X-Gitlab-Encoding X-Gitlab-File-Name X-Gitlab-File-Path X-Gitlab-Last-Commit-Id X-Gitlab-Ref X-Gitlab-Size]

lib/gitlab/middleware/compressed_json.rb

+# frozen_string_literal: true
+
+module Gitlab
+  module Middleware
+    class CompressedJson
+      COLLECTOR_PATH = '/api/v4/error_tracking/collector'
+      MAXIMUM_BODY_SIZE = 200.kilobytes.to_i
+
+      def initialize(app)
+        @app = app
+      end
+
+      def call(env)
+        if compressed_et_request?(env)
+          input = extract(env['rack.input'])
+
+          if input.length > MAXIMUM_BODY_SIZE
+            return too_large
+          end
+
+          env.delete('HTTP_CONTENT_ENCODING')
+          env['CONTENT_LENGTH'] = input.length
+          env['rack.input'] = StringIO.new(input)
+        end
+
+        @app.call(env)
+      end
+
+      def compressed_et_request?(env)
+        env['REQUEST_METHOD'] == 'POST' &&
+          env['HTTP_CONTENT_ENCODING'] == 'gzip' &&
+          env['CONTENT_TYPE'] == 'application/json' &&
+          env['PATH_INFO'].start_with?((File.join(relative_url, COLLECTOR_PATH)))
+      end
+
+      def too_large
+        [413, { 'Content-Type' => 'text/plain' }, ['Payload Too Large']]
+      end
+
+      def relative_url
+        File.join('', Gitlab.config.gitlab.relative_url_root).chomp('/')
+      end
+
+      def extract(input)
+        Zlib::GzipReader.new(input).read(MAXIMUM_BODY_SIZE + 1)
+      end
+    end
+  end
+end

spec/lib/gitlab/middleware/compressed_json_spec.rb

+# frozen_string_literal: true
+
+require 'spec_helper'
+
+RSpec.describe Gitlab::Middleware::CompressedJson do
+  let_it_be(:decompressed_input) { '{"foo": "bar"}' }
+  let_it_be(:input) { ActiveSupport::Gzip.compress(decompressed_input) }
+
+  let(:app) { double(:app) }
+  let(:middleware) { described_class.new(app) }
+  let(:env) do
+    {
+      'HTTP_CONTENT_ENCODING' => 'gzip',
+      'REQUEST_METHOD' => 'POST',
+      'CONTENT_TYPE' => 'application/json',
+      'PATH_INFO' => path,
+      'rack.input' => StringIO.new(input)
+    }
+  end
+
+  shared_examples 'decompress middleware' do
+    it 'replaces input with a decompressed content' do
+      expect(app).to receive(:call)
+
+      middleware.call(env)
+
+      expect(env['rack.input'].read).to eq(decompressed_input)
+      expect(env['CONTENT_LENGTH']).to eq(decompressed_input.length)
+      expect(env['HTTP_CONTENT_ENCODING']).to be_nil
+    end
+  end
+
+  describe '#call' do
+    context 'with collector route' do
+      let(:path) { '/api/v4/error_tracking/collector/1/store'}
+
+      it_behaves_like 'decompress middleware'
+    end
+
+    context 'with collector route under relative url' do
+      let(:path) { '/gitlab/api/v4/error_tracking/collector/1/store'}
+
+      before do
+        stub_config_setting(relative_url_root: '/gitlab')
+      end
+
+      it_behaves_like 'decompress middleware'
+    end
+
+    context 'with some other route' do
+      let(:path) { '/api/projects/123' }
+
+      it 'keeps the original input' do
+        expect(app).to receive(:call)
+
+        middleware.call(env)
+
+        expect(env['rack.input'].read).to eq(input)
+        expect(env['HTTP_CONTENT_ENCODING']).to eq('gzip')
+      end
+    end
+
+    context 'payload is too large' do
+      let(:body_limit) { Gitlab::Middleware::CompressedJson::MAXIMUM_BODY_SIZE }
+      let(:decompressed_input) { 'a' * (body_limit + 100) }
+      let(:input) { ActiveSupport::Gzip.compress(decompressed_input) }
+      let(:path) { '/api/v4/error_tracking/collector/1/envelope'}
+
+      it 'reads only limited size' do
+        expect(middleware.call(env))
+          .to eq([413, { 'Content-Type' => 'text/plain' }, ['Payload Too Large']])
+      end
+    end
+  end
+end

spec/requests/api/error_tracking/collector_spec.rb

@@ -122,6 +122,20 @@ RSpec.describe API::ErrorTracking::Collector do
       it_behaves_like 'bad request'
     end
 
+    context 'gzip body' do
+      let(:headers) do
+        {
+          'X-Sentry-Auth' => "Sentry sentry_key=#{client_key.public_key}",
+          'HTTP_CONTENT_ENCODING' => 'gzip',
+          'CONTENT_TYPE' => 'application/json'
+        }
+      end
+
+      let(:raw_event) { ActiveSupport::Gzip.compress(fixture_file('error_tracking/parsed_event.json')) }
+
+      it_behaves_like 'successful request'
+    end
+
     context 'sentry_key as param and empty headers' do
       let(:url) { "/error_tracking/collector/api/#{project.id}/store?sentry_key=#{sentry_key}" }
       let(:headers) { {} }