Merge branch 'cablett-service-desk-delivered-to-fallback' into 'master'

Handle Received header fallback for missing Delivered-To

See merge request gitlab-org/gitlab!81489

config/feature_flags/development/use_received_header_for_incoming_emails.yml

+---
+name: use_received_header_for_incoming_emails
+introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/81489
+rollout_issue_url:
+milestone: '14.9'
+type: development
+group: group::certify
+default_enabled: true

lib/gitlab/email/receiver.rb

@@ -8,6 +8,8 @@ module Gitlab
     class Receiver
       include Gitlab::Utils::StrongMemoize
 
+      RECEIVED_HEADER_REGEX = /for\s+\<(.+)\>/.freeze
+
       def initialize(raw)
         @raw = raw
       end
@@ -37,6 +39,8 @@ module Gitlab
           delivered_to: delivered_to.map(&:value),
           envelope_to: envelope_to.map(&:value),
           x_envelope_to: x_envelope_to.map(&:value),
+          # reduced down to what looks like an email in the received headers
+          received_recipients: recipients_from_received_headers,
           meta: {
             client_id: "email/#{mail.from.first}",
             project: handler&.project&.full_path
@@ -82,7 +86,8 @@ module Gitlab
         find_key_from_references ||
           find_key_from_delivered_to_header ||
           find_key_from_envelope_to_header ||
-          find_key_from_x_envelope_to_header
+          find_key_from_x_envelope_to_header ||
+          find_first_key_from_received_headers
       end
 
       def ensure_references_array(references)
@@ -117,6 +122,10 @@ module Gitlab
         Array(mail[:x_envelope_to])
       end
 
+      def received
+        Array(mail[:received])
+      end
+
       def find_key_from_delivered_to_header
         delivered_to.find do |header|
           key = email_class.key_from_address(header.value)
@@ -138,6 +147,21 @@ module Gitlab
         end
       end
 
+      def find_first_key_from_received_headers
+        return unless ::Feature.enabled?(:use_received_header_for_incoming_emails, default_enabled: :yaml)
+
+        recipients_from_received_headers.find do |email|
+          key = email_class.key_from_address(email)
+          break key if key
+        end
+      end
+
+      def recipients_from_received_headers
+        strong_memoize :emails_from_received_headers do
+          received.map { |header| header.value[RECEIVED_HEADER_REGEX, 1] }.compact
+        end
+      end
+
       def ignore_auto_reply!
         if auto_submitted? || auto_replied?
           raise AutoGeneratedEmailError

spec/fixtures/emails/missing_delivered_to_header.eml

+Return-Path: <jake@example.com>
+Received: from myserver.example.com ([unix socket]) by myserver (Cyrus v2.2.13-Debian-2.2.13-19+squeeze3) with LMTPA; Thu, 13 Jun 2013 17:03:50 -0400
+Received: by 10.0.0.1 with HTTP; Thu, 13 Jun 2013 14:03:48 -0700
+Received: from blabla.google.com (blabla.google.com. [1.1.1.1])
+        by bla.google.com with SMTPS id something.1.1.1.1.1.1.1
+        for <incoming+gitlabhq/gitlabhq+auth_token@appmail.example.com>
+        (Google Transport Security);
+        Mon, 21 Feb 2022 14:41:58 -0800 (PST)
+Received: from mail.example.com (mail.example.com [IPv6:2607:f8b0:4001:c03::234]) by myserver.example.com (8.14.3/8.14.3/Debian-9.4) with ESMTP id r5DL3nFJ016967 (version=TLSv1/SSLv3 cipher=RC4-SHA bits=128 verify=NOT) for <incoming+gitlabhq/gitlabhq@example.com>; Thu, 13 Jun 2013 17:03:50 -0400
+From: "jake@example.com" <jake@example.com>
+To: "support@example.com" <support@example.com>
+Subject: Insert hilarious subject line here
+Date: Tue, 26 Nov 2019 14:22:41 +0000
+Message-ID: <7e2296f83dbf4de388cbf5f56f52c11f@EXDAG29-1.EXCHANGE.INT>
+Accept-Language: de-DE, en-US
+Content-Language: de-DE
+X-MS-Has-Attach:
+X-MS-TNEF-Correlator:
+x-ms-exchange-transport-fromentityheader: Hosted
+x-originating-ip: [62.96.54.178]
+Content-Type: multipart/alternative;
+	boundary="_000_7e2296f83dbf4de388cbf5f56f52c11fEXDAG291EXCHANGEINT_"
+MIME-Version: 1.0
+
+--_000_7e2296f83dbf4de388cbf5f56f52c11fEXDAG291EXCHANGEINT_
+Content-Type: text/plain; charset="iso-8859-1"
+Content-Transfer-Encoding: quoted-printable
+
+
+
+--_000_7e2296f83dbf4de388cbf5f56f52c11fEXDAG291EXCHANGEINT_
+Content-Type: text/html; charset="iso-8859-1"
+Content-Transfer-Encoding: quoted-printable
+
+Look, a message with no Delivered-To header! Let's fallback to Received: in case it's there.

spec/fixtures/emails/valid_note_on_issuable.eml

 Return-Path: <jake@adventuretime.ooo>
 Received: from iceking.adventuretime.ooo ([unix socket]) by iceking (Cyrus v2.2.13-Debian-2.2.13-19+squeeze3) with LMTPA; Thu, 13 Jun 2013 17:03:50 -0400
-Received: from mail-ie0-x234.google.com (mail-ie0-x234.google.com [IPv6:2607:f8b0:4001:c03::234]) by iceking.adventuretime.ooo (8.14.3/8.14.3/Debian-9.4) with ESMTP id r5DL3nFJ016967 (version=TLSv1/SSLv3 cipher=RC4-SHA bits=128 verify=NOT) for <incoming+gitlabhq/gitlabhq@appmail.adventuretime.ooo>; Thu, 13 Jun 2013 17:03:50 -0400
+Received: from mail-ie0-x234.google.com (mail-ie0-x234.google.com [IPv6:2607:f8b0:4001:c03::234]) by iceking.adventuretime.ooo (8.14.3/8.14.3/Debian-9.4) with ESMTP id r5DL3nFJ016967 (version=TLSv1/SSLv3 cipher=RC4-SHA bits=128 verify=NOT) for <incoming+gitlabhq-gitlabhq-project_id-auth_token-issue-issue_iid@appmail.adventuretime.ooo>; Thu, 13 Jun 2013 17:03:50 -0400
 Received: by mail-ie0-f180.google.com with SMTP id f4so21977375iea.25 for <incoming+gitlabhq-gitlabhq-project_id-auth_token-issue-issue_iid@appmail.adventuretime.ooo>; Thu, 13 Jun 2013 14:03:48 -0700
 Received: by 10.0.0.1 with HTTP; Thu, 13 Jun 2013 14:03:48 -0700
 Date: Thu, 13 Jun 2013 17:03:48 -0400

spec/lib/gitlab/email/receiver_spec.rb

@@ -32,12 +32,21 @@ RSpec.describe Gitlab::Email::Receiver do
 
       metadata = receiver.mail_metadata
 
-      expect(metadata.keys).to match_array(%i(mail_uid from_address to_address mail_key references delivered_to envelope_to x_envelope_to meta))
+      expect(metadata.keys).to match_array(%i(mail_uid from_address to_address mail_key references delivered_to envelope_to x_envelope_to meta received_recipients))
       expect(metadata[:meta]).to include(client_id: 'email/jake@example.com', project: project.full_path)
       expect(metadata[meta_key]).to eq(meta_value)
     end
   end
 
+  shared_examples 'failed receive' do
+    it 'adds metric event' do
+      expect(::Gitlab::Metrics::BackgroundTransaction).to receive(:current).and_return(metric_transaction)
+      expect(metric_transaction).to receive(:add_event).with('email_receiver_error', { error: expected_error.name })
+
+      expect { receiver.execute }.to raise_error(expected_error)
+    end
+  end
+
   context 'when the email contains a valid email address in a header' do
     before do
       stub_incoming_email_setting(enabled: true, address: "incoming+%{key}@appmail.example.com")
@@ -74,14 +83,25 @@ RSpec.describe Gitlab::Email::Receiver do
 
       it_behaves_like 'successful receive'
     end
-  end
 
-  shared_examples 'failed receive' do
-    it 'adds metric event' do
-      expect(::Gitlab::Metrics::BackgroundTransaction).to receive(:current).and_return(metric_transaction)
-      expect(metric_transaction).to receive(:add_event).with('email_receiver_error', { error: expected_error.name })
+    context 'when all other headers are missing' do
+      let(:email_raw) { fixture_file('emails/missing_delivered_to_header.eml') }
+      let(:meta_key) { :received_recipients }
+      let(:meta_value) { ['incoming+gitlabhq/gitlabhq+auth_token@appmail.example.com', 'incoming+gitlabhq/gitlabhq@example.com'] }
 
-      expect { receiver.execute }.to raise_error(expected_error)
+      context 'when use_received_header_for_incoming_emails is enabled' do
+        it_behaves_like 'successful receive'
+      end
+
+      context 'when use_received_header_for_incoming_emails is disabled' do
+        let(:expected_error) { Gitlab::Email::UnknownIncomingEmail }
+
+        before do
+          stub_feature_flags(use_received_header_for_incoming_emails: false)
+        end
+
+        it_behaves_like 'failed receive'
+      end
     end
   end