Commit 8906caba authored by Douwe Maan's avatar Douwe Maan

Changes and stuff.

parent a428838d
...@@ -274,3 +274,5 @@ gem "newrelic_rpm" ...@@ -274,3 +274,5 @@ gem "newrelic_rpm"
gem 'octokit', '3.7.0' gem 'octokit', '3.7.0'
gem "mail_room", github: "DouweM/mail_room", branch: "sidekiq" gem "mail_room", github: "DouweM/mail_room", branch: "sidekiq"
gem 'email_reply_parser'
...@@ -163,6 +163,7 @@ GEM ...@@ -163,6 +163,7 @@ GEM
dotenv (0.9.0) dotenv (0.9.0)
dropzonejs-rails (0.7.1) dropzonejs-rails (0.7.1)
rails (> 3.1) rails (> 3.1)
email_reply_parser (0.5.8)
email_spec (1.6.0) email_spec (1.6.0)
launchy (~> 2.1) launchy (~> 2.1)
mail (~> 2.2) mail (~> 2.2)
...@@ -780,6 +781,7 @@ DEPENDENCIES ...@@ -780,6 +781,7 @@ DEPENDENCIES
diffy (~> 3.0.3) diffy (~> 3.0.3)
doorkeeper (= 2.1.3) doorkeeper (= 2.1.3)
dropzonejs-rails dropzonejs-rails
email_reply_parser
email_spec (~> 1.6.0) email_spec (~> 1.6.0)
enumerize enumerize
factory_girl_rails factory_girl_rails
......
...@@ -146,7 +146,7 @@ class Notify < ActionMailer::Base ...@@ -146,7 +146,7 @@ class Notify < ActionMailer::Base
if reply_key if reply_key
headers['X-GitLab-Reply-Key'] = reply_key headers['X-GitLab-Reply-Key'] = reply_key
headers['Reply-To'] = Gitlab.config.reply_by_email.address.gsub('%{reply_key}', reply_key) headers['Reply-To'] = Gitlab::ReplyByEmail.reply_address(reply_key)
end end
mail(headers) mail(headers)
...@@ -165,6 +165,10 @@ class Notify < ActionMailer::Base ...@@ -165,6 +165,10 @@ class Notify < ActionMailer::Base
headers['In-Reply-To'] = message_id(model) headers['In-Reply-To'] = message_id(model)
headers['References'] = message_id(model) headers['References'] = message_id(model)
if headers[:subject]
headers[:subject].prepend('Re: ')
end
mail_new_thread(model, headers) mail_new_thread(model, headers)
end end
...@@ -173,8 +177,6 @@ class Notify < ActionMailer::Base ...@@ -173,8 +177,6 @@ class Notify < ActionMailer::Base
end end
def reply_key def reply_key
return nil unless Gitlab.config.reply_by_email.enabled @reply_key ||= Gitlab::ReplyByEmail.reply_key
@reply_key ||= SecureRandom.hex(16)
end end
end end
...@@ -6,8 +6,8 @@ class SentNotification < ActiveRecord::Base ...@@ -6,8 +6,8 @@ class SentNotification < ActiveRecord::Base
validate :project, :recipient, :reply_key, presence: true validate :project, :recipient, :reply_key, presence: true
validate :reply_key, uniqueness: true validate :reply_key, uniqueness: true
validates :noteable_id, presence: true, if: ->(n) { n.noteable_type.present? && n.noteable_type != 'Commit' } validates :noteable_id, presence: true, unless: :for_commit?
validates :commit_id, presence: true, if: ->(n) { n.noteable_type == 'Commit' } validates :commit_id, presence: true, if: :for_commit?
def self.for(reply_key) def self.for(reply_key)
find_by(reply_key: reply_key) find_by(reply_key: reply_key)
...@@ -19,11 +19,9 @@ class SentNotification < ActiveRecord::Base ...@@ -19,11 +19,9 @@ class SentNotification < ActiveRecord::Base
def noteable def noteable
if for_commit? if for_commit?
project.commit(commit_id) project.commit(commit_id) rescue nil
else else
super super
end end
rescue
nil
end end
end end
...@@ -4,7 +4,7 @@ class EmailReceiverWorker ...@@ -4,7 +4,7 @@ class EmailReceiverWorker
sidekiq_options queue: :incoming_email sidekiq_options queue: :incoming_email
def perform(raw) def perform(raw)
return unless Gitlab.config.reply_by_email.enabled return unless Gitlab::ReplyByEmail.enabled?
# begin # begin
Gitlab::EmailReceiver.new(raw).process Gitlab::EmailReceiver.new(raw).process
......
...@@ -153,7 +153,7 @@ Settings.gitlab['restricted_signup_domains'] ||= [] ...@@ -153,7 +153,7 @@ Settings.gitlab['restricted_signup_domains'] ||= []
# Reply by email # Reply by email
# #
Settings['reply_by_email'] ||= Settingslogic.new({}) Settings['reply_by_email'] ||= Settingslogic.new({})
Settings.reply_by_email['enabled'] = false if Settings.gravatar['enabled'].nil? Settings.reply_by_email['enabled'] = false if Settings.reply_by_email['enabled'].nil?
# #
# Gravatar # Gravatar
......
# Taken mostly from Discourse's Email::HtmlCleaner
module Gitlab
# HtmlCleaner cleans up the extremely dirty HTML that many email clients
# generate by stripping out any excess divs or spans, removing styling in
# the process (which also makes the html more suitable to be parsed as
# Markdown).
class EmailHtmlCleaner
# Elements to hoist all children out of
HTML_HOIST_ELEMENTS = %w(div span font table tbody th tr td)
# Node types to always delete
HTML_DELETE_ELEMENT_TYPES = [
Nokogiri::XML::Node::DTD_NODE,
Nokogiri::XML::Node::COMMENT_NODE,
]
# Private variables:
# @doc - nokogiri document
# @out - same as @doc, but only if trimming has occured
def initialize(html)
if html.is_a?(String)
@doc = Nokogiri::HTML(html)
else
@doc = html
end
end
class << self
# EmailHtmlCleaner.trim(inp, opts={})
#
# Arguments:
# inp - Either a HTML string or a Nokogiri document.
# Options:
# :return => :doc, :string
# Specify the desired return type.
# Defaults to the type of the input.
# A value of :string is equivalent to calling get_document_text()
# on the returned document.
def trim(inp, opts={})
cleaner = EmailHtmlCleaner.new(inp)
opts[:return] ||= (inp.is_a?(String) ? :string : :doc)
if opts[:return] == :string
cleaner.output_html
else
cleaner.output_document
end
end
# EmailHtmlCleaner.get_document_text(doc)
#
# Get the body portion of the document, including html, as a string.
def get_document_text(doc)
body = doc.xpath('//body')
if body
body.inner_html
else
doc.inner_html
end
end
end
def output_document
@out ||= begin
doc = @doc
trim_process_node doc
add_newlines doc
doc
end
end
def output_html
EmailHtmlCleaner.get_document_text(output_document)
end
private
def add_newlines(doc)
# Replace <br> tags with a markdown \n
doc.xpath('//br').each do |br|
br.replace(new_linebreak_node doc, 2)
end
# Surround <p> tags with newlines, to help with line-wise postprocessing
# and ensure markdown paragraphs
doc.xpath('//p').each do |p|
p.before(new_linebreak_node doc)
p.after(new_linebreak_node doc, 2)
end
end
def new_linebreak_node(doc, count=1)
Nokogiri::XML::Text.new("\n" * count, doc)
end
def trim_process_node(node)
if should_hoist?(node)
hoisted = trim_hoist_element node
hoisted.each { |child| trim_process_node child }
elsif should_delete?(node)
node.remove
else
if children = node.children
children.each { |child| trim_process_node child }
end
end
node
end
def trim_hoist_element(element)
hoisted = []
element.children.each do |child|
element.before(child)
hoisted << child
end
element.remove
hoisted
end
def should_hoist?(node)
return false unless node.element?
HTML_HOIST_ELEMENTS.include? node.name
end
def should_delete?(node)
return true if HTML_DELETE_ELEMENT_TYPES.include? node.type
return true if node.element? && node.name == 'head'
return true if node.text? && node.text.strip.blank?
false
end
end
end
# Inspired in great part by Discourse's Email::Receiver
module Gitlab module Gitlab
class EmailReceiver class EmailReceiver
class ProcessingError < StandardError; end
class EmailUnparsableError < ProcessingError; end
class EmptyEmailError < ProcessingError; end
class UserNotFoundError < ProcessingError; end
class UserNotAuthorizedLevelError < ProcessingError; end
class NoteableNotFoundError < ProcessingError; end
class AutoGeneratedEmailError < ProcessingError; end
class SentNotificationNotFound < ProcessingError; end
class InvalidNote < ProcessingError; end
def initialize(raw) def initialize(raw)
@raw = raw @raw = raw
end end
def message def message
@message ||= Mail::Message.new(@raw) @message ||= Mail::Message.new(@raw)
rescue Encoding::UndefinedConversionError, Encoding::InvalidByteSequenceError => e
raise EmailUnparsableError, e
end end
def process def process
return unless message && sent_notification raise EmptyEmailError if @raw.blank?
raise AutoGeneratedEmailError if message.header.to_s =~ /auto-(generated|replied)/
raise SentNotificationNotFound unless sent_notification
author = sent_notification.recipient
raise UserNotFoundError unless author
project = sent_notification.project
Notes::CreateService.new( raise UserNotAuthorizedLevelError unless author.can?(:create_note, project)
sent_notification.project,
sent_notification.recipient, raise NoteableNotFoundError unless sent_notification.noteable
note: message.text_part.to_s,
body = parse_body(message)
note = Notes::CreateService.new(
project,
author,
note: body,
noteable_type: sent_notification.noteable_type, noteable_type: sent_notification.noteable_type,
noteable_id: sent_notification.noteable_id, noteable_id: sent_notification.noteable_id,
commit_id: sent_notification.commit_id commit_id: sent_notification.commit_id
).execute ).execute
unless note.persisted?
raise InvalidNote, note.errors.full_messages.join("\n")
end
end end
private private
def reply_key def reply_key
address = Gitlab.config.reply_by_email.address reply_key = nil
return nil unless address message.to.each do |address|
reply_key = Gitlab::ReplyByEmail.reply_key_from_address(address)
break if reply_key
end
reply_key
end
def sent_notification
return nil unless reply_key
regex = Regexp.escape(address) SentNotification.for(reply_key)
regex = regex.gsub(Regexp.escape('%{reply_key}'), "(.*)") end
regex = Regexp.new(regex)
address = message.to.find { |address| address =~ regex } def parse_body(message)
return nil unless address body = select_body(message)
match = address.match(regex) encoding = body.encoding
raise EmptyEmailError if body.strip.blank?
return nil unless match && match[1].present? body = discourse_email_trimmer(body)
raise EmptyEmailError if body.strip.blank?
match[1] body = EmailReplyParser.parse_reply(body)
raise EmptyEmailError if body.strip.blank?
body.force_encoding(encoding).encode("UTF-8")
end end
def sent_notification def select_body(message)
return nil unless reply_key html = nil
text = nil
SentNotification.for(reply_key) if message.multipart?
html = fix_charset(message.html_part)
text = fix_charset(message.text_part)
elsif message.content_type =~ /text\/html/
html = fix_charset(message)
end
# prefer plain text
return text if text
if html
body = EmailHtmlCleaner.new(html).output_html
else
body = fix_charset(message)
end
# Certain trigger phrases that means we didn't parse correctly
if body =~ /(Content\-Type\:|multipart\/alternative|text\/plain)/
raise EmptyEmailError
end
body
end
# Force encoding to UTF-8 on a Mail::Message or Mail::Part
def fix_charset(object)
return nil if object.nil?
if object.charset
object.body.decoded.force_encoding(object.charset.gsub(/utf8/i, "UTF-8")).encode("UTF-8").to_s
else
object.body.to_s
end
rescue
nil
end
REPLYING_HEADER_LABELS = %w(From Sent To Subject Reply To Cc Bcc Date)
REPLYING_HEADER_REGEX = Regexp.union(REPLYING_HEADER_LABELS.map { |label| "#{label}:" })
def discourse_email_trimmer(body)
lines = body.scrub.lines.to_a
range_end = 0
lines.each_with_index do |l, idx|
break if l =~ /\A\s*\-{3,80}\s*\z/ ||
# This one might be controversial but so many reply lines have years, times and end with a colon.
# Let's try it and see how well it works.
(l =~ /\d{4}/ && l =~ /\d:\d\d/ && l =~ /\:$/) ||
(l =~ /On \w+ \d+,? \d+,?.*wrote:/)
# Headers on subsequent lines
break if (0..2).all? { |off| lines[idx+off] =~ REPLYING_HEADER_REGEX }
# Headers on the same line
break if REPLYING_HEADER_LABELS.count { |label| l.include?(label) } >= 3
range_end = idx
end
lines[0..range_end].join.strip
end end
end end
end end
module Gitlab
module ReplyByEmail
class << self
def enabled?
config.enabled &&
config.address &&
config.address.include?("%{reply_key}")
end
def reply_key
return nil unless enabled?
SecureRandom.hex(16)
end
def reply_address(reply_key)
config.address.gsub('%{reply_key}', reply_key)
end
def reply_key_from_address(address)
return unless address_regex
match = address.match(address_regex)
return unless match
match[1]
end
private
def config
Gitlab.config.reply_by_email
end
def address_regex
@address_regex ||= begin
wildcard_address = config.address
return nil unless wildcard_address
regex = Regexp.escape(wildcard_address)
regex = regex.gsub(Regexp.escape('%{reply_key}'), "(.+)")
Regexp.new(regex).freeze
end
end
end
end
end
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment