Merge pull request #868 from SaitoWu/bugfix/encoding

Bugfix/encoding

app/helpers/commits_helper.rb

@@ -58,7 +58,7 @@ module CommitsHelper
       next if line.match(/^\-\-\- a/)
       next if line.match(/^\+\+\+ b/)
 
-      full_line = html_escape(line.gsub(/\n/, '')).force_encoding("UTF-8")
+      full_line = html_escape(line.gsub(/\n/, ''))
 
       if line.match(/^@@ -/)
         type = "match"

app/views/refs/_tree.html.haml

@@ -42,9 +42,9 @@
         .readme
           - if content.name =~ /\.(md|markdown)$/i
             = preserve do
-              = markdown(content.data.detect_encoding!)
+              = markdown(content.data)
           - else
-            = simple_format(content.data.detect_encoding!)
+            = simple_format(content.data)
 
 - if params[:path]
   - history_path = tree_file_project_ref_path(@project, @ref, params[:path])

app/views/refs/_tree_file.html.haml

@@ -13,7 +13,7 @@
       #tree-readme-holder
         .readme
           = preserve do
-            = markdown(file.data.detect_encoding!)
+            = markdown(file.data)
     - else
       .view_file_content
         - unless file.empty?

config/initializers/gitlabhq/20_grit_ext.rb

 require 'grit'
 require 'pygments'
 
+Grit::Git.git_timeout = GIT_OPTS["git_timeout"]
+Grit::Git.git_max_size = GIT_OPTS["git_max_size"]
+
 Grit::Blob.class_eval do
   include Linguist::BlobHelper
-end
 
-#monkey patch raw_object from string
-Grit::GitRuby::Internal::RawObject.class_eval do
-  def content
-    @content
+  def data
+    @data ||= @repo.git.cat_file({:p => true}, id)
+    Gitlab::Encode.utf8 @data
   end
 end
 
-
 Grit::Diff.class_eval do
   def old_path
-    Gitlab::Encode.utf8 a_path
+    Gitlab::Encode.utf8 @a_path
   end
 
   def new_path
-    Gitlab::Encode.utf8 b_path
+    Gitlab::Encode.utf8 @b_path
   end
-end
 
-Grit::Git.git_timeout = GIT_OPTS["git_timeout"]
-Grit::Git.git_max_size = GIT_OPTS["git_max_size"]
+  def diff
+    Gitlab::Encode.utf8 @diff
+  end
+end

lib/gitlab/encode.rb

@@ -5,21 +5,29 @@ module Gitlab
     extend self
 
     def utf8 message
+      # return nil if message is nil
       return nil unless message
 
-      detect = CharlockHolmes::EncodingDetector.detect(message) rescue {}
+      message.force_encoding("utf-8")
+      # return message if message type is binary
+      detect = CharlockHolmes::EncodingDetector.detect(message)
+      return message if detect[:type] == :binary
 
-      # It's better to default to UTF-8 as sometimes it's wrongly detected as another charset
-      if detect[:encoding] && detect[:confidence] == 100
-        CharlockHolmes::Converter.convert(message, detect[:encoding], 'UTF-8')
-      else
-        message
-      end.force_encoding("utf-8")
+      # if message is utf-8 encoding, just return it
+      return message if message.valid_encoding?
 
-    # Prevent app from crash cause of 
-    # encoding errors
+      # if message is not utf-8 encoding, convert it
+      if detect[:encoding]
+        message.force_encoding(detect[:encoding])
+        message.encode!("utf-8", detect[:encoding], :undef => :replace, :replace => "", :invalid => :replace)
+      end
+
+      # ensure message encoding is utf8
+      message.valid_encoding? ? message : raise
+
+    # Prevent app from crash cause of encoding errors
     rescue
-      "--broken encoding: #{encoding}"
+      "--broken encoding: #{detect[:encoding]}"
     end
 
     def detect_encoding message

lib/graph_commit.rb

@@ -96,13 +96,13 @@ class GraphCommit
     h[:parents] = self.parents.collect do |p|
       [p.id,0,0]
     end
-    h[:author]  = author.name.force_encoding("UTF-8")
+    h[:author]  = Gitlab::Encode.utf8(author.name)
     h[:time]    = time
     h[:space]   = space
     h[:refs]    = refs.collect{|r|r.name}.join(" ") unless refs.nil?
     h[:id]      = sha
     h[:date]    = date
-    h[:message] = message.force_encoding("UTF-8")
+    h[:message] = Gitlab::Encode.utf8(message)
     h[:login]   = author.email
     h
   end