Support UTF-8 encoded CSV

/reviewed-on nexedi/cloudooo!18

cloudooo/handler/ffmpeg/tests/testFfmpegAllAudioFormat.py

@@ -48,5 +48,3 @@ class TestAllSupportedFormat(TestCase):
     """Tests if ffmpeg convets midi file"""
     self.runConversionList(join('data', 'test.ogg'), "ogg", "midi", "audio/rtp-midi")
 
-def test_suite():
-  return make_suite(TestAllSupportedFormat)

cloudooo/handler/ffmpeg/tests/testFfmpegAllVideoFormat.py

@@ -50,5 +50,3 @@ class TestAllSupportedFormat(TestCase):
   def testAllSupportedFormat(self):
     self.runConversionList(self.ConversionScenarioList())
 
-def test_suite():
-  return make_suite(TestAllSupportedFormat)

cloudooo/handler/ffmpeg/tests/testFfmpegHandler.py

@@ -69,5 +69,3 @@ class TestHandler(HandlerTestCase):
     self.assertEquals(file_format, 'audio/x-wav')
 
 
-def test_suite():
-  return make_suite(TestHandler)

cloudooo/handler/ffmpeg/tests/testFfmpegInterface.py

@@ -45,5 +45,3 @@ class TestInterface(unittest.TestCase):
         ('metadata_dict',))
 
 
-def test_suite():
-  return make_suite(TestInterface)

cloudooo/handler/ffmpeg/tests/testFfmpegServer.py

@@ -86,5 +86,3 @@ class TestServer(TestCase):
     """Test if metadata is inserted correctly into video files"""
     self.runUpdateMetadataList(self.UpdateMetadataScenarioList())
 
-def test_suite():
-  return make_suite(TestServer)

cloudooo/handler/imagemagick/tests/testImagemagickHandler.py

@@ -61,5 +61,3 @@ class TestHandler(HandlerTestCase):
     self.assertRaises(NotImplementedError, handler.setMetadata)
 
 
-def test_suite():
-  return make_suite(TestHandler)

cloudooo/handler/imagemagick/tests/testImagemagickServer.py

@@ -74,5 +74,3 @@ class TestServer(TestCase):
     self.runFaultGetMetadataList(self.FaultGetMetadataScenarioList())
 
 
-def test_suite():
-  return make_suite(TestServer)

cloudooo/handler/ooo/application/openoffice.py

@@ -138,16 +138,16 @@ class OpenOffice(Application):
       removeDirectory(self.path_user_installation)
     # Create command with all parameters to start the instance
     self.command = [join(self.office_binary_path, self._bin_soffice),
-         '-headless',
-         '-invisible',
-         '-nocrashreport',
-         '-nologo',
-         '-nodefault',
-         '-norestore',
-         '-nofirststartwizard',
-         '-accept=socket,host=%s,port=%d;urp;' % (self.hostname, self.port),
+         '--headless',
+         '--invisible',
+         '--nocrashreport',
+         '--nologo',
+         '--nodefault',
+         '--norestore',
+         '--nofirststartwizard',
+         '--accept=socket,host=%s,port=%d;urp;' % (self.hostname, self.port),
          '-env:UserInstallation=file://%s' % self.path_user_installation,
-         '-language=%s' % self.default_language,
+         '--language=%s' % self.default_language,
          ]
     # To run soffice.bin, several environment variables should be set.
     env = self.environment_dict.copy()

cloudooo/handler/ooo/handler.py

@@ -54,9 +54,6 @@ class Handler(object):
 
   def __init__(self, base_folder_url, data, source_format, **kw):
     """Creates document in file system and loads it in OOo."""
-    self.document = FileSystemDocument(base_folder_url,
-                                      data,
-                                      source_format)
     self.zip = kw.get('zip', False)
     self.uno_path = kw.get("uno_path", None)
     self.office_binary_path = kw.get("office_binary_path", None)
@@ -67,6 +64,22 @@ class Handler(object):
       self.uno_path = environ.get("uno_path")
     if not self.office_binary_path:
       self.office_binary_path = environ.get("office_binary_path")
+    self._createDocument(base_folder_url, data, source_format)
+
+  def _createDocument(self, base_folder_url, data, source_format):
+    if source_format == 'csv':
+      # Cloudooo expect utf-8 encoded csv, but also tolerate latin9 for
+      # backward compatibility.
+      # The heuristic is "if it's not utf-8", let's assume it's iso-8859-15.
+      try:
+        unicode(data, 'utf-8')
+      except UnicodeDecodeError:
+        data = unicode(data, 'iso-8859-15').encode('utf-8')
+        logger.warn("csv data is not utf-8, assuming iso-8859-15")
+    self.document = FileSystemDocument(
+         base_folder_url,
+         data,
+         source_format)
 
   def _getCommand(self, *args, **kw):
     """Transforms all parameters passed in a command"""

cloudooo/handler/ooo/helper/unoconverter.py

@@ -28,8 +28,10 @@
 ##############################################################################
 
 import sys
+import csv
+import codecs
 import helper_util
-from os.path import dirname
+from os.path import dirname, splitext
 from tempfile import mktemp
 from base64 import decodestring, encodestring
 from getopt import getopt, GetoptError
@@ -144,6 +146,26 @@ class UnoConverter(object):
     else:
       return ()
 
+  def _getPropertyToImport(self, source_url):
+    """Create the property for import filter, according to the extension of the file."""
+    _, extension = splitext(source_url)
+    if extension == '.csv':
+      # https://wiki.openoffice.org/wiki/Documentation/DevGuide/Spreadsheets/Filter_Options
+
+      # Try to sniff the csv delimiter
+      with codecs.open(source_url, 'rb', 'utf-8', errors="ignore") as csvfile:
+        try:
+          dialect = csv.Sniffer().sniff(csvfile.read(1024))
+          delimiter = ord(dialect.delimiter)
+        except csv.Error:
+          delimiter = ord(',')
+
+      return (
+        self._createProperty("FilterName", "Text - txt - csv (StarCalc)"),
+        self._createProperty("FilterOptions", "{delimiter},34,UTF-8".format(**locals())), )
+
+    return ()
+
   def _load(self):
     """Create one document with basic properties
     refresh argument tells to uno environment to
@@ -154,7 +176,11 @@ class UnoConverter(object):
                                                     self.office_binary_path)
     desktop = service_manager.createInstance("com.sun.star.frame.Desktop")
     uno_url = self.systemPathToFileUrl(self.document_url)
-    uno_document = desktop.loadComponentFromURL(uno_url, "_blank", 0, ())
+    uno_document = desktop.loadComponentFromURL(
+        uno_url,
+        "_blank",
+        0,
+        self._getPropertyToImport(self.document_url))
     if not uno_document:
       raise AttributeError("This document can not be loaded or is empty")
     if self.refresh:

cloudooo/handler/ooo/tests/data/csv_ascii.csv

+test;1234

cloudooo/handler/ooo/tests/data/csv_latin9.csv

+Jrome;1

cloudooo/handler/ooo/tests/data/csv_semicolon.csv

+a a,1
+b b,2;x

cloudooo/handler/ooo/tests/data/csv_utf8.csv

+Jérome,ジェローム
+नमस्ते,여보세요

cloudooo/handler/ooo/tests/data/tsv.tsv

+a	b
+1,3	c
\ No newline at end of file

cloudooo/handler/ooo/tests/testOooAllFormat.py

@@ -62,5 +62,3 @@ class TestAllFormats(TestCase):
     for extension in extension_list:
         self._testConvertFile(input_url, source_format, extension[0], None)
 
-def test_suite():
-  return make_suite(TestAllFormats)

cloudooo/handler/ooo/tests/testOooAllFormatERP5Compatibility.py

@@ -75,5 +75,3 @@ class TestAllFormatsERP5Compatibility(TestCase):
       self.fail('Failed Conversions:\n' + message)
 
 
-def test_suite():
-  return make_suite(TestAllFormatsERP5Compatibility)

cloudooo/handler/ooo/tests/testOooApplication.py

@@ -60,5 +60,3 @@ class TestApplication(unittest.TestCase):
     self.assertEquals(self.application.pid(), None)
 
 
-def test_suite():
-  return make_suite(TestApplication)

cloudooo/handler/ooo/tests/testOooFileSystemDocument.py

@@ -130,5 +130,3 @@ class TestFileSystemDocument(unittest.TestCase):
                 sorted(['logo.gif', 'test.htm']))
 
 
-def test_suite():
-  return make_suite(TestFileSystemDocument)

cloudooo/handler/ooo/tests/testOooFilter.py

@@ -54,5 +54,3 @@ class TestFilter(unittest.TestCase):
     self.assertTrue(self.filter.isPreferred())
 
 
-def test_suite():
-  return make_suite(TestFilter)

cloudooo/handler/ooo/tests/testOooGranulator.py

@@ -174,5 +174,3 @@ class TestOOGranulator(HandlerTestCase):
     oogranulator = OOGranulator(data, 'odt')
     self.assertEquals(['Title 1', 1], oogranulator.getChapterItem(1))
 
-def test_suite():
-  return make_suite(TestOOGranulator)

cloudooo/handler/ooo/tests/testOooHandler.py

@@ -685,5 +685,3 @@ class TestHandler(HandlerTestCase):
         ('image/x-ms-bmp', 'BMP - Windows Bitmap'),
         ('text/html', 'HTML Document (Impress)') ])
 
-def test_suite():
-  return make_suite(TestHandler)

cloudooo/handler/ooo/tests/testOooHighLoad.py

@@ -65,5 +65,3 @@ class TestHighLoad(TestCase):
     self.assertTrue(all(result_list))
 
 
-def test_suite():
-  return make_suite(TestHighLoad)

cloudooo/handler/ooo/tests/testOooInterface.py

@@ -181,5 +181,3 @@ class TestInterface(TestCase):
     self.assertEquals(sorted(ILockable.names()), sorted(lockable_method_list))
 
 
-def test_suite():
-  return make_suite(TestInterface)

cloudooo/handler/ooo/tests/testOooLegacyInterface.py

@@ -64,5 +64,3 @@ class TestLegacyInterface(TestCase):
     self.assertEquals(self._getFileType(response_dict['data']),
                       'application/vnd.oasis.opendocument.text')
 
-def test_suite():
-  return make_suite(TestLegacyInterface)

cloudooo/handler/ooo/tests/testOooMimemapper.py

@@ -306,5 +306,3 @@ class TestMimeMapper(HandlerTestCase):
     self.assertEquals(filtername, "impress_html_Export")
 
 
-def test_suite():
-  return make_suite(TestMimeMapper)

cloudooo/handler/ooo/tests/testOooMonitorInit.py

@@ -72,5 +72,3 @@ class TestMonitorInit(HandlerTestCase):
                                  True)
 
 
-def test_suite():
-  return make_suite(TestMonitorInit)

cloudooo/handler/ooo/tests/testOooMonitorMemory.py

@@ -107,5 +107,3 @@ class TestMonitorMemory(unittest.TestCase):
     self.assertEquals(type(memory_usage_int), IntType)
 
 
-def test_suite():
-  return make_suite(TestMonitorMemory)

cloudooo/handler/ooo/tests/testOooMonitorRequest.py

@@ -60,5 +60,3 @@ class TestMonitorRequest(HandlerTestCase):
     monitor_request.terminate()
 
 
-def test_suite():
-  return make_suite(TestMonitorRequest)

cloudooo/handler/ooo/tests/testOooMonitorTimeout.py

@@ -91,5 +91,3 @@ class TestMonitorTimeout(unittest.TestCase):
       openoffice.release()
 
 
-def test_suite():
-  return make_suite(TestMonitorTimeout)

cloudooo/handler/ooo/tests/testOooOdfDocument.py

@@ -65,5 +65,3 @@ class TestOdfDocument(HandlerTestCase):
                     'document-content'))
 
 
-def test_suite():
-  return make_suite(TestOdfDocument)

cloudooo/handler/ooo/tests/testOooOpenOffice.py

@@ -117,5 +117,3 @@ class TestOpenOffice(HandlerTestCase):
     self.assertFalse(second_openoffice.status())
 
 
-def test_suite():
-  return make_suite(TestOpenOffice)

cloudooo/handler/ooo/tests/testOooUnoConverter.py

@@ -87,5 +87,3 @@ class TestUnoConverter(HandlerTestCase):
     self.assertEquals(exists(output_url), False)
 
 
-def test_suite():
-  return make_suite(TestUnoConverter)

cloudooo/handler/ooo/tests/testOooUnoMimemapper.py

@@ -118,5 +118,3 @@ class TestUnoMimeMapper(HandlerTestCase):
     openoffice.start()
 
 
-def test_suite():
-  return make_suite(TestUnoMimeMapper)

cloudooo/handler/ooo/tests/testOooUtil.py

@@ -62,9 +62,3 @@ class TestUtil(unittest.TestCase):
     self.assertEquals(mimetypes.types_map.get(".3gp"), "video/3gpp")
 
 
-def test_suite():
-  return make_suite(TestUtil)
-
-if "__main__" == __name__:
-  suite = unittest.TestLoader().loadTestsFromTestCase(TestUtil)
-  unittest.TextTestRunner(verbosity=2).run(suite)

cloudooo/handler/pdf/tests/testPdfHandler.py

@@ -78,5 +78,3 @@ class TestHandler(HandlerTestCase):
     self.assertEquals(get("text/plain;ignored=param"), [])
     self.assertEquals(get("text/plain;charset=UTF-8;ignored=param"), [])
 
-def test_suite():
-  return make_suite(TestHandler)

cloudooo/handler/pdf/tests/testPdfServer.py

@@ -86,5 +86,3 @@ class TestServer(TestCase):
     """Test if metadata is inserted correctly in pdf file"""
     self.runUpdateMetadataList(self.UpdateMetadataScenarioList())
 
-def test_suite():
-  return make_suite(TestServer)

cloudooo/handler/wkhtmltopdf/tests/testWkhtmltopdfHandler.py

@@ -90,5 +90,3 @@ class TestHandler(HandlerTestCase):
     # Unhandled mimetypes
     self.assertEquals(get("application/pdf;ignored=param"), [])
 
-def test_suite():
-  return make_suite(TestHandler)

cloudooo/handler/wkhtmltopdf/tests/testWkhtmltopdfServer.py

@@ -49,5 +49,3 @@ class TestServer(TestCase):
       (open(join('data', 'test_with_png_dataurl.html')).read(), 'html', 'xyz'),
     ]
 
-def test_suite():
-  return make_suite(TestServer)

cloudooo/handler/x2t/tests/testX2tHandler.py

@@ -133,5 +133,3 @@ class TestHandler(HandlerTestCase):
       [("application/vnd.openxmlformats-officedocument.presentationml.presentation", "PowerPoint 2007 Presentation"),
 	('application/vnd.oasis.opendocument.presentation', 'ODF Presentation Document')])
 
-def test_suite():
-  return make_suite(TestHandler)

cloudooo/handler/x2t/tests/testX2tServer.py

@@ -51,5 +51,3 @@ class TestServer(TestCase):
       (open(join('data', 'test.xlsx')).read(), 'xlsx', 'xyz'),
     ]
 
-def test_suite():
-  return make_suite(TestServer)

cloudooo/tests/runHandlerUnitTest.py

@@ -70,15 +70,12 @@ def run():
   config = ConfigParser()
   config.read(server_cloudooo_conf)
   module = __import__(test_name)
-  if not hasattr(module, "test_suite"):
-    exit("No test suite to run, exiting immediately")
 
   DAEMON = getattr(module, 'DAEMON', False)
   OPENOFFICE = getattr(module, 'OPENOFFICE', False)
 
   TestRunner = backportUnittest.TextTestRunner
-  suite = unittest.TestSuite()
-  suite.addTest(module.test_suite())
+  suite = unittest.defaultTestLoader.loadTestsFromModule(module)
 
   if DAEMON:
     log_file = '%s/cloudooo_test.log' % config.get('app:main',