Commit 3ff2d0a7 authored by Jérome Perrin's avatar Jérome Perrin

Support UTF-8 encoded CSV

/reviewed-on !18
parents 4912750b 3bdab08f
...@@ -48,5 +48,3 @@ class TestAllSupportedFormat(TestCase): ...@@ -48,5 +48,3 @@ class TestAllSupportedFormat(TestCase):
"""Tests if ffmpeg convets midi file""" """Tests if ffmpeg convets midi file"""
self.runConversionList(join('data', 'test.ogg'), "ogg", "midi", "audio/rtp-midi") self.runConversionList(join('data', 'test.ogg'), "ogg", "midi", "audio/rtp-midi")
def test_suite():
return make_suite(TestAllSupportedFormat)
...@@ -50,5 +50,3 @@ class TestAllSupportedFormat(TestCase): ...@@ -50,5 +50,3 @@ class TestAllSupportedFormat(TestCase):
def testAllSupportedFormat(self): def testAllSupportedFormat(self):
self.runConversionList(self.ConversionScenarioList()) self.runConversionList(self.ConversionScenarioList())
def test_suite():
return make_suite(TestAllSupportedFormat)
...@@ -69,5 +69,3 @@ class TestHandler(HandlerTestCase): ...@@ -69,5 +69,3 @@ class TestHandler(HandlerTestCase):
self.assertEquals(file_format, 'audio/x-wav') self.assertEquals(file_format, 'audio/x-wav')
def test_suite():
return make_suite(TestHandler)
...@@ -45,5 +45,3 @@ class TestInterface(unittest.TestCase): ...@@ -45,5 +45,3 @@ class TestInterface(unittest.TestCase):
('metadata_dict',)) ('metadata_dict',))
def test_suite():
return make_suite(TestInterface)
...@@ -86,5 +86,3 @@ class TestServer(TestCase): ...@@ -86,5 +86,3 @@ class TestServer(TestCase):
"""Test if metadata is inserted correctly into video files""" """Test if metadata is inserted correctly into video files"""
self.runUpdateMetadataList(self.UpdateMetadataScenarioList()) self.runUpdateMetadataList(self.UpdateMetadataScenarioList())
def test_suite():
return make_suite(TestServer)
...@@ -61,5 +61,3 @@ class TestHandler(HandlerTestCase): ...@@ -61,5 +61,3 @@ class TestHandler(HandlerTestCase):
self.assertRaises(NotImplementedError, handler.setMetadata) self.assertRaises(NotImplementedError, handler.setMetadata)
def test_suite():
return make_suite(TestHandler)
...@@ -74,5 +74,3 @@ class TestServer(TestCase): ...@@ -74,5 +74,3 @@ class TestServer(TestCase):
self.runFaultGetMetadataList(self.FaultGetMetadataScenarioList()) self.runFaultGetMetadataList(self.FaultGetMetadataScenarioList())
def test_suite():
return make_suite(TestServer)
...@@ -138,16 +138,16 @@ class OpenOffice(Application): ...@@ -138,16 +138,16 @@ class OpenOffice(Application):
removeDirectory(self.path_user_installation) removeDirectory(self.path_user_installation)
# Create command with all parameters to start the instance # Create command with all parameters to start the instance
self.command = [join(self.office_binary_path, self._bin_soffice), self.command = [join(self.office_binary_path, self._bin_soffice),
'-headless', '--headless',
'-invisible', '--invisible',
'-nocrashreport', '--nocrashreport',
'-nologo', '--nologo',
'-nodefault', '--nodefault',
'-norestore', '--norestore',
'-nofirststartwizard', '--nofirststartwizard',
'-accept=socket,host=%s,port=%d;urp;' % (self.hostname, self.port), '--accept=socket,host=%s,port=%d;urp;' % (self.hostname, self.port),
'-env:UserInstallation=file://%s' % self.path_user_installation, '-env:UserInstallation=file://%s' % self.path_user_installation,
'-language=%s' % self.default_language, '--language=%s' % self.default_language,
] ]
# To run soffice.bin, several environment variables should be set. # To run soffice.bin, several environment variables should be set.
env = self.environment_dict.copy() env = self.environment_dict.copy()
......
...@@ -54,9 +54,6 @@ class Handler(object): ...@@ -54,9 +54,6 @@ class Handler(object):
def __init__(self, base_folder_url, data, source_format, **kw): def __init__(self, base_folder_url, data, source_format, **kw):
"""Creates document in file system and loads it in OOo.""" """Creates document in file system and loads it in OOo."""
self.document = FileSystemDocument(base_folder_url,
data,
source_format)
self.zip = kw.get('zip', False) self.zip = kw.get('zip', False)
self.uno_path = kw.get("uno_path", None) self.uno_path = kw.get("uno_path", None)
self.office_binary_path = kw.get("office_binary_path", None) self.office_binary_path = kw.get("office_binary_path", None)
...@@ -67,6 +64,22 @@ class Handler(object): ...@@ -67,6 +64,22 @@ class Handler(object):
self.uno_path = environ.get("uno_path") self.uno_path = environ.get("uno_path")
if not self.office_binary_path: if not self.office_binary_path:
self.office_binary_path = environ.get("office_binary_path") self.office_binary_path = environ.get("office_binary_path")
self._createDocument(base_folder_url, data, source_format)
def _createDocument(self, base_folder_url, data, source_format):
if source_format == 'csv':
# Cloudooo expect utf-8 encoded csv, but also tolerate latin9 for
# backward compatibility.
# The heuristic is "if it's not utf-8", let's assume it's iso-8859-15.
try:
unicode(data, 'utf-8')
except UnicodeDecodeError:
data = unicode(data, 'iso-8859-15').encode('utf-8')
logger.warn("csv data is not utf-8, assuming iso-8859-15")
self.document = FileSystemDocument(
base_folder_url,
data,
source_format)
def _getCommand(self, *args, **kw): def _getCommand(self, *args, **kw):
"""Transforms all parameters passed in a command""" """Transforms all parameters passed in a command"""
......
...@@ -28,8 +28,10 @@ ...@@ -28,8 +28,10 @@
############################################################################## ##############################################################################
import sys import sys
import csv
import codecs
import helper_util import helper_util
from os.path import dirname from os.path import dirname, splitext
from tempfile import mktemp from tempfile import mktemp
from base64 import decodestring, encodestring from base64 import decodestring, encodestring
from getopt import getopt, GetoptError from getopt import getopt, GetoptError
...@@ -144,6 +146,26 @@ class UnoConverter(object): ...@@ -144,6 +146,26 @@ class UnoConverter(object):
else: else:
return () return ()
def _getPropertyToImport(self, source_url):
"""Create the property for import filter, according to the extension of the file."""
_, extension = splitext(source_url)
if extension == '.csv':
# https://wiki.openoffice.org/wiki/Documentation/DevGuide/Spreadsheets/Filter_Options
# Try to sniff the csv delimiter
with codecs.open(source_url, 'rb', 'utf-8', errors="ignore") as csvfile:
try:
dialect = csv.Sniffer().sniff(csvfile.read(1024))
delimiter = ord(dialect.delimiter)
except csv.Error:
delimiter = ord(',')
return (
self._createProperty("FilterName", "Text - txt - csv (StarCalc)"),
self._createProperty("FilterOptions", "{delimiter},34,UTF-8".format(**locals())), )
return ()
def _load(self): def _load(self):
"""Create one document with basic properties """Create one document with basic properties
refresh argument tells to uno environment to refresh argument tells to uno environment to
...@@ -154,7 +176,11 @@ class UnoConverter(object): ...@@ -154,7 +176,11 @@ class UnoConverter(object):
self.office_binary_path) self.office_binary_path)
desktop = service_manager.createInstance("com.sun.star.frame.Desktop") desktop = service_manager.createInstance("com.sun.star.frame.Desktop")
uno_url = self.systemPathToFileUrl(self.document_url) uno_url = self.systemPathToFileUrl(self.document_url)
uno_document = desktop.loadComponentFromURL(uno_url, "_blank", 0, ()) uno_document = desktop.loadComponentFromURL(
uno_url,
"_blank",
0,
self._getPropertyToImport(self.document_url))
if not uno_document: if not uno_document:
raise AttributeError("This document can not be loaded or is empty") raise AttributeError("This document can not be loaded or is empty")
if self.refresh: if self.refresh:
......
Jérome,ジェローム
नमस्ते,여보세요
a b
1,3 c
\ No newline at end of file
...@@ -62,5 +62,3 @@ class TestAllFormats(TestCase): ...@@ -62,5 +62,3 @@ class TestAllFormats(TestCase):
for extension in extension_list: for extension in extension_list:
self._testConvertFile(input_url, source_format, extension[0], None) self._testConvertFile(input_url, source_format, extension[0], None)
def test_suite():
return make_suite(TestAllFormats)
...@@ -75,5 +75,3 @@ class TestAllFormatsERP5Compatibility(TestCase): ...@@ -75,5 +75,3 @@ class TestAllFormatsERP5Compatibility(TestCase):
self.fail('Failed Conversions:\n' + message) self.fail('Failed Conversions:\n' + message)
def test_suite():
return make_suite(TestAllFormatsERP5Compatibility)
...@@ -60,5 +60,3 @@ class TestApplication(unittest.TestCase): ...@@ -60,5 +60,3 @@ class TestApplication(unittest.TestCase):
self.assertEquals(self.application.pid(), None) self.assertEquals(self.application.pid(), None)
def test_suite():
return make_suite(TestApplication)
...@@ -130,5 +130,3 @@ class TestFileSystemDocument(unittest.TestCase): ...@@ -130,5 +130,3 @@ class TestFileSystemDocument(unittest.TestCase):
sorted(['logo.gif', 'test.htm'])) sorted(['logo.gif', 'test.htm']))
def test_suite():
return make_suite(TestFileSystemDocument)
...@@ -54,5 +54,3 @@ class TestFilter(unittest.TestCase): ...@@ -54,5 +54,3 @@ class TestFilter(unittest.TestCase):
self.assertTrue(self.filter.isPreferred()) self.assertTrue(self.filter.isPreferred())
def test_suite():
return make_suite(TestFilter)
...@@ -174,5 +174,3 @@ class TestOOGranulator(HandlerTestCase): ...@@ -174,5 +174,3 @@ class TestOOGranulator(HandlerTestCase):
oogranulator = OOGranulator(data, 'odt') oogranulator = OOGranulator(data, 'odt')
self.assertEquals(['Title 1', 1], oogranulator.getChapterItem(1)) self.assertEquals(['Title 1', 1], oogranulator.getChapterItem(1))
def test_suite():
return make_suite(TestOOGranulator)
...@@ -685,5 +685,3 @@ class TestHandler(HandlerTestCase): ...@@ -685,5 +685,3 @@ class TestHandler(HandlerTestCase):
('image/x-ms-bmp', 'BMP - Windows Bitmap'), ('image/x-ms-bmp', 'BMP - Windows Bitmap'),
('text/html', 'HTML Document (Impress)') ]) ('text/html', 'HTML Document (Impress)') ])
def test_suite():
return make_suite(TestHandler)
...@@ -65,5 +65,3 @@ class TestHighLoad(TestCase): ...@@ -65,5 +65,3 @@ class TestHighLoad(TestCase):
self.assertTrue(all(result_list)) self.assertTrue(all(result_list))
def test_suite():
return make_suite(TestHighLoad)
...@@ -181,5 +181,3 @@ class TestInterface(TestCase): ...@@ -181,5 +181,3 @@ class TestInterface(TestCase):
self.assertEquals(sorted(ILockable.names()), sorted(lockable_method_list)) self.assertEquals(sorted(ILockable.names()), sorted(lockable_method_list))
def test_suite():
return make_suite(TestInterface)
...@@ -64,5 +64,3 @@ class TestLegacyInterface(TestCase): ...@@ -64,5 +64,3 @@ class TestLegacyInterface(TestCase):
self.assertEquals(self._getFileType(response_dict['data']), self.assertEquals(self._getFileType(response_dict['data']),
'application/vnd.oasis.opendocument.text') 'application/vnd.oasis.opendocument.text')
def test_suite():
return make_suite(TestLegacyInterface)
...@@ -306,5 +306,3 @@ class TestMimeMapper(HandlerTestCase): ...@@ -306,5 +306,3 @@ class TestMimeMapper(HandlerTestCase):
self.assertEquals(filtername, "impress_html_Export") self.assertEquals(filtername, "impress_html_Export")
def test_suite():
return make_suite(TestMimeMapper)
...@@ -72,5 +72,3 @@ class TestMonitorInit(HandlerTestCase): ...@@ -72,5 +72,3 @@ class TestMonitorInit(HandlerTestCase):
True) True)
def test_suite():
return make_suite(TestMonitorInit)
...@@ -107,5 +107,3 @@ class TestMonitorMemory(unittest.TestCase): ...@@ -107,5 +107,3 @@ class TestMonitorMemory(unittest.TestCase):
self.assertEquals(type(memory_usage_int), IntType) self.assertEquals(type(memory_usage_int), IntType)
def test_suite():
return make_suite(TestMonitorMemory)
...@@ -60,5 +60,3 @@ class TestMonitorRequest(HandlerTestCase): ...@@ -60,5 +60,3 @@ class TestMonitorRequest(HandlerTestCase):
monitor_request.terminate() monitor_request.terminate()
def test_suite():
return make_suite(TestMonitorRequest)
...@@ -91,5 +91,3 @@ class TestMonitorTimeout(unittest.TestCase): ...@@ -91,5 +91,3 @@ class TestMonitorTimeout(unittest.TestCase):
openoffice.release() openoffice.release()
def test_suite():
return make_suite(TestMonitorTimeout)
...@@ -65,5 +65,3 @@ class TestOdfDocument(HandlerTestCase): ...@@ -65,5 +65,3 @@ class TestOdfDocument(HandlerTestCase):
'document-content')) 'document-content'))
def test_suite():
return make_suite(TestOdfDocument)
...@@ -117,5 +117,3 @@ class TestOpenOffice(HandlerTestCase): ...@@ -117,5 +117,3 @@ class TestOpenOffice(HandlerTestCase):
self.assertFalse(second_openoffice.status()) self.assertFalse(second_openoffice.status())
def test_suite():
return make_suite(TestOpenOffice)
This diff is collapsed.
...@@ -87,5 +87,3 @@ class TestUnoConverter(HandlerTestCase): ...@@ -87,5 +87,3 @@ class TestUnoConverter(HandlerTestCase):
self.assertEquals(exists(output_url), False) self.assertEquals(exists(output_url), False)
def test_suite():
return make_suite(TestUnoConverter)
...@@ -118,5 +118,3 @@ class TestUnoMimeMapper(HandlerTestCase): ...@@ -118,5 +118,3 @@ class TestUnoMimeMapper(HandlerTestCase):
openoffice.start() openoffice.start()
def test_suite():
return make_suite(TestUnoMimeMapper)
...@@ -62,9 +62,3 @@ class TestUtil(unittest.TestCase): ...@@ -62,9 +62,3 @@ class TestUtil(unittest.TestCase):
self.assertEquals(mimetypes.types_map.get(".3gp"), "video/3gpp") self.assertEquals(mimetypes.types_map.get(".3gp"), "video/3gpp")
def test_suite():
return make_suite(TestUtil)
if "__main__" == __name__:
suite = unittest.TestLoader().loadTestsFromTestCase(TestUtil)
unittest.TextTestRunner(verbosity=2).run(suite)
...@@ -78,5 +78,3 @@ class TestHandler(HandlerTestCase): ...@@ -78,5 +78,3 @@ class TestHandler(HandlerTestCase):
self.assertEquals(get("text/plain;ignored=param"), []) self.assertEquals(get("text/plain;ignored=param"), [])
self.assertEquals(get("text/plain;charset=UTF-8;ignored=param"), []) self.assertEquals(get("text/plain;charset=UTF-8;ignored=param"), [])
def test_suite():
return make_suite(TestHandler)
...@@ -86,5 +86,3 @@ class TestServer(TestCase): ...@@ -86,5 +86,3 @@ class TestServer(TestCase):
"""Test if metadata is inserted correctly in pdf file""" """Test if metadata is inserted correctly in pdf file"""
self.runUpdateMetadataList(self.UpdateMetadataScenarioList()) self.runUpdateMetadataList(self.UpdateMetadataScenarioList())
def test_suite():
return make_suite(TestServer)
...@@ -90,5 +90,3 @@ class TestHandler(HandlerTestCase): ...@@ -90,5 +90,3 @@ class TestHandler(HandlerTestCase):
# Unhandled mimetypes # Unhandled mimetypes
self.assertEquals(get("application/pdf;ignored=param"), []) self.assertEquals(get("application/pdf;ignored=param"), [])
def test_suite():
return make_suite(TestHandler)
...@@ -49,5 +49,3 @@ class TestServer(TestCase): ...@@ -49,5 +49,3 @@ class TestServer(TestCase):
(open(join('data', 'test_with_png_dataurl.html')).read(), 'html', 'xyz'), (open(join('data', 'test_with_png_dataurl.html')).read(), 'html', 'xyz'),
] ]
def test_suite():
return make_suite(TestServer)
...@@ -133,5 +133,3 @@ class TestHandler(HandlerTestCase): ...@@ -133,5 +133,3 @@ class TestHandler(HandlerTestCase):
[("application/vnd.openxmlformats-officedocument.presentationml.presentation", "PowerPoint 2007 Presentation"), [("application/vnd.openxmlformats-officedocument.presentationml.presentation", "PowerPoint 2007 Presentation"),
('application/vnd.oasis.opendocument.presentation', 'ODF Presentation Document')]) ('application/vnd.oasis.opendocument.presentation', 'ODF Presentation Document')])
def test_suite():
return make_suite(TestHandler)
...@@ -51,5 +51,3 @@ class TestServer(TestCase): ...@@ -51,5 +51,3 @@ class TestServer(TestCase):
(open(join('data', 'test.xlsx')).read(), 'xlsx', 'xyz'), (open(join('data', 'test.xlsx')).read(), 'xlsx', 'xyz'),
] ]
def test_suite():
return make_suite(TestServer)
...@@ -70,15 +70,12 @@ def run(): ...@@ -70,15 +70,12 @@ def run():
config = ConfigParser() config = ConfigParser()
config.read(server_cloudooo_conf) config.read(server_cloudooo_conf)
module = __import__(test_name) module = __import__(test_name)
if not hasattr(module, "test_suite"):
exit("No test suite to run, exiting immediately")
DAEMON = getattr(module, 'DAEMON', False) DAEMON = getattr(module, 'DAEMON', False)
OPENOFFICE = getattr(module, 'OPENOFFICE', False) OPENOFFICE = getattr(module, 'OPENOFFICE', False)
TestRunner = backportUnittest.TextTestRunner TestRunner = backportUnittest.TextTestRunner
suite = unittest.TestSuite() suite = unittest.defaultTestLoader.loadTestsFromModule(module)
suite.addTest(module.test_suite())
if DAEMON: if DAEMON:
log_file = '%s/cloudooo_test.log' % config.get('app:main', log_file = '%s/cloudooo_test.log' % config.get('app:main',
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment