Add support of OnlyOffice document (as ziplike file)

Originaly, the y format (docy, xlsy, ppty) is a file extension that is use by an OnlyOffice document.

An OnlyOffice document is composed by : 

- a file with extension in (docy, xlsy, ppty)
- a separate media folder containing media files (Ex: media/image1.png)

Several problems : 

- Two documents cannot be in the same folder
- How to send / get from Cloudooo to ERP5 ?
- The document cannot be stored easily in ERP5 (embedded files ? seperate images linked to the document by predecessors ??)
- How to download an OnlyOffice document from ERP5 ? How to load it on OfficeJS ?

To solve, we embbed all the files composing the document into one zip (like docx, odt, ...) : 

- Two documents can be in the same folder
- We send / get from Cloudooo as usual with portal transform
- The document can be stored in File data in ERP5 like any other docx document.
- document.getData allows to download the document data and JSZip can be used to extract easily the body in OfficeJS.

To keep backward compatibility, a docy can be : 
- the OnlyOffice document body (`startsWith("DOCY;v2;")`)
- can be a zip file (`startsWith("PK\x03\x04")`) containing one file at the root of the archive + optional media folder

/reviewed-on !8

cloudooo/handler/x2t/handler.py

+##############################################################################
+#
+# Copyright (c) 2009-2011 Nexedi SA and Contributors. All Rights Reserved.
+#                    Gabriel M. Monnerat <gabriel@tiolive.com>
+#
+# WARNING: This program as such is intended to be used by professional
+# programmers who take the whole responsibility of assessing all potential
+# consequences resulting from its eventual inadequacies and bugs
+# End users who are looking for a ready-to-use solution with commercial
+# guarantees and support are strongly adviced to contract a Free Software
+# Service Company
+#
+# This program is Free Software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License
+# as published by the Free Software Foundation; either version 2
+# of the License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.
+#
+##############################################################################
+from xml.etree import ElementTree
+from subprocess import Popen, PIPE
+from tempfile import NamedTemporaryFile, mktemp
+import sys
+import os
+
+from zope.interface import implements
+
+from cloudooo.interfaces.handler import IHandler
+from cloudooo.file import File
+from cloudooo.util import logger, zipTree, unzip
+
+AVS_OFFICESTUDIO_FILE_UNKNOWN = "0"
+AVS_OFFICESTUDIO_FILE_DOCUMENT_DOCX = "65"
+AVS_OFFICESTUDIO_FILE_PRESENTATION_PPTX = "129"
+AVS_OFFICESTUDIO_FILE_PRESENTATION_PPSX = "132"
+AVS_OFFICESTUDIO_FILE_SPREADSHEET_XLSX = "257"
+AVS_OFFICESTUDIO_FILE_CROSSPLATFORM_PDF = "513"
+AVS_OFFICESTUDIO_FILE_TEAMLAB_DOCY = "4097"
+AVS_OFFICESTUDIO_FILE_TEAMLAB_XLSY = "4098"
+AVS_OFFICESTUDIO_FILE_TEAMLAB_PPTY = "4099"
+AVS_OFFICESTUDIO_FILE_CANVAS_WORD = "8193"
+AVS_OFFICESTUDIO_FILE_CANVAS_SPREADSHEET = "8194"
+AVS_OFFICESTUDIO_FILE_CANVAS_PRESENTATION = "8195"
+AVS_OFFICESTUDIO_FILE_OTHER_HTMLZIP = "2051"
+AVS_OFFICESTUDIO_FILE_OTHER_ZIP = "2057"
+
+format_code_map = {
+  "docy": AVS_OFFICESTUDIO_FILE_CANVAS_WORD,
+  "docx": AVS_OFFICESTUDIO_FILE_DOCUMENT_DOCX,
+  "xlsy": AVS_OFFICESTUDIO_FILE_CANVAS_SPREADSHEET,
+  "xlsx": AVS_OFFICESTUDIO_FILE_SPREADSHEET_XLSX,
+  "ppty": AVS_OFFICESTUDIO_FILE_CANVAS_PRESENTATION,
+  "pptx": AVS_OFFICESTUDIO_FILE_PRESENTATION_PPTX,
+}
+
+yformat_tuple = ("docy", "xlsy", "ppty")
+
+class Handler(object):
+  """ImageMagic Handler is used to handler images."""
+
+  implements(IHandler)
+
+  def __init__(self, base_folder_url, data, source_format, **kw):
+    """
+    base_folder_url(string)
+      The requested url for data base folder
+    data(string)
+      The opened and readed file into a string
+    source_format(string)
+      The source format of the inputed file
+    """
+    self.base_folder_url = base_folder_url
+    self.file = File(base_folder_url, data, source_format)
+    self.environment = kw.get("env", {})
+
+  def convert(self, destination_format=None, **kw):
+    """ Convert the inputed file to output as format that were informed """
+    source_format = self.file.source_format
+    logger.debug("x2t convert: %s > %s" % (source_format, destination_format))
+
+    # init vars and xml configuration file
+    in_format = format_code_map[source_format]
+    out_format = format_code_map[destination_format]
+    root_dir = self.file.directory_name
+    input_dir = os.path.join(root_dir, "input");
+    output_dir = os.path.join(root_dir, "output");
+    final_file_name = os.path.join(root_dir, "document.%s" % destination_format)
+    input_file_name = self.file.getUrl()
+    output_file_name = final_file_name
+    config_file_name = os.path.join(root_dir, "config.xml")
+
+    if source_format in yformat_tuple:
+      os.mkdir(input_dir)
+      unzip(self.file.getUrl(), input_dir)
+      for _, _, files in os.walk(input_dir):
+        input_file_name, = files
+        break
+      input_file_name = os.path.join(input_dir, input_file_name)
+    if destination_format in yformat_tuple:
+      os.mkdir(output_dir)
+      output_file_name = os.path.join(output_dir, "body.txt")
+
+    config_file = open(config_file_name, "w")
+
+    config = {
+      # 'm_sKey': 'from',
+      'm_sFileFrom': input_file_name,
+      'm_nFormatFrom': in_format,
+      'm_sFileTo': output_file_name,
+      'm_nFormatTo': out_format,
+      # 'm_bPaid': 'true',
+      # 'm_bEmbeddedFonts': 'false',
+      # 'm_bFromChanges': 'false',
+      # 'm_sFontDir': '/usr/share/fonts',
+      # 'm_sThemeDir': '/var/www/onlyoffice/documentserver/FileConverterService/presentationthemes',
+    }
+    root = ElementTree.Element('root')
+    for key, value in config.items():
+      ElementTree.SubElement(root, key).text = value
+    ElementTree.ElementTree(root).write(config_file, encoding='utf-8', xml_declaration=True, default_namespace=None, method="xml")
+    config_file.close()
+
+    # run convertion binary
+    p = Popen(
+      ["x2t", config_file.name],
+      stdout=PIPE,
+      stderr=PIPE,
+      close_fds=True,
+      env=self.environment,
+    )
+    stdout, stderr = p.communicate()
+    if p.returncode != 0:
+      raise RuntimeError("x2t: exit code %d != 0\n+ %s\n> stdout: %s\n> stderr: %s@ x2t xml:\n%s" % (p.returncode, " ".join(["x2t", config_file.name]), stdout, stderr, "  " + open(config_file.name).read().replace("\n", "\n  ")))
+
+    if destination_format in yformat_tuple:
+      zipTree(
+        final_file_name,
+        (output_file_name, ""),
+        (os.path.join(os.path.dirname(output_file_name), "media"), ""),
+      )
+
+    self.file.reload(final_file_name)
+    try:
+      return self.file.getContent()
+    finally:
+      self.file.trash()
+
+  def getMetadata(self, base_document=False):
+    """Returns a dictionary with all metadata of document.
+    along with the metadata.
+    """
+    raise NotImplementedError
+
+  def setMetadata(self, metadata={}):
+    """Returns image with new metadata.
+    Keyword arguments:
+    metadata -- expected an dictionary with metadata.
+    """
+    raise NotImplementedError

cloudooo/handler/x2t/tests/testX2tHandler.py

+##############################################################################
+#
+# Copyright (c) 2009-2010 Nexedi SA and Contributors. All Rights Reserved.
+#                    Gabriel M. Monnerat <gabriel@tiolive.com>
+#
+# WARNING: This program as such is intended to be used by professional
+# programmers who take the whole responsibility of assessing all potential
+# consequences resulting from its eventual inadequacies and bugs
+# End users who are looking for a ready-to-use solution with commercial
+# guarantees and support are strongly adviced to contract a Free Software
+# Service Company
+#
+# This program is Free Software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License
+# as published by the Free Software Foundation; either version 2
+# of the License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.
+#
+##############################################################################
+
+
+import magic
+import os.path
+from zipfile import ZipFile
+from cStringIO import StringIO
+from cloudooo.handler.x2t.handler import Handler
+from cloudooo.tests.handlerTestCase import HandlerTestCase, make_suite
+
+class TestHandler(HandlerTestCase):
+
+  def afterSetUp(self):
+    self.kw = dict(env=dict(PATH=self.env_path))
+
+  def testConvertXlsx(self):
+    """Test conversion of xlsx to xlsy and back"""
+    y_data = Handler(self.tmp_url, open("data/test.xlsx").read(), "xlsx", **self.kw).convert("xlsy")
+    y_body_data = ZipFile(StringIO(y_data)).open("body.txt").read()
+    self.assertTrue(y_body_data.startswith("XLSY;v2;5883;"), "%r... does not start with 'XLSY;v2;5883;'" % (y_body_data[:20],))
+
+    x_data = Handler(self.tmp_url, y_data, "xlsy", **self.kw).convert("xlsx")
+    # magic inspired by https://github.com/minad/mimemagic/pull/19/files
+    self.assertIn("xl/", x_data[:2000])
+
+  def testConvertXlsy(self):
+    """Test conversion of xlsy to xlsx and back"""
+    x_data = Handler(self.tmp_url, open("data/test.xlsy").read(), "xlsy", **self.kw).convert("xlsx")
+    self.assertIn("xl/", x_data[:2000])
+
+    y_data = Handler(self.tmp_url, x_data, "xlsx", **self.kw).convert("xlsy")
+    y_body_data = ZipFile(StringIO(y_data)).open("body.txt").read()
+    self.assertTrue(y_body_data.startswith("XLSY;v2;10579;"), "%r... does not start with 'XLSY;v2;10579;'" % (y_body_data[:20],))
+
+  def testConvertDocx(self):
+    """Test conversion of docx to docy and back"""
+    y_data = Handler(self.tmp_url, open("data/test_with_image.docx").read(), "docx", **self.kw).convert("docy")
+    y_zip = ZipFile(StringIO(y_data))
+    y_body_data = y_zip.open("body.txt").read()
+    self.assertTrue(y_body_data.startswith("DOCY;v5;2795;"), "%r... does not start with 'DOCY;v5;2795;'" % (y_body_data[:20],))
+    y_zip.open("media/image1.png")
+
+    x_data = Handler(self.tmp_url, y_data, "docy", **self.kw).convert("docx")
+    # magic inspired by https://github.com/minad/mimemagic/pull/19/files
+    self.assertIn("word/", x_data[:2000])
+
+  def testConvertDocy(self):
+    """Test conversion of docy to docx and back"""
+    x_data = Handler(self.tmp_url, open("data/test_with_image.docy").read(), "docy", **self.kw).convert("docx")
+    self.assertIn("word/", x_data[:2000])
+
+    y_data = Handler(self.tmp_url, x_data, "docx", **self.kw).convert("docy")
+    y_zip = ZipFile(StringIO(y_data))
+    y_body_data = y_zip.open("body.txt").read()
+    self.assertTrue(y_body_data.startswith("DOCY;v5;7519;"), "%r... does not start with 'DOCY;v5;7519;'" % (y_body_data[:20],))
+    y_zip.open("media/image1.png")
+
+  def testgetMetadataFromImage(self):
+    """Test getMetadata not implemented form yformats"""
+    handler = Handler(self.tmp_url, "", "xlsy", **self.kw)
+    self.assertRaises(NotImplementedError, handler.getMetadata)
+
+  def testsetMetadata(self):
+    """Test setMetadata not implemented for yformats"""
+    handler = Handler(self.tmp_url, "", "xlsy", **self.kw)
+    self.assertRaises(NotImplementedError, handler.setMetadata)
+
+
+def test_suite():
+  return make_suite(TestHandler)

cloudooo/handler/x2t/tests/testX2tServer.py

+##############################################################################
+#
+# Copyright (c) 2016 Nexedi SA and Contributors. All Rights Reserved.
+#
+# WARNING: This program as such is intended to be used by professional
+# programmers who take the whole responsibility of assessing all potential
+# consequences resulting from its eventual inadequacies and bugs
+# End users who are looking for a ready-to-use solution with commercial
+# guarantees and support are strongly adviced to contract a Free Software
+# Service Company
+#
+# This program is Free Software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License
+# as published by the Free Software Foundation; either version 2
+# of the License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.
+#
+##############################################################################
+from os.path import join
+from cloudooo.tests.cloudoooTestCase import TestCase, make_suite
+
+
+class TestServer(TestCase):
+  """Test XmlRpc Server. Needs cloudooo server started"""
+
+  def ConversionScenarioList(self):
+    return [
+      (join('data', 'test.xlsx'), "xlsx", "xlsy", "application/zip"),
+      (join('data', 'test.xlsy'), "xlsy", "xlsx", "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet"),
+      (join('data', 'test_with_image.docx'), "docx", "docy", "application/zip"),
+      # Here, the content type should be application/vnd.openxmlformats-officedocument.wordprocessingml.document
+      # but "magic" seems to not detect it correctly. However, the document can be read correctly by LibreOffice.
+      (join('data', 'test_with_image.docy'), "docy", "docx", "application/octet-stream"),
+    ]
+
+  def testConvertHtmltoPdf(self):
+    """Converts xlsx,docx to their y format and y to x"""
+    self.runConversionList(self.ConversionScenarioList())
+
+  def FaultConversionScenarioList(self):
+    return [
+      # Test to verify if server fail when a empty string is sent
+      ('', '', ''),
+      # Try convert one xlsx for a invalid format
+      (open(join('data', 'test.xlsx')).read(), 'xlsx', 'xyz'),
+    ]
+
+def test_suite():
+  return make_suite(TestServer)

cloudooo/mime.types

@@ -56,6 +56,7 @@ application/vnd.oasis.opendocument.database
 application/vnd.oasis.opendocument.image                                        odi
 application/vnd.ms-word.document.macroEnabled.12                                docm
 application/vnd.openxmlformats-officedocument.wordprocessingml.document         docx
+application/x-asc-text                                                          docy
 application/vnd.openxmlformats-officedocument.wordprocessingml.template         dotx
 application/vnd.ms-powerpoint.template.macroEnabled.12                          potm
 application/vnd.openxmlformats-officedocument.presentationml.template           potx
@@ -64,10 +65,12 @@ application/vnd.ms-powerpoint.slideshow.macroEnabled.12
 application/vnd.openxmlformats-officedocument.presentationml.slideshow          ppsx
 application/vnd.ms-powerpoint.presentation.macroEnabled.12                      pptm
 application/vnd.openxmlformats-officedocument.presentationml.presentation       pptx
+application/x-asc-presentation                                                  ppty
 application/vnd.ms-excel.addin.macroEnabled.12                                  xlam
 application/vnd.ms-excel.sheet.binary.macroEnabled.12                           xlsb
 application/vnd.ms-excel.sheet.macroEnabled.12                                  xlsm
 application/vnd.openxmlformats-officedocument.spreadsheetml.sheet               xlsx
+application/x-asc-spreadsheet                                                   xlsy
 application/vnd.ms-excel.template.macroEnabled.12                               xltm
 application/vnd.openxmlformats-officedocument.spreadsheetml.template            xltx
 application/x-msmetafile                                                        wmf

cloudooo/util.py

@@ -29,6 +29,8 @@
 import logging
 import mimetypes
 import pkg_resources
+import os
+from zipfile import ZipFile, ZIP_DEFLATED
 
 logger = logging.getLogger('Cloudooo')
 
@@ -98,3 +100,36 @@ def convertStringToBool(string):
     return False
   else:
     return None
+
+def zipTree(destination, *tree_path_list):
+  """
+    destination may be a path or a StringIO
+
+    tree_path_list is a list that may contain a path or a couple(path, archive_root)
+  """
+  def archive(arg, archive_root):
+    archive_name = os.path.join(archive_root, os.path.basename(arg))
+    if os.path.islink(arg):
+      pass  # XXX logger.warn("zipTree: symlink %r ignored\n" % arg)
+    elif os.path.isdir(arg):
+      for r, _, ff in os.walk(arg):
+        zfile.write(r, archive_name)
+        for f in ff:
+          archive(os.path.join(r, f), archive_name)
+    elif os.path.isfile(arg):
+      zfile.write(arg, archive_name)
+    else:
+      pass  # XXX logger.warn("zipTree: unknown %r ignored\n" % arg)
+  zfile = ZipFile(destination, "w", ZIP_DEFLATED)
+  for tree_path in tree_path_list:
+    if isinstance(tree_path, tuple):
+      archive(*tree_path)
+    else:
+      archive(tree_path, os.path.dirname(tree_path))
+  zfile.close()
+  return destination
+
+def unzip(source, destination):
+  zipfile = ZipFile(source)
+  zipfile.extractall(destination)
+  zipfile.close()