XMLExportImport: WIP support pickle protocol 3 🚧

product/ERP5/Document/BusinessTemplate.py

@@ -78,7 +78,7 @@ from OFS.Traversable import NotFound
 from OFS import SimpleItem
 from OFS.Image import Pdata
 import coverage
-from io import BytesIO
+from io import BytesIO, StringIO
 from copy import deepcopy
 from zExceptions import BadRequest
 from Products.ERP5Type.XMLExportImport import exportXML, customImporters
@@ -347,10 +347,11 @@ class BusinessTemplateArchive(object):
     try:
       write = self._writeFile
     except AttributeError:
-      if not isinstance(obj, (bytes, str)):
+      if hasattr(obj, 'read'):
         obj.seek(0)
         obj = obj.read()
-      elif not isinstance(obj, bytes):
+        #import pdb; pdb.set_trace()
+      if not isinstance(obj, bytes):
         obj = obj.encode('utf-8')
       self.revision.hash(path, obj)
       self._writeString(obj, path)
@@ -865,9 +866,9 @@ class ObjectTemplateItem(BaseTemplateItem):
           obj = self.removeProperties(obj, 1, keep_workflow_history = True)
           transaction.savepoint(optimistic=True)
 
-        f = BytesIO()
+        f = StringIO()
         exportXML(obj._p_jar, obj._p_oid, f)
-        bta.addObject(f, key, path=path)
+        bta.addObject(f.getvalue().encode(), key, path=path)
 
       if catalog_method_template_item:
         # add all datas specific to catalog inside one file

product/ERP5Type/XMLExportImport/__init__.py

@@ -48,7 +48,7 @@ from lxml import etree
 from lxml.etree import Element, SubElement
 from xml_marshaller.xml_marshaller import Marshaller
 from OFS.Image import Pdata
-from base64 import standard_b64encode
+from base64 import standard_b64encode, encodebytes
 from hashlib import sha1
 from Products.ERP5Type.Utils import ensure_list
 #from zLOG import LOG
@@ -223,7 +223,6 @@ def Folder_asXML(object, omit_xml_declaration=True, root=None):
 
 ## The code below was initially from OFS.XMLExportImport
 from six import string_types as basestring
-from base64 import encodestring
 from ZODB.serialize import referencesf
 from ZODB.ExportImport import TemporaryFile, export_end_marker
 from ZODB.utils import p64
@@ -253,7 +252,7 @@ def reorderPickle(jar, p):
                         new_oid=storage.new_oid):
 
         "Remap a persistent id to an existing ID and create a ghost for it."
-
+        print('persistent_load1', ooid)
         if isinstance(ooid, tuple): ooid, klass = ooid
         else: klass=None
 
@@ -270,34 +269,43 @@ def reorderPickle(jar, p):
     unpickler.persistent_load=persistent_load
 
     newp=BytesIO()
-    pickler=OrderedPickler(newp,1)
+    pickler = OrderedPickler(newp, 3)
     pickler.persistent_id=persistent_id
 
     classdef = unpickler.load()
     obj = unpickler.load()
     pickler.dump(classdef)
     pickler.dump(obj)
+
+    if 0: # debug
+      debugp = BytesIO()
+      debugpickler = OrderedPickler(debugp, 3)
+      debugpickler.persistent_id = persistent_id
+      debugpickler.dump(obj)
+      import pickletools
+      print(debugp.getvalue())
+      print(pickletools.dis(debugp.getvalue()))
+
     p=newp.getvalue()
     return obj, p
 
 def _mapOid(id_mapping, oid):
     idprefix = str(u64(oid))
     id = id_mapping[idprefix]
-    old_aka = encodestring(oid)[:-1]
-    aka=encodestring(p64(long_(id)))[:-1]  # Rebuild oid based on mapped id
+    old_aka = encodebytes(oid)[:-1]
+    aka=encodebytes(p64(long_(id)))[:-1]  # Rebuild oid based on mapped id
     id_mapping.setConvertedAka(old_aka, aka)
     return idprefix+'.', id, aka
 
 def XMLrecord(oid, plen, p, id_mapping):
     # Proceed as usual
-    q=ppml.ToXMLUnpickler
-    f=BytesIO(p)
-    u=q(f)
+    f = BytesIO(p)
+    u = ppml.ToXMLUnpickler(f)
     u.idprefix, id, aka = _mapOid(id_mapping, oid)
-    p=u.load(id_mapping=id_mapping).__str__(4)
+    p = u.load(id_mapping=id_mapping).__str__(4)
     if f.tell() < plen:
         p=p+u.load(id_mapping=id_mapping).__str__(4)
-    String='  <record id="%s" aka="%s">\n%s  </record>\n' % (id, aka, p)
+    String='  <record id="%s" aka="%s">\n%s  </record>\n' % (id, aka.decode(), p)
     return String
 
 def exportXML(jar, oid, file=None):
@@ -334,15 +342,21 @@ def exportXML(jar, oid, file=None):
 
     # Do real export
     if file is None:
-        file = TemporaryFile()
+        file = TemporaryFile(mode='w')
     elif isinstance(file, basestring):
-        file = open(file, 'w+b')
+        file = open(file, 'w')
     write = file.write
     write('<?xml version="1.0"?>\n<ZopeData>\n')
     for oid in reordered_oid_list:
         p = getReorderedPickle(oid)
         write(XMLrecord(oid, len(p), p, id_mapping))
     write('</ZopeData>\n')
+    if 0:
+      try:
+        print(file.getvalue())
+      except AttributeError:
+        pass
+      import pdb; pdb.set_trace()
     return file
 
 class zopedata: