Skip to content

erp5
erp5
Commit 064843e1 authored by Jérome Perrin's avatar Jérome Perrin
Browse files
Options

XMLExportImport: WIP support pickle protocol 3 🚧

parent 8ef18ff0
Pipeline #27222 canceled with stage
    in 0 seconds
    Hide whitespace changes
    Inline Side-by-side
    Showing with 236 additions and 69 deletions
    product/ERP5/Document/BusinessTemplate.py
    View file @ 064843e1
    ......@@ -78,7 +78,7 @@ from OFS.Traversable import NotFound
    from OFS import SimpleItem
    from OFS.Image import Pdata
    import coverage
    from io import BytesIO
    from io import BytesIO, StringIO
    from copy import deepcopy
    from zExceptions import BadRequest
    from Products.ERP5Type.XMLExportImport import exportXML, customImporters
    ......@@ -347,10 +347,11 @@ class BusinessTemplateArchive(object):
    try:
    write = self._writeFile
    except AttributeError:
    if not isinstance(obj, (bytes, str)):
    if hasattr(obj, 'read'):
    obj.seek(0)
    obj = obj.read()
    elif not isinstance(obj, bytes):
    #import pdb; pdb.set_trace()
    if not isinstance(obj, bytes):
    obj = obj.encode('utf-8')
    self.revision.hash(path, obj)
    self._writeString(obj, path)
    ......@@ -865,9 +866,9 @@ class ObjectTemplateItem(BaseTemplateItem):
    obj = self.removeProperties(obj, 1, keep_workflow_history = True)
    transaction.savepoint(optimistic=True)
    f = BytesIO()
    f = StringIO()
    exportXML(obj._p_jar, obj._p_oid, f)
    bta.addObject(f, key, path=path)
    bta.addObject(f.getvalue().encode(), key, path=path)
    if catalog_method_template_item:
    # add all datas specific to catalog inside one file
    ......
    product/ERP5Type/XMLExportImport/__init__.py
    View file @ 064843e1
    ......@@ -48,7 +48,7 @@ from lxml import etree
    from lxml.etree import Element, SubElement
    from xml_marshaller.xml_marshaller import Marshaller
    from OFS.Image import Pdata
    from base64 import standard_b64encode
    from base64 import standard_b64encode, encodebytes
    from hashlib import sha1
    from Products.ERP5Type.Utils import ensure_list
    #from zLOG import LOG
    ......@@ -223,7 +223,6 @@ def Folder_asXML(object, omit_xml_declaration=True, root=None):
    ## The code below was initially from OFS.XMLExportImport
    from six import string_types as basestring
    from base64 import encodebytes
    from ZODB.serialize import referencesf
    from ZODB.ExportImport import TemporaryFile, export_end_marker
    from ZODB.utils import p64
    ......@@ -253,7 +252,7 @@ def reorderPickle(jar, p):
    new_oid=storage.new_oid):
    "Remap a persistent id to an existing ID and create a ghost for it."
    print('persistent_load1', ooid)
    if isinstance(ooid, tuple): ooid, klass = ooid
    else: klass=None
    ......@@ -270,13 +269,23 @@ def reorderPickle(jar, p):
    unpickler.persistent_load=persistent_load
    newp=BytesIO()
    pickler=OrderedPickler(newp,1)
    pickler = OrderedPickler(newp, 3)
    pickler.persistent_id=persistent_id
    classdef = unpickler.load()
    obj = unpickler.load()
    pickler.dump(classdef)
    pickler.dump(obj)
    if 0: # debug
    debugp = BytesIO()
    debugpickler = OrderedPickler(debugp, 3)
    debugpickler.persistent_id = persistent_id
    debugpickler.dump(obj)
    import pickletools
    print(debugp.getvalue())
    print(pickletools.dis(debugp.getvalue()))
    p=newp.getvalue()
    return obj, p
    ......@@ -290,14 +299,13 @@ def _mapOid(id_mapping, oid):
    def XMLrecord(oid, plen, p, id_mapping):
    # Proceed as usual
    q=ppml.ToXMLUnpickler
    f=BytesIO(p)
    u=q(f)
    f = BytesIO(p)
    u = ppml.ToXMLUnpickler(f)
    u.idprefix, id, aka = _mapOid(id_mapping, oid)
    p=u.load(id_mapping=id_mapping).__str__(4)
    p = u.load(id_mapping=id_mapping).__str__(4)
    if f.tell() < plen:
    p=p+u.load(id_mapping=id_mapping).__str__(4)
    String=' <record id="%s" aka="%s">\n%s </record>\n' % (id, aka, p)
    String=' <record id="%s" aka="%s">\n%s </record>\n' % (id, aka.decode(), p)
    return String
    def exportXML(jar, oid, file=None):
    ......@@ -334,15 +342,21 @@ def exportXML(jar, oid, file=None):
    # Do real export
    if file is None:
    file = TemporaryFile()
    file = TemporaryFile(mode='w')
    elif isinstance(file, basestring):
    file = open(file, 'w+b')
    file = open(file, 'w')
    write = file.write
    write('<?xml version="1.0"?>\n<ZopeData>\n')
    for oid in reordered_oid_list:
    p = getReorderedPickle(oid)
    write(XMLrecord(oid, len(p), p, id_mapping))
    write('</ZopeData>\n')
    if 0:
    try:
    print(file.getvalue())
    except AttributeError:
    pass
    import pdb; pdb.set_trace()
    return file
    class zopedata:
    ......
    product/ERP5Type/XMLExportImport/ppml.py
    View file @ 064843e1
    ......@@ -159,6 +159,10 @@ class Long(Scalar):
    return result
    class String(Scalar):
    def tag_name(self):
    return self.__class__.__name__.lower()
    encoding = None
    def __init__(self, v, mapping, encoding=''):
    ......@@ -174,7 +178,7 @@ class String(Scalar):
    # be converted.
    encoding = 'base64'
    v = base64.encodebytes(self._v)[:-1]
    self._v = self.mapping.convertBase64(v)
    self._v = self.mapping.convertBase64(v).decode()
    else:
    encoding, self._v = convert(self._v)
    self.encoding = encoding
    ......@@ -184,7 +188,7 @@ class String(Scalar):
    encoding = '' # JPS repr is default encoding
    else:
    encoding = ' encoding="%s"' % encoding
    name=self.__class__.__name__.lower()
    name = self.tag_name()
    result = '<%s%s%s>%s</%s>' % (name, id, encoding, v, name)
    if hasattr(self, 'id'):
    # The value is Immutable - let us add it the the immutable mapping
    ......@@ -193,8 +197,14 @@ class String(Scalar):
    return '%s%s\n' % (' '*indent, result)
    class Unicode(String):
    def value(self):
    return self._v.encode('utf-8')
    def tag_name(self):
    if six.PY3:
    return 'string'
    return super(Unicode, self).tag_name()
    class Bytes(String):
    pass
    class Wrapper:
    def __init__(self, v, mapping):
    ......@@ -480,88 +490,169 @@ class ToXMLUnpickler(Unpickler):
    def load_persid(self):
    pid = self.readline()[:-1]
    self.append(self.persistent_load(String(pid, self.id_mapping)))
    dispatch[PERSID] = load_persid
    if six.PY2:
    dispatch[PERSID] = load_persid
    dispatch[PERSID[0]] = load_persid
    def load_binpersid(self):
    pid = self.stack.pop()
    self.append(self.persistent_load(pid))
    if six.PY2:
    dispatch[BINPERSID] = load_binpersid
    dispatch[BINPERSID[0]] = load_binpersid
    def load_none(self):
    self.append(none)
    dispatch[NONE] = load_none
    if six.PY2:
    dispatch[NONE] = load_none
    dispatch[NONE[0]] = load_none
    def load_int(self):
    self.append(Int(int(self.readline()[:-1]), self.id_mapping))
    dispatch[INT] = load_int
    if six.PY2:
    dispatch[INT] = load_int
    dispatch[INT[0]] = load_int
    def load_binint(self):
    self.append(Int(mloads('i' + self.read(4)), self.id_mapping))
    dispatch[BININT] = load_binint
    self.append(Int(mloads(b'i' + self.read(4)), self.id_mapping))
    if six.PY2:
    dispatch[BININT] = load_binint
    dispatch[BININT[0]] = load_binint
    def load_binint1(self):
    self.append(Int(mloads('i' + self.read(1) + '\000\000\000'), self.id_mapping))
    dispatch[BININT1] = load_binint1
    self.append(Int(mloads(b'i' + self.read(1) + b'\000\000\000'), self.id_mapping))
    if six.PY2:
    dispatch[BININT1] = load_binint1
    dispatch[BININT1[0]] = load_binint1
    def load_binint2(self):
    self.append(Int(mloads('i' + self.read(2) + '\000\000'), self.id_mapping))
    dispatch[BININT2] = load_binint2
    self.append(Int(mloads(b'i' + self.read(2) + b'\000\000'), self.id_mapping))
    if six.PY2:
    dispatch[BININT2] = load_binint2
    dispatch[BININT2[0]] = load_binint2
    def load_long(self):
    self.append(Long(long_(self.readline()[:-1], 0), self.id_mapping))
    dispatch[LONG] = load_long
    if six.PY2:
    dispatch[LONG] = load_long
    dispatch[LONG[0]] = load_long
    def load_float(self):
    self.append(Float(float(self.readline()[:-1]), self.id_mapping))
    dispatch[FLOAT] = load_float
    if six.PY2:
    dispatch[FLOAT] = load_float
    dispatch[FLOAT[0]] = load_float
    def load_binfloat(self, unpack=struct.unpack):
    self.append(Float(unpack('>d', self.read(8))[0], self.id_mapping))
    dispatch[BINFLOAT] = load_binfloat
    if six.PY2:
    dispatch[BINFLOAT] = load_binfloat
    dispatch[BINFLOAT[0]] = load_binfloat
    def load_string(self):
    self.append(String(eval(self.readline()[:-1],
    {'__builtins__': {}}), self.id_mapping)) # Let's be careful
    dispatch[STRING] = load_string
    if six.PY2:
    dispatch[STRING] = load_string
    dispatch[STRING[0]] = load_string
    def load_binstring(self):
    len = mloads('i' + self.read(4))
    len = mloads(b'i' + self.read(4))
    self.append(String(self.read(len), self.id_mapping))
    dispatch[BINSTRING] = load_binstring
    if six.PY2:
    dispatch[BINSTRING] = load_binstring
    dispatch[BINSTRING[0]] = load_binstring
    def load_unicode(self):
    self.append(Unicode(six.text_type(eval(self.readline()[:-1],
    line = self.readline()
    self.append(Unicode(six.text_type(eval(line[:-1],
    {'__builtins__': {}})), self.id_mapping)) # Let's be careful
    dispatch[UNICODE] = load_unicode
    if six.PY2:
    dispatch[UNICODE] = load_unicode
    dispatch[UNICODE[0]] = load_unicode
    def load_binunicode(self):
    len = mloads('i' + self.read(4))
    len = mloads(b'i' + self.read(4))
    self.append(Unicode(six.text_type(self.read(len), 'utf-8'), self.id_mapping))
    dispatch[BINUNICODE] = load_binunicode
    if six.PY2:
    dispatch[BINUNICODE] = load_binunicode
    dispatch[BINUNICODE[0]] = load_binunicode
    def load_short_binstring(self):
    len = mloads('i' + self.read(1) + '\000\000\000')
    len = mloads(b'i' + self.read(1) + b'\000\000\000')
    self.append(String(self.read(len), self.id_mapping))
    dispatch[SHORT_BINSTRING] = load_short_binstring
    if six.PY2:
    dispatch[SHORT_BINSTRING] = load_short_binstring
    dispatch[SHORT_BINSTRING[0]] = load_short_binstring
    def load_binbytes(self):
    len = mloads(b'i' + self.read(4))
    self.append(Bytes(self.read(len), self.id_mapping))
    if six.PY2:
    dispatch[BINBYTES] = load_binbytes
    dispatch[BINBYTES[0]] = load_binbytes
    def load_short_binbytes(self):
    len = mloads(b'i' + self.read(1) + b'\000\000\000')
    self.append(Bytes(self.read(len), self.id_mapping))
    if six.PY2:
    dispatch[SHORT_BINBYTES] = load_short_binbytes
    dispatch[SHORT_BINBYTES[0]] = load_short_binbytes
    def load_tuple(self):
    k = self.marker()
    #LOG('load_tuple, k',0,k)
    #LOG('load_tuple, stack[k+1:]',0,self.stack[k+1:])
    self.stack[k:] = [Tuple(self.id_mapping, v=self.stack[k+1:])]
    dispatch[TUPLE] = load_tuple
    if six.PY2:
    dispatch[TUPLE] = load_tuple
    dispatch[TUPLE[0]] = load_tuple
    def load_tuple1(self):
    k = self.marker()
    self.stack[-1] = Tuple(self.id_mapping, v=(self.stack[-1],))
    if six.PY2:
    dispatch[TUPLE1] = load_tuple1
    dispatch[TUPLE1[0]] = load_tuple1
    def load_tuple2(self):
    k = self.marker()
    self.stack[-2:] = [Tuple(self.id_mapping, v=(self.stack[-2], self.stack[-1]))]
    if six.PY2:
    dispatch[TUPLE2] = load_tuple2
    dispatch[TUPLE2[0]] = load_tuple2
    def load_tuple3(self):
    k = self.marker()
    self.stack[-3:] = [Tuple(self.id_mapping, v=(self.stack[-3], self.stack[-2], self.stack[-1]))]
    if six.PY2:
    dispatch[TUPLE3] = load_tuple3
    dispatch[TUPLE3[0]] = load_tuple3
    def load_empty_tuple(self):
    self.stack.append(Tuple(self.id_mapping))
    dispatch[EMPTY_TUPLE] = load_empty_tuple
    if six.PY2:
    dispatch[EMPTY_TUPLE] = load_empty_tuple
    dispatch[EMPTY_TUPLE[0]] = load_empty_tuple
    def load_empty_list(self):
    self.stack.append(List(self.id_mapping))
    dispatch[EMPTY_LIST] = load_empty_list
    if six.PY2:
    dispatch[EMPTY_LIST] = load_empty_list
    dispatch[EMPTY_LIST[0]] = load_empty_list
    def load_empty_dictionary(self):
    self.stack.append(Dictionary(self.id_mapping))
    dispatch[EMPTY_DICT] = load_empty_dictionary
    if six.PY2:
    dispatch[EMPTY_DICT] = load_empty_dictionary
    dispatch[EMPTY_DICT[0]] = load_empty_dictionary
    def load_list(self):
    k = self.marker()
    self.stack[k:] = [List(self.id_mapping, v=self.stack[k+1:])]
    dispatch[LIST] = load_list
    if six.PY2:
    dispatch[LIST] = load_list
    dispatch[LIST[0]] = load_list
    def load_dict(self):
    k = self.marker()
    ......@@ -572,17 +663,21 @@ class ToXMLUnpickler(Unpickler):
    value = items[i+1]
    d[key] = value
    self.stack[k:] = [d]
    dispatch[DICT] = load_dict
    if six.PY2:
    dispatch[DICT] = load_dict
    dispatch[DICT[0]] = load_dict
    def load_inst(self):
    k = self.marker()
    args = Tuple(self.id_mapping, v=self.stack[k+1:])
    del self.stack[k:]
    module = self.readline()[:-1]
    name = self.readline()[:-1]
    module = self.readline()[:-1].decode()
    name = self.readline()[:-1].decode()
    value=Object(Global(module, name, self.id_mapping), args, self.id_mapping)
    self.append(value)
    dispatch[INST] = load_inst
    if six.PY2:
    dispatch[INST] = load_inst
    dispatch[INST[0]] = load_inst
    def load_obj(self):
    stack = self.stack
    ......@@ -593,13 +688,29 @@ class ToXMLUnpickler(Unpickler):
    del stack[k:]
    value=Object(klass,args, self.id_mapping)
    self.append(value)
    dispatch[OBJ] = load_obj
    if six.PY2:
    dispatch[OBJ] = load_obj
    dispatch[OBJ[0]] = load_obj
    def load_newobj(self):
    # TODO: not really sure of this one, maybe we need
    # a NewObj instead of Object
    args = self.stack.pop()
    cls = self.stack[-1]
    obj = Object(cls, args, self.id_mapping)
    self.stack[-1] = obj
    print('load_newobj', self.stack)
    if six.PY2:
    dispatch[NEWOBJ] = load_newobj
    dispatch[NEWOBJ[0]] = load_newobj
    def load_global(self):
    module = self.readline()[:-1]
    name = self.readline()[:-1]
    module = self.readline()[:-1].decode()
    name = self.readline()[:-1].decode()
    self.append(Global(module, name, self.id_mapping))
    dispatch[GLOBAL] = load_global
    if six.PY2:
    dispatch[GLOBAL] = load_global
    dispatch[GLOBAL[0]] = load_global
    def load_reduce(self):
    stack = self.stack
    ......@@ -610,38 +721,51 @@ class ToXMLUnpickler(Unpickler):
    value=Object(callable, arg_tup, self.id_mapping)
    self.append(value)
    dispatch[REDUCE] = load_reduce
    if six.PY2:
    dispatch[REDUCE] = load_reduce
    dispatch[REDUCE[0]] = load_reduce
    idprefix=''
    def load_get(self):
    self.append(Get(self.idprefix+self.readline()[:-1], self.id_mapping))
    dispatch[GET] = load_get
    if six.PY2:
    dispatch[GET] = load_get
    dispatch[GET[0]] = load_get
    def load_binget(self):
    i = mloads('i' + self.read(1) + '\000\000\000')
    i = mloads(b'i' + self.read(1) + b'\000\000\000')
    self.append(Get(self.idprefix+repr(i), self.id_mapping))
    dispatch[BINGET] = load_binget
    if six.PY2:
    dispatch[BINGET] = load_binget
    dispatch[BINGET[0]] = load_binget
    def load_long_binget(self):
    i = mloads('i' + self.read(4))
    i = mloads(b'i' + self.read(4))
    self.append(Get(self.idprefix+repr(i), self.id_mapping))
    dispatch[LONG_BINGET] = load_long_binget
    if six.PY2:
    dispatch[LONG_BINGET] = load_long_binget
    dispatch[LONG_BINGET[0]] = load_long_binget
    def load_put(self):
    self.stack[-1].id=self.idprefix+self.readline()[:-1]
    dispatch[PUT] = load_put
    if six.PY2:
    dispatch[PUT] = load_put
    dispatch[PUT[0]] = load_put
    def load_binput(self):
    i = mloads('i' + self.read(1) + '\000\000\000')
    #LOG('load_binput', 0, 'self.stack = %r, self.idprefix+`i` = %r' % (self.stack, self.idprefix+`i`))
    i = mloads(b'i' + self.read(1) + b'\000\000\000')
    self.stack[-1].id=self.idprefix+repr(i)
    dispatch[BINPUT] = load_binput
    if six.PY2:
    dispatch[BINPUT] = load_binput
    dispatch[BINPUT[0]] = load_binput
    def load_long_binput(self):
    i = mloads('i' + self.read(4))
    i = mloads(b'i' + self.read(4))
    self.stack[-1].id=self.idprefix+repr(i)
    dispatch[LONG_BINPUT] = load_long_binput
    if six.PY2:
    dispatch[LONG_BINPUT] = load_long_binput
    dispatch[LONG_BINPUT[0]] = load_long_binput
    class LogCall:
    def __init__(self, func):
    ......@@ -651,8 +775,8 @@ class ToXMLUnpickler(Unpickler):
    #LOG('LogCall', 0, 'self.stack = %r, func = %s' % (context.stack, self.func.__name__))
    return self.func(context)
    #for code in dispatch.keys():
    # dispatch[code] = LogCall(dispatch[code])
    # for code in dispatch.keys():
    # dispatch[code] = LogCall(dispatch[code])
    def ToXMLload(file):
    return ToXMLUnpickler(file).load()
    ......@@ -711,7 +835,7 @@ def save_string(self, tag, data):
    a = data[1]
    v = b''.join(data[2:])
    encoding = a.get('encoding', 'repr') # JPS: repr is default encoding
    if encoding is not '':
    if encoding != '':
    v = unconvert(encoding, v)
    if self.binary:
    l = len(v)
    ......@@ -731,6 +855,27 @@ def save_string(self, tag, data):
    v = STRING + repr(v) + '\n'
    return save_put(self, v, a)
    def save_bytes(self, tag, data):
    a = data[1]
    v = b''.join(data[2:])
    encoding = a.get('encoding', 'repr')
    assert encoding == 'base64'
    if encoding is not '':
    v = unconvert(encoding, v)
    if self.binary:
    l = len(v)
    if l < 256:
    op = SHORT_BINBYTES
    v = op + six.int2byte(l) + v
    else:
    op = BINBYTES
    v = op + struct.pack('<i', l) + v
    else:
    # XXX used ??? seems wrong
    v = BYTES + repr(v) + '\n'
    return save_put(self, v, a)
    def save_unicode(self, tag, data):
    binary=self.binary
    v=b''
    ......@@ -820,6 +965,12 @@ def save_global(self, tag, data):
    def save_persis(self, tag, data):
    v = data[2]
    if self.binary:
    if isinstance(v, list):
    # XXX with binpersid we have:
    # ['bytes', {'encoding': 'base64'}, b'AAAAAAAAAAI=']]
    # but probably because something was wrong
    import pdb; pdb.set_trace()
    _, _, v = v
    return v + BINPERSID
    else:
    return PERSID + v
    ......@@ -864,6 +1015,7 @@ class xmlPickler(NoBlanks, xyap):
    'int': save_int,
    'long': save_long,
    'float': save_float,
    'bytes': save_bytes,
    'string': save_string,
    'unicode': save_unicode,
    'reference': save_reference,
    ......
    Markdown is supported
    0% or
    You are about to add 0 people to the discussion. Proceed with caution.
    Finish editing this message first!
    Please register or to comment
    GitLab Nexedi Edition | About GitLab | About Nexedi | 沪ICP备2021021310号-2 | 沪ICP备2021021310号-7