diff --git a/bt5/erp5_core_test/TestTemplateItem/portal_components/test.erp5.testXMLPickle.py b/bt5/erp5_core_test/TestTemplateItem/portal_components/test.erp5.testXMLPickle.py index c89ee988ea5a187d84c2bf01601f49f47e46b489..5d6c25ed575b6a4c23a529ff7dc97e4f483427b3 100644 --- a/bt5/erp5_core_test/TestTemplateItem/portal_components/test.erp5.testXMLPickle.py +++ b/bt5/erp5_core_test/TestTemplateItem/portal_components/test.erp5.testXMLPickle.py @@ -31,7 +31,7 @@ import pickle import re import xml.parsers.pyexpat from StringIO import StringIO -from Shared.DC.xml import ppml +from Products.ERP5Type.XMLExportImport import ppml class DummyClass: diff --git a/product/ERP5/Document/BusinessTemplate.py b/product/ERP5/Document/BusinessTemplate.py index bad3899e8b60ac30bbd7aed62d9555ddf40602d7..94c128180d3c1c35d9abe4b670742793a5530113 100644 --- a/product/ERP5/Document/BusinessTemplate.py +++ b/product/ERP5/Document/BusinessTemplate.py @@ -71,16 +71,13 @@ from Products.ERP5Type.dynamic.portal_type_class import synchronizeDynamicModule from Products.ERP5Type.Core.PropertySheet import PropertySheet as PropertySheetDocument from Products.ERP5Type.TransactionalVariable import getTransactionalVariable from OFS.Traversable import NotFound -from OFS import SimpleItem, XMLExportImport +from OFS import SimpleItem from OFS.Image import Pdata from cStringIO import StringIO from copy import deepcopy from zExceptions import BadRequest -import OFS.XMLExportImport -from Products.ERP5Type.patches.ppml import importXML -customImporters={ - XMLExportImport.magic: importXML, - } +from Products.ERP5Type.XMLExportImport import exportXML +from OFS.ObjectManager import customImporters from Products.ERP5Type.Workflow import WorkflowHistoryList from zLOG import LOG, WARNING, INFO from warnings import warn @@ -858,7 +855,7 @@ class ObjectTemplateItem(BaseTemplateItem): transaction.savepoint(optimistic=True) f = StringIO() - XMLExportImport.exportXML(obj._p_jar, obj._p_oid, f) + exportXML(obj._p_jar, obj._p_oid, f) bta.addObject(f, key, path=path) if catalog_method_template_item: @@ -1015,8 +1012,8 @@ class ObjectTemplateItem(BaseTemplateItem): pass #LOG('Business Template', 0, 'Compiling %s...' % (name,)) - from Shared.DC.xml import ppml - from OFS.XMLExportImport import start_zopedata, save_record, save_zopedata + from Products.ERP5Type.XMLExportImport import (ppml, + start_zopedata, save_record, save_zopedata) import xml.parsers.expat outfile=StringIO() try: @@ -1069,10 +1066,10 @@ class ObjectTemplateItem(BaseTemplateItem): new_object = self._objects[path] new_io = StringIO() old_io = StringIO() - OFS.XMLExportImport.exportXML(new_object._p_jar, new_object._p_oid, new_io) + exportXML(new_object._p_jar, new_object._p_oid, new_io) new_obj_xml = new_io.getvalue() try: - OFS.XMLExportImport.exportXML(old_object._p_jar, old_object._p_oid, old_io) + exportXML(old_object._p_jar, old_object._p_oid, old_io) old_obj_xml = old_io.getvalue() except (ImportError, UnicodeDecodeError), e: # module is already # removed etc. @@ -6167,8 +6164,8 @@ Business Template is a set of definitions, such as skins, portal types and categ new_object = new_item.removeProperties(new_object, 1) installed_object = installed_item.removeProperties(installed_object, 1) # XML Export in memory - OFS.XMLExportImport.exportXML(new_object._p_jar, new_object._p_oid, f1) - OFS.XMLExportImport.exportXML(installed_object._p_jar, + exportXML(new_object._p_jar, new_object._p_oid, f1) + exportXML(installed_object._p_jar, installed_object._p_oid, f2) new_obj_xml = f1.getvalue() f1.close() @@ -6503,6 +6500,8 @@ Business Template is a set of definitions, such as skins, portal types and categ 'Products.ERP5Type.interfaces.json_representable', 'Products.ERP5Type.mixin.json_representable', 'Products.ERP5Type.XMLExportImport', + 'Products.ERP5Type.XMLExportImport.ppml', + 'Products.ERP5Type.XMLExportImport.xyap', 'Products.ERP5Type.mixin.property_translatable', 'Products.ERP5Type.Error', 'Products.ERP5Type.Errors', diff --git a/product/ERP5/tests/utils.py b/product/ERP5/tests/utils.py index 38656c18ff4125040406fdad700eb6106ddbed57..68e44111f717df0001b642c3aa40824cc3de90e5 100644 --- a/product/ERP5/tests/utils.py +++ b/product/ERP5/tests/utils.py @@ -32,7 +32,7 @@ import xml.dom.minidom from urllib import url2pathname from ZODB.DemoStorage import DemoStorage from ZODB import DB -from OFS.XMLExportImport import importXML +from Products.ERP5Type.XMLExportImport import importXML if int(os.environ.get('erp5_report_new_simulation_failures') or 0): newSimulationExpectedFailure = lambda test: test diff --git a/product/ERP5Type/XMLExportImport/OFSXMLExportImport.py b/product/ERP5Type/XMLExportImport/OFSXMLExportImport.py new file mode 100644 index 0000000000000000000000000000000000000000..069388c5da08a61992807137836b497855086965 --- /dev/null +++ b/product/ERP5Type/XMLExportImport/OFSXMLExportImport.py @@ -0,0 +1,13 @@ +############################################################################## +# +# Copyright (c) 2001,2002 Zope Foundation and Contributors. +# Copyright (c) 2002,2005 Nexedi SARL and Contributors. All Rights Reserved. +# +# This software is subject to the provisions of the Zope Public License, +# Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution. +# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED +# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS +# FOR A PARTICULAR PURPOSE +# +############################################################################## diff --git a/product/ERP5Type/XMLExportImport.py b/product/ERP5Type/XMLExportImport/__init__.py similarity index 57% rename from product/ERP5Type/XMLExportImport.py rename to product/ERP5Type/XMLExportImport/__init__.py index 31e54525f2a84f8e0ad71023aacfac481701062b..b0486b2875f8891b24f9b1fe72a82d871f53b17c 100644 --- a/product/ERP5Type/XMLExportImport.py +++ b/product/ERP5Type/XMLExportImport/__init__.py @@ -1,7 +1,16 @@ # -*- coding: utf-8 -*- ############################################################################## # -# Copyright (c) 2002-2003 Nexedi SARL and Contributors. All Rights Reserved. +# Copyright (c) 2001,2002 Zope Foundation and Contributors. +# +# This software is subject to the provisions of the Zope Public License, +# Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution. +# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED +# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS +# FOR A PARTICULAR PURPOSE +# +# Copyright (c) 2002-2005 Nexedi SARL and Contributors. All Rights Reserved. # Sebastien Robin <seb@nexedi.com> # # WARNING: This program as such is intended to be used by professional @@ -27,8 +36,8 @@ # ############################################################################## +## The code below was initially in ERP5Type/XMLExportImport.py from Acquisition import aq_base, aq_inner - from collections import OrderedDict from cStringIO import StringIO from zodbpickle.pickle import Pickler @@ -38,10 +47,9 @@ from lxml import etree from lxml.etree import Element, SubElement from xml_marshaller.xml_marshaller import Marshaller from OFS.Image import Pdata -from zLOG import LOG from base64 import standard_b64encode - from hashlib import sha1 +#from zLOG import LOG MARSHALLER_NAMESPACE_URI = 'http://www.erp5.org/namespaces/marshaller' marshaller = Marshaller(namespace_uri=MARSHALLER_NAMESPACE_URI, @@ -207,3 +215,191 @@ def Folder_asXML(object, omit_xml_declaration=True, root=None): return etree.tostring(root, encoding='utf-8', xml_declaration=xml_declaration, pretty_print=True) + +## The code below was initially from OFS.XMLExportImport +from base64 import encodestring +from ZODB.serialize import referencesf +from ZODB.ExportImport import TemporaryFile, export_end_marker +from ZODB.utils import p64 +from ZODB.utils import u64 +from functools import partial +from inspect import getargspec +from types import TupleType +from OFS import ObjectManager +from . import ppml + +magic='<?xm' # importXML(jar, file, clue)} + +def reorderPickle(jar, p): + try: + from ZODB._compat import Unpickler, Pickler + except ImportError: # BBB: ZODB 3.10 + from ZODB.ExportImport import Unpickler, Pickler + from ZODB.ExportImport import Ghost, persistent_id + + oids = {} + storage = jar._storage + new_oid = storage.new_oid + store = storage.store + + def persistent_load(ooid, + Ghost=Ghost, + oids=oids, wrote_oid=oids.has_key, + new_oid=storage.new_oid): + + "Remap a persistent id to an existing ID and create a ghost for it." + + if type(ooid) is TupleType: ooid, klass = ooid + else: klass=None + + try: + Ghost=Ghost() + Ghost.oid=ooid + except TypeError: + Ghost=Ghost(ooid) + return Ghost + + # Reorder pickle by doing I/O + pfile = StringIO(p) + unpickler=Unpickler(pfile) + unpickler.persistent_load=persistent_load + + newp=StringIO() + pickler=OrderedPickler(newp,1) + pickler.persistent_id=persistent_id + + classdef = unpickler.load() + obj = unpickler.load() + pickler.dump(classdef) + pickler.dump(obj) + p=newp.getvalue() + return obj, p + +def _mapOid(id_mapping, oid): + idprefix = str(u64(oid)) + id = id_mapping[idprefix] + old_aka = encodestring(oid)[:-1] + aka=encodestring(p64(long(id)))[:-1] # Rebuild oid based on mapped id + id_mapping.setConvertedAka(old_aka, aka) + return idprefix+'.', id, aka + +def XMLrecord(oid, plen, p, id_mapping): + # Proceed as usual + q=ppml.ToXMLUnpickler + f=StringIO(p) + u=q(f) + u.idprefix, id, aka = _mapOid(id_mapping, oid) + p=u.load(id_mapping=id_mapping).__str__(4) + if f.tell() < plen: + p=p+u.load(id_mapping=id_mapping).__str__(4) + String=' <record id="%s" aka="%s">\n%s </record>\n' % (id, aka, p) + return String + +def exportXML(jar, oid, file=None): + # For performance reasons, exportXML does not use 'XMLrecord' anymore to map + # oids. This requires to initialize MinimalMapping.marked_reference before + # any string output, i.e. in ppml.Reference.__init__ + # This also fixed random failures when DemoStorage is used, because oids + # can have values that have a shorter representation in 'repr' instead of + # 'base64' (see ppml.convert) and ppml.String does not support this. + load = jar._storage.load + if 'version' in getargspec(load).args: # BBB: ZODB<5 (TmpStore) + load = partial(load, version='') + pickle_dict = {oid: None} + max_cache = [1e7] # do not cache more than 10MB of pickle data + def getReorderedPickle(oid): + p = pickle_dict[oid] + if p is None: + p = load(oid)[0] + p = reorderPickle(jar, p)[1] + if len(p) < max_cache[0]: + max_cache[0] -= len(p) + pickle_dict[oid] = p + return p + + # Sort records and initialize id_mapping + id_mapping = ppml.MinimalMapping() + reordered_oid_list = [oid] + for oid in reordered_oid_list: + _mapOid(id_mapping, oid) + for oid in referencesf(getReorderedPickle(oid)): + if oid not in pickle_dict: + pickle_dict[oid] = None + reordered_oid_list.append(oid) + + # Do real export + if file is None: + file = TemporaryFile() + elif isinstance(file, basestring): + file = open(file, 'w+b') + write = file.write + write('<?xml version="1.0"?>\n<ZopeData>\n') + for oid in reordered_oid_list: + p = getReorderedPickle(oid) + write(XMLrecord(oid, len(p), p, id_mapping)) + write('</ZopeData>\n') + return file + +class zopedata: + def __init__(self, parser, tag, attrs): + self.file=parser.file + write=self.file.write + write('ZEXP') + + def append(self, data): + file=self.file + write=file.write + pos=file.tell() + file.seek(pos) + write(data) + +def start_zopedata(parser, tag, data): + return zopedata(parser, tag, data) + +def save_zopedata(parser, tag, data): + file=parser.file + write=file.write + pos=file.tell() + file.seek(pos) + write(export_end_marker) + +def save_record(parser, tag, data): + file=parser.file + write=file.write + pos=file.tell() + file.seek(pos) + a=data[1] + if a.has_key('id'): oid=a['id'] + oid=p64(int(oid)) + v='' + for x in data[2:]: + v=v+x + l=p64(len(v)) + v=oid+l+v + return v + +import xml.parsers.expat +def importXML(jar, file, clue=''): + if type(file) is str: + file=open(file, 'rb') + outfile=TemporaryFile() + data=file.read() + F=ppml.xmlPickler() + F.end_handlers['record'] = save_record + F.end_handlers['ZopeData'] = save_zopedata + F.start_handlers['ZopeData'] = start_zopedata + F.binary=1 + F.file=outfile + # <patch> + # Our BTs XML files don't declare encoding but have accented chars in them + # So we have to declare an encoding but not use unicode, so the unpickler + # can deal with the utf-8 strings directly + p=xml.parsers.expat.ParserCreate('utf-8') + p.returns_unicode = False + # </patch> + p.CharacterDataHandler=F.handle_data + p.StartElementHandler=F.unknown_starttag + p.EndElementHandler=F.unknown_endtag + r=p.Parse(data) + outfile.seek(0) + return jar.importFile(outfile,clue) diff --git a/product/ERP5Type/patches/ppml.py b/product/ERP5Type/XMLExportImport/ppml.py similarity index 84% rename from product/ERP5Type/patches/ppml.py rename to product/ERP5Type/XMLExportImport/ppml.py index 309a2f50a8402ca9ba276337f6780362b585e7e0..73afcd96fae2609e1d2f6dbd4ee5c37c6f4dc823 100644 --- a/product/ERP5Type/patches/ppml.py +++ b/product/ERP5Type/XMLExportImport/ppml.py @@ -1,27 +1,54 @@ -# -*- coding: utf-8 -*- ############################################################################## # -# Copyright (c) 2001 Zope Corporation and Contributors. All Rights Reserved. +# Copyright (c) 2001,2002 Zope Corporation and Contributors. All Rights Reserved. # Copyright (c) 2002,2005 Nexedi SARL and Contributors. All Rights Reserved. # # This software is subject to the provisions of the Zope Public License, -# Version 2.0 (ZPL). A copy of the ZPL should accompany this distribution. +# Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution. # THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED # WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED # WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS # FOR A PARTICULAR PURPOSE # ############################################################################## +"""Provide conversion between Python pickles and XML +""" +from pickle import * +import struct +import base64 import re +from marshal import loads as mloads +from .xyap import NoBlanks +from .xyap import xyap -# Import everything right now, not after -# or new patch will not work -from Shared.DC.xml.ppml import * -from Shared.DC.xml import ppml - +import re from marshal import dumps as mdumps -from zLOG import LOG +#from zLOG import LOG + +binary = re.compile('[^\x1f-\x7f]').search + +def escape(s, encoding='repr'): + if binary(s) and isinstance(s, str): + s = base64.encodestring(s)[:-1] + encoding = 'base64' + elif '>' in s or '<' in s or '&' in s: + if not ']]>' in s: + s = '<![CDATA[' + s + ']]>' + encoding = 'cdata' + else: + s = s.replace('&', '&') + s = s.replace('>', '>') + s = s.replace('<', '<') + return encoding, s + +def unescape(s, encoding): + if encoding == 'base64': + return base64.decodestring(s) + else: + s = s.replace('<', '<') + s = s.replace('>', '>') + return s.replace('&', '&') # For converting to a more readable expression. reprs = {} @@ -38,15 +65,12 @@ reprs2={} reprs2['<'] = "\\074" reprs2['>'] = "\\076" reprs2['&'] = "\\046" - reprs_re = re.compile('|'.join(re.escape(k) for k in reprs.keys())) def sub_reprs(m): return reprs[m.group(0)] - reprs2_re = re.compile('|'.join(re.escape(k) for k in reprs2.keys())) def sub_reprs2(m): return reprs2[m.group(0)] - def convert(S): new = '' ### patch begin: if the input string is a valid utf8 string, only @@ -69,8 +93,6 @@ def convert(S): return 'repr', reprs2_re.sub(sub_reprs2, new) return 'repr', new -ppml.convert = convert - # For optimization. def unconvert(encoding,S): if encoding == 'base64': @@ -78,10 +100,7 @@ def unconvert(encoding,S): else: return eval("'" + S.replace('\n', '') + "'") -ppml.unconvert = unconvert - class Global: - def __init__(self, module, name, mapping): self.module=module self.name=name @@ -95,10 +114,14 @@ class Global: return '%s<%s%s name="%s" module="%s"/>\n' % ( ' '*indent, name, id, self.name, self.module) -ppml.Global = Global +class Immutable: + def __init__(self, value): + self.value = value -class Scalar: + def getValue(self): + return self.value +class Scalar: def __init__(self, v, mapping): self._v=v self.mapping = mapping @@ -116,17 +139,14 @@ class Scalar: self.mapping.setImmutable(self.id, Immutable(value = result)) return result -ppml.Scalar = Scalar - -class Immutable: - def __init__(self, value): - self.value = value - - def getValue(self): - return self.value +class Long(Scalar): + def value(self): + result = str(self._v) + if result[-1:] == 'L': + return result[:-1] + return result class String(Scalar): - encoding = None def __init__(self, v, mapping, encoding=''): @@ -160,16 +180,11 @@ class String(Scalar): self.mapping.setImmutable(self.id, Immutable(value = result)) return '%s%s\n' % (' '*indent, result) -ppml.String = String - class Unicode(String): def value(self): return self._v.encode('utf-8') -ppml.Unicode = Unicode - class Wrapper: - def __init__(self, v, mapping): self._v=v self.mapping = mapping @@ -189,10 +204,7 @@ class Wrapper: v=v.__str__(indent+2) return '%s<%s%s>\n%s%s</%s>\n' % (i, name, id, v, i, name) -ppml.Wrapper = Wrapper - class Collection: - def __init__(self, mapping): self.mapping = mapping @@ -208,14 +220,15 @@ class Collection: else: return '%s<%s%s/>\n' % (i, name, id) -ppml.Collection = Collection - class Dictionary(Collection): def __init__(self, mapping): self.mapping = mapping self._d=[] + def __len__(self): return len(self._d) + def __setitem__(self, k, v): self._d.append((k,v)) + def value(self, indent): #self._d.sort(lambda a, b: cmp(a[0]._v, b[0]._v)) # Sort the sequence by key JPS Improvement ind = ' ' * indent @@ -230,10 +243,7 @@ class Dictionary(Collection): for i in self._d ) -ppml.Dictionary = Dictionary - class Sequence(Collection): - def __init__(self, mapping, v=None): if not v: v=[] self._subs=v @@ -251,26 +261,10 @@ class Sequence(Collection): v.__str__(indent) for v in self._subs) -ppml.Sequence = Sequence - -class Persistent(Wrapper): - +class none: def __str__(self, indent=0): - id = '' - if hasattr(self, 'id'): - if self.mapping.isMarked(self.id): id=' id="%s"' % self.mapping[self.id] - name=self.__class__.__name__.lower() - v=self._v - i=' '*indent - if isinstance(v,String): - return '%s<%s%s> %s </%s>\n' % (i, name, id, v.__str__(map_value=1)[:-1], name) - elif isinstance(v,Scalar): - return '%s<%s%s> %s </%s>\n' % (i, name, id, str(v)[:-1], name) - else: - v=v.__str__(indent+2) - return '%s<%s%s>\n%s%s</%s>\n' % (i, name, id, v, i, name) - -ppml.Persistent = Persistent + return ' ' * indent + '<none/>\n' +none = none() class Reference(Scalar): def __init__(self, v, mapping): @@ -288,9 +282,7 @@ class Reference(Scalar): value = '<%s id="%s"/>' % (name, self.mapping[v]) return '%s%s\n' % (' '*indent, value) -ppml.Reference = Reference Get = Reference -ppml.Get = Get class Object(Sequence): def __init__(self, klass, args, mapping): @@ -299,10 +291,33 @@ class Object(Sequence): def __setstate__(self, v): self.append(State(v, self.mapping)) -ppml.Object = Object +class Int(Scalar): pass +class Float(Scalar): pass +class List(Sequence): pass +class Tuple(Sequence): pass +class Key(Wrapper): pass +class Value(Wrapper): pass +class Klass(Wrapper): pass +class State(Wrapper): pass +class Pickle(Wrapper): pass + +class Persistent(Wrapper): + def __str__(self, indent=0): + id = '' + if hasattr(self, 'id'): + if self.mapping.isMarked(self.id): id=' id="%s"' % self.mapping[self.id] + name=self.__class__.__name__.lower() + v=self._v + i=' '*indent + if isinstance(v,String): + return '%s<%s%s> %s </%s>\n' % (i, name, id, v.__str__(map_value=1)[:-1], name) + elif isinstance(v,Scalar): + return '%s<%s%s> %s </%s>\n' % (i, name, id, str(v)[:-1], name) + else: + v=v.__str__(indent+2) + return '%s<%s%s>\n%s%s</%s>\n' % (i, name, id, v, i, name) blanck_line_expression = re.compile('^ +$') - class NoBlanks: """ This allows to ignore at least whitespaces between elements and also @@ -350,10 +365,7 @@ class NoBlanks: self.append(data) -ppml.NoBlanks = NoBlanks - class IdentityMapping: - def __init__(self): self.resetMapping() self.immutable = {} @@ -385,11 +397,7 @@ class IdentityMapping: def hasImmutable(self, k): return self.immutable.has_key(k) - -ppml.IdentityMapping = IdentityMapping - class MinimalMapping(IdentityMapping): - def resetMapping(self): self.mapped_id = {} self.mapped_core_id = {} @@ -443,30 +451,7 @@ class MinimalMapping(IdentityMapping): def __str__(self, a): return "Error here" -ppml.MinimalMapping = MinimalMapping - -class List(Sequence): pass -class Tuple(Sequence): pass - -class Klass(Wrapper): pass -class State(Wrapper): pass -class Pickle(Wrapper): pass - -class Int(Scalar): pass -class Float(Scalar): pass - -class Key(Wrapper): pass -class Value(Wrapper): pass - -class Long(Scalar): - def value(self): - result = str(self._v) - if result[-1:] == 'L': - return result[:-1] - return result - class ToXMLUnpickler(Unpickler): - def load(self, id_mapping=None): if id_mapping is None: self.id_mapping = IdentityMapping() @@ -657,7 +642,58 @@ class ToXMLUnpickler(Unpickler): #for code in dispatch.keys(): # dispatch[code] = LogCall(dispatch[code]) -ppml.ToXMLUnpickler = ToXMLUnpickler +def ToXMLload(file): + return ToXMLUnpickler(file).load() + +def ToXMLloads(str): + from StringIO import StringIO + file = StringIO(str) + return ToXMLUnpickler(file).load() + +def name(self, tag, data): + return ''.join(data[2:]).strip() + +def start_pickle(self, tag, attrs): + self._pickleids = {} + return [tag, attrs] + +def save_int(self, tag, data): + if self.binary: + v = int(name(self, tag, data)) + if v >= 0: + if v <= 0xff: + return BININT1 + chr(v) + if v <= 0xffff: + return '%c%c%c' % (BININT2, v & 0xff, v >> 8) + hb = v >> 31 + if hb == 0 or hb == -1: + return BININT + struct.pack('<i', v) + return INT + name(self, tag, data) + '\n' + +def save_float(self, tag, data): + if self.binary: + return BINFLOAT + struct.pack('>d', float(name(self, tag, data))) + else: + return FLOAT + name(self, tag, data) + '\n' + +def save_put(self, v, attrs): + id = attrs.get('id', '') + if id: + prefix = id.rfind('.') + if prefix >= 0: + id = id[prefix + 1:] + elif id[0] == 'i': + id = id[1:] + if self.binary: + id = int(id) + if id < 256: + id = BINPUT + chr(id) + else: + id = LONG_BINPUT + struct.pack('<i', id) + else: + id = PUT + repr(id) + '\n' + return v + id + return v def save_string(self, tag, data): binary=self.binary @@ -680,8 +716,6 @@ def save_string(self, tag, data): else: v="S'"+v+"'\012" return save_put(self, v, a) -ppml.save_string = save_string - def save_unicode(self, tag, data): binary=self.binary v='' @@ -699,7 +733,50 @@ def save_unicode(self, tag, data): else: v=UNICODE+"'"+v+"'\012" return save_put(self, v, a) -ppml.save_unicode = save_unicode +def save_tuple(self, tag, data): + T = data[2:] + if not T: + return EMPTY_TUPLE + return save_put(self, MARK + ''.join(T) + TUPLE, data[1]) + +def save_list(self, tag, data): + L = data[2:] + if self.binary: + v = save_put(self, EMPTY_LIST, data[1]) + if L: + v = v + MARK + ''.join(L) + APPENDS + else: + v = save_put(self, MARK + LIST, data[1]) + if L: + v = APPEND.join(L) + APPEND + return v + +def save_dict(self, tag, data): + D = data[2:] + if self.binary: + v = save_put(self, EMPTY_DICT, data[1]) + if D: + v = v + MARK + ''.join(D) + SETITEMS + else: + v = save_put(self, MARK + DICT, data[1]) + if D: + v = v + SETITEM.join(D) + SETITEM + return v + +def save_reference(self, tag, data): + a = data[1] + id = a['id'] + prefix = id.rfind('.') + if prefix >= 0: + id = id[prefix + 1:] + if self.binary: + id = int(id) + if id < 256: + return BINGET + chr(id) + else: + return LONG_BINGET + struct.pack('<i', i) + else: + return GET + repr(id) + '\n' def save_object(self, tag, data): if len(data)==5: @@ -720,7 +797,16 @@ def save_object(self, tag, data): v=v+'R' return v -ppml.save_object = save_object +def save_global(self, tag, data): + a = data[1] + return save_put(self, GLOBAL + a['module'] + '\n' + a['name'] + '\n', a) + +def save_persis(self, tag, data): + v = data[2] + if self.binary: + return v + BINPERSID + else: + return PERSID + v def save_pickle_start(self, tag, attrs): return [tag, attrs] @@ -778,41 +864,3 @@ class xmlPickler(NoBlanks, xyap): 'persistent': save_persis, } -# FIXME: Leo: Do we still need to replace ppml.xmlPickler now that we're -# using our own xmlPickler in our own importXML function below? Do we support -# any other use of xmlPickler except for Business Template export/import? -ppml.xmlPickler = xmlPickler - -class Tuple(Sequence): pass - -ppml.Tuple = Tuple - -# Copied from OFS.XMLExportImport.importXML (of Zope 2.12) -# Imported and used directly by Products.ERP5.Document.BusinessTemplate -from OFS.XMLExportImport import save_record, save_zopedata, start_zopedata -from tempfile import TemporaryFile -import xml.parsers.expat -def importXML(jar, file, clue=''): - if type(file) is str: - file=open(file, 'rb') - outfile=TemporaryFile() - data=file.read() - F=xmlPickler() - F.end_handlers['record'] = save_record - F.end_handlers['ZopeData'] = save_zopedata - F.start_handlers['ZopeData'] = start_zopedata - F.binary=1 - F.file=outfile - # <patch> - # Our BTs XML files don't declare encoding but have accented chars in them - # So we have to declare an encoding but not use unicode, so the unpickler - # can deal with the utf-8 strings directly - p=xml.parsers.expat.ParserCreate('utf-8') - p.returns_unicode = False - # </patch> - p.CharacterDataHandler=F.handle_data - p.StartElementHandler=F.unknown_starttag - p.EndElementHandler=F.unknown_endtag - r=p.Parse(data) - outfile.seek(0) - return jar.importFile(outfile,clue) diff --git a/product/ERP5Type/XMLExportImport/xyap.py b/product/ERP5Type/XMLExportImport/xyap.py new file mode 100644 index 0000000000000000000000000000000000000000..469ffeb04b2c233f277706ebe7338a77ba28864a --- /dev/null +++ b/product/ERP5Type/XMLExportImport/xyap.py @@ -0,0 +1,117 @@ +"""Yet another XML parser + +This is meant to be very simple: + + - stack based + + - The parser has a table of start handlers and end handlers. + + - start tag handlers are called with the parser instance, tag names + and attributes. The result is placed on the stack. The default + handler places a special object on the stack (uh, a list, with the + tag name and attributes as the first two elements. ;) + + - end tag handlers are called with the object on the parser, the tag + name, and top of the stack right after it has been removed. The + result is appended to the object on the top of the stack. + +Note that namespace attributes should recieve some special handling. +Oh well. +""" + +import string +import xml.parsers.expat + + +class xyap: + start_handlers = {} + end_handlers = {} + + def __init__(self): + top = [] + self._stack = _stack = [top] + self.push = _stack.append + self.append = top.append + + def handle_data(self, data): + self.append(data) + + def unknown_starttag(self, tag, attrs): + if isinstance(attrs, list): + attrs = dict(attrs) + start = self.start_handlers + if tag in start: + tag = start[tag](self, tag, attrs) + else: + tag = [tag, attrs] + self.push(tag) + self.append = tag.append + + def unknown_endtag(self, tag): + _stack = self._stack + top = _stack.pop() + append = self.append = _stack[-1].append + end = self.end_handlers + if tag in end: + top = end[tag](self, tag, top) + append(top) + +class NoBlanks: + + def handle_data(self, data): + if data.strip(): + self.append(data) + + +def struct(self, tag, data): + r = {} + for k, v in data[2:]: + r[k] = v + return r + +_nulljoin = "".join + +def name(self, tag, data): + return _nulljoin(data[2:]).strip() + +def tuplef(self, tag, data): + return tuple(data[2:]) + +class XYap(xyap): + def __init__(self): + self._parser = xml.parsers.expat.ParserCreate() + self._parser.StartElementHandler = self.unknown_starttag + self._parser.EndElementHandler = self.unknown_endtag + self._parser.CharacterDataHandler = self.handle_data + xyap.__init__(self) + +class xmlrpc(NoBlanks, XYap): + end_handlers = { + 'methodCall': tuplef, + 'methodName': name, + 'params': tuplef, + 'param': lambda self, tag, data: data[2], + 'value': lambda self, tag, data: data[2], + 'i4': + lambda self, tag, data, atoi=string.atoi, name=name: + atoi(name(self, tag, data)), + 'int': + lambda self, tag, data, atoi=string.atoi, name=name: + atoi(name(self, tag, data)), + 'boolean': + lambda self, tag, data, atoi=string.atoi, name=name: + atoi(name(self, tag, data)), + 'string': lambda self, tag, data, join=string.join: + join(data[2:], ''), + 'double': + lambda self, tag, data, atof=string.atof, name=name: + atof(name(self, tag, data)), + 'float': + lambda self, tag, data, atof=string.atof, name=name: + atof(name(self, tag, data)), + 'struct': struct, + 'member': tuplef, + 'name': name, + 'array': lambda self, tag, data: data[2], + 'data': lambda self, tag, data: data[2:], + } diff --git a/product/ERP5Type/ZopePatch.py b/product/ERP5Type/ZopePatch.py index b03d41d49350affd3868e3a419ed854c76362ff1..7dd23c1be501fe6e7dd71c32c1d3859a6b9164ab 100644 --- a/product/ERP5Type/ZopePatch.py +++ b/product/ERP5Type/ZopePatch.py @@ -40,8 +40,6 @@ if WITH_LEGACY_WORKFLOW: from Products.ERP5Type.patches import WorkflowTool from Products.ERP5Type.patches import WorkflowTool from Products.ERP5Type.patches import DynamicType -from Products.ERP5Type.patches import XMLExportImport -from Products.ERP5Type.patches import ppml from Products.ERP5Type.patches import Expression from Products.ERP5Type.patches import sqltest from Products.ERP5Type.patches import sqlvar diff --git a/product/ERP5Type/mixin/json_representable.py b/product/ERP5Type/mixin/json_representable.py index 21f5baefb8ed8519bcab98991135d12d908fdb45..39a591e4f1992c62dbd9954cd54857d6237174c9 100644 --- a/product/ERP5Type/mixin/json_representable.py +++ b/product/ERP5Type/mixin/json_representable.py @@ -36,7 +36,7 @@ except ImportError: warnings.warn("Please install xmltodict, it is needed by json_representable mixin", DeprecationWarning) import zope.interface -from OFS import XMLExportImport +from Products.ERP5Type import XMLExportImport from StringIO import StringIO from AccessControl import ClassSecurityInfo from Products.ERP5Type.interfaces.json_representable import IJSONRepresentable diff --git a/product/ERP5Type/patches/ObjectManager.py b/product/ERP5Type/patches/ObjectManager.py index 28f5947f5d8c0d6ff687af5b509f916c7c8b12b3..08de9ded87814909e37cadce2466de714506388b 100644 --- a/product/ERP5Type/patches/ObjectManager.py +++ b/product/ERP5Type/patches/ObjectManager.py @@ -12,10 +12,13 @@ # ############################################################################## -# Import: add rename feature and make _importObjectFromFile return the object -from OFS.ObjectManager import ObjectManager, customImporters -from App.version_txt import getZopeVersion +from Products.ERP5Type.XMLExportImport import magic, importXML +customImporters = {magic: importXML} + +import OFS.ObjectManager +OFS.ObjectManager.customImporters = customImporters +# Import: add rename feature and make _importObjectFromFile return the object def ObjectManager_importObjectFromFile(self, filepath, verify=1, set_owner=1, id=None, suppress_events=False): #LOG('_importObjectFromFile, filepath',0,filepath) # locate a valid connection @@ -41,4 +44,4 @@ def ObjectManager_importObjectFromFile(self, filepath, verify=1, set_owner=1, id ob.manage_changeOwnershipType(explicit=0) return ob -ObjectManager._importObjectFromFile=ObjectManager_importObjectFromFile +OFS.ObjectManager.ObjectManager._importObjectFromFile=ObjectManager_importObjectFromFile diff --git a/product/ERP5Type/patches/XMLExportImport.py b/product/ERP5Type/patches/XMLExportImport.py deleted file mode 100644 index d38c9dd663eb6e5d14dfcb169a0cb5631961d035..0000000000000000000000000000000000000000 --- a/product/ERP5Type/patches/XMLExportImport.py +++ /dev/null @@ -1,144 +0,0 @@ -############################################################################## -# -# Copyright (c) 2001 Zope Corporation and Contributors. All Rights Reserved. -# Copyright (c) 2002,2005 Nexedi SARL and Contributors. All Rights Reserved. -# -# This software is subject to the provisions of the Zope Public License, -# Version 2.0 (ZPL). A copy of the ZPL should accompany this distribution. -# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED -# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED -# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS -# FOR A PARTICULAR PURPOSE -# -############################################################################## - -# Make sure the xml export will be ordered - -from functools import partial -from inspect import getargspec -from ZODB.utils import u64, p64 -from Shared.DC.xml import ppml -from base64 import encodestring -from cStringIO import StringIO -from ZODB.serialize import referencesf -from ZODB.ExportImport import TemporaryFile -from types import TupleType -from OFS import ObjectManager, XMLExportImport -from ..XMLExportImport import OrderedPickler - -from logging import getLogger -log = getLogger(__name__) - -def reorderPickle(jar, p): - try: - from ZODB._compat import Unpickler, Pickler - except ImportError: # BBB: ZODB 3.10 - from ZODB.ExportImport import Unpickler, Pickler - from ZODB.ExportImport import Ghost, persistent_id - - oids = {} - storage = jar._storage - new_oid = storage.new_oid - store = storage.store - - def persistent_load(ooid, - Ghost=Ghost, - oids=oids, wrote_oid=oids.has_key, - new_oid=storage.new_oid): - - "Remap a persistent id to an existing ID and create a ghost for it." - - if type(ooid) is TupleType: ooid, klass = ooid - else: klass=None - - try: - Ghost=Ghost() - Ghost.oid=ooid - except TypeError: - Ghost=Ghost(ooid) - return Ghost - - # Reorder pickle by doing I/O - pfile = StringIO(p) - unpickler=Unpickler(pfile) - unpickler.persistent_load=persistent_load - - newp=StringIO() - pickler=OrderedPickler(newp,1) - pickler.persistent_id=persistent_id - - classdef = unpickler.load() - obj = unpickler.load() - pickler.dump(classdef) - pickler.dump(obj) - p=newp.getvalue() - return obj, p - -def _mapOid(id_mapping, oid): - idprefix = str(u64(oid)) - id = id_mapping[idprefix] - old_aka = encodestring(oid)[:-1] - aka=encodestring(p64(long(id)))[:-1] # Rebuild oid based on mapped id - id_mapping.setConvertedAka(old_aka, aka) - return idprefix+'.', id, aka - -def XMLrecord(oid, plen, p, id_mapping): - # Proceed as usual - q=ppml.ToXMLUnpickler - f=StringIO(p) - u=q(f) - u.idprefix, id, aka = _mapOid(id_mapping, oid) - p=u.load(id_mapping=id_mapping).__str__(4) - if f.tell() < plen: - p=p+u.load(id_mapping=id_mapping).__str__(4) - String=' <record id="%s" aka="%s">\n%s </record>\n' % (id, aka, p) - return String - -XMLExportImport.XMLrecord = XMLrecord - -def exportXML(jar, oid, file=None): - # For performance reasons, exportXML does not use 'XMLrecord' anymore to map - # oids. This requires to initialize MinimalMapping.marked_reference before - # any string output, i.e. in ppml.Reference.__init__ - # This also fixed random failures when DemoStorage is used, because oids - # can have values that have a shorter representation in 'repr' instead of - # 'base64' (see ppml.convert) and ppml.String does not support this. - load = jar._storage.load - if 'version' in getargspec(load).args: # BBB: ZODB<5 (TmpStore) - load = partial(load, version='') - pickle_dict = {oid: None} - max_cache = [1e7] # do not cache more than 10MB of pickle data - def getReorderedPickle(oid): - p = pickle_dict[oid] - if p is None: - p = load(oid)[0] - p = reorderPickle(jar, p)[1] - if len(p) < max_cache[0]: - max_cache[0] -= len(p) - pickle_dict[oid] = p - return p - - # Sort records and initialize id_mapping - id_mapping = ppml.MinimalMapping() - reordered_oid_list = [oid] - for oid in reordered_oid_list: - _mapOid(id_mapping, oid) - for oid in referencesf(getReorderedPickle(oid)): - if oid not in pickle_dict: - pickle_dict[oid] = None - reordered_oid_list.append(oid) - - # Do real export - if file is None: - file = TemporaryFile() - elif isinstance(file, basestring): - file = open(file, 'w+b') - write = file.write - write('<?xml version="1.0"?>\n<ZopeData>\n') - for oid in reordered_oid_list: - p = getReorderedPickle(oid) - write(XMLrecord(oid, len(p), p, id_mapping)) - write('</ZopeData>\n') - return file - -ObjectManager.exportXML = XMLExportImport.exportXML = exportXML