Skip to content

Z
Zope
Commit 8462954a authored by joey's avatar joey
Browse files
Options

Updated ppml file with xmlUnpickler and xmlPickler for conversion 

  from xml to pickles.
parent 2fc80782
Show whitespace changes
Inline Side-by-side
Showing with 358 additions and 422 deletions
utilities/ppml.py
View file @ 8462954a
......@@ -91,6 +91,56 @@ __version__ = "1.9" # Code version
from pickle import *
from string import replace
import struct
import base64
import string
import xmllib, pickle
# Create a list of all the characters
L = map(chr,range(256))
# Create an empty dictionary
d = {}
# Create a dictionary d that maps each character to its
# repr form
for c in L:
d[c] = repr(c)[1:-1]
# Modify values in the dictionary
d['<'] = "\\074"
d['>'] = "\\076"
d['&'] = "\\046"
d['\n'] = "\\n\n"
d['\t'] = "\\t"
d['\\'] = "\\"
d['\r'] = "\\r"
d["'"] = "\\'"
# Function convert takes a string and converts it to either
# repr or base64 format
def convert(S):
new = ''
encoding = 'repr'
new = string.join(map(lambda s: d[s], S), '')
if len(new) > (1.4*len(S)):
encoding = 'base64'
new = base64.encodestring(S)
return encoding, new
# Function unconvert takes a encoding and a string and
# returns the original string
def unconvert(encoding,S):
original = ''
if encoding == 'base64':
original = base64.decodestring(S)
else:
x = string.replace(S, '\n', '')
original = eval("'"+x+"'")
return original
class Global:
......@@ -107,7 +157,8 @@ class Global:
class Scalar:
def __init__(self, v): self._v=v
def __init__(self, v):
self._v=v
def value(self): return self._v
......@@ -131,7 +182,19 @@ class Long(Scalar):
def value(self): return str(self._v)[:-1]
class Float(Scalar): pass
class String(Scalar):
def value(self): return xmlstr(self._v)
def __init__(self, v, encoding=''):
encoding, v = convert(v)
self.encoding=encoding
self._v=v
def value(self): return self._v
def __str__(self,indent=0):
if hasattr(self,'id'):id=' id="%s"' % self.id
else: id=''
if hasattr(self, 'encoding'):encoding=' encoding="%s"' % self.encoding
else: encoding=''
name=string.lower(self.__class__.__name__)
return '%s<%s%s%s>%s</%s>\n' % (
' '*indent, name, id, encoding, self.value(), name)
class Wrapper:
......@@ -146,7 +209,7 @@ class Wrapper:
v=self._v
i=' '*indent
if isinstance(v,Scalar):
return '%s<%s%s>%s</%s>\n' % (i, name, id, str(v)[:-1], name)
return '%s<%s%s> %s </%s>\n' % (i, name, id, str(v)[:-1], name)
else:
v=v.__str__(indent+2)
return '%s<%s%s>\n%s%s</%s>\n' % (i, name, id, v, i, name)
......@@ -326,8 +389,7 @@ class ToXMLUnpickler(Unpickler):
del self.stack[k:]
module = self.readline()[:-1]
name = self.readline()[:-1]
klass = self.find_class(module, name)
value=Object(Global(module, klass), args)
value=Object(Global(module, name), args)
self.append(value)
dispatch[INST] = load_inst
......@@ -389,354 +451,8 @@ class ToXMLUnpickler(Unpickler):
self.stack[-1].id=self.idprefix+`i`
dispatch[LONG_BINPUT] = load_long_binput
def ToXMLload(file):
return ToXMLUnpickler(file).load()
def ToXMLloads(str):
file = StringIO(str)
return ToXMLUnpickler(file).load()
# The rest is used for testing only
class C:
def __cmp__(self, other):
return cmp(self.__dict__, other.__dict__)
def test():
fn = 'out'
c = C()
c.foo = 1
c.bar = 2
x = [0, 1, 2, 3]
y = ('abc', 'abc', c, c)
x.append(y)
x.append(y)
x.append(5)
x.append(None)
f = open(fn, 'w')
F = Pickler(f,1)
F.dump(x)
f.close()
f = open(fn, 'r')
U = ToXMLUnpickler(f)
U.idprefix='42.'
x2 = U.load()
print x
print x2
if __name__ == '__main__':
test()
"""Provide conversion between Python pickles and XML
"""
__version__ = "1.9" # Code version
from pickle import *
from string import replace
class Global:
def __init__(self, module, name):
self.module=module
self.name=name
def __str__(self, indent=0):
if hasattr(self, 'id'): id=' id="%s"' % self.id
else: id=''
name=string.lower(self.__class__.__name__)
return '%s<%s%s name="%s" module="%s"/>\n' % (
' '*indent, name, id, self.name, self.module)
class Scalar:
def __init__(self, v): self._v=v
def value(self): return self._v
def __str__(self, indent=0):
if hasattr(self, 'id'): id=' id="%s"' % self.id
else: id=''
name=string.lower(self.__class__.__name__)
return '%s<%s%s>%s</%s>\n' % (
' '*indent, name, id, self.value(), name)
def xmlstr(v):
v=`v`
if v[:1]=='\'':
v=string.replace(v,'"','\\"')
v=replace(v,'%','\\045')
v=replace(v,'&','\\046')
return v[1:-1]
class Int(Scalar): pass
class Long(Scalar):
def value(self): return str(self._v)[:-1]
class Float(Scalar): pass
class String(Scalar):
def value(self): return xmlstr(self._v)
class Wrapper:
def __init__(self, v): self._v=v
def value(self): return self._v
def __str__(self, indent=0):
if hasattr(self, 'id'): id=' id="%s"' % self.id
else: id=''
name=string.lower(self.__class__.__name__)
v=self._v
i=' '*indent
if isinstance(v,Scalar):
return '%s<%s%s>%s</%s>\n' % (i, name, id, str(v)[:-1], name)
else:
v=v.__str__(indent+2)
return '%s<%s%s>\n%s%s</%s>\n' % (i, name, id, v, i, name)
class Collection:
def __str__(self, indent=0):
if hasattr(self, 'id'): id=' id="%s"' % self.id
else: id=''
name=string.lower(self.__class__.__name__)
i=' '*indent
if self:
return '%s<%s%s>\n%s%s</%s>\n' % (
i, name, id, self.value(indent+2), i, name)
else:
return '%s<%s%s/>\n' % (i, name, id)
class Key(Wrapper): pass
class Value(Wrapper): pass
class Dictionary(Collection):
def __init__(self): self._d=[]
def __len__(self): return len(self._d)
def __setitem__(self, k, v): self._d.append((k,v))
def value(self, indent):
return string.join(
map(lambda i, ind=' '*indent, indent=indent+4:
'%s<item>\n'
'%s'
'%s'
'%s</item>\n'
%
(ind,
Key(i[0]).__str__(indent),
Value(i[1]).__str__(indent),
ind),
self._d
),
'')
class Sequence(Collection):
def __init__(self, v=None):
if not v: v=[]
self._subs=v
def __len__(self): return len(self._subs)
def append(self, v): self._subs.append(v)
def value(self, indent):
return string.join(map(
lambda v, indent=indent: v.__str__(indent),
self._subs),'')
class List(Sequence): pass
class Tuple(Sequence): pass
class Klass(Wrapper): pass
class State(Wrapper): pass
class Pickle(Wrapper): pass
class Persistent(Wrapper): pass
class none:
def __str__(self, indent=0): return ' '*indent+'<none/>\n'
none=none()
class Reference(Scalar):
def __init__(self, v): self._v=v
def __str__(self, indent=0):
v=self._v
name=string.lower(self.__class__.__name__)
return '%s<%s id="%s"/>\n' % (' '*indent,name,v)
Get=Reference
class Object(Sequence):
def __init__(self, klass, args):
self._subs=[Klass(klass), args]
def __setstate__(self, v): self.append(State(v))
class ToXMLUnpickler(Unpickler):
def load(self): return Pickle(Unpickler.load(self))
dispatch = {}
dispatch.update(Unpickler.dispatch)
def persistent_load(self, v):
return Persistent(v)
def load_persid(self):
pid = self.readline()[:-1]
self.append(self.persistent_load(String(pid)))
dispatch[PERSID] = load_persid
def load_none(self):
self.append(none)
dispatch[NONE] = load_none
def load_int(self):
self.append(Int(string.atoi(self.readline()[:-1])))
dispatch[INT] = load_int
def load_binint(self):
self.append(Int(mloads('i' + self.read(4))))
dispatch[BININT] = load_binint
def load_binint1(self):
self.append(Int(mloads('i' + self.read(1) + '\000\000\000')))
dispatch[BININT1] = load_binint1
def load_binint2(self):
self.append(Int(mloads('i' + self.read(2) + '\000\000')))
dispatch[BININT2] = load_binint2
def load_long(self):
self.append(Long(string.atol(self.readline()[:-1], 0)))
dispatch[LONG] = load_long
def load_float(self):
self.append(Float(string.atof(self.readline()[:-1])))
dispatch[FLOAT] = load_float
def load_binfloat(self, unpack=struct.unpack):
self.append(Float(unpack('>d', self.read(8))[0]))
dispatch['G'] = load_binfloat
def load_string(self):
self.append(String(eval(self.readline()[:-1],
{'__builtins__': {}}))) # Let's be careful
dispatch[STRING] = load_string
def load_binstring(self):
len = mloads('i' + self.read(4))
self.append(String(self.read(len)))
dispatch[BINSTRING] = load_binstring
def load_short_binstring(self):
len = mloads('i' + self.read(1) + '\000\000\000')
self.append(String(self.read(len)))
dispatch[SHORT_BINSTRING] = load_short_binstring
def load_tuple(self):
k = self.marker()
self.stack[k:] = [Tuple(self.stack[k+1:])]
dispatch[TUPLE] = load_tuple
def load_empty_tuple(self):
self.stack.append(Tuple())
dispatch[EMPTY_TUPLE] = load_empty_tuple
def load_empty_list(self):
self.stack.append(List())
dispatch[EMPTY_LIST] = load_empty_list
def load_empty_dictionary(self):
self.stack.append(Dictionary())
dispatch[EMPTY_DICT] = load_empty_dictionary
def load_list(self):
k = self.marker()
self.stack[k:] = [List(self.stack[k+1:])]
dispatch[LIST] = load_list
def load_dict(self):
k = self.marker()
d = Dictionary()
items = self.stack[k+1:]
for i in range(0, len(items), 2):
key = items[i]
value = items[i+1]
d[key] = value
self.stack[k:] = [d]
dispatch[DICT] = load_dict
def load_inst(self):
k = self.marker()
args = Tuple(self.stack[k+1:])
del self.stack[k:]
module = self.readline()[:-1]
name = self.readline()[:-1]
klass = self.find_class(module, name)
value=Object(Global(module, klass), args)
self.append(value)
dispatch[INST] = load_inst
def load_obj(self):
stack = self.stack
k = self.marker()
klass = stack[k + 1]
del stack[k + 1]
args = Tuple(stack[k + 1:])
del stack[k:]
value=Object(klass,args)
self.append(value)
dispatch[OBJ] = load_obj
def load_global(self):
module = self.readline()[:-1]
name = self.readline()[:-1]
self.append(Global(module, name))
dispatch[GLOBAL] = load_global
def load_reduce(self):
stack = self.stack
callable = stack[-2]
arg_tup = stack[-1]
del stack[-2:]
value=Object(callable, arg_tup)
self.append(value)
dispatch[REDUCE] = load_reduce
idprefix=''
def load_get(self):
self.append(Get(self.idprefix+self.readline()[:-1]))
dispatch[GET] = load_get
def load_binget(self):
i = mloads('i' + self.read(1) + '\000\000\000')
self.append(Get(self.idprefix+`i`))
dispatch[BINGET] = load_binget
def load_long_binget(self):
i = mloads('i' + self.read(4))
self.append(Get(self.idprefix+`i`))
dispatch[LONG_BINGET] = load_long_binget
def load_put(self):
self.stack[-1].id=self.idprefix+self.readline()[:-1]
dispatch[PUT] = load_put
def load_binput(self):
i = mloads('i' + self.read(1) + '\000\000\000')
self.stack[-1].id=self.idprefix+`i`
dispatch[BINPUT] = load_binput
def load_long_binput(self):
i = mloads('i' + self.read(4))
self.stack[-1].id=self.idprefix+`i`
dispatch[LONG_BINPUT] = load_long_binput
def ToXMLload(file):
return ToXMLUnpickler(file).load()
......@@ -746,33 +462,253 @@ def ToXMLloads(str):
return ToXMLUnpickler(file).load()
class XYap:
start_handlers={}
end_handlers={}
def __init__(self):
xmllib.XMLParser.__init__(self)
top=[]
self._stack=_stack=[top]
self.push=_stack.append
self.append=top.append
def handle_data(self, data): self.append(data)
def unknown_starttag(self, tag, attrs):
start=self.start_handlers
if start.has_key(tag): tag = start[tag](self, tag, attrs)
else: tag = [tag, attrs]
self.push(tag)
self.append=tag.append
def unknown_endtag(self, tag):
_stack=self._stack
top=_stack[-1]
del _stack[-1]
append=self.append=_stack[-1].append
end=self.end_handlers
if end.has_key(tag): top=end[tag](self, tag, top)
append(top)
class NoBlanks:
def handle_data(self, data):
if string.strip(data): self.append(data)
def name(self, tag, data, join=string.join, strip=string.strip):
return strip(join(data[2:],''))
def start_pickle(self, tag, attr):
self._pickleids={}
return [tag,attr]
def end_string(self, tag, data):
v=data[2]
a=data[1]
if a is not None:
v=unconvert(a,v)
if a.has_key('id'): self._pickleids[a['id']]=v
return v
def end_none(self,tag,data):
return None
def end_reference(self, tag, data):
return self._pickleids[data[1]['id']]
def end_list(self, tag, data):
v=data[2:]
a=data[1]
if a.has_key('id'): self._pickleids[data[1]['id']]=v
return v
def end_tuple(self, tag, data):
v=tuple(data[2:])
a=data[1]
if a.has_key('id'): self._pickleids[data[1]['id']]=v
return v
def end_dictionary(self, tag, data):
D={}
a=data[1]
for k, v in data[2:]: D[k]=v
if a.has_key('id'): self._pickleids[a['id']]=D
return D
def end_item(self, tag, data):
v=data[2:]
return v
class xmlUnpickler(NoBlanks, XYap, xmllib.XMLParser):
start_handlers={'pickle': start_pickle}
end_handlers={
'int':
lambda self,tag,data,atoi=string.atoi,name=name:
atoi(name(self, tag, data)),
'boolean':
lambda self,tag,data,atoi=string.atoi,name=name:
atoi(name(self, tag, data)),
'string': end_string ,
'double':
lambda self,tag,data,atof=string.atof,name=name:
atof(name(self, tag, data)),
'float':
lambda self,tag,data,atof=string.atof,name=name:
atof(name(self, tag, data)),
'none': end_none,
'list': end_list,
'tuple': end_tuple,
'dictionary': end_dictionary,
'key': lambda self, tag, data: data[2],
'value': lambda self, tag, data: data[2],
'item': end_item,
'reference': end_reference,
'state': lambda self, tag, data: data[2],
'klass': lambda self, tag, data: data[2],
}
def save_none(self, tag, data):
return 'N'
def save_int(self, tag, data):
v='I'+name(self, tag, data)+'\012'
return v
def save_float(self, tag, data):
v='F'+name(self, tag, data)+'\012'
return v
def save_string(self, tag, data):
v=data[2] #data[3] with encoding
a=data[1]
encoding=None # data[2] with encoding
if encoding is not None:
v=base.unconvert(encoding,v)
v="S'"+v+"'\012"+"p"+a["id"]+"\012"
return v
def save_tuple(self, tag, data):
T=data[2:]
a=data[1]
v=''
for x in T:
v=v+x
if a.has_key('id'): v='('+v+'tp'+a['id']+'\012'
return v
def save_list(self, tag, data):
L=data[2:]
a=data[1]
v='(lp'+a['id']+'\012'
x=0
while x<len(L):
v=v+L[x]+'a'
x=x+1
return v
def save_dict(self, tag, data):
D=data[2:]
a=data[1]
v='(dp'+a['id']+'\012'
x=0
while x<len(D):
v=v+D[x]+'s'
x=x+1
if a.has_key('id'): self._pickleids[a['id']]=D
return v
def save_item(self, tag, data):
v=''
for x in data[2:]:
v=v+x
return v
def save_pickle(self, tag, data):
v=data[2]+'.'
return v
def save_reference(self, tag, data):
v='g'+data[1]['id']+'\012'
return v
def save_object(self, tag, data):
a=data[1]
v='(c'
for x in data[2:]:
v=v+x
v=v+'p'+a['id']+'\012'+'b'
if a.has_key('id'): self._pickleids[a['id']]=v
return v
def save_global(self, tag, data):
a=data[1]
if a.has_key('id'):
v=a['module']+'\012'+a['name']+'\012op'+a['id']+'\012'
self._pickleids[a['id']]=v
else: v=a['module']+'\012'+a['name']+'\012o'
return v
def save_persis(self, tag, data):
v=data[2]
v=v+'Q'
return v
class xmlPickler(xmlUnpickler):
start_handlers={'pickle':start_pickle}
end_handlers={
'pickle': save_pickle,
'none': save_none,
'int': save_int,
'float': save_float,
'string': save_string,
'reference': save_reference,
'tuple': save_tuple,
'list': save_list,
'dictionary': save_dict,
'item': save_item,
'value': lambda self, tag, data: data[2],
'key' : lambda self, tag, data: data[2],
'object': save_object,
'klass': lambda self, tag, data: data[2],
'state': lambda self, tag, data: data[2],
'global': save_global,
'persistent': save_persis,
}
# The rest is used for testing only
class C:
def __cmp__(self, other):
return cmp(self.__dict__, other.__dict__)
def test():
fn = 'out'
c = C()
c.foo = 1
c.bar = 2
x = [0, 1, 2, 3]
y = ('abc', 'abc', c, c)
c=C()
c.foo=1
c.bar=2
x=[0,1,2,3]
y=('abc','abc',c,c)
x.append(y)
x.append(y)
x.append(5)
x.append(None)
f = open(fn, 'w')
F = Pickler(f,1)
F.dump(x)
f.close()
f = open(fn, 'r')
U = ToXMLUnpickler(f)
U.idprefix='42.'
x2 = U.load()
print x
print x2
print x, '\012'
f=pickle.dumps(x)
print f, '\012'
p=ToXMLloads(f)
print p, '\012'
F=xmlPickler()
data=string.split(str(p),'\n')
for l in data:
F.feed(l)
F.close
r=F._stack
print r, '\012'
print r[0][0], '\012'
print pickle.loads(r[0][0]), '\012'
if __name__ == '__main__':
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment
GitLab Nexedi Edition | About GitLab | About Nexedi | 沪ICP备2021021310号-2 | 沪ICP备2021021310号-7