Skip to content

Z
Zope
Commit ac18b517 authored by Fred Drake's avatar Fred Drake
Browse files
Options

Jim explained that these are old and no longer relevant.

parent 1f97e260
Expand all Show whitespace changes
Inline Side-by-side
Showing with 0 additions and 1465 deletions
utilities/ExtensionBuilder.py deleted 100755 → 0
View file @ 1f97e260
This diff is collapsed.
utilities/FS.py deleted 100644 → 0
View file @ 1f97e260
##############################################################################
#
# Copyright (c) 2001 Zope Corporation and Contributors. All Rights Reserved.
#
# This software is subject to the provisions of the Zope Public License,
# Version 2.0 (ZPL). A copy of the ZPL should accompany this distribution.
# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
# FOR A PARTICULAR PURPOSE
#
##############################################################################
import struct, tempfile, string, time, pickle, os, sys
from struct import pack, unpack
from cStringIO import StringIO
class FS:
"""FileStorage 'report' writer for converting BoboPOS to FileStorage
"""
def __init__(self, fname, file):
file.seek(0,2)
self._input_size=file.tell()
file.seek(0)
self.__name__=fname
self._file=open(fname,'w+b')
self._file.write('FS21')
self._index={}
self._indexpos=self._index.get
self._tindex=[]
self._tappend=self._tindex.append
self._tfile=tempfile.TemporaryFile()
self._pos=4
self._progress=None
def rpt(self, pos, oid, start, tname, user, t, p, first, newtrans):
if pos is None:
self.tpc_finish()
sys.stderr.write(' %% 100 \n')
return
else:
progress=' %% %.1f \r' % (pos*100.0/self._input_size)
if progress != self._progress:
sys.stderr.write(progress)
self._progress=progress
if newtrans:
try: string.atof(tname)
except:
# Ugh, we have a weird tname. We'll just ignore the transaction
# boundary and merge transactions
if first:
# But we can't ignore the first one, so we'll hack in a
# bogus start date
self.tpc_begin('100', user, t)
else:
if not first: self.tpc_finish()
self.tpc_begin(tname, user, t)
self.store(oid, p)
def store(self, oid, data):
old=self._indexpos(oid, 0)
pnv=None
tfile=self._tfile
write=tfile.write
pos=self._pos
here=tfile.tell()+pos+self._thl
self._tappend(oid, here)
serial=self._serial
data=fixpickle(data, oid)
write(pack(">8s8s8s8sH8s",
p64(oid+1),serial,p64(old),p64(pos),
0,p64(len(data))
)
)
write(data)
def tpc_begin(self, tname, user, desc):
del self._tindex[:] # Just to be sure!
self._tfile.seek(0)
t=string.atof(tname)
y,m,d,h,mi=time.gmtime(t)[:5]
s=t%60
self._serial=struct.pack(
">II",
(((((y-1900)*12)+m-1)*31+d-1)*24+h)*60+mi,
long(s * (1L << 32) / 60)
)
# Ugh, we have to record the transaction header length
# so that we can get version pointers right.
self._thl=23+len(user)+len(desc)
# And we have to save the data used to compute the
# header length. It's unlikely that this stuff would
# change, but if it did, it would be a disaster.
self._ud=user, desc
def tpc_finish(self):
file=self._file
write=file.write
tfile=self._tfile
dlen=tfile.tell()
tfile.seek(0)
id=self._serial
user, desc = self._ud
self._ud=None
tlen=self._thl
pos=self._pos
tl=tlen+dlen
stl=p64(tl)
write(pack(
">8s" "8s" "c" "H" "H" "H"
,id, stl, ' ', len(user), len(desc), 0,
))
if user: write(user)
if desc: write(desc)
cp(tfile, file, dlen)
write(stl)
self._pos=pos+tl+8
tindex=self._tindex
index=self._index
for oid, pos in tindex: index[oid]=pos
del tindex[:]
class ZEXP:
"""Zope Export format 'report' writer
"""
def __init__(self, fname, file):
file.seek(0,2)
self._input_size=file.tell()
file.seek(0)
self.__name__=fname
self._file=open(fname,'w+b')
self._file.write('ZEXP')
self._pos=4
self._progress=None
def rpt(self, pos, oid, start, tname, user, t, p, first, newtrans):
write=self._file.write
if pos is None:
write('\377'*16) # end marker
sys.stderr.write(' %% 100 \n')
return
else:
progress=' %% %.1f \r' % (pos*100.0/self._input_size)
if progress != self._progress:
sys.stderr.write(progress)
self._progress=progress
data=fixpickle(p, oid)
l=len(data)
write(p64(oid+1)+p64(l))
write(data)
self._pos=self._pos+l+16
t32 = 1L << 32
def p64(v, pack=struct.pack):
if v < t32: h=0
else:
h=v/t32
v=v%t32
return pack(">II", h, v)
def cp(f1, f2, l):
read=f1.read
write=f2.write
n=8192
while l > 0:
if n > l: n=l
d=read(n)
write(d)
l = l - len(d)
class Ghost: pass
class Global:
__safe_for_unpickling__=1
def __init__(self, m, n):
self._module, self._name = m, n
def __call__(self, *args):
return Inst(self, args)
def __basicnew__(self):
return Inst(self, None)
def _global(m, n):
if m[:8]=='BoboPOS.':
if m=='BoboPOS.PickleDictionary' and n=='Root':
m='ZODB.conversionhack'
n='hack'
elif m=='BoboPOS.PersistentMapping': m='Persistence'
elif m=='BoboPOS.cPickleJar' and n=='ec':
m=n='ExtensionClass'
else:
raise 'Unexpected BoboPOS class', (m, n)
elif m=='PersistentMapping': m='Persistence'
return Global(m,n)
class Inst:
_state=None
def __init__(self, c, args):
self._cls=c
self._args=args
def __setstate__(self, state): self._state=state
from pickle import INST, GLOBAL, MARK, BUILD, OBJ, REDUCE
InstanceType=type(Ghost())
class Unpickler(pickle.Unpickler):
dispatch={}
dispatch.update(pickle.Unpickler.dispatch)
def load_inst(self):
k = self.marker()
args = tuple(self.stack[k+1:])
del self.stack[k:]
module = self.readline()[:-1]
name = self.readline()[:-1]
klass = _global(module, name)
value=Inst(klass, args)
self.append(value)
dispatch[INST] = load_inst
def load_global(self):
module = self.readline()[:-1]
name = self.readline()[:-1]
klass = _global(module, name)
self.append(klass)
dispatch[GLOBAL] = load_global
def persistent_load(self, oid,
TupleType=type(()), Ghost=Ghost, p64=p64):
"Remap object ids from ZODB 2 stype to ZODB 3 style"
if type(oid) is TupleType:
oid, klass = oid
oid = p64(oid+1), klass
else:
oid = p64(oid+1)
Ghost=Ghost()
Ghost.oid=oid
return Ghost
class Pickler(pickle.Pickler):
dispatch={}
dispatch.update(pickle.Pickler.dispatch)
def persistent_id(self, object, Ghost=Ghost):
if hasattr(object, '__class__') and object.__class__ is Ghost:
return object.oid
def save_inst(self, object):
d = id(object)
cls = object.__class__
memo = self.memo
write = self.write
save = self.save
if cls is Global:
memo_len = len(memo)
write(GLOBAL + object._module + '\n' + object._name + '\n' +
self.put(memo_len))
memo[d] = (memo_len, object)
return
self.save_inst(object._cls)
save(object._args)
memo_len = len(memo)
write(REDUCE + self.put(memo_len))
memo[d] = (memo_len, object)
stuff=object._state
save(stuff)
write(BUILD)
dispatch[InstanceType] = save_inst
def save_global(self, object, name = None):
write = self.write
memo = self.memo
if (name is None):
name = object._name
module=object._module
memo_len = len(memo)
write(GLOBAL + module + '\n' + name + '\n' +
self.put(memo_len))
memo[id(object)] = (memo_len, object)
dispatch[type(Ghost)] = save_global
def fixpickle(p, oid):
pfile=StringIO(p)
unpickler=Unpickler(pfile)
newp=StringIO()
pickler=Pickler(newp,1)
pickler.dump(unpickler.load())
state=unpickler.load()
if oid==-1: state={'_container': state}
pickler.dump(state)
p=newp.getvalue()
return p
utilities/bbb.py deleted 100644 → 0
View file @ 1f97e260
##############################################################################
#
# Copyright (c) 2001 Zope Corporation and Contributors. All Rights Reserved.
#
# This software is subject to the provisions of the Zope Public License,
# Version 2.0 (ZPL). A copy of the ZPL should accompany this distribution.
# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
# FOR A PARTICULAR PURPOSE
#
##############################################################################
"""Read and (re-)format BoboPOS 2 database files
"""
import struct, string, sys, time, os
try: from cStringIO import StringIO
except: from StringIO import StringIO
ppml=None
file__version__=3.0
packed_version='SDBMV'+struct.pack(">f",file__version__)
def error(message, fatal=0, exc=0):
if exc:
sys.stderr.write('%s: %s' % (sys.exc_info()[0], sys.exc_info()[1]))
sys.stderr.write("\n%s\n" % message)
if fatal: sys.exit(fatal)
def _read_and_report(file, rpt=None, fromEnd=0, both=0, n=99999999, show=0,
forgive=0, export=0):
"""\
Read a file's index up to the given time.
"""
first=1
seek=file.seek
read=file.read
unpack=struct.unpack
split=string.split
join=string.join
find=string.find
seek(0,2)
file_size=file.tell()
if not export:
seek(0)
h=read(len(packed_version))
if h != packed_version:
if h[:4]=='FS21':
error("The input file is a ZODB File Storage\n"
"This script only works with ZODB 2 (BoboPOS) "
"data or export files."
,1)
else:
error("The input file is not a ZODB 2 database file.\n"
"This script only works with ZODB 2 (BoboPOS) "
"data or export files."
,1)
gmtime=time.gmtime
if fromEnd: pos=file_size
else: pos=newpos=(not export) and len(packed_version)
tlast=0
err=0
tnamelast=None
while 1:
if fromEnd:
seek(pos-4)
l=unpack(">i", read(4))[0]
if l==0:
b=pos
p=pos-4
while l==0:
p=p-4
seek(p)
l=unpack(">i", read(4))[0]
pos=p+4
error("nulls skipped from %s to %s" % (pos,b))
p=pos-l
if p < 0:
error('Corrupted data before %s' % pos)
if show > 0:
p=pos-show
if p < 0: p=0
seek(p)
p=read(pos-p)
else:
p=''
error(p,1)
pos=p
else:
pos=newpos
seek(pos)
h=read(24) # 24=header_size
if not h: break
if len(h) != 24: break
oid,prev,start,tlen,plen=unpack(">iidii",h)
if prev < 0 or (prev and (prev >= pos)):
error('Bad previous record pointer (%s) at %s' % (prev, pos))
if show > 0: error(read(show))
if not forgive:
err=1
break
if start < tlast:
error('record time stamps are not chronological at %s' % pos)
if show > 0: error(read(show))
if not forgive:
err=1
break
if plen > tlen or plen < 0 or oid < -999:
error('Corrupted data record at %s' % pos)
if show > 0: error(read(show))
err=1
break
newpos=pos+tlen
if newpos > file_size:
error('Truncated data record at %s' % pos)
if show > 0: error(read(show))
if not forgive:
err=1
break
seek(newpos-4)
if read(4) != h[16:20]:
__traceback_info__=pos, oid,prev,start,tlen,plen
error('Corrupted data record at %s' % pos)
if show > 0:
seek(pos+24)
error(read(show))
err=1
break
tlast=start-100
if rpt is None: continue
n=n-1
if n < 1: break
seek(pos+24)
if plen > 0:
p=read(plen)
if p[-1:] != '.':
error('Corrupted pickle at %s %s %s' % (pos,plen,len(p)))
if show > 0:
seek(pos+24)
error(read(show))
err=1
break
else: p=''
t=split(read(tlen-plen-28),'\t')
tname, user = (t+[''])[:2]
t=join(t[2:],'\t')
start,f=divmod(start,1)
y,m,d,h,mn,s=gmtime(start)[:6]
s=s+f
start="%.4d-%.2d-%.2d %.2d:%.2d:%.3f" % (y,m,d,h,mn,s)
rpt(pos,oid,start,tname,user,t,p,first,tname!=tnamelast)
first=0
tnamelast=tname
if err and both and not fromEnd:
_read_and_report(file, rpt, 1, 0, n, show)
rpt(None, None, None, None, None, None, None, None, None)
def none(*ignored): pass
def positions(pos, *ignored):
if pos is not None: sys.stdout.write("%s\n" % pos)
def oids(pos, oid, *ignored):
if pos is not None: sys.stdout.write("%s\n" % oid)
def tab_delimited(*args):
sys.stdout.write("%s\n" % string.join(args[:6],'\t'))
def undo_log(pos, oid, start, tname, user, t, p, first, newtrans):
if not newtrans: return
sys.stdout.write("%s:\t%s\t%s\t%s\n" % (pos, start, user, t))
reports={
'none': (none,
('Read a database file checking for errors',
'but producing no output')
),
'oids': (oids,
('Read the database and output object ids',)),
'positions': (positions,
('Read the database and output record positions',)),
'tab_delimited': (tab_delimited,
('Output record meta-data in tab-delimited format',)),
'undo': (undo_log,
(
'Output a transaction summary that shows the position of',
'each transaction. This is useful for undoing ',
'transactions from the OS command line when',
'some programming error has caused objects to get to',
'a state where Zope can\'t start up.',
'',
'Eventually, there will be an undo utility for undoing',
'individual transactions. For now, you can simply',
'truncate the file at the position of a problem',
'transaction to return the database to the state it',
'was in before the transaction',
)),
}
def main(argv):
import getopt
items=reports.items()
items.sort()
usage="""Usage: python %s [options] filename
where filename is the name of the database file.
options:
-r report
Specify an output report.
The valid reports are:\n\n\t\t%s
-e
Read the file from back to front
-l n
Show only n records
-b
If an error is encountered while reading from front,
ret reading from the back.
-s n
If a corrupted data record is found, show the first n
bytes of the corrupted record.
-f filename
Convert to ZODB 3 File-Storage format
-p path
Add a directory to the Python path.
-x
The input file is a ZODB 2 export file.
""" % (sys.argv[0],
string.join(map(
lambda i:
("%s --\n\t\t\t%s" % (i[0], string.join(i[1][1],'\n\t\t\t'))),
items),
',\n\n\t\t'))
sys.path.append(os.path.split(sys.argv[0])[0])
try:
opts, args = getopt.getopt(argv,'r:ebl:s:f:p:x')
filename,=args
except: error(usage,1,1)
try: file=open(filename,'rb')
except: error('Coud not open %s' % filename,1,1)
rpt=none
fromEnd=0
both=0
n=99999999
show=0
export=0
convert=0
for o, v in opts:
o=o[1:]
if o=='r':
try: rpt=reports[v][0]
except: error('Invalid report: %s' % v, 1)
elif o=='l':
try: n=string.atoi(v)
except: error('The number of records, %s, shuld ne an integer'
% v, 1)
elif o=='s':
try: show=string.atoi(v)
except: error('The number of bytes, %s, shuld ne an integer'
% v, 1)
elif o=='e': fromEnd=1
elif o=='x': export=1
elif o=='f': convert=1
elif o=='b': both=1
elif o=='p':
if v=='-':
v=os.path.join(
os.path.split(sys.argv[0])[0],
'..','lib','python')
sys.path.insert(0,v)
print sys.path
else: error('Unrecognized option: -%s' % o, 1)
if convert:
import FS
if export:
rpt=FS.ZEXP(v, file).rpt
else:
rpt=FS.FS(v, file).rpt
_read_and_report(file, rpt, fromEnd, both, n, show,
forgive=1, export=export)
if __name__=='__main__': main(sys.argv[1:])
utilities/fixbbbts.py deleted 100644 → 0
View file @ 1f97e260
import sys
__doc__="""Fix BoboPOS time stamps
If a system has a problem with it's clock setting, it may cause
database records to be written with time stamps in the future. This
causes problems when the clock is fixed or when the data are moved to
a system that doesn't have a broken clock.
The database has a requirement that records should be chronologically
ordered and that times not be in the future.
This copies a database, restamping the times as it goes.
%s [options] file1 file2
Copy file1 to file2 restamping the records.
options:
-o offset
Records that are later offset seconds in the past
are moved back to offset seconds in the past plus
some small offset chosen so that times are not the same and
are chronological.
""" % sys.argv[0]
InvalidFormat='Format Error'
Corrupted='Data Corruption'
def main():
import getopt, string, struct, time
file__version__=3.0
packed_version='SDBMV'+struct.pack(">f",file__version__)
try:
opts, args = getopt.getopt(sys.argv[1:], 'o:')
file1, file2 = args
offset=86400
for o, v in opts:
if o=='-o':
offset=string.atoi(v)
except:
print __doc__
print "%s: %s" % sys.exc_info()[:2]
start=time.time()-offset
next=start+0.001
input=open(file1,'rb')
read=input.read
output=open(file2,'wb')
write=output.write
pack=struct.pack
unpack=struct.unpack
h=read(len(packed_version))
if h != packed_version:
raise InvalidFormat, 'This is not a BoboPOS file'
write(h)
pos=len(h)
while 1:
h=read(24)
if not h: break
if len(h) < 24: raise Corrupted, pos
oid, prev, t, tlen, plen = unpack(">iidii", h)
if start is None or t > start:
t=next
next=next+0.001
start=None
if plen > tlen or tlen < 28: raise Corrupted, pos
write(pack(">iidii", oid, prev, t, tlen, plen))
l=tlen-28
s=8196
while l > 0:
if s > l: s=l
d=read(s)
if not d: raise Corrupted, pos
write(d)
l=l-len(d)
d=read(4)
if d != h[16:20]: raise Corrupted, pos
write(d)
pos=pos+tlen
if __name__=='__main__': main()
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment
GitLab Nexedi Edition | About GitLab | About Nexedi | 沪ICP备2021021310号-2 | 沪ICP备2021021310号-7