Skip to content

Z
ZEO
Commit 1ff59e88 authored by Tim Peters's avatar Tim Peters
Browse files
Options

The ZODB4 -> ZODB3 conversion utility is no longer needed. 

Removing the subtree rooted at ZODB/src/ZODB/zodb4/.
parent a2833f3a
Expand all Hide whitespace changes
Inline Side-by-side
Showing with 0 additions and 1623 deletions
src/ZODB/zodb4/__init__.py deleted 100644 → 0
View file @ a2833f3a
# This directory contains a Python package.
src/ZODB/zodb4/conversion.py deleted 100644 → 0
View file @ a2833f3a
##############################################################################
#
# Copyright (c) 2004 Zope Corporation and Contributors.
# All Rights Reserved.
#
# This software is subject to the provisions of the Zope Public License,
# Version 2.0 (ZPL). A copy of the ZPL should accompany this distribution.
# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
# FOR A PARTICULAR PURPOSE.
#
##############################################################################
"""Nitty-gritty conversion of a ZODB 4 FileStorage to a ZODB 3 FileStorage."""
import sys
from cPickle import Pickler, Unpickler
from cStringIO import StringIO
from ZODB.FileStorage import FileStorage
from ZODB.zodb4 import z4iterator
errors = {}
skipped = 0
class Conversion:
def __init__(self, input_path, output_path):
"""Initialize a ZODB4->ZODB3 FileStorage converter."""
self.instore = IterableFileIterator(input_path)
self.outstore = FileStorage(output_path)
def run(self):
self.instore._read_metadata()
self.outstore.copyTransactionsFrom(self.instore)
self.outstore.close()
self.instore.close()
if errors:
sys.stderr.write(error_explanation)
sys.stderr.write("%s database records skipped\n" % skipped)
class IterableFileIterator(z4iterator.FileIterator):
def iterator(self):
return self
def __iter__(self):
baseiter = z4iterator.FileIterator.__iter__(self)
for txn in baseiter:
yield DataRecordConvertingTxn(txn)
class DataRecordConvertingTxn(object):
def __init__(self, txn):
self._txn = txn
self.user = str8(txn.user)
self.description = str8(txn.description)
def __getattr__(self, name):
return getattr(self._txn, name)
def __iter__(self):
global skipped
for record in self._txn:
record.tid = record.serial
# transform the data record format
# (including persistent references)
sio = StringIO(record.data)
up = Unpickler(sio)
up.persistent_load = PersistentIdentifier
try:
classmeta = up.load()
state = up.load()
except Exception, v:
v = str(v)
if v not in errors:
if not errors:
sys.stderr.write("Pickling errors:\n\n")
sys.stderr.write('\t'+v+'\n\n')
errors[v] = True
skipped += 1
continue
sio = StringIO()
p = Pickler(sio, 1)
p.persistent_id = get_persistent_id
p.dump(classmeta)
p.dump(state)
record.data = sio.getvalue()
yield record
error_explanation = """
There were errors while copying data records.
If the errors were import errors, then this is because modules
referenced by the database couldn't be found. You might be able to
fix this by getting the necessary modules. It's possible that the
affected objects aren't used any more, in which case, it doesn't
matter whether they were copied. (We apologise for the lame import
errors that don't show full dotted module names.)
If errors looked something like:
"('object.__new__(SomeClass) is not safe, use
persistent.Persistent.__new__()', <function _reconstructor at
0x4015ccdc>, (<class 'somemodule.SomeClass'>, <type 'object'>,
None))",
then the error arises from data records for objects whos classes
changed from being non-persistent to being persistent.
If other errors were reported, it would be a good idea to ask about
them on zope3-dev.
In any case, keep your original data file in case you decide to rerun
the conversion.
"""
class PersistentIdentifier:
def __init__(self, ident):
if isinstance(ident, tuple):
self._oid, (self._class, args) = ident
if args:
# we have args from __getnewargs__(), but can just
# lose them since they're an optimization to allow
# ghost construction
self._class = None
else:
assert isinstance(ident, str)
self._oid = ident
self._class = None
def get_persistent_id(ob):
if isinstance(ob, PersistentIdentifier):
if ob._class is None:
return ob._oid
else:
return ob._oid, ob._class
else:
return None
def str8(s):
# convert unicode strings to 8-bit strings
if isinstance(s, unicode):
# Should we use UTF-8 or ASCII? Not sure.
return s.encode("ascii")
else:
return s
src/ZODB/zodb4/main.py deleted 100644 → 0
View file @ a2833f3a
##############################################################################
#
# Copyright (c) 2004 Zope Corporation and Contributors.
# All Rights Reserved.
#
# This software is subject to the provisions of the Zope Public License,
# Version 2.0 (ZPL). A copy of the ZPL should accompany this distribution.
# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
# FOR A PARTICULAR PURPOSE.
#
##############################################################################
"""Script to convert a ZODB 4 file storage to a ZODB 3 file storage.
This is needed since Zope 3 is being changed to use ZODB 3 instead of
ZODB 4.
"""
import getopt
import os
import sys
try:
__file__
except NameError:
__file__ = os.path.realpath(sys.argv[0])
here = os.path.dirname(__file__)
topdir = os.path.dirname(os.path.dirname(here))
# Make sure that if we're run as a script, we can import the ZODB
# package and our sibling modules.
try:
import ZODB.zodb4
except ImportError:
sys.path.append(topdir)
import ZODB.zodb4
from ZODB.lock_file import LockFile
from ZODB.zodb4 import conversion
class ConversionApp:
def __init__(self, name=None, args=None):
if name is None:
name = os.path.basename(sys.argv[0])
if args is None:
args = sys.argv[1:]
self.name = name
self.verbosity = 0
self.dbfile = None
self.parse_args(args)
def run(self):
# Load server-independent site config
from zope.configuration import xmlconfig
context = xmlconfig.file('site.zcml', execute=True)
if not os.path.exists(self.dbfile):
self.error("input database does not exist: %s" % self.dbfile)
base, ext = os.path.splitext(self.dbfile)
if ext != ".fs":
base = self.dbfile
self.dbindex = self.dbfile + ".index"
self.bakfile = base + ".fs4"
self.bakindex = self.bakfile + ".index"
if os.path.exists(self.bakfile):
self.error("backup database already exists: %s\n"
"please move aside and try again" % self.bakfile)
if os.path.exists(self.bakindex):
self.error("backup database index already exists: %s\n"
"please move aside and try again" % self.bakindex)
self.convert()
# XXX the conversion script leaves an invalid index behind. Why?
os.remove(self.dbindex)
def convert(self):
lock = LockFile(self.bakfile + ".lock")
try:
# move the ZODB 4 database to be the backup
os.rename(self.dbfile, self.bakfile)
if os.path.exists(self.dbindex):
try:
os.rename(self.dbindex, self.bakindex)
except:
# we couldn't rename *both*, so try to make sure we
# don't rename either
os.rename(self.bakfile, self.dbfile)
raise
# go:
converter = conversion.Conversion(self.bakfile, self.dbfile)
converter.run()
finally:
lock.close()
def parse_args(self, args):
opts, args = getopt.getopt(args, "v", ["verbose"])
for opt, arg in opts:
if opt in ("-v", "--verbose"):
self.verbosity += 1
if len(args) == 0:
# use default location for Data.fs
self.dbfile = os.path.join(topdir, "Data.fs")
elif len(args) == 1:
self.dbfile = args[0]
else:
self.error("too many command-line arguments", rc=2)
def error(self, message, rc=1):
print >>sys.stderr, "%s: %s" % (self.name, message)
sys.exit(rc)
def main():
ConversionApp().run()
if __name__ == "__main__":
main()
src/ZODB/zodb4/tests/__init__.py deleted 100644 → 0
View file @ a2833f3a
# This directory contains a Python package.
src/ZODB/zodb4/tests/test_z4utils.py deleted 100644 → 0
View file @ a2833f3a
##############################################################################
#
# Copyright (c) 2001, 2002 Zope Corporation and Contributors.
# All Rights Reserved.
#
# This software is subject to the provisions of the Zope Public License,
# Version 2.0 (ZPL). A copy of the ZPL should accompany this distribution.
# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
# FOR A PARTICULAR PURPOSE.
#
##############################################################################
"""Test the routines to convert between long and 64-bit strings"""
# originally zodb.tests.test_utils
import random
import unittest
NUM = 100
from ZODB.zodb4.z4utils import p64, u64
class TestUtils(unittest.TestCase):
small = [random.randrange(1, 1L<<32, int=long)
for i in range(NUM)]
large = [random.randrange(1L<<32, 1L<<64, int=long)
for i in range(NUM)]
all = small + large
def test_LongToStringToLong(self):
for num in self.all:
s = p64(num)
n2 = u64(s)
self.assertEquals(num, n2, "u64() failed")
def test_KnownConstants(self):
self.assertEquals("\000\000\000\000\000\000\000\001", p64(1))
self.assertEquals("\000\000\000\001\000\000\000\000", p64(1L<<32))
self.assertEquals(u64("\000\000\000\000\000\000\000\001"), 1)
self.assertEquals(u64("\000\000\000\001\000\000\000\000"), 1L<<32)
def test_suite():
return unittest.makeSuite(TestUtils)
if __name__ == "__main__":
unittest.main(defaultTest="test_suite")
src/ZODB/zodb4/z4base.py deleted 100644 → 0
View file @ a2833f3a
##############################################################################
#
# Copyright (c) 2001 Zope Corporation and Contributors.
# All Rights Reserved.
#
# This software is subject to the provisions of the Zope Public License,
# Version 2.0 (ZPL). A copy of the ZPL should accompany this distribution.
# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
# FOR A PARTICULAR PURPOSE
#
##############################################################################
"""Convenience function extracted from ZODB4's zodb.storage.base."""
def splitrefs(refstr, oidlen=8):
# refstr is a packed string of reference oids. Always return a list of
# oid strings. Most storages use fixed oid lengths of 8 bytes, but if
# the oids in refstr are a different size, use oidlen to specify. This
# does /not/ support variable length oids in refstr.
if not refstr:
return []
num, extra = divmod(len(refstr), oidlen)
fmt = '%ds' % oidlen
assert extra == 0, refstr
return list(struct.unpack('>' + (fmt * num), refstr))
src/ZODB/zodb4/z4errors.py deleted 100644 → 0
View file @ a2833f3a
##############################################################################
#
# Copyright (c) 2003 Zope Corporation and Contributors.
# All Rights Reserved.
#
# This software is subject to the provisions of the Zope Public License,
# Version 2.0 (ZPL). A copy of the ZPL should accompany this distribution.
# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
# FOR A PARTICULAR PURPOSE.
#
##############################################################################
"""FileStorage-specific exceptions."""
from ZODB.zodb4.z4interfaces import _fmt_oid, POSError
# originally from zodb.storage.interfaces
class StorageError(POSError):
"""Base class for storage based exceptions."""
class StorageSystemError(StorageError):
"""Panic! Internal storage error!"""
# originally zodb.storage.file.errors
class FileStorageError(StorageError):
pass
class PackError(FileStorageError):
pass
class FileStorageFormatError(FileStorageError):
"""Invalid file format
The format of the given file is not valid.
"""
class CorruptedError(FileStorageError, StorageSystemError):
"""Corrupted file storage."""
class CorruptedDataError(CorruptedError):
def __init__(self, oid=None, buf=None, pos=None):
self.oid = oid
self.buf = buf
self.pos = pos
def __str__(self):
if self.oid:
msg = "Error reading oid %s. Found %r" % (_fmt_oid(self.oid),
self.buf)
else:
msg = "Error reading unknown oid. Found %r" % self.buf
if self.pos:
msg += " at %d" % self.pos
return msg
class FileStorageQuotaError(FileStorageError, StorageSystemError):
"""File storage quota exceeded."""
src/ZODB/zodb4/z4format.py deleted 100644 → 0
View file @ a2833f3a
This diff is collapsed.
src/ZODB/zodb4/z4interfaces.py deleted 100644 → 0
View file @ a2833f3a
This diff is collapsed.
src/ZODB/zodb4/z4iterator.py deleted 100644 → 0
View file @ a2833f3a
##############################################################################
#
# Copyright (c) 2001 Zope Corporation and Contributors.
# All Rights Reserved.
#
# This software is subject to the provisions of the Zope Public License,
# Version 2.0 (ZPL). A copy of the ZPL should accompany this distribution.
# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
# FOR A PARTICULAR PURPOSE.
#
##############################################################################
"""Iterator support for ZODB 4 databases."""
from cPickle import loads
from struct import unpack
from ZODB.zodb4.z4interfaces import ZERO
from ZODB.zodb4.z4utils import u64, splitrefs
from ZODB.zodb4.z4format import FileStorageFormatter
from ZODB.zodb4.z4format import TRANS_HDR, TRANS_HDR_LEN, DATA_HDR
from ZODB.zodb4.z4format import DATA_HDR_LEN, DATA_VERSION_HDR_LEN
# originally from zodb.storage.file.main
class FileIterator(FileStorageFormatter):
"""Iterate over the transactions in a FileStorage file."""
_ltid = ZERO
## implements(IStorageIterator)
def __init__(self, file):
# - removed start and stop arguments
if isinstance(file, str):
file = open(file, 'rb')
self._file = file
self._read_metadata()
self._file.seek(0,2)
self._file_size = self._file.tell()
self._pos = self._metadata_size
def close(self):
file = self._file
if file is not None:
self._file = None
file.close()
def __iter__(self):
if self._file is None:
# A closed iterator. XXX: Is IOError the best we can do? For
# now, mimic a read on a closed file.
raise IOError("iterator is closed")
file = self._file
seek = file.seek
read = file.read
pos = self._pos
while True:
# Read the transaction record
seek(pos)
h = read(TRANS_HDR_LEN)
if len(h) < TRANS_HDR_LEN:
break
tid, tl, status, ul, dl, el = unpack(TRANS_HDR,h)
if el < 0:
el = (1L<<32) - el
if tid <= self._ltid:
warn("%s time-stamp reduction at %s", self._file.name, pos)
self._ltid = tid
if pos+(tl+8) > self._file_size or status=='c':
# Hm, the data were truncated or the checkpoint flag wasn't
# cleared. They may also be corrupted,
# in which case, we don't want to totally lose the data.
warn("%s truncated, possibly due to damaged records at %s",
self._file.name, pos)
break
if status not in ' p':
warn('%s has invalid status, %s, at %s', self._file.name,
status, pos)
if tl < (TRANS_HDR_LEN+ul+dl+el):
# We're in trouble. Find out if this is bad data in
# the middle of the file, or just a turd that Win 9x
# dropped at the end when the system crashed. Skip to
# the end and read what should be the transaction
# length of the last transaction.
seek(-8, 2)
rtl = u64(read(8))
# Now check to see if the redundant transaction length is
# reasonable:
if self._file_size - rtl < pos or rtl < TRANS_HDR_LEN:
logger.critical('%s has invalid transaction header at %s',
self._file.name, pos)
warn("It appears that there is invalid data at the end of "
"the file, possibly due to a system crash. %s "
"truncated to recover from bad data at end.",
self._file.name)
break
else:
warn('%s has invalid transaction header at %s',
self._file.name, pos)
break
tpos = pos
tend = tpos+tl
pos = tpos+(TRANS_HDR_LEN+ul+dl+el)
# user and description are utf-8 encoded strings
user = read(ul).decode('utf-8')
description = read(dl).decode('utf-8')
e = {}
if el:
try:
e = loads(read(el))
# XXX can we do better?
except:
pass
result = RecordIterator(tid, status, user, description, e, pos,
tend, file, tpos)
pos = tend
# Read the (intentionally redundant) transaction length
seek(pos)
l = u64(read(8))
if l != tl:
warn("%s redundant transaction length check failed at %s",
self._file.name, pos)
break
pos += 8
yield result
class RecordIterator(FileStorageFormatter):
"""Iterate over data records for a transaction in a FileStorage."""
## implements(ITransactionRecordIterator, ITransactionAttrs)
def __init__(self, tid, status, user, desc, ext, pos, tend, file, tpos):
self.tid = tid
self.status = status
self.user = user
self.description = desc
self._extension = ext
self._pos = pos
self._tend = tend
self._file = file
self._tpos = tpos
def __iter__(self):
pos = self._pos
while pos < self._tend:
# Read the data records for this transaction
h = self._read_data_header(pos)
dlen = h.recordlen()
if pos + dlen > self._tend or h.tloc != self._tpos:
warn("%s data record exceeds transaction record at %s",
file.name, pos)
return
pos += dlen
prev_txn = None
if h.plen:
refsdata = self._file.read(h.nrefs * 8)
refs = splitrefs(refsdata)
data = self._file.read(h.plen)
else:
if not h.back:
# If the backpointer is 0, then this transaction
# undoes the object creation. It either aborts
# the version that created the object or undid the
# transaction that created it. Return None
# for data and refs because the backpointer has
# the real data and refs.
data = None
refs = None
else:
data, refs, _s, tid = self._loadBackTxn(h.oid, h.back)
prev_txn = self.getTxnFromData(h.oid, h.back)
yield Record(h.oid, h.serial, h.version, data, prev_txn, refs)
class Record:
"""An abstract database record."""
## implements(IDataRecord)
def __init__(self, oid, serial, version, data, data_txn, refs):
self.oid = oid
self.serial = serial
self.version = version
self.data = data
self.data_txn = data_txn
self.refs = refs
src/ZODB/zodb4/z4utils.py deleted 100644 → 0
View file @ a2833f3a
##############################################################################
#
# Copyright (c) 2001, 2002 Zope Corporation and Contributors.
# All Rights Reserved.
#
# This software is subject to the provisions of the Zope Public License,
# Version 2.0 (ZPL). A copy of the ZPL should accompany this distribution.
# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
# FOR A PARTICULAR PURPOSE.
#
##############################################################################
# originally zodb.utils
import struct
def p64(v):
"""Pack an integer or long into a 8-byte string"""
return struct.pack(">Q", v)
def u64(v):
"""Unpack an 8-byte string into a 64-bit long integer."""
return struct.unpack(">Q", v)[0]
def cp(f1, f2, l):
read = f1.read
write = f2.write
n = 8192
while l > 0:
if n > l:
n = l
d = read(n)
if not d:
break
write(d)
l = l - len(d)
# originally from zodb.storage.base
def splitrefs(refstr, oidlen=8):
# refstr is a packed string of reference oids. Always return a list of
# oid strings. Most storages use fixed oid lengths of 8 bytes, but if
# the oids in refstr are a different size, use oidlen to specify. This
# does /not/ support variable length oids in refstr.
if not refstr:
return []
num, extra = divmod(len(refstr), oidlen)
fmt = '%ds' % oidlen
assert extra == 0, refstr
return list(struct.unpack('>' + (fmt * num), refstr))
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment
GitLab Nexedi Edition | About GitLab | About Nexedi | 沪ICP备2021021310号-2 | 沪ICP备2021021310号-7