debugged export/import, version commit/abort and mostly undo

src/ZODB/Connection.py

@@ -84,22 +84,22 @@
 ##############################################################################
 """Database connection support
 
-$Id: Connection.py,v 1.5 1999/05/12 15:55:43 jim Exp $"""
-__version__='$Revision: 1.5 $'[11:-2]
+$Id: Connection.py,v 1.6 1999/05/18 15:55:09 jim Exp $"""
+__version__='$Revision: 1.6 $'[11:-2]
 
 from cPickleCache import PickleCache
-from bpthread import allocate_lock
-from POSException import ConflictError
+from POSException import ConflictError, ExportError
 from cStringIO import StringIO
 from cPickle import Unpickler, Pickler
 from ExtensionClass import Base
+import Transaction, string, ExportImport 
 
 ExtensionKlass=Base.__class__
 
 class HelperClass: pass
 ClassType=type(HelperClass)
 
-class Connection:
+class Connection(ExportImport.ExportImport):
     """Object managers for individual object space.
 
     An object space is a version of collection of objects.  In a
@@ -117,10 +117,9 @@ class Connection:
         self._version=version
         self._cache=cache=PickleCache(self, cache_size, cache_deactivate_after)
         self._incrgc=cache.incrgc
-        self._invalidated={}
-        lock=allocate_lock()
-        self._a=lock.acquire
-        self._r=lock.release
+        self._invalidated=d={}
+        self._invalid=d.has_key
+        self._committed=[]
 
     def _breakcr(self):
         try: del self._cache
@@ -217,11 +216,7 @@ class Connection:
         Any objects modified since the last transaction are invalidated.
         """     
         self._db=odb
-        cache=self._cache
-        for oid in self._invalidated.keys():
-            if cache.has_key(oid):
-                cache[oid]._p_deactivate()
-        self._invalidated.clear()
+        self._cache.invalidate(self._invalidated)
 
         return self
 
@@ -232,7 +227,8 @@ class Connection:
 
     def commit(self, object, transaction):
         oid=object._p_oid
-        if self._invalidated.has_key(oid): raise ConflictError, oid
+        invalid=self._invalid
+        if invalid(oid) or invalid(None): raise ConflictError, oid
         self._invalidating.append(oid)
         plan=self._planToStore
         stack=[]
@@ -283,7 +279,7 @@ class Connection:
                 oid=object._p_oid
                 try: serial=object._p_serial
                 except: serial='\0'*8
-                if self._invalidated.has_key(oid): raise ConflictError, oid
+                if invalid(oid): raise ConflictError, oid
                 klass = object.__class__
 
                 if klass is ExtensionKlass:
@@ -320,9 +316,6 @@ class Connection:
 
         return topoid
 
-    def commitVersion(self, destination=''):
-        raise 'Not Implemented Yet!'
-
     def db(self): return self._db
 
     def getVersion(self): return self._version
@@ -334,9 +327,7 @@ class Connection:
         it.  The object data will be actually invalidated at certain
         transaction boundaries.
         """
-        self._a()
         self._invalidated[oid]=1
-        self._r()
 
     def modifiedInVersion(self, o):
         return self._db.modifiedInVersion(o._p_oid)
@@ -347,11 +338,8 @@ class Connection:
         # Note, we no longer mess with the object's state
         # flag, _p_changed.  This is the object's job.
         oid=object._p_oid
-        self._a()
-        if self._invalidated.has_key(oid):
-            self._r()
-            raise ConflictError, oid
-        self._r()
+        invalid=self._invalid
+        if invalid(oid) or invalid(None): raise ConflictError, oid
         p, serial = self._storage.load(oid, self._version)
         file=StringIO(p)
         unpickler=Unpickler(file)
@@ -368,45 +356,25 @@ class Connection:
     def tpc_abort(self, transaction):
         self._storage.tpc_abort(transaction)
         cache=self._cache
-        invalidated=self._invalidated
-        for oid in invalidated.keys():
-            if cache.has_key(oid):
-                cache[oid]._p_deactivate()
-        invalidated.clear()
-        for oid in self._invalidating:
-            cache[oid]._p_deactivate()
+        cache.invalidate(self._invalidated)
+        cache.invalidate(self._invalidating)
 
     def tpc_begin(self, transaction):
+        if self._invalid(None): # Some nitwit invalidated everything!
+            raise ConflictError, oid 
         self._invalidating=[]
         self._storage.tpc_begin(transaction)
 
     def tpc_finish(self, transaction):
         self._storage.tpc_finish(transaction, self.tpc_finish_)
-        cache=self._cache
-        invalidated=self._invalidated
-        for oid in invalidated.keys():
-            if cache.has_key(oid):
-                cache[oid]._p_deactivate()
-        invalidated.clear()
+        self._cache.invalidate(self._invalidated)
 
     def tpc_finish_(self):
         invalidate=self._db.invalidate
         for oid in self._invalidating: invalidate(oid, self)
 
-    def exportFile(self, oid, file=None):
-        pass # Not implemented yet
-
-    def importFile(self, file):
-        pass # Not implemented yet
-
-    ######################################################################
-    # BoboPOS 2 compat.
-
-    def export_file(self, o, file=None): return self.exportFile(o._p_oid, file)
-
-    import_file=importFile
-
 class tConnection(Connection):
 
     def close(self):
         self._breakcr()
+

src/ZODB/DB.py

@@ -84,13 +84,16 @@
 ##############################################################################
 """Database objects
 
-$Id: DB.py,v 1.5 1999/05/12 15:55:43 jim Exp $"""
-__version__='$Revision: 1.5 $'[11:-2]
+$Id: DB.py,v 1.6 1999/05/18 15:55:09 jim Exp $"""
+__version__='$Revision: 1.6 $'[11:-2]
 
-import cPickle, cStringIO, sys
+import cPickle, cStringIO, sys, POSException
 from Connection import Connection
 from bpthread import allocate_lock
 from Transaction import Transaction
+from referencesf import referencesf
+
+StringType=type('')
 
 class DB:
     """The Object Database
@@ -117,6 +120,7 @@ class DB:
 
         """
         self._storage=storage
+        storage.registerDB(self, None)
         try: storage.load('\0\0\0\0\0\0\0\0','')
         except:
             import PersistentMapping
@@ -199,7 +203,7 @@ class DB:
         finally: self._r()
 
     def abortVersion(self, version):
-        raise 'Not Yet Implemented'
+        AbortVersion(self, version)
 
     def cacheDetail(self):
         """Return information on objects in the various caches
@@ -271,7 +275,7 @@ class DB:
     def close(self): self._storage.close()
 
     def commitVersion(self, source, destination=''):
-        raise 'Not yet implemented'
+        CommitVersion(self, source, destination)
 
     def exportFile(self, oid, file=None):
         raise 'Not yet implemented'
@@ -294,7 +298,8 @@ class DB:
     def importFile(self, file):
         raise 'Not yet implemented'
 
-    def invalidate(self, oid, connection=None, rc=sys.getrefcount):
+    def invalidate(self, oid, connection=None, version='',
+                   rc=sys.getrefcount):
         """Invalidate references to a given oid.
 
         This is used to indicate that one of the connections has committed a
@@ -303,7 +308,6 @@ class DB:
         connection.
         """
         if connection is not None: version=connection._version
-        else: version=''
         self._a()
         try:
             pools,pooll=self._pools
@@ -327,6 +331,11 @@ class DB:
                 self._temps=t
         finally: self._r()
 
+    def invalidateMany(self, oids=None, version=''):
+        if oids is None: self.invalidate(None, version=version)
+        else:
+            for oid in oids: self.invalidate(oid, version=version)
+
     def objectCount(self): return len(self._storage)
         
     def open(self, version='', transaction=None, temporary=0, force=None,
@@ -344,6 +353,9 @@ class DB:
         Note that the connection pool is managed as a stack, to increate the
         likelihood that the connection's stack will include useful objects.
         """
+        if type(version) is not StringType:
+            raise POSException.Unimplemented, 'temporary versions'
+        
         self._a()
         try:
 
@@ -430,7 +442,7 @@ class DB:
         finally: self._r()
         
     def pack(self, t):
-        self._storage.pack(t,referencesf,-1)
+        self._storage.pack(t,referencesf)
                            
     def setCacheDeactivateAfter(self, v): self._cache_deactivate_after=v
     def setCacheSize(self, v): self._cache_size=v
@@ -449,20 +461,37 @@ class DB:
     def versionEmpty(self, version):
         return self._storage.versionEmpty(version)
 
-def referencesf(p,rootl,
-                Unpickler=cPickle.Unpickler,
-                StringIO=cStringIO.StringIO):
-    u=Unpickler(StringIO(p))
-    u.persistent_load=rootl
-    u.noload()
-    try: u.noload()
-    except:
-        # Hm.  We failed to do second load.  Maybe there wasn't a
-        # second pickle.  Let's check:
-        f=StringIO(p)
-        u=Unpickler(f)
-        u.persistent_load=[]
-        u.noload()
-        if len(p) > f.tell(): raise ValueError, 'Error unpickling, %s' % p
+class CommitVersion:
+    """An object that will see to version commit
 
+    in cooperation with a transaction manager.
+    """
+    def __init__(self, db, version, dest=''):
+        self._db=db
+        s=db._storage
+        self._version=version
+        self._dest=dest
+        self.tpc_abort=s.tpc_abort
+        self.tpc_begin=s.tpc_begin
+        self.tpc_finish=s.tpc_finish
+        get_transaction().register(self)
+
+    def abort(self, reallyme, t): pass
+
+    def commit(self, reallyme, t):
+        db=self._db
+        dest=self._dest
+        for oid in db._storage.commitVersion(self._version, dest, t):
+            db.invalidate(oid, version=dest)
+    
+class AbortVersion(CommitVersion):
+    """An object that will see to version abortion
+
+    in cooperation with a transaction manager.
+    """
 
+    def commit(self, reallyme, t):
+        db=self._db
+        version=self._version
+        for oid in db._storage.abortVersion(version, t):
+            db.invalidate(oid, version=version)

src/ZODB/ExportImport.py

+##############################################################################
+# 
+# Zope Public License (ZPL) Version 1.0
+# -------------------------------------
+# 
+# Copyright (c) Digital Creations.  All rights reserved.
+# 
+# This license has been certified as Open Source(tm).
+# 
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met:
+# 
+# 1. Redistributions in source code must retain the above copyright
+#    notice, this list of conditions, and the following disclaimer.
+# 
+# 2. Redistributions in binary form must reproduce the above copyright
+#    notice, this list of conditions, and the following disclaimer in
+#    the documentation and/or other materials provided with the
+#    distribution.
+# 
+# 3. Digital Creations requests that attribution be given to Zope
+#    in any manner possible. Zope includes a "Powered by Zope"
+#    button that is installed by default. While it is not a license
+#    violation to remove this button, it is requested that the
+#    attribution remain. A significant investment has been put
+#    into Zope, and this effort will continue if the Zope community
+#    continues to grow. This is one way to assure that growth.
+# 
+# 4. All advertising materials and documentation mentioning
+#    features derived from or use of this software must display
+#    the following acknowledgement:
+# 
+#      "This product includes software developed by Digital Creations
+#      for use in the Z Object Publishing Environment
+#      (http://www.zope.org/)."
+# 
+#    In the event that the product being advertised includes an
+#    intact Zope distribution (with copyright and license included)
+#    then this clause is waived.
+# 
+# 5. Names associated with Zope or Digital Creations must not be used to
+#    endorse or promote products derived from this software without
+#    prior written permission from Digital Creations.
+# 
+# 6. Modified redistributions of any form whatsoever must retain
+#    the following acknowledgment:
+# 
+#      "This product includes software developed by Digital Creations
+#      for use in the Z Object Publishing Environment
+#      (http://www.zope.org/)."
+# 
+#    Intact (re-)distributions of any official Zope release do not
+#    require an external acknowledgement.
+# 
+# 7. Modifications are encouraged but must be packaged separately as
+#    patches to official Zope releases.  Distributions that do not
+#    clearly separate the patches from the original work must be clearly
+#    labeled as unofficial distributions.  Modifications which do not
+#    carry the name Zope may be packaged in any form, as long as they
+#    conform to all of the clauses above.
+# 
+# 
+# Disclaimer
+# 
+#   THIS SOFTWARE IS PROVIDED BY DIGITAL CREATIONS ``AS IS'' AND ANY
+#   EXPRESSED OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+#   IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+#   PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL DIGITAL CREATIONS OR ITS
+#   CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+#   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+#   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF
+#   USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+#   ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+#   OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
+#   OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+#   SUCH DAMAGE.
+# 
+# 
+# This software consists of contributions made by Digital Creations and
+# many individuals on behalf of Digital Creations.  Specific
+# attributions are listed in the accompanying credits file.
+# 
+##############################################################################
+
+import POSException, string
+
+from utils import p64, u64
+from referencesf import referencesf
+from cStringIO import StringIO
+from cPickle import Pickler, Unpickler
+
+class ExportImport:
+
+    def exportFile(self, oid, file=None):
+
+        if file is None: file=TemporaryFile()
+        elif type(file) is StringType: file=open(file,'w+b')
+        write=file.write
+        write('ZEXP')
+        version=self._version
+        ref=referencesf
+        oids=[oid]
+        done_oids={}
+        done=done_oids.has_key
+        load=self._storage.load
+        while oids:
+            oid=oids[0]
+            del oids[0]
+            try: p, serial = load(oid, version)
+            except: pass # Ick, a broken reference
+            else:
+                ref(p, oids)
+                write(oid)
+                write(p64(len(p)))
+                write(p)
+        write(export_end_marker)
+        return file
+
+    def importFile(self, file, clue=''):
+        # This is tricky, because we need to work in a transaction!
+
+        if type(file) is StringType:
+            file_name=file
+            file=open(file,'rb')
+        else:
+            try: file_name=file.name
+            except: file_name='(unknown)'
+
+        t=get_transaction().sub()
+
+        t.note('import into %s from %s' % (self.db().getName(), file_name))
+        if clue: t.note(clue)
+
+        storage=self._storage
+        new_oid=storage.new_oid
+        
+        read=file.read
+        if read(4) != 'ZEXP':
+            raise POSException.ExportError, 'Invalid export header'
+
+        oids={}
+        wrote_oid=oids.has_key
+        new_oid=storage.new_oid
+        store=storage.store
+
+        def persistent_load(ooid,
+                            Ghost=Ghost, StringType=StringType,
+                            atoi=string.atoi, TupleType=type(()),
+                            oids=oids, wrote_oid=wrote_oid, new_oid=new_oid):
+        
+            "Remap a persistent id to a new ID and create a ghost for it."
+
+            if type(ooid) is TupleType: ooid, klass = ooid
+            else: klass=None
+
+            if wrote_oid(ooid): oid=oids[ooid]
+            else:
+                if klass is None: oid=new_oid()
+                else: oid=new_oid(), klass
+                oids[ooid]=oid
+
+            Ghost=Ghost()
+            Ghost.oid=oid
+            return Ghost
+
+        version=self._version
+        return_oid=None
+
+        storage.tpc_begin(t)
+        try:
+            while 1:
+                h=read(16)
+                if h==export_end_marker: break
+                if len(h) != 16: raise ExportError, 'Truncated export file'
+                l=u64(h[8:16])
+                p=read(l)
+                if len(p) != l:  raise ExportError, 'Truncated export file'
+
+                ooid=h[:8]
+                if oids:
+                    oid=oids[ooid]
+                    if type(oid) is TupleType: oid=oid[0]
+                else: oids[ooid]=return_oid=oid=new_oid()
+
+                pfile=StringIO(p)
+                unpickler=Unpickler(pfile)
+                unpickler.persistent_load=persistent_load
+
+                newp=StringIO()
+                pickler=Pickler(newp,1)
+                pickler.persistent_id=persistent_id
+
+                pickler.dump(unpickler.load())
+                pickler.dump(unpickler.load())
+                p=newp.getvalue()
+                plen=len(p)
+
+                store(oid, None, p, version, t)
+                
+        except:
+            storage.tpc_abort(t)
+            raise
+        else:
+            storage.tpc_finish(t)
+            if return_oid is not None: return self[return_oid]
+            
+
+    ######################################################################
+    # BoboPOS 2 compat.
+
+    def export_file(self, o, file=None): return self.exportFile(o._p_oid, file)
+
+    import_file=importFile
+
+StringType=type('')
+
+def TemporaryFile():
+    # This is sneaky suicide
+    global TemporaryFile
+    import tempfile
+    TemporaryFile=tempfile.TemporaryFile
+    return TemporaryFile()
+
+export_end_marker='\377'*16
+
+class Ghost: pass
+
+def persistent_id(object, Ghost=Ghost):
+    if hasattr(object, '__class__') and object.__class__ is Ghost:
+        return object.oid
+    

src/ZODB/POSException.py

@@ -84,8 +84,8 @@
 ##############################################################################
 '''BoboPOS-defined exceptions
 
-$Id: POSException.py,v 1.2 1999/05/10 23:15:56 jim Exp $'''
-__version__='$Revision: 1.2 $'[11:-2]
+$Id: POSException.py,v 1.3 1999/05/18 15:55:10 jim Exp $'''
+__version__='$Revision: 1.3 $'[11:-2]
 
 
 class POSError(Exception):
@@ -129,3 +129,16 @@ class StorageSystemError(StorageError):
     """Panic! Internal storage error!
     """
 
+class ExportError(POSError):
+    """An export file doesn't have the right format.
+    """
+    pass
+
+class Unimplemented(POSError):
+    """An unimplemented feature was used
+    """
+    pass
+
+class Unsupported(POSError):
+    """An feature that is unsupported bt the storage was used.
+    """

src/ZODB/Transaction.py

@@ -84,8 +84,8 @@
 ##############################################################################
 """Transaction management
 
-$Id: Transaction.py,v 1.6 1999/05/12 15:55:44 jim Exp $"""
-__version__='$Revision: 1.6 $'[11:-2]
+$Id: Transaction.py,v 1.7 1999/05/18 15:55:10 jim Exp $"""
+__version__='$Revision: 1.7 $'[11:-2]
 
 import time, sys, struct
 from struct import pack
@@ -100,30 +100,50 @@ class Transaction:
     _connections=None
     _extension=None
 
-    def __init__(self,
-                 time=time.time, pack=struct.pack, gmtime=time.gmtime):
+    def __init__(self, id=None):
+        self._id=id
         self._objects=[]
         self._append=self._objects.append
-        self._note=self._user=self._description=''
+
+    def _init(self):
+        self._objects=[]
+        self._append=self._objects.append
+        self.user=self.description=''
         if self._connections:
             for c in self._connections.values(): c.close()
             del self._connections
 
-    def __str__(self): return "%.3f\t%s" % (self.time,self._note)
+    def sub(self):
+        r=self.__class__()
+        r.user=self.user
+        r.description=self.description
+        r._extention=self._extension
+        return r
+        
+    def __str__(self): return "%.3f\t%s" % (self._id, self.user)
 
     def abort(self, freeme=1):
-        'Abort the transaction.'
+        '''Abort the transaction.
+
+        This is called from the application.  This means that we haven\'t
+        entered two-phase commit yet, so no tpc_ messages are sent.
+        '''
         t=v=tb=None
         try:
+            # Abort the objects
             for o in self._objects:
                 try:
-                    if hasattr(o,'_p_jar'): o=o._p_jar
-                    if hasattr(o,'tpc_abort'): o.tpc_abort(self)
-                except: t,v,tb=sys.exc_info()
+                    j=getattr(o, '_p_jar', o)
+                    j.abort(o, self)
+                except:
+                    if t is None:
+                        t,v,tb=sys.exc_info()
+        
             if t is not None: raise t,v,tb
+
         finally:
             tb=None
-            if freeme: free_transaction()
+            if self._id is not None and freeme: free_transaction()
 
     def begin(self, info=None):
         '''Begin a new transaction.
@@ -131,40 +151,61 @@ class Transaction:
         This aborts any transaction in progres.
         '''
         if self._objects: self.abort(0)
-        self.__init__()
+        self._init()
         if info:
             info=split(info,'\t')
             self.user=strip(info[0])
-            self.description=strip(join(info,'\t'))
+            self.description=strip(join(info[1:],'\t'))
 
     def commit(self):
         'Finalize the transaction'
         
         t=v=tb=None
+        jars={}
+        objects=self._objects
         try:
             try:
-                for o in self._objects:
-                    if hasattr(o,'_p_jar'):
-                        j=o._p_jar
+                while objects:
+                    o=objects[-1]
+                    j=getattr(o, '_p_jar', o)
+                    i=id(j)
+                    if not jars.has_key(i):
+                        jars[i]=j
                         j.tpc_begin(self)
                     j.commit(o,self)
-                    elif hasattr(o,'tpc_begin'):
-                        o.tpc_begin(self)
+                    del objects[-1]
             except:
                 t,v,tb=sys.exc_info()
-                self.abort()
+
+                # Ugh, we got an got an error during commit, so we
+                # have to clean up.
+
+                # First, we have to abort any uncommitted objects.
+                for o in objects:
+                    try:
+                        j=getattr(o, '_p_jar', o)
+                        j.abort(o, self)
+                    except: pass
+
+                # Then, we unwind TPC for the jars that began it.
+                for j in jars.values():
+                    try: j.tpc_abort(self) # This should never fail
+                    except: pass
+
                 raise t,v,tb
 
-            for o in self._objects:
+            for j in jars.values():
                 try:
-                    if hasattr(o,'_p_jar'): o=o._p_jar
-                    if hasattr(o,'tpc_finish'): o.tpc_finish(self)
-                except: t,v,tb=sys.exc_info()
+                    j.tpc_finish(self) # This should never fail
+                except:
+                    if t is None:
+                        t,v,tb=sys.exc_info()
+                        
             if t is not None: raise t,v,tb
 
         finally:
             tb=None
-            free_transaction()
+            if self._id is not None: free_transaction()
 
     def register(self,object):
         'Register the given object for transaction control.'
@@ -191,11 +232,18 @@ class Transaction:
 
 try:
     import thread
+
+except:
+    _t=Transaction(None)
+    def get_transaction(_t=_t): return _t
+    def free_transaction(_t=_t): _t.__init__()
+
+else:
     _t={}
     def get_transaction(_id=thread.get_ident, _t=_t):
         id=_id()
         try: t=_t[id]
-        except KeyError: _t[id]=t=Transaction()
+        except KeyError: _t[id]=t=Transaction(id)
         return t
 
     def free_transaction(_id=thread.get_ident, _t=_t):
@@ -205,11 +253,6 @@ try:
 
     del thread
 
-except:
-    _t=Transaction()
-    def get_transaction(_t=_t): return _t
-    def free_transaction(_t=_t): _t.__init__()
-
 del _t
 
 import __main__ 

src/ZODB/referencesf.py

+##############################################################################
+# 
+# Zope Public License (ZPL) Version 1.0
+# -------------------------------------
+# 
+# Copyright (c) Digital Creations.  All rights reserved.
+# 
+# This license has been certified as Open Source(tm).
+# 
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met:
+# 
+# 1. Redistributions in source code must retain the above copyright
+#    notice, this list of conditions, and the following disclaimer.
+# 
+# 2. Redistributions in binary form must reproduce the above copyright
+#    notice, this list of conditions, and the following disclaimer in
+#    the documentation and/or other materials provided with the
+#    distribution.
+# 
+# 3. Digital Creations requests that attribution be given to Zope
+#    in any manner possible. Zope includes a "Powered by Zope"
+#    button that is installed by default. While it is not a license
+#    violation to remove this button, it is requested that the
+#    attribution remain. A significant investment has been put
+#    into Zope, and this effort will continue if the Zope community
+#    continues to grow. This is one way to assure that growth.
+# 
+# 4. All advertising materials and documentation mentioning
+#    features derived from or use of this software must display
+#    the following acknowledgement:
+# 
+#      "This product includes software developed by Digital Creations
+#      for use in the Z Object Publishing Environment
+#      (http://www.zope.org/)."
+# 
+#    In the event that the product being advertised includes an
+#    intact Zope distribution (with copyright and license included)
+#    then this clause is waived.
+# 
+# 5. Names associated with Zope or Digital Creations must not be used to
+#    endorse or promote products derived from this software without
+#    prior written permission from Digital Creations.
+# 
+# 6. Modified redistributions of any form whatsoever must retain
+#    the following acknowledgment:
+# 
+#      "This product includes software developed by Digital Creations
+#      for use in the Z Object Publishing Environment
+#      (http://www.zope.org/)."
+# 
+#    Intact (re-)distributions of any official Zope release do not
+#    require an external acknowledgement.
+# 
+# 7. Modifications are encouraged but must be packaged separately as
+#    patches to official Zope releases.  Distributions that do not
+#    clearly separate the patches from the original work must be clearly
+#    labeled as unofficial distributions.  Modifications which do not
+#    carry the name Zope may be packaged in any form, as long as they
+#    conform to all of the clauses above.
+# 
+# 
+# Disclaimer
+# 
+#   THIS SOFTWARE IS PROVIDED BY DIGITAL CREATIONS ``AS IS'' AND ANY
+#   EXPRESSED OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+#   IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+#   PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL DIGITAL CREATIONS OR ITS
+#   CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+#   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+#   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF
+#   USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+#   ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+#   OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
+#   OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+#   SUCH DAMAGE.
+# 
+# 
+# This software consists of contributions made by Digital Creations and
+# many individuals on behalf of Digital Creations.  Specific
+# attributions are listed in the accompanying credits file.
+# 
+##############################################################################
+"""Provide a function that can find object references in pickles
+"""
+import cPickle, cStringIO
+
+def referencesf(p,rootl,
+                Unpickler=cPickle.Unpickler,
+                StringIO=cStringIO.StringIO,
+                tt=type(()),
+                type=type):
+
+    u=Unpickler(StringIO(p))
+    l=len(rootl)
+    u.persistent_load=rootl
+    u.noload()
+    try: u.noload()
+    except:
+        # Hm.  We failed to do second load.  Maybe there wasn't a
+        # second pickle.  Let's check:
+        f=StringIO(p)
+        u=Unpickler(f)
+        u.persistent_load=[]
+        u.noload()
+        if len(p) > f.tell(): raise ValueError, 'Error unpickling, %s' % p
+
+    # References may have class info, so we need to
+    # check for wrapped references.
+    for i in range(l, len(rootl)):
+        v=rootl[i]
+        if v:
+            if type(v) is tt: v=v[0]
+            rootl[i]=v
+                

src/ZODB/utils.py

+##############################################################################
+# 
+# Zope Public License (ZPL) Version 1.0
+# -------------------------------------
+# 
+# Copyright (c) Digital Creations.  All rights reserved.
+# 
+# This license has been certified as Open Source(tm).
+# 
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met:
+# 
+# 1. Redistributions in source code must retain the above copyright
+#    notice, this list of conditions, and the following disclaimer.
+# 
+# 2. Redistributions in binary form must reproduce the above copyright
+#    notice, this list of conditions, and the following disclaimer in
+#    the documentation and/or other materials provided with the
+#    distribution.
+# 
+# 3. Digital Creations requests that attribution be given to Zope
+#    in any manner possible. Zope includes a "Powered by Zope"
+#    button that is installed by default. While it is not a license
+#    violation to remove this button, it is requested that the
+#    attribution remain. A significant investment has been put
+#    into Zope, and this effort will continue if the Zope community
+#    continues to grow. This is one way to assure that growth.
+# 
+# 4. All advertising materials and documentation mentioning
+#    features derived from or use of this software must display
+#    the following acknowledgement:
+# 
+#      "This product includes software developed by Digital Creations
+#      for use in the Z Object Publishing Environment
+#      (http://www.zope.org/)."
+# 
+#    In the event that the product being advertised includes an
+#    intact Zope distribution (with copyright and license included)
+#    then this clause is waived.
+# 
+# 5. Names associated with Zope or Digital Creations must not be used to
+#    endorse or promote products derived from this software without
+#    prior written permission from Digital Creations.
+# 
+# 6. Modified redistributions of any form whatsoever must retain
+#    the following acknowledgment:
+# 
+#      "This product includes software developed by Digital Creations
+#      for use in the Z Object Publishing Environment
+#      (http://www.zope.org/)."
+# 
+#    Intact (re-)distributions of any official Zope release do not
+#    require an external acknowledgement.
+# 
+# 7. Modifications are encouraged but must be packaged separately as
+#    patches to official Zope releases.  Distributions that do not
+#    clearly separate the patches from the original work must be clearly
+#    labeled as unofficial distributions.  Modifications which do not
+#    carry the name Zope may be packaged in any form, as long as they
+#    conform to all of the clauses above.
+# 
+# 
+# Disclaimer
+# 
+#   THIS SOFTWARE IS PROVIDED BY DIGITAL CREATIONS ``AS IS'' AND ANY
+#   EXPRESSED OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+#   IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+#   PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL DIGITAL CREATIONS OR ITS
+#   CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+#   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+#   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF
+#   USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+#   ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+#   OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
+#   OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+#   SUCH DAMAGE.
+# 
+# 
+# This software consists of contributions made by Digital Creations and
+# many individuals on behalf of Digital Creations.  Specific
+# attributions are listed in the accompanying credits file.
+# 
+##############################################################################
+
+import TimeStamp, time, struct
+
+t32 = 1L << 32
+
+def p64(v, pack=struct.pack):
+    if v < t32: h=0
+    else:
+        h=v/t32
+        v=v%t32
+    return pack(">II", h, v)
+
+def u64(v, unpack=struct.unpack):
+    h, v = unpack(">ii", v)
+    if v < 0: v=t32-v
+    if h:
+        if h < 0: h=t32-h
+        v=h*t32+v
+    return v
+
+def cp(f1, f2, l):
+    read=f1.read
+    write=f2.write
+    n=8192
+    
+    while l > 0:
+        if n > l: n=l
+        d=read(n)
+        write(d)
+        l = l - len(d)
+
+
+def newTimeStamp(old=None,
+                 TimeStamp=TimeStamp.TimeStamp,
+                 time=time.time, gmtime=time.gmtime):
+    t=time()
+    ts=TimeStamp(gmtime(t)[:5]+(t%60,))
+    if old is not None: return ts.laterThan(than)
+    
+