New Feature:

- There's a new utility script, strip_versions that strips version
  data from storages. This is needed to prepare databases containing
  version records for using ZODB 3.9, which no-longer supports
  versions.

NEWS.txt

-Next Release
-===========
+Whats new in ZODB 3.8.2
+=======================
+
+New Feature:
+
+- There's a new utility script, strip_versions that strips version
+  data from storages. This is needed to prepare databases containing
+  version records for using ZODB 3.9, which no-longer supports
+  versions.
 
 Bugs Fixed:
 

setup.py

@@ -56,6 +56,7 @@ except ImportError:
                    "src/ZODB/scripts/fstail.py",
                    "src/ZODB/scripts/fstest.py",
                    "src/ZODB/scripts/repozo.py",
+                   "src/ZODB/scripts/strip_versions.py",
                    "src/ZEO/scripts/zeopack.py",
                    "src/ZEO/scripts/runzeo.py",
                    "src/ZEO/scripts/zeopasswd.py",
@@ -71,6 +72,7 @@ else:
     fsrefs = ZODB.scripts.fsrefs:main
     fstail = ZODB.scripts.fstail:Main
     repozo = ZODB.scripts.repozo:main
+    strip_versions =  ZODB.scripts.strip_versions:main
     zeopack = ZEO.scripts.zeopack:main
     runzeo = ZEO.runzeo:main
     zeopasswd = ZEO.zeopasswd:main

src/ZODB/scripts/strip_versions.py

+##############################################################################
+#
+# Copyright Zope Foundation and Contributors.
+# All Rights Reserved.
+#
+# This software is subject to the provisions of the Zope Public License,
+# Version 2.1 (ZPL).  A copy of the ZPL should accompany this distribution.
+# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
+# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
+# FOR A PARTICULAR PURPOSE.
+#
+##############################################################################
+
+import optparse
+import sys
+import ZODB.BaseStorage
+import ZODB.config
+import ZODB.FileStorage
+
+parser = optparse.OptionParser("""%prog input output
+
+Remove version records by copying input to output, stripping any
+version records seen in input.
+
+The input and output arguments are file-storage file names.  If the -c
+option is used, then the input and output arguments must be storage
+configuration files.
+""")
+parser.add_option(
+    "-c", "--config", dest="config", action="store_true",
+    help=
+    'Treat the input and output names as configuration file names,'
+    ' rather than file-storage file names')
+
+class Record:
+
+    def __init__(self, transaction, it):
+        self.__transaction = transaction
+        self.__it = it
+
+    def __getattr__(self, name):
+        return getattr(self.__transaction, name)
+
+    def __iter__(self):
+        it = self.__it
+        for record in self.__transaction:
+            if record.version:
+                it.versioned += 1
+                continue  # Strip version records
+            it.copied += 1
+            if hasattr(record, 'data_txn'):
+                record.data_txn = None
+            yield record
+
+class Iterator:
+
+    versioned = copied = 0
+
+    def __init__(self, iterator):
+        self.close = iterator.close
+        def it():
+            for transaction in iterator:
+                yield Record(transaction, self)
+        self.__it = it()
+
+    def next(self):
+        return self.__it.next()
+
+    def __iter__(self):
+        return self
+    iterator = __iter__
+
+def main(args=None):
+    if args is None:
+        args = sys.argv[1:]
+
+    (options, args) = parser.parse_args(args)
+
+    if len(args) != 2:
+        parser.parse_args(['-h'])
+
+    input, output = args
+    toclose = []
+    if options.config:
+        input_storage = ZODB.config.storageFromFile(open(input))
+        it = input_storage.iterator()
+        toclose.append(input_storage)
+        output_storage = ZODB.config.storageFromFile(open(output))
+    else:
+        it  = ZODB.FileStorage.FileIterator(input)
+        output_storage = ZODB.FileStorage.FileStorage(output)
+    toclose.append(output_storage)
+
+    it = Iterator(it)
+    ZODB.BaseStorage.copy(it, output_storage)
+
+    for s in toclose:
+        s.close()
+
+    print 'Copied', it.copied, 'records.'
+    print 'Removed', it.versioned, 'versioned records.'

src/ZODB/scripts/strip_versions.test

+Strip version records from a FileStorage database.
+==================================================
+
+Older releases of ZODB supported versions.  ZODB versions 3.9 and
+later do not.  The strip_versions script strips version records from
+old data files.
+
+We have a demo data file that has some version data in it:
+
+>>> open('old.fs', 'wb').write("""\
+... RlMyMQOAAADPZRciAAAAAAAAAJwgAAAAGQAAaW5pdGlhbCBkYXRhYmFzZSBjcmVhdGlvbgAAAAAA
+... AAAAA4AAAM9lFyIAAAAAAAAAAAAAAAAAAAAEAAAAAAAAAAAAQihjcGVyc2lzdGVudC5tYXBwaW5n
+... ClBlcnNpc3RlbnRNYXBwaW5nCnEBTnQufXECVQpfY29udGFpbmVycQN9cQRzLgAAAAAAAACcA4AA
+... AM9mXgAAAAAAAAAA/iAAAAAAAAAAAAAAAAAAAAOAAADPZl4AAAAAAAAAADQAAAAAAAAAqAAAAAAA
+... AAAAAFRjcGVyc2lzdGVudC5tYXBwaW5nClBlcnNpc3RlbnRNYXBwaW5nCnEBLn1xAihVAWEoVQgA
+... AAAAAAAAAXEDaAF0UVUKX2NvbnRhaW5lcnEEfXEFdS4AAAAAAAAAAQOAAADPZl4AAAAAAAAAAAAA
+... AAAAAAAAqAAAAAAAAAAAAD9jcGVyc2lzdGVudC5tYXBwaW5nClBlcnNpc3RlbnRNYXBwaW5nCnEB
+... Ln1xAlUKX2NvbnRhaW5lcnEDfXEEcy4AAAAAAAAA/gOAAADPZ30zAAAAAAAAARIgAAAAAAAAAAAA
+... AAAAAAADgAAAz2d9MwAAAAAAAAC/AAAAAAAAAa4AAAAAAAAAAABoY3BlcnNpc3RlbnQubWFwcGlu
+... ZwpQZXJzaXN0ZW50TWFwcGluZwpxAS59cQIoVQFhKFUIAAAAAAAAAAFxA2gBdFFVCl9jb250YWlu
+... ZXJxBH1xBVUBYihVCAAAAAAAAAACcQZoAXRRdS4AAAAAAAAAAgOAAADPZ30zAAAAAAAAAAAAAAAA
+... AAABrgAAAAAAAAAAAD9jcGVyc2lzdGVudC5tYXBwaW5nClBlcnNpc3RlbnRNYXBwaW5nCnEBLn1x
+... AlUKX2NvbnRhaW5lcnEDfXEEcy4AAAAAAAABEgOAAADPaWndAAAAAAAAAJggAAAAAAAAAAAAAAAA
+... AAEDgAAAz2lp3QAAAAAAAAE9AAAAAAAAAsgAAgAAAAAAAABFAAAAAAAAAT0AAAAAAAAAAHYxY3Bl
+... cnNpc3RlbnQubWFwcGluZwpQZXJzaXN0ZW50TWFwcGluZwpxAS59cQJVCl9jb250YWluZXJxA31x
+... BFUBMUsBc3MuAAAAAAAAAJgDgAAAz2p4zAAAAAAAAAEoIAAAAAAAAAAAAAAAAAABA4AAAM9qeMwA
+... AAAAAAAC3wAAAAAAAANoAAIAAAAAAAAAWgAAAAAAAAE9AAAAAAAAAt92MWNwZXJzaXN0ZW50Lm1h
+... cHBpbmcKUGVyc2lzdGVudE1hcHBpbmcKcQEufXECKFUBYShVCAAAAAAAAAADcQNoAXRRVQpfY29u
+... dGFpbmVycQR9cQVVATFLAXN1LgAAAAAAAAADA4AAAM9qeMwAAAAAAAAAAAAAAAAAAANoAAIAAAAA
+... AAAAPwAAAAAAAAAAAAAAAAAAA392MWNwZXJzaXN0ZW50Lm1hcHBpbmcKUGVyc2lzdGVudE1hcHBp
+... bmcKcQEufXECVQpfY29udGFpbmVycQN9cQRzLgAAAAAAAAEoA4AAAM9sJXcAAAAAAAAAmCAAAAAA
+... AAAAAAAAAAAAAgOAAADPbCV3AAAAAAAAAlcAAAAAAAAEmAACAAAAAAAAAEUAAAAAAAACVwAAAAAA
+... AAAAdjJjcGVyc2lzdGVudC5tYXBwaW5nClBlcnNpc3RlbnRNYXBwaW5nCnEBLn1xAlUKX2NvbnRh
+... aW5lcnEDfXEEVQExSwFzcy4AAAAAAAAAmAOAAADPbTXMAAAAAAAAASggAAAAAAAAAAAAAAAAAAID
+... gAAAz201zAAAAAAAAASvAAAAAAAABTgAAgAAAAAAAABaAAAAAAAAAlcAAAAAAAAEr3YyY3BlcnNp
+... c3RlbnQubWFwcGluZwpQZXJzaXN0ZW50TWFwcGluZwpxAS59cQIoVQpfY29udGFpbmVycQN9cQRV
+... ATFLAXNVAWIoVQgAAAAAAAAABHEFaAF0UXUuAAAAAAAAAAQDgAAAz201zAAAAAAAAAAAAAAAAAAA
+... BTgAAgAAAAAAAAA/AAAAAAAAAAAAAAAAAAAFT3YyY3BlcnNpc3RlbnQubWFwcGluZwpQZXJzaXN0
+... ZW50TWFwcGluZwpxAS59cQJVCl9jb250YWluZXJxA31xBHMuAAAAAAAAASgDgAAAz26rZgAAAAAA
+... AAB7IAAAAAAAAAAAAAAAAAAEA4AAAM9uq2YAAAAAAAAF5QAAAAAAAAZoAAAAAAAAAAAAAAAAAAAA
+... AAXlAAAAAAAAAAIDgAAAz26rZgAAAAAAAAVPAAAAAAAABmgAAAAAAAAAAAAAAAAAAAAABU8AAAAA
+... AAAAewOAAADPcGhEAAAAAAAAAK0gAAAAAAAAAAAAAAAAAAIDgAAAz3BoRAAAAAAAAAaxAAAAAAAA
+... BusAAgAAAAAAAABaAAAAAAAABrEAAAAAAAAAAHYzY3BlcnNpc3RlbnQubWFwcGluZwpQZXJzaXN0
+... ZW50TWFwcGluZwpxAS59cQIoVQpfY29udGFpbmVycQN9cQRVATFLAnNVAWIoVQgAAAAAAAAABHEF
+... aAF0UXUuAAAAAAAAAK0DgAAAz3GSVQAAAAAAAAE8IAAAAAAAAAAAAAAAAAACA4AAAM9xklUAAAAA
+... AAAHAgAAAAAAAAegAAIAAAAAAAAAbgAAAAAAAAaxAAAAAAAABwJ2M2NwZXJzaXN0ZW50Lm1hcHBp
+... bmcKUGVyc2lzdGVudE1hcHBpbmcKcQEufXECKFUBYyhVCAAAAAAAAAAFcQNoAXRRVQFiKFUIAAAA
+... AAAAAARxBGgBdFFVCl9jb250YWluZXJxBX1xBlUBMUsCc3UuAAAAAAAAAAUDgAAAz3GSVQAAAAAA
+... AAAAAAAAAAAAB6AAAgAAAAAAAAA/AAAAAAAAAAAAAAAAAAAHt3YzY3BlcnNpc3RlbnQubWFwcGlu
+... ZwpQZXJzaXN0ZW50TWFwcGluZwpxAS59cQJVCl9jb250YWluZXJxA31xBHMuAAAAAAAAATwDgAAA
+... z3L6MwAAAAAAAAB7IAAAAAAAAAAAAAAAAAAFA4AAAM9y+jMAAAAAAAAIYQAAAAAAAAjkAAAAAAAA
+... AAAAAAAAAAAAAAAAAAAAAAAAAAIDgAAAz3L6MwAAAAAAAAe3AAAAAAAACOQAAAAAAAAAAAAAAAAA
+... AAAABrEAAAAAAAAAew==""".decode('base64'))
+
+Let's strip the versions:
+
+    >>> import ZODB.scripts.strip_versions
+    >>> ZODB.scripts.strip_versions.main('old.fs new.fs'.split())
+    Copied 9 records.
+    Removed 9 versioned records.
+
+    >>> import ZODB.FileStorage
+    >>> db = ZODB.DB(ZODB.FileStorage.FileStorage('new.fs'))
+
+    >>> conn = db.open()
+    >>> oids = sorted(db._storage._index)
+    >>> import ZODB.utils
+    >>> for oid in oids:
+    ...     try:
+    ...         ob = conn.get(oid)
+    ...     except ZODB.POSException.POSKeyError:
+    ...         continue
+    ...     print ZODB.utils.u64(oid), sorted(ob.items())
+    0 []
+    1 []
+    2 [('1', 1)]
+    4 []
+
+    >>> db.close()
+
+The example above assumed the old and new databases were both file
+storages. If we want to use some other storage formats, we need to
+specify the old and new databases using storage configuration files.
+If the -c (--config) option is used, then the arguments are treated as
+configuration file names.
+
+    >>> import os
+    >>> open('old.conf', 'w').write("""
+    ... <filestorage>
+    ...    path old.fs
+    ...    read-only true
+    ... </filestorage>
+    ... """)
+    >>> open('new.conf', 'w').write("""
+    ... <filestorage>
+    ...    path new2.fs
+    ... </filestorage>
+    ... """)
+
+    >>> ZODB.scripts.strip_versions.main('-c old.conf new.conf'.split())
+    Copied 9 records.
+    Removed 9 versioned records.
+
+    >>> db = ZODB.DB(ZODB.FileStorage.FileStorage('new.fs'))
+    >>> conn = db.open()
+    >>> oids = sorted(db._storage._index)
+    >>> import ZODB.utils
+    >>> for oid in oids:
+    ...     try:
+    ...         ob = conn.get(oid)
+    ...     except ZODB.POSException.POSKeyError:
+    ...         continue
+    ...     print ZODB.utils.u64(oid), sorted(ob.items())
+    0 []
+    1 []
+    2 [('1', 1)]
+    4 []
+
+    >>> db.close()

src/ZODB/scripts/tests.py

@@ -17,9 +17,12 @@ $Id$
 """
 import unittest
 from zope.testing import doctest
+import ZODB.tests.util
 
 def test_suite():
     return unittest.TestSuite((
-        doctest.DocFileSuite('referrers.txt'),
+        doctest.DocFileSuite('referrers.txt', 'strip_versions.test',
+            setUp=ZODB.tests.util.setUp, tearDown=ZODB.tests.util.tearDown,
+            ),
         ))