tests: Adjust testdata FileStorage for current Python on the fly

FileStorage/py2 uses `FS21` magic in file header, whereas
FileStorage/py3 uses `FS30` magic:

    https://github.com/zopefoundation/ZODB/blob/0e72b8b13657/src/ZODB/_compat.py#L39
    https://github.com/zopefoundation/ZODB/blob/0e72b8b13657/src/ZODB/_compat.py#L74

And if, upon opening the database, file magic does not match to what ZODB
expects, open is rejected:

    https://github.com/zopefoundation/ZODB/blob/0e72b8b13657/src/ZODB/FileStorage/FileStorage.py#L88
    https://github.com/zopefoundation/ZODB/blob/0e72b8b13657/src/ZODB/FileStorage/FileStorage.py#L1625-L1630

This is done with the idea for a database, that was written from
Python2, to be rejected to be opened from Python3 and vice-versa because
strings/bytes semantics changed in between py23.

As the result, many zodbtools tests currently fail on py3 when they try
to access prepared FileStorage database in testdata, because that
database was originally prepared on py2. Here is, for example, how
test_zodbdump fails:

    ___________________________ test_zodbdump[zext-raw] ____________________________

    zext = <function zext.<locals>._ at 0x7f28530bf9d0>, pretty = 'raw'

        @mark.parametrize('pretty', ('raw', 'zpickledis'))
        def test_zodbdump(zext, pretty):
            tdir  = dirname(__file__)
            zkind = '_!zext' if zext.disabled else ''
    >       stor  = FileStorage('%s/testdata/1%s.fs' % (tdir, zkind), read_only=True)

    zodbtools/test/test_dump.py:41:
    _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
    ../ZODB/src/ZODB/FileStorage/FileStorage.py:315: in __init__
        self._pos, self._oid, tid = read_index(
    _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _

    file = <_io.BufferedReader name='/home/kirr/src/wendelin/z/zodbtools/zodbtools/test/testdata/1.fs'>
    name = '/home/kirr/src/wendelin/z/zodbtools/zodbtools/test/testdata/1.fs'
    index = <ZODB.fsIndex.fsIndex object at 0x7f2852fee2b0>, tindex = {}
    stop = b'\xff\xff\xff\xff\xff\xff\xff\xff'
    ltid = b'\x00\x00\x00\x00\x00\x00\x00\x00', start = 4
    maxoid = b'\x00\x00\x00\x00\x00\x00\x00\x00', recover = 0, read_only = True

        def read_index(file, name, index, tindex, stop=b'\377'*8,
                       ltid=z64, start=4, maxoid=z64, recover=0, read_only=0):
            """Scan the file storage and update the index."""
            ...
            if file_size:
                if file_size < start:
                    raise FileStorageFormatError(file.name)
                seek(0)
                if read(4) != packed_version:
    >               raise FileStorageFormatError(name)
    E               ZODB.FileStorage.FileStorage.FileStorageFormatError: /home/kirr/src/wendelin/z/zodbtools/zodbtools/test/testdata/1.fs

    ../ZODB/src/ZODB/FileStorage/FileStorage.py:1630: FileStorageFormatError

Since zodbtools primarily work on raw data without decoding stored
pickles, unlike Zope or ERP5, it should not be a problem for zodbtools
to work on py3 with the database that was prepared on py2.

-> Adjust all tests to use FileStorage data generated on the fly based
on original files in testdata/ but with FileStorage header being
rewritten to match current python.

zodbtools/test/test_analyze.py

 # -*- coding: utf-8 -*-
-# Copyright (C) 2019 Nexedi SA and Contributors.
+# Copyright (C) 2019-2022 Nexedi SA and Contributors.
 #
 # This program is free software: you can Use, Study, Modify and Redistribute
 # it under the terms of the GNU General Public License version 3, or (at your
@@ -18,14 +18,18 @@
 # See https://www.nexedi.com/licensing for rationale and options.
 
 from zodbtools.zodbanalyze import analyze, report
+from zodbtools.test.testutil import fs1_testdata_py23
 import os.path
 
 
-def test_zodbanalyze(capsys):
+def test_zodbanalyze(tmpdir, capsys):
+    tfs1 = fs1_testdata_py23(tmpdir,
+                    os.path.join(os.path.dirname(__file__), "testdata", "1.fs"))
+
     for use_dbm in (False, True):
         report(
             analyze(
-                os.path.join(os.path.dirname(__file__), "testdata", "1.fs"),
+                tfs1,
                 use_dbm=use_dbm,
                 delta_fs=False,
                 tidmin=None,
@@ -40,7 +44,7 @@ def test_zodbanalyze(capsys):
     # csv output
     report(
         analyze(
-            os.path.join(os.path.dirname(__file__), "testdata", "1.fs"),
+            tfs1,
             use_dbm=False,
             delta_fs=False,
             tidmin=None,
@@ -61,7 +65,7 @@ __main__.Object,56,1880,54.366686%,33.571429,9,303,47,1577
     # empty range
     report(
         analyze(
-            os.path.join(os.path.dirname(__file__), "testdata", "1.fs"),
+            tfs1,
             use_dbm=False,
             delta_fs=False,
             tidmin="ffffffffffffffff",

zodbtools/test/test_dump.py

@@ -30,15 +30,16 @@ from io import BytesIO
 
 from os.path import dirname
 
-from zodbtools.test.testutil import zext_supported
+from zodbtools.test.testutil import zext_supported, fs1_testdata_py23
 from pytest import mark, raises, xfail
 
 # verify zodbdump output against golden
 @mark.parametrize('pretty', ('raw', 'zpickledis'))
-def test_zodbdump(zext, pretty):
+def test_zodbdump(tmpdir, zext, pretty):
     tdir  = dirname(__file__)
     zkind = '_!zext' if zext.disabled else ''
-    stor  = FileStorage('%s/testdata/1%s.fs' % (tdir, zkind), read_only=True)
+    tfs1  = fs1_testdata_py23(tmpdir, '%s/testdata/1%s.fs' % (tdir, zkind))
+    stor  = FileStorage(tfs1, read_only=True)
 
     with open('%s/testdata/1%s.zdump.%s.ok' % (tdir, zkind, pretty), 'rb') as f:
         dumpok = f.read()

zodbtools/test/test_restore.py

@@ -22,6 +22,7 @@ from __future__ import print_function
 
 from zodbtools.zodbrestore import zodbrestore
 from zodbtools.util import storageFromURL, readfile
+from zodbtools.test.testutil import fs1_testdata_py23
 
 from os.path import dirname
 from tempfile import mkdtemp
@@ -30,9 +31,7 @@ from golang import func, defer
 
 # verify zodbrestore.
 @func
-def test_zodbrestore(zext):
-    tmpd = mkdtemp('', 'zodbrestore.')
-    defer(lambda: rmtree(tmpd))
+def test_zodbrestore(tmpdir, zext):
     zkind = '_!zext' if zext.disabled else ''
 
     # restore from testdata/1.zdump.ok and verify it gives result that is
@@ -43,12 +42,12 @@ def test_zodbrestore(zext):
         zdump = open("%s/1%s.zdump.raw.ok" % (tdata, zkind), 'rb')
         defer(zdump.close)
 
-        stor = storageFromURL('%s/2.fs' % tmpd)
+        stor = storageFromURL('%s/2.fs' % tmpdir)
         defer(stor.close)
 
         zodbrestore(stor, zdump)
     _()
 
-    zfs1 = readfile("%s/1%s.fs" % (tdata, zkind))
-    zfs2 = readfile("%s/2.fs" % tmpd)
+    zfs1 = readfile(fs1_testdata_py23(tmpdir, "%s/1%s.fs" % (tdata, zkind)))
+    zfs2 = readfile("%s/2.fs" % tmpdir)
     assert zfs1 == zfs2

zodbtools/test/testutil.py

 #!/usr/bin/env python
 # -*- coding: utf-8 -*-
-# Copyright (C) 2019  Nexedi SA and Contributors.
+# Copyright (C) 2019-2022  Nexedi SA and Contributors.
 #                          Kirill Smelkov <kirr@nexedi.com>
 #
 # This program is free software: you can Use, Study, Modify and Redistribute
@@ -27,6 +27,10 @@ import transaction
 from tempfile import mkdtemp
 from shutil import rmtree
 from golang import func, defer
+from six import PY3
+from os.path import basename
+
+from zodbtools.util import readfile, writefile
 
 # zext_supported checks whether ZODB supports txn.extension_bytes .
 _zext_supported_memo = None
@@ -61,3 +65,19 @@ def _zext_supported():
 
     assert last_txn.extension == {'a': 'b'}
     return hasattr(last_txn, 'extension_bytes')
+
+
+# fs1_testdata_py23 prepares and returns path to temprary FileStorage prepared
+# from testdata with header adjusted to work on current Python.
+def fs1_testdata_py23(tmpdir, path):
+    data  = readfile(path)
+    index = readfile(path + ".index")
+    assert data[:4] == b"FS21"      # FileStorage magic for Python2
+    if PY3:
+        data = b"FS30" + data[4:]   # FileStorage magic for Python3
+
+    path_ = "%s/%s" % (tmpdir, basename(path))
+
+    writefile(path_, data)
+    writefile("%s.index" % path_, index)
+    return path_