Commit e796a2b6 authored by Kirill Smelkov's avatar Kirill Smelkov

X move test DB generation code to zodbtools/test

parent 3b1f7c94
...@@ -18,22 +18,13 @@ ...@@ -18,22 +18,13 @@
# #
# See COPYING file for full licensing terms. # See COPYING file for full licensing terms.
# See https://www.nexedi.com/licensing for rationale and options. # See https://www.nexedi.com/licensing for rationale and options.
# TODO move gen .fs & .index part to zodbtools and use it there for tests
# NOTE as of 14 Mar 2017 FileStorage cannot commit transactions with non-ASCII
# metadata - so it is not tested
"""generate reference database and index for tests""" """generate reference database and index for tests"""
from ZODB.FileStorage import FileStorage from ZODB.FileStorage import FileStorage
from ZODB import DB from zodbtools.test.gen_testdata import gen_testdb
from ZODB.POSException import UndoError
from persistent import Persistent
import transaction
from zodbtools.util import escapeqq from zodbtools.util import escapeqq
import struct import struct
import time
import random
import logging
# convert numeric oid to/from str # convert numeric oid to/from str
def p64(num): def p64(num):
...@@ -45,143 +36,13 @@ def unpack64(packed): ...@@ -45,143 +36,13 @@ def unpack64(packed):
def hex64(packed): def hex64(packed):
return '0x%016x' % unpack64(packed) return '0x%016x' % unpack64(packed)
# make time.time() predictable
_xtime = time.mktime(time.strptime("04 Jan 1979", "%d %b %Y"))
def xtime():
global _xtime
_xtime += 1.1
return _xtime
time.time = xtime
# prepare transaction for a commit
def precommit(user, description, extension):
txn = transaction.get()
txn.user = user
txn.description = description
txn.extension = extension
return txn
def commit(user, description, extension):
txn = precommit(user, description, extension)
txn.commit()
class Object(Persistent):
# .value
def __init__(self, value):
self.value = value
def __getstate__(self):
return self.value
def __setstate__(self, state):
self.value = state
# prepare extension dictionary for subject
alnum = "0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ"
def ext(subj):
d = {"x-generator": "zodb/py2 (%s)" % subj}
# also add some random 'x-cookie'
cooklen = 5
cookie = ""
for _ in range(cooklen):
cookie += random.choice(alnum)
xcookie = "x-cookie" + random.choice(alnum)
d[xcookie] = cookie
# shufle extension dict randomly - to likely trigger different ordering on save
keyv = d.keys()
random.shuffle(keyv)
ext = {}
for key in keyv:
ext[key] = d[key]
return ext
def main(): def main():
logging.basicConfig()
outfs = "testdata/1.fs" outfs = "testdata/1.fs"
gen_testdb(outfs)
# generate random changes to objects hooked to top-level root by a/b/c/... key
random.seed(0)
namev = [_ for _ in "abcdefg"]
Niter = 2
for i in range(Niter):
stor = FileStorage(outfs, create=(i == 0))
db = DB(stor)
conn = db.open()
root = conn.root()
assert root._p_oid == p64(0), `root._p_oid`
for j in range(25):
name = random.choice(namev)
if name in root:
obj = root[name]
else:
root[name] = obj = Object(None)
obj.value = "%s%i.%i" % (name, i, j)
commit(u"user%i.%i" % (i,j), u"step %i.%i" % (i, j), ext(name))
# undo a transaction one step before a latest one a couple of times
for j in range(2):
# XXX undoLog, despite what its interface says:
# https://github.com/zopefoundation/ZODB/blob/2490ae09/src/ZODB/interfaces.py#L472
# just returns log of all transactions in specified range:
# https://github.com/zopefoundation/ZODB/blob/2490ae09/src/ZODB/FileStorage/FileStorage.py#L1008
# https://github.com/zopefoundation/ZODB/blob/2490ae09/src/ZODB/FileStorage/FileStorage.py#L2103
# so we retry undoing next log's txn on conflict.
for ul in db.undoLog(1, 20):
try:
db.undo(ul["id"])
commit(u"root%i.%i\nYour\nMagesty " % (i, j),
u"undo %i.%i\nmore detailed description\n\nzzz ..." % (i, j) + "\t"*(i+j),
ext("undo %s" % ul["id"]))
except UndoError:
transaction.abort()
continue
break
# delete an object
name = random.choice(root.keys())
obj = root[name]
root[name] = Object("%s%i*" % (name, i))
# NOTE user/ext are kept empty on purpose - to also test this case
commit(u"", u"predelete %s" % unpack64(obj._p_oid), {})
# XXX obj in db could be changed by above undo, but ZODB does not automatically
# propagate undo changes to live objects - so obj._p_serial can be stale.
# Get serial via history.
obj_tid_lastchange = db.history(obj._p_oid)[0]['tid']
txn = precommit(u"root%i\nYour\nRoyal\nMagesty' " % i +
''.join(chr(_) for _ in range(32)), # <- NOTE all control characters
u"delete %i\nalpha beta gamma'delta\"lambda\n\nqqq ..." % i,
ext("delete %s" % unpack64(obj._p_oid)))
stor.tpc_begin(txn)
stor.deleteObject(obj._p_oid, obj_tid_lastchange, txn)
stor.tpc_vote(txn)
# TODO different txn status vvv
# XXX vvv it does the thing, but py fs iterator treats this txn as EOF
#if i != Niter-1:
# stor.tpc_finish(txn)
stor.tpc_finish(txn)
# close db & rest not to get conflict errors after we touched stor
# directly a bit. everything will be reopened on next iteration.
conn.close()
db.close()
stor.close()
# dump to go what to expect # dump to go what to expect
stor = FileStorage(outfs, read_only=True)
with open("ztestdata_expect_test.go", "w") as f: with open("ztestdata_expect_test.go", "w") as f:
def emit(v): def emit(v):
print >>f, v print >>f, v
...@@ -197,8 +58,6 @@ def main(): ...@@ -197,8 +58,6 @@ def main():
emit("}") emit("}")
# database records # database records
stor = FileStorage(outfs, read_only=True)
emit("\nvar _1fs_dbEntryv = [...]dbEntry{") emit("\nvar _1fs_dbEntryv = [...]dbEntry{")
txnLenPrev = -1 txnLenPrev = -1
for txn in stor.iterator(): # txn is TransactionRecord for txn in stor.iterator(): # txn is TransactionRecord
......
...@@ -47,6 +47,8 @@ Dump format: ...@@ -47,6 +47,8 @@ Dump format:
quote: quote string with " with non-printable and control characters \-escaped quote: quote string with " with non-printable and control characters \-escaped
hashfunc: one of sha1, sha256, sha512 ... hashfunc: one of sha1, sha256, sha512 ...
TODO also protect txn record by hash.
*/ */
package zodbtools package zodbtools
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment