Commit 339f1884 authored by Kirill Smelkov's avatar Kirill Smelkov

wcfs: tests: Always start tDB with ZBigFile pre-created before WCFS startup

This should hopefully exercise codepaths in wcfs.go a bit more for
mistakes similar to a7bf0311 (wcfs: Fix crash if on invalidation
handledδZ needs to access ZODB) where the code on server side forgets to
put zhead's transaction into context.

Currently, because watching @tail is disallowed, this leads to panic triggered by test_wcfs_watch_setup:

    @at0 (03e59e3e606b89bb) -> @at1 (03e59e3e610692bb) -> @at2 (03e59e3e612a5811) -> @at3 (03e59e3e614fa9cc) -> @at4 (03e59e3e6189c3ee) -> @at5 (03e59e3e61af0baa)

    C: setup watch f<0000000000000002> @at0 (03e59e3e606b89bb)
    #  pinok: {0: @at0 (03e59e3e606b89bb), 2: @at0 (03e59e3e606b89bb), 3: @at0 (03e59e3e606b89bb), 5: @at0 (03e59e3e606b89bb)}
    panic: at out of bounds: at: @03e59e3e606b89bb,  (tail, head] = (@03e59e3e606b89bb, @03e59e3e61af0baa]

    goroutine 187 [running]:
    lab.nexedi.com/nexedi/wendelin.core/wcfs/internal/zdata.panicf(...)
            /home/kirr/src/neo/src/lab.nexedi.com/nexedi/wendelin.core/wcfs/internal/zdata/misc.go:47
    lab.nexedi.com/nexedi/wendelin.core/wcfs/internal/zdata.(*ΔFtail).BlkRevAt(0xc000077d40, {0x969718, 0xc000062940}, 0xc0003060c0, 0x4174f4, 0x3e59e3e606b89bb)
            /home/kirr/src/neo/src/lab.nexedi.com/nexedi/wendelin.core/wcfs/internal/zdata/δftail.go:1077 +0xa45
    main.(*WatchLink).setupWatch(0xc000108050, {0x969718, 0xc000062940}, 0x2, 0x3e59e3e606b89bb)
            /home/kirr/src/neo/src/lab.nexedi.com/nexedi/wendelin.core/wcfs/wcfs.go:1754 +0xe3f
    main.(*WatchLink)._handleWatch(0x0, {0x969718, 0xc000062940}, {0xc00001c812, 0xa00000})
            /home/kirr/src/neo/src/lab.nexedi.com/nexedi/wendelin.core/wcfs/wcfs.go:1973 +0x65
    main.(*WatchLink).handleWatch(0x74039b, {0x969718, 0xc000062940}, 0xc0000a4280, {0xc00001c812, 0x28})
            /home/kirr/src/neo/src/lab.nexedi.com/nexedi/wendelin.core/wcfs/wcfs.go:1955 +0x10c
    main.(*WatchLink)._serve.func3({0x969718, 0xc000062940})
            /home/kirr/src/neo/src/lab.nexedi.com/nexedi/wendelin.core/wcfs/wcfs.go:1944 +0x3c
    lab.nexedi.com/kirr/go123/xsync.(*WorkGroup).Go.func1()
            /home/kirr/src/neo/src/lab.nexedi.com/kirr/go123/xsync/xsync.go:86 +0x68
    created by lab.nexedi.com/kirr/go123/xsync.(*WorkGroup).Go
            /home/kirr/src/neo/src/lab.nexedi.com/kirr/go123/xsync/xsync.go:83 +0x92
    >>> Change history by file:

    f<0000000000000002>:
                                    0 1 2 3 4 5 6 7
                                    a b c d e f g h
            @at0 (03e59e3e606b89bb)
            @at1 (03e59e3e610692bb)     2
            @at2 (03e59e3e612a5811)     2 3 4 5
            @at3 (03e59e3e614fa9cc) 0   2     5
            @at4 (03e59e3e6189c3ee)     2   4 5
            @at5 (03e59e3e61af0baa)       3   5

However next we will anyway need to allow to setup watches @tail, and so
we will be fixing this and other errors in followup commits.

NOTE: we don't loose coverage for the case when ZBigFile is created after wcfs
startup due to test_wcfs_watch_2files, where that scenario is tested.

ΔFtail/ΔBtail tests also exercise ZBigFile/BTree epochs
(creation/deletion) well.
parent c07d771b
...@@ -403,10 +403,6 @@ class tDB(tWCFS): ...@@ -403,10 +403,6 @@ class tDB(tWCFS):
dbclose(t.root) dbclose(t.root)
defer(_) defer(_)
# start wcfs after testdb is created
super(tDB, t).__init__()
# ZBigFile(s) scheduled for commit # ZBigFile(s) scheduled for commit
t._changed = {} # ZBigFile -> {} blk -> data t._changed = {} # ZBigFile -> {} blk -> data
...@@ -414,17 +410,6 @@ class tDB(tWCFS): ...@@ -414,17 +410,6 @@ class tDB(tWCFS):
t.tail = t.root._p_jar.db().storage.lastTransaction() t.tail = t.root._p_jar.db().storage.lastTransaction()
t.dFtail = [] # of DF; head = dFtail[-1].rev t.dFtail = [] # of DF; head = dFtail[-1].rev
# fh(.wcfs/zhead) + history of zhead read from there
t._wc_zheadfh = open(t.wc.mountpoint + "/.wcfs/zhead")
# whether head/ ZBigFile(s) blocks were ever accessed via wcfs.
# this is updated only explicitly via ._blkheadaccess() .
t._blkaccessedViaHead = {} # ZBigFile -> set(blk)
# tracked opened tFiles & tWatchLinks
t._files = set()
t._wlinks = set()
# ID of the thread which created tDB # ID of the thread which created tDB
# ( transaction plays dirty games with threading.local and we have to # ( transaction plays dirty games with threading.local and we have to
# check the thread is the same when .root is used ) # check the thread is the same when .root is used )
...@@ -434,13 +419,27 @@ class tDB(tWCFS): ...@@ -434,13 +419,27 @@ class tDB(tWCFS):
if not _with_old_data: if not _with_old_data:
t.root['!file'] = t.nonzfile = Persistent() t.root['!file'] = t.nonzfile = Persistent()
t.root['zfile'] = t.zfile = ZBigFile(blksize) t.root['zfile'] = t.zfile = ZBigFile(blksize)
t.at0 = t.commit() t.at0 = t._commit()
else: else:
t.at0 = tAt(t, t.tail) t.at0 = tAt(t, t.tail)
t.nonzfile = t.root['!file'] t.nonzfile = t.root['!file']
t.zfile = t.root['zfile'] t.zfile = t.root['zfile']
# start wcfs after testdb is created
super(tDB, t).__init__()
# fh(.wcfs/zhead) + history of zhead read from there
t._wc_zheadfh = open(t.wc.mountpoint + "/.wcfs/zhead")
# whether head/ ZBigFile(s) blocks were ever accessed via wcfs.
# this is updated only explicitly via ._blkheadaccess() .
t._blkaccessedViaHead = {} # ZBigFile -> set(blk)
# tracked opened tFiles & tWatchLinks
t._files = set()
t._wlinks = set()
@property @property
def head(t): def head(t):
return t.dFtail[-1].rev return t.dFtail[-1].rev
...@@ -1801,6 +1800,8 @@ def test_wcfs_no_pin_twice(): ...@@ -1801,6 +1800,8 @@ def test_wcfs_no_pin_twice():
# verify watching for 2 files over single watch link. # verify watching for 2 files over single watch link.
#
# NOTE this test also verifies how wcfs handles ZBigFile created after wcfs startup.
@func @func
def test_wcfs_watch_2files(): def test_wcfs_watch_2files():
t = tDB(); zf1 = t.zfile t = tDB(); zf1 = t.zfile
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment