X Hit FUSE-releated deadlock

See details in added comments.

The bug shows itself as e.g.

I0311 14:31:56.593078    6433 wcfs.go:787] zwatcher: zevent: commit @03d673b3f16ffe99 δ: [0000000000000003]
D0311 14:31:56.593190    6421 wcfs/client/wcfs.cpp:615] wcfs /tmp/wcfs/96ae146745cf6b7225160689a6e8ed97d24aebea: conn13 @03d673b3ef93e6cc: resync -> @03d673b3f16ffe99:
D0311 14:31:56.593262    6421 wcfs/client/wcfs.cpp:252] wcfs /tmp/wcfs/96ae146745cf6b7225160689a6e8ed97d24aebea: headWait @03d673b3f16ffe99:
wcfs: 14:31:56.593426 tx 0:     NOTIFY_RETRIEVE_CACHE, {> 544: i9 [0 +131072)}
wcfs: 14:31:56.593449 Response: NOTIFY_RETRIEVE_CACHE: OK

wcfs: 14:31:56.593466 rx 392: LOOKUP i1 ["head"] 5b			<-- NOTE stuck

wcfs: 14:31:56.593504 rx 544: NOTIFY_REPLY i9 {[0 +131072)}  131072b
wcfs: 14:31:56.593552 tx 0:     NOTIFY_RETRIEVE_CACHE, {> 545: i9 [131072 +131072)}
wcfs: 14:31:56.593571 Response: NOTIFY_RETRIEVE_CACHE: OK
wcfs: 14:31:56.593636 rx 545: NOTIFY_REPLY i9 {[131072 +131072)}  131072b
wcfs: 14:31:56.593744 tx 0:     NOTIFY_RETRIEVE_CACHE, {> 546: i9 [262144 +131072)}
wcfs: 14:31:56.593761 Response: NOTIFY_RETRIEVE_CACHE: OK
wcfs: 14:31:56.593809 rx 546: NOTIFY_REPLY i9 {[262144 +131072)}  131072b
wcfs: 14:31:56.593840 tx 0:     NOTIFY_RETRIEVE_CACHE, {> 547: i9 [393216 +131072)}
wcfs: 14:31:56.593856 Response: NOTIFY_RETRIEVE_CACHE: OK
wcfs: 14:31:56.593922 rx 547: NOTIFY_REPLY i9 {[393216 +131072)}  131072b
wcfs: 14:31:56.593988 tx 0:     NOTIFY_RETRIEVE_CACHE, {> 548: i9 [524288 +131072)}
wcfs: 14:31:56.594005 Response: NOTIFY_RETRIEVE_CACHE: OK
wcfs: 14:31:56.594062 rx 548: NOTIFY_REPLY i9 {[524288 +131072)}  131072b
wcfs: 14:31:56.594116 tx 0:     NOTIFY_RETRIEVE_CACHE, {> 549: i9 [655360 +131072)}
wcfs: 14:31:56.594127 Response: NOTIFY_RETRIEVE_CACHE: OK
wcfs: 14:31:56.594180 rx 549: NOTIFY_REPLY i9 {[655360 +131072)}  131072b
wcfs: 14:31:56.594241 tx 0:     NOTIFY_RETRIEVE_CACHE, {> 550: i9 [786432 +131072)}
wcfs: 14:31:56.594257 Response: NOTIFY_RETRIEVE_CACHE: OK
wcfs: 14:31:56.594299 rx 550: NOTIFY_REPLY i9 {[786432 +131072)}  131072b
wcfs: 14:31:56.594337 tx 0:     NOTIFY_RETRIEVE_CACHE, {> 551: i9 [917504 +131072)}
wcfs: 14:31:56.594348 Response: NOTIFY_RETRIEVE_CACHE: OK
wcfs: 14:31:56.594384 rx 551: NOTIFY_REPLY i9 {[917504 +131072)}  131072b
wcfs: 14:31:56.594450 tx 0:     NOTIFY_RETRIEVE_CACHE, {> 552: i9 [1048576 +131072)}
wcfs: 14:31:56.594464 Response: NOTIFY_RETRIEVE_CACHE: OK
wcfs: 14:31:56.594520 rx 552: NOTIFY_REPLY i9 {[1048576 +131072)}  131072b
wcfs: 14:31:56.594558 tx 0:     NOTIFY_RETRIEVE_CACHE, {> 553: i9 [1179648 +131072)}
wcfs: 14:31:56.594568 Response: NOTIFY_RETRIEVE_CACHE: OK
wcfs: 14:31:56.594622 rx 553: NOTIFY_REPLY i9 {[1179648 +131072)}  131072b
wcfs: 14:31:56.594666 tx 0:     NOTIFY_RETRIEVE_CACHE, {> 554: i9 [1310720 +131072)}
wcfs: 14:31:56.594677 Response: NOTIFY_RETRIEVE_CACHE: OK
wcfs: 14:31:56.594805 rx 554: NOTIFY_REPLY i9 {[1310720 +131072)}  131072b
wcfs: 14:31:56.594839 tx 0:     NOTIFY_RETRIEVE_CACHE, {> 555: i9 [1441792 +131072)}
wcfs: 14:31:56.594849 Response: NOTIFY_RETRIEVE_CACHE: OK
wcfs: 14:31:56.594885 rx 555: NOTIFY_REPLY i9 {[1441792 +131072)}  131072b
wcfs: 14:31:56.594924 tx 0:     NOTIFY_RETRIEVE_CACHE, {> 556: i9 [1572864 +131072)}
wcfs: 14:31:56.594941 Response: NOTIFY_RETRIEVE_CACHE: OK
wcfs: 14:31:56.594989 rx 556: NOTIFY_REPLY i9 {[1572864 +131072)}  131072b
wcfs: 14:31:56.595029 tx 0:     NOTIFY_RETRIEVE_CACHE, {> 557: i9 [1703936 +131072)}
wcfs: 14:31:56.595048 Response: NOTIFY_RETRIEVE_CACHE: OK
wcfs: 14:31:56.595086 rx 557: NOTIFY_REPLY i9 {[1703936 +131072)}  131072b
wcfs: 14:31:56.595140 tx 0:     NOTIFY_RETRIEVE_CACHE, {> 558: i9 [1835008 +131072)}
wcfs: 14:31:56.595151 Response: NOTIFY_RETRIEVE_CACHE: OK
wcfs: 14:31:56.595201 rx 558: NOTIFY_REPLY i9 {[1835008 +131072)}  131072b
wcfs: 14:31:56.595256 tx 0:     NOTIFY_RETRIEVE_CACHE, {> 559: i9 [1966080 +131072)}
wcfs: 14:31:56.595268 Response: NOTIFY_RETRIEVE_CACHE: OK
wcfs: 14:31:56.595336 rx 559: NOTIFY_REPLY i9 {[1966080 +131072)}  131072b
wcfs: 14:31:56.595396 tx 0:     NOTIFY_RETRIEVE_CACHE, {> 560: i9 [2097152 +0)}
wcfs: 14:31:56.595405 Response: NOTIFY_RETRIEVE_CACHE: OK
wcfs: 14:31:56.595413 rx 560: NOTIFY_REPLY i9 {[2097152 +0)}

wcfs: 14:31:56.595509 rx 393: LOOKUP i1 ["@03d673b3ef93e6cc"] 18b	<-- NOTE stuck

wcfs/wcfs.go

@@ -1042,6 +1042,20 @@ func (f *BigFile) invalidateBlk(ctx context.Context, blk int64) (err error) {
 	if int64(len(blkdata)) == blksize {
 		func() {
 			// store retrieved data back to OS cache for file @<rev>/file[blk]
+			//
+			// FIXME there is deadlock here:
+			//
+			// T1: Lookup "head"
+			//	-> nodefs.rawBridge.lookupLock.Lock
+			//	-> zheadMu.RLock
+			//	   `-> block (waiting for handleδZ to WUnlock zheadMu)
+			//
+			// T2 (this thread): Lookup "@<revX>"
+			//	-> nodefs.rawBridge.lookupLock.Lock
+			//	   `-> block (waiting for T1)
+			//
+			// see https://github.com/hanwen/go-fuse/commit/d0fca860
+			// for when rawBridge.lookupLock was introduced.
 			blkrev, _ := f.LastBlkRev(ctx, blk, f.head.zconn.At())
 			frev, funlock, err := groot.lockRevFile(blkrev, f.zfile.POid())
 			if err != nil {