Merge branch 't' into t2

* t: (129 commits)
  X go.mod: v↑ *
  go/zodb/btree: Change V<op> family to also provide visited node key coverage on visit callback
  go/zodb/btree: Add KeyRange type
  go/zodb/btree: Introduce constants for min/max key value
  go/zodb/btree: tests: Don't forget to close storage
  go/zodb/btree: Cosmetics
  .
  .
  .
  .
  .
  .
  .
  X Start reworking BTree to provide keycov on visit callback
  X go/neo: Fix credentials parsing with go1.17
  fixup! Y client: Fix URI scheme to move credentials out of query
  go/internal/xtesting: Add missing X
  go/zodb/{fs1,zeo}: ~staticcheck
  go/zodb/btree: Fix missing return on data-consistency error
  go/zodb, go/zodb/btree: Fix go generate after rename on zodbtools side
  ...

go/go.mod

@@ -3,24 +3,36 @@ module lab.nexedi.com/kirr/neo/go
 go 1.14
 
 require (
-	github.com/DataDog/czlib v0.0.0-20160811164712-4bc9a24e37f2
+	github.com/DataDog/czlib v0.0.0-20210322182103-8087f4e14ae7
 	github.com/cznic/mathutil v0.0.0-20181122101859-297441e03548 // indirect
 	github.com/cznic/strutil v0.0.0-20181122101858-275e90344537
-	github.com/fsnotify/fsnotify v1.4.10-0.20200417215612-7f4cf4dd2b52
-	github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b
-	github.com/gwenn/gosqlite v0.0.0-20200521090053-24878be1a237
-	github.com/kisielk/og-rek v1.0.1-0.20180928202415-8b25c4cefd6c
+	github.com/davecgh/go-spew v1.1.1 // indirect
+	github.com/fsnotify/fsnotify v1.5.1
+	github.com/golang/glog v1.0.0
+	github.com/golang/protobuf v1.4.3 // indirect
+	github.com/google/go-cmp v0.5.4 // indirect
+	github.com/gwenn/gosqlite v0.0.0-20211101095637-b18efb2e44c8
+	github.com/gwenn/yacr v0.0.0-20211101095056-492fb0c571bc // indirect
+	github.com/kisielk/og-rek v1.2.0
+	github.com/kr/text v0.2.0 // indirect
 	github.com/kylelemons/godebug v1.1.0
-	github.com/philhofer/fwd v1.0.0 // indirect
+	github.com/philhofer/fwd v1.1.1
 	github.com/pkg/errors v0.9.1
 	github.com/remyoudompheng/bigfft v0.0.0-20200410134404-eec4a21b6bb0 // indirect
-	github.com/shamaton/msgpack v1.1.1
-	github.com/soheilhy/cmux v0.1.4
+	github.com/shamaton/msgpack v1.2.1
+	github.com/soheilhy/cmux v0.1.5
 	github.com/someonegg/gocontainer v1.0.0
-	github.com/stretchr/testify v1.6.1
-	github.com/tinylib/msgp v1.1.3-0.20200327023543-e88e92c0ccca
-	golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208
-	google.golang.org/grpc v1.32.0 // indirect
-	google.golang.org/grpc/examples v0.0.0-20200915000551-32e7099cccac // indirect
-	lab.nexedi.com/kirr/go123 v0.0.0-20200915142026-a281a51cf49b
+	github.com/stretchr/testify v1.7.0
+	github.com/tinylib/msgp v1.1.6
+	github.com/ttacon/chalk v0.0.0-20160626202418-22c06c80ed31 // indirect
+	golang.org/x/net v0.0.0-20211111160137-58aab5ef257a // indirect
+	golang.org/x/sync v0.0.0-20210220032951-036812b2e83c // indirect
+	golang.org/x/sys v0.0.0-20211111213525-f221eed1c01e // indirect
+	golang.org/x/text v0.3.7 // indirect
+	google.golang.org/genproto v0.0.0-20210226172003-ab064af71705 // indirect
+	google.golang.org/grpc v1.36.0 // indirect
+	google.golang.org/grpc/examples v0.0.0-20210301210255-fc8f38cccf75 // indirect
+	gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c // indirect
+	gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b // indirect
+	lab.nexedi.com/kirr/go123 v0.0.0-20210906140734-c9eb28d9e408
 )

go/go.sum

 cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
 cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
-cloud.google.com/go v0.38.0/go.mod h1:990N+gfupTy94rShfmMCWGDn0LpTmnzTp2qbd1dvSRU=
-cloud.google.com/go v0.44.1/go.mod h1:iSa0KzasP4Uvy3f1mN/7PiObzGgflwredwwASm/v6AU=
-cloud.google.com/go v0.44.2/go.mod h1:60680Gw3Yr4ikxnPRS/oxxkBccT6SA1yMk63TGekxKY=
-cloud.google.com/go v0.45.1/go.mod h1:RpBamKRgapWJb87xiFSdk4g1CME7QZg3uwTez+TSTjc=
-cloud.google.com/go v0.46.3/go.mod h1:a6bKKbmY7er1mI7TEI4lsAkts/mkhTSZK8w33B4RAg0=
-cloud.google.com/go v0.50.0/go.mod h1:r9sluTvynVuxRIOHXQEHMFffphuXHOMZMycpNR5e6To=
-cloud.google.com/go v0.52.0/go.mod h1:pXajvRH/6o3+F9jDHZWQ5PbGhn+o8w9qiu/CffaVdO4=
-cloud.google.com/go v0.53.0/go.mod h1:fp/UouUEsRkN6ryDKNW/Upv/JBKnv6WDthjR6+vze6M=
-cloud.google.com/go v0.54.0/go.mod h1:1rq2OEkV3YMf6n/9ZvGWI3GWw0VoqH/1x2nd8Is/bPc=
-cloud.google.com/go v0.56.0/go.mod h1:jr7tqZxxKOVYizybht9+26Z/gUq7tiRzu+ACVAMbKVk=
-cloud.google.com/go v0.57.0/go.mod h1:oXiQ6Rzq3RAkkY7N6t3TcE6jE+CIBBbA36lwQ1JyzZs=
-cloud.google.com/go v0.62.0/go.mod h1:jmCYTdRCQuc1PHIIJ/maLInMho30T/Y0M4hTdTShOYc=
-cloud.google.com/go v0.63.0/go.mod h1:GmezbQc7T2snqkEXWfZ0sy0VfkB/ivI2DdtJL2DEmlg=
-cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o=
-cloud.google.com/go/bigquery v1.3.0/go.mod h1:PjpwJnslEMmckchkHFfq+HTD2DmtT67aNFKH1/VBDHE=
-cloud.google.com/go/bigquery v1.4.0/go.mod h1:S8dzgnTigyfTmLBfrtrhyYhwRxG72rYxvftPBK2Dvzc=
-cloud.google.com/go/bigquery v1.5.0/go.mod h1:snEHRnqQbz117VIFhE8bmtwIDY80NLUZUMb4Nv6dBIg=
-cloud.google.com/go/bigquery v1.7.0/go.mod h1://okPTzCYNXSlb24MZs83e2Do+h+VXtc4gLoIoXIAPc=
-cloud.google.com/go/bigquery v1.8.0/go.mod h1:J5hqkt3O0uAFnINi6JXValWIb1v0goeZM77hZzJN/fQ=
-cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE=
-cloud.google.com/go/datastore v1.1.0/go.mod h1:umbIZjpQpHh4hmRpGhH4tLFup+FVzqBi1b3c64qFpCk=
-cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I=
-cloud.google.com/go/pubsub v1.1.0/go.mod h1:EwwdRX2sKPjnvnqCa270oGRyludottCI76h+R3AArQw=
-cloud.google.com/go/pubsub v1.2.0/go.mod h1:jhfEVHT8odbXTkndysNHCcx0awwzvfOlguIAii9o8iA=
-cloud.google.com/go/pubsub v1.3.1/go.mod h1:i+ucay31+CNRpDW4Lu78I4xXG+O1r/MAHgjpRVR+TSU=
-cloud.google.com/go/storage v1.0.0/go.mod h1:IhtSnM/ZTZV8YYJWCY8RULGVqBDmpoyjwiyrjsg+URw=
-cloud.google.com/go/storage v1.5.0/go.mod h1:tpKbwo567HUNpVclU5sGELwQWBDZ8gh0ZeosJ0Rtdos=
-cloud.google.com/go/storage v1.6.0/go.mod h1:N7U0C8pVQ/+NIKOBQyamJIeKQKkZ+mxpohlUTyfDhBk=
-cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RXyy7KQOVs=
-cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0=
 crawshaw.io/iox v0.0.0-20181124134642-c51c3df30797/go.mod h1:sXBiorCo8c46JlQV3oXPKINnZ8mcqnye1EkVkqsectk=
+crawshaw.io/sqlite v0.3.2 h1:N6IzTjkiw9FItHAa0jp+ZKC6tuLzXqAYIv+ccIWos1I=
 crawshaw.io/sqlite v0.3.2/go.mod h1:igAO5JulrQ1DbdZdtVq48mnZUBAPOeFzer7VhDWNtW4=
-dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU=
 github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
-github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo=
 github.com/DataDog/czlib v0.0.0-20160811164712-4bc9a24e37f2 h1:/Xf0UwdF1dlv6cNu0q1TWTQgfSsALgXrdHNtBMj7ybU=
 github.com/DataDog/czlib v0.0.0-20160811164712-4bc9a24e37f2/go.mod h1:xK7es18l1aQtR1g0i6RhEQ78ckvRAWiwlG0vBFud/Jk=
+github.com/DataDog/czlib v0.0.0-20210322182103-8087f4e14ae7 h1:6ZJZdzkbvKb6HRXmZ12ICZ0IbqfR+0Cd2C0IutWHHIA=
+github.com/DataDog/czlib v0.0.0-20210322182103-8087f4e14ae7/go.mod h1:ROY4muaTWpoeQAx/oUkvxe9zKCmgU5xDGXsfEbA+omc=
 github.com/bmizerany/assert v0.0.0-20160611221934-b7ed37b82869 h1:DDGfHa7BWjL4YnC6+E63dPcxHo2sUxDIu8g3QgEJdRY=
 github.com/bmizerany/assert v0.0.0-20160611221934-b7ed37b82869/go.mod h1:Ekp36dRnpXw/yCqJaO+ZrUyxD+3VXMFFr56k5XYrpB4=
 github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
-github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI=
-github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI=
-github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU=
 github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
 github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc=
+github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk=
+github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
 github.com/cznic/mathutil v0.0.0-20181122101859-297441e03548 h1:iwZdTE0PVqJCos1vaoKsclOGD3ADKpshg3SRtYBbwso=
 github.com/cznic/mathutil v0.0.0-20181122101859-297441e03548/go.mod h1:e6NPNENfs9mPDVNRekM7lKScauxd5kXTr1Mfyig6TDM=
 github.com/cznic/strutil v0.0.0-20181122101858-275e90344537 h1:MZRmHqDBd0vxNwenEbKSQqRVT24d3C05ft8kduSwlqM=
 github.com/cznic/strutil v0.0.0-20181122101858-275e90344537/go.mod h1:AHHPPPXTw0h6pVabbcbyGRK1DckRn7r/STdZEeIDzZc=
-github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
+github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
 github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
 github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98=
+github.com/envoyproxy/go-control-plane v0.9.9-0.20201210154907-fd9021fe5dad/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk=
 github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c=
 github.com/fsnotify/fsnotify v1.4.10-0.20200417215612-7f4cf4dd2b52 h1:0NmERxogGTU8hgzOhRKNoKivtBZkDW29GeuJtK9e0sc=
 github.com/fsnotify/fsnotify v1.4.10-0.20200417215612-7f4cf4dd2b52/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ=
-github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU=
-github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8=
-github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8=
+github.com/fsnotify/fsnotify v1.5.1 h1:mZcQUHVQUQWoPXXtuf9yuEXKudkV2sx1E06UadKWpgI=
+github.com/fsnotify/fsnotify v1.5.1/go.mod h1:T3375wBYaZdLLcVNkcVbzGHY7f1l/uK5T5Ai1i3InKU=
 github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b h1:VKtxabqXZkF25pY9ekfRL6a582T4P37/31XEstQ5p58=
 github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
-github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
-github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
-github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
+github.com/golang/glog v1.0.0 h1:nfP3RFugxnNRyKgeWd4oI1nYvXpxrx8ck8ZrcizshdQ=
+github.com/golang/glog v1.0.0/go.mod h1:EWib/APOK0SL3dFbYqvxE3UYd8E6s1ouQ7iEp/0LWV4=
 github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
-github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
-github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFUx0Y=
-github.com/golang/mock v1.4.0/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw=
-github.com/golang/mock v1.4.1/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw=
-github.com/golang/mock v1.4.3/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw=
-github.com/golang/mock v1.4.4/go.mod h1:l3mdAwkq5BuhzHwde/uurv3sEJeZMXNpwsxVWU71h+4=
 github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
-github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
 github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
 github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw=
-github.com/golang/protobuf v1.3.4/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw=
-github.com/golang/protobuf v1.3.5/go.mod h1:6O5/vntMXwX2lRkT1hjjk0nAC1IDOTvTlVgjlRvqsdk=
 github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8=
 github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA=
 github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs=
 github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w=
 github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0=
 github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8=
-github.com/golang/protobuf v1.4.2 h1:+Z5KGCizgyZCbGh1KZqA0fcLLkwbsjIzS4aV2v7wJX0=
 github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI=
-github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
-github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
+github.com/golang/protobuf v1.4.3 h1:JjCZWpVbqXDqFVmTfYWEVTMIYrL/NPdPSCHPJ0T/raM=
+github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI=
 github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M=
 github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
 github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
 github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
-github.com/google/go-cmp v0.4.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
-github.com/google/go-cmp v0.5.1 h1:JFrFEBb2xKufg6XkJsJr+WbKb4FQlURi5RUcBveYu9k=
-github.com/google/go-cmp v0.5.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
-github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs=
-github.com/google/martian/v3 v3.0.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0=
-github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc=
-github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc=
-github.com/google/pprof v0.0.0-20191218002539-d4f498aebedc/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM=
-github.com/google/pprof v0.0.0-20200212024743-f11f1df84d12/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM=
-github.com/google/pprof v0.0.0-20200229191704-1ebb73c60ed3/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM=
-github.com/google/pprof v0.0.0-20200430221834-fc25d7d30c6d/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM=
-github.com/google/pprof v0.0.0-20200708004538-1a94d8640e99/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM=
-github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI=
-github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg=
-github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk=
-github.com/gwenn/gosqlite v0.0.0-20200521090053-24878be1a237 h1:RY/HYWiI6K/HL8ePZYpgvMGf4wX3/FgGxgE8kPh8vDM=
-github.com/gwenn/gosqlite v0.0.0-20200521090053-24878be1a237/go.mod h1:WBYs9HfQGOYDCz7rFwMk7aHkbTTB0cUkQe3pZQARvIg=
-github.com/gwenn/yacr v0.0.0-20200110180258-a66d8c42d0ff h1:xv97Y/sC4qXwoZQ9HEhgoeDN/7j4JfA5qBWLNxihZQo=
+github.com/google/go-cmp v0.5.4 h1:L8R9j+yAqZuZjsqh/z+F1NCffTKKLShY6zXTItVIZ8M=
+github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
+github.com/gwenn/gosqlite v0.0.0-20201008200117-82e079acf5b6 h1:GUdDtRtiCm0bipN/Hc03rqJl++aX2wdWs+F3pGCaPTU=
+github.com/gwenn/gosqlite v0.0.0-20201008200117-82e079acf5b6/go.mod h1:WBYs9HfQGOYDCz7rFwMk7aHkbTTB0cUkQe3pZQARvIg=
+github.com/gwenn/gosqlite v0.0.0-20211101095637-b18efb2e44c8 h1:sWkgaGez8CNa2KHGBTTop16/mC03VP6MDqPKfvhEmCU=
+github.com/gwenn/gosqlite v0.0.0-20211101095637-b18efb2e44c8/go.mod h1:WBYs9HfQGOYDCz7rFwMk7aHkbTTB0cUkQe3pZQARvIg=
 github.com/gwenn/yacr v0.0.0-20200110180258-a66d8c42d0ff/go.mod h1:5SNcBGxZ5OaJAMJCSI/x3V7SGsvXqbwnwP/sHZLgYsw=
-github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
-github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
-github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc=
-github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU=
-github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk=
-github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
-github.com/kisielk/og-rek v1.0.1-0.20180928202415-8b25c4cefd6c h1:JKB9XSrj+dsCcfOZANbOTgDf3XLZVmX+rgFlKmuIZ+8=
-github.com/kisielk/og-rek v1.0.1-0.20180928202415-8b25c4cefd6c/go.mod h1:Fo0gkB8QTO7j5Ha+ObtDMW5GtL5UoY66JUU8ZoLJeLQ=
-github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
-github.com/kr/pretty v0.2.0 h1:s5hAObm+yFO5uHYt5dYjxi2rXrsnmRpJx4OYvIWUaQs=
+github.com/gwenn/yacr v0.0.0-20200112083327-bbe82c1f4d60 h1:JX4Yy6S9U/f3Jix82M58NLIAFeW/UjBFVrnYn5GS4X8=
+github.com/gwenn/yacr v0.0.0-20200112083327-bbe82c1f4d60/go.mod h1:Ps/gikIXcn2rRmeP0HQ9EvUYJrfrjAi51Wg8acsrkP0=
+github.com/gwenn/yacr v0.0.0-20211101095056-492fb0c571bc h1:AUv494HF3D9ht26o89DuJjqM9QDHwZeYCNU/JSU5jqI=
+github.com/gwenn/yacr v0.0.0-20211101095056-492fb0c571bc/go.mod h1:Ps/gikIXcn2rRmeP0HQ9EvUYJrfrjAi51Wg8acsrkP0=
+github.com/kisielk/og-rek v1.1.0 h1:u10TvQbPtrlY/6H4+BiFsBywwSVTGFsx0YOVtpx3IbI=
+github.com/kisielk/og-rek v1.1.0/go.mod h1:6ihsOSzSAxR/65S3Bn9zNihoEqRquhDQZ2c6I2+MG3c=
+github.com/kisielk/og-rek v1.2.0 h1:CTvDIin+YnetsSQAYbe+QNAxXU3B50C5hseEz8xEoJw=
+github.com/kisielk/og-rek v1.2.0/go.mod h1:6ihsOSzSAxR/65S3Bn9zNihoEqRquhDQZ2c6I2+MG3c=
 github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI=
+github.com/kr/pretty v0.2.1 h1:Fmg33tUaq4/8ym9TJN1x7sLJnHVwhP33CNkpYV/7rwI=
+github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI=
 github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
-github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE=
 github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
+github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
+github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
 github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc=
 github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw=
-github.com/philhofer/fwd v1.0.0 h1:UbZqGr5Y38ApvM/V/jEljVxwocdweyH+vmYvRPBnbqQ=
-github.com/philhofer/fwd v1.0.0/go.mod h1:gk3iGcWd9+svBvR0sR+KPcfE+RNWozjowpeBVG3ZVNU=
+github.com/philhofer/fwd v1.1.1 h1:GdGcTjf5RNAxwS4QLsiMzJYj5KEvPJD3Abr261yRQXQ=
+github.com/philhofer/fwd v1.1.1/go.mod h1:gk3iGcWd9+svBvR0sR+KPcfE+RNWozjowpeBVG3ZVNU=
 github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
 github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
@@ -137,259 +87,112 @@ github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZN
 github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
 github.com/remyoudompheng/bigfft v0.0.0-20200410134404-eec4a21b6bb0 h1:OdAsTTz6OkFY5QxjkYwrChwuRruF69c169dPK26NUlk=
 github.com/remyoudompheng/bigfft v0.0.0-20200410134404-eec4a21b6bb0/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo=
-github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
-github.com/shamaton/msgpack v1.1.1 h1:wZcK/aIifWvsDl6jXo+c/tRYo9Vnb8abWCossDbj2Ww=
-github.com/shamaton/msgpack v1.1.1/go.mod h1:ibiaNQRTCUISAYkkyOpaSCEBiCAxXe6u6Mu1sQ6945U=
+github.com/shamaton/msgpack v1.2.1 h1:40cwW7YAEdOIxcxIsUkAxSMUyYWZUyNiazI5AyiBntI=
+github.com/shamaton/msgpack v1.2.1/go.mod h1:ibiaNQRTCUISAYkkyOpaSCEBiCAxXe6u6Mu1sQ6945U=
 github.com/soheilhy/cmux v0.1.4 h1:0HKaf1o97UwFjHH9o5XsHUOF+tqmdA7KEzXLpiyaw0E=
 github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM=
+github.com/soheilhy/cmux v0.1.5-0.20210205191134-5ec6847320e5 h1:GJTW+uNMIV1RKwox+T4aN0/sQlYRg78uHZf2H0aBcDw=
+github.com/soheilhy/cmux v0.1.5-0.20210205191134-5ec6847320e5/go.mod h1:T7TcVDs9LWfQgPlPsdngu6I6QIoyIFZDDC6sNE1GqG0=
+github.com/soheilhy/cmux v0.1.5 h1:jjzc5WVemNEDTLwv9tlmemhC73tI08BNOIGwBOo10Js=
+github.com/soheilhy/cmux v0.1.5/go.mod h1:T7TcVDs9LWfQgPlPsdngu6I6QIoyIFZDDC6sNE1GqG0=
 github.com/someonegg/gocontainer v1.0.0 h1:9MMUFbQf7g+g9sMG4ggBHPDS1+Iz+wd9Ee/O4BNRdw0=
 github.com/someonegg/gocontainer v1.0.0/go.mod h1:zGJcXRK0ikzEYPFKTaFXi6UU/ulNuJypfADX4UQGtMw=
 github.com/someonegg/gox v1.0.0/go.mod h1:pngAcWxBFnyYM4oY+h9Rgv0WaikLkSfY5dBYuabUnBE=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
-github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
-github.com/stretchr/testify v1.6.1 h1:hDPOHmpOpP40lSULcqw7IrRb/u7w6RpDC9399XyoNd0=
-github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
-github.com/tinylib/msgp v1.1.3-0.20200327023543-e88e92c0ccca h1:RhRT3kgpvYL7aEUQoWwmhRTvObmqRJPV3rEMkr1qTfM=
-github.com/tinylib/msgp v1.1.3-0.20200327023543-e88e92c0ccca/go.mod h1:+d+yLhGm8mzTaHzB+wgMYrodPfmZrzkirds8fDWklFE=
-github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
-github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
-github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
+github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA=
+github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY=
+github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
+github.com/tinylib/msgp v1.1.5 h1:2gXmtWueD2HefZHQe1QOy9HVzmFrLOVvsXwXBQ0ayy0=
+github.com/tinylib/msgp v1.1.5/go.mod h1:eQsjooMTnV42mHu917E26IogZ2930nFyBQdofk10Udg=
+github.com/tinylib/msgp v1.1.6 h1:i+SbKraHhnrf9M5MYmvQhFnbLhAXSDWF8WWsuyRdocw=
+github.com/tinylib/msgp v1.1.6/go.mod h1:75BAfg2hauQhs3qedfdDZmWAPcFMAvJE5b9rGOMufyw=
+github.com/ttacon/chalk v0.0.0-20160626202418-22c06c80ed31/go.mod h1:onvgF043R+lC5RZ8IT9rBXDaEDnpnw/Cl+HFiw+v/7Q=
 github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
-go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU=
-go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8=
-go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=
-go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=
-go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
-golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
-golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
 golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
 golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
 golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
-golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
-golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8=
-golang.org/x/exp v0.0.0-20190829153037-c13cbed26979/go.mod h1:86+5VVa7VpoJ4kLfm080zCjGlMRFzhUhsZKEZO7MGek=
-golang.org/x/exp v0.0.0-20191030013958-a1ab85dbe136/go.mod h1:JXzH8nQsPlswgeRAPE3MuO9GYsAcnJvJ4vnMwN/5qkY=
-golang.org/x/exp v0.0.0-20191129062945-2f5052295587/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4=
-golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4=
-golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4=
-golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM=
-golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU=
-golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js=
-golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0=
 golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
 golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU=
-golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
 golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
-golang.org/x/lint v0.0.0-20190409202823-959b441ac422/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
-golang.org/x/lint v0.0.0-20190909230951-414d861bb4ac/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
-golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
-golang.org/x/lint v0.0.0-20191125180803-fdd1cda4f05f/go.mod h1:5qLYkcX4OjUUV8bRuDixDT3tpyyb+LUpUlRWLxfhWrs=
-golang.org/x/lint v0.0.0-20200130185559-910be7a94367/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY=
-golang.org/x/lint v0.0.0-20200302205851-738671d3881b/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY=
-golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE=
-golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o=
-golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc=
-golang.org/x/mod v0.1.0/go.mod h1:0QHyrYULN0/3qlju5TqG8bIK38QM8yzMo5ekMj3DlcY=
-golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg=
-golang.org/x/mod v0.1.1-0.20191107180719-034126e5016b/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg=
-golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
-golang.org/x/mod v0.3.0 h1:RM4zey1++hCTbCVQfnWeKs9/IEsaBLA8vTkd0WVtmH4=
 golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
+golang.org/x/mod v0.4.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
+golang.org/x/mod v0.4.1/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
 golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
 golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
-golang.org/x/net v0.0.0-20190501004415-9ce7a6920f09/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
-golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
-golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks=
 golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
-golang.org/x/net v0.0.0-20190628185345-da137c7871d7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
-golang.org/x/net v0.0.0-20190724013045-ca1201d0de80/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
-golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
-golang.org/x/net v0.0.0-20200114155413-6afb5195e5aa/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
-golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
-golang.org/x/net v0.0.0-20200222125558-5a598a2470a0/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
-golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
-golang.org/x/net v0.0.0-20200301022130-244492dfa37a/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
-golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
-golang.org/x/net v0.0.0-20200501053045-e0ff5e5a1de5/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
-golang.org/x/net v0.0.0-20200506145744-7e3656a0809f/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
-golang.org/x/net v0.0.0-20200513185701-a91f0712d120/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
-golang.org/x/net v0.0.0-20200520182314-0ba52f642ac2/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
-golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
-golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
-golang.org/x/net v0.0.0-20200822124328-c89045814202 h1:VvcQYSHwXgi7W+TpUR6A9g6Up98WAHf3f/ulnJ62IyA=
-golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
+golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
+golang.org/x/net v0.0.0-20201202161906-c7110b5ffcbb/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
+golang.org/x/net v0.0.0-20210226172049-e18ecbb05110 h1:qWPm9rbaAMKs8Bq/9LRpbMqxWRVUAQwMI9fVrssnTfw=
+golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
+golang.org/x/net v0.0.0-20211111160137-58aab5ef257a h1:c83jeVQW0KGKNaKBRfelNYNHaev+qawl9yaA825s8XE=
+golang.org/x/net v0.0.0-20211111160137-58aab5ef257a/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
 golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
-golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
-golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
-golang.org/x/oauth2 v0.0.0-20191202225959-858c2ad4c8b6/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
 golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
 golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
-golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
-golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
-golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
-golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208 h1:qwRHBd0NqMbJxfbotnDhm2ByMI1Shq4Y6oRJo21SGJA=
-golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20210220032951-036812b2e83c h1:5KslGYwFpkhGh+Q16bwMP3cOontH8FOep7tGV86Y7SQ=
+golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
-golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20191001151750-bb3f8db39f24/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20191228213918-04cbcbbfeed8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20200113162924-86b910548bc1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20200122134326-e047566fdf82/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20200212091648-12a6c2dcc1e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20200302150141-5c8b2ff67527/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd h1:xhmwyvizuTgC2qz7ZlMluP20uW+C3Rm0FD/WLDX8884=
-golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20200331124033-c3d80250170d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20200501052902-10377860bb8e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20200511232937-7e40ca221e25/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20200515095857-1151b9dac4a9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20200523222454-059865788121/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20200803210538-64077c9b5642 h1:B6caxRw+hozq68X2MY7jEpZh/cr4/aHLv9xU8Kkadrw=
-golang.org/x/sys v0.0.0-20200803210538-64077c9b5642/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
-golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg=
+golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210301091718-77cc2087c03b h1:kHlr0tATeLRMEiZJu5CknOw/E8V6h69sXXQFGoPtjcc=
+golang.org/x/sys v0.0.0-20210301091718-77cc2087c03b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20211111213525-f221eed1c01e h1:zeJt6jBtVDK23XK9QXcmG0FvO0elikp0dYZQZOeL1y0=
+golang.org/x/sys v0.0.0-20211111213525-f221eed1c01e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
-golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
-golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
-golang.org/x/text v0.3.3 h1:cokOdA+Jmi5PJGXLlLllQSgYigAEfHXJAERHVMaCc2k=
 golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
-golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
-golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
-golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
+golang.org/x/text v0.3.5 h1:i6eZZ+zk0SOf0xgBpEpPD18qWcJda6q1sxt3S0kzyUQ=
+golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
+golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
+golang.org/x/text v0.3.7 h1:olpwvP2KacW1ZWvsR7uQhoyTYvKAupfQrRGBFM352Gk=
+golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
 golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY=
 golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
-golang.org/x/tools v0.0.0-20190312151545-0bb0c0a6e846/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
-golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
-golang.org/x/tools v0.0.0-20190425150028-36563e24a262/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
-golang.org/x/tools v0.0.0-20190506145303-2d16b83fe98c/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
 golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
-golang.org/x/tools v0.0.0-20190606124116-d0a3d012864b/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
-golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
-golang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
-golang.org/x/tools v0.0.0-20190816200558-6889da9d5479/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
-golang.org/x/tools v0.0.0-20190911174233-4f2ddba30aff/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
-golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
-golang.org/x/tools v0.0.0-20191113191852-77e3bb0ad9e7/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
-golang.org/x/tools v0.0.0-20191115202509-3a792d9c32b2/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
 golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
-golang.org/x/tools v0.0.0-20191125144606-a911d9008d1f/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
-golang.org/x/tools v0.0.0-20191130070609-6e064ea0cf2d/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
-golang.org/x/tools v0.0.0-20191216173652-a0e659d51361/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
-golang.org/x/tools v0.0.0-20191227053925-7b8e75db28f4/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
-golang.org/x/tools v0.0.0-20200117161641-43d50277825c/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
-golang.org/x/tools v0.0.0-20200122220014-bf1340f18c4a/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
-golang.org/x/tools v0.0.0-20200130002326-2f3ba24bd6e7/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
-golang.org/x/tools v0.0.0-20200204074204-1cc6d1ef6c74/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
-golang.org/x/tools v0.0.0-20200207183749-b753a1ba74fa/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
-golang.org/x/tools v0.0.0-20200212150539-ea181f53ac56/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
-golang.org/x/tools v0.0.0-20200224181240-023911ca70b2/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
-golang.org/x/tools v0.0.0-20200227222343-706bc42d1f0d/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
-golang.org/x/tools v0.0.0-20200304193943-95d2e580d8eb/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw=
-golang.org/x/tools v0.0.0-20200312045724-11d5b4c81c7d/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw=
-golang.org/x/tools v0.0.0-20200331025713-a30bf2db82d4/go.mod h1:Sl4aGygMT6LrqrWclx+PTx3U+LnKx/seiNR+3G19Ar8=
-golang.org/x/tools v0.0.0-20200501065659-ab2804fb9c9d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
-golang.org/x/tools v0.0.0-20200512131952-2bc93b1c0c88/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
-golang.org/x/tools v0.0.0-20200515010526-7d3b6ebf133d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
-golang.org/x/tools v0.0.0-20200618134242-20370b0cb4b2/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
-golang.org/x/tools v0.0.0-20200729194436-6467de6f59a7/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA=
-golang.org/x/tools v0.0.0-20200804011535-6c149bb5ef0d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA=
-golang.org/x/tools v0.0.0-20200806022845-90696ccdc692/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA=
-golang.org/x/tools v0.0.0-20200915031644-64986481280e h1:tfSNPIxC48Azhz4nLSPskz/yE9R6ftFRK8pfgfqWUAc=
-golang.org/x/tools v0.0.0-20200915031644-64986481280e/go.mod h1:z6u4i615ZeAfBE4XtMziQW1fSVJXACjjbWkB/mvPzlU=
+golang.org/x/tools v0.0.0-20201022035929-9cf592e881e9/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
+golang.org/x/tools v0.0.0-20210108195828-e2f9c7f1fc8e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
+golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0=
 golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE=
 golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
-google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE=
-google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M=
-google.golang.org/api v0.8.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg=
-google.golang.org/api v0.9.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg=
-google.golang.org/api v0.13.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI=
-google.golang.org/api v0.14.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI=
-google.golang.org/api v0.15.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI=
-google.golang.org/api v0.17.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE=
-google.golang.org/api v0.18.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE=
-google.golang.org/api v0.19.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE=
-google.golang.org/api v0.20.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE=
-google.golang.org/api v0.22.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE=
-google.golang.org/api v0.24.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE=
-google.golang.org/api v0.28.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE=
-google.golang.org/api v0.29.0/go.mod h1:Lcubydp8VUV7KeIHD9z2Bys/sm/vGKnG1UHuDBSrHWM=
-google.golang.org/api v0.30.0/go.mod h1:QGmEvQ87FHZNiUVJkT14jQNYJ4ZJjdRF23ZXz5138Fc=
 google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=
 google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
-google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
-google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0=
-google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc=
-google.golang.org/appengine v1.6.6/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc=
 google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc=
-google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
-google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
-google.golang.org/genproto v0.0.0-20190425155659-357c62f0e4bb/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
-google.golang.org/genproto v0.0.0-20190502173448-54afdca5d873/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
-google.golang.org/genproto v0.0.0-20190801165951-fa694d86fc64/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc=
 google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc=
-google.golang.org/genproto v0.0.0-20190911173649-1774047e7e51/go.mod h1:IbNlFCBrqXvoKpeg0TB2l7cyZUmoaFKYIwrEpbDKLA8=
-google.golang.org/genproto v0.0.0-20191108220845-16a3f7862a1a/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc=
-google.golang.org/genproto v0.0.0-20191115194625-c23dd37a84c9/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc=
-google.golang.org/genproto v0.0.0-20191216164720-4f79533eabd1/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc=
-google.golang.org/genproto v0.0.0-20191230161307-f3c370f40bfb/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc=
-google.golang.org/genproto v0.0.0-20200115191322-ca5a22157cba/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc=
-google.golang.org/genproto v0.0.0-20200122232147-0452cf42e150/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc=
-google.golang.org/genproto v0.0.0-20200204135345-fa8e72b47b90/go.mod h1:GmwEX6Z4W5gMy59cAlVYjN9JhxgbQH6Gn+gFDQe2lzA=
-google.golang.org/genproto v0.0.0-20200212174721-66ed5ce911ce/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
-google.golang.org/genproto v0.0.0-20200224152610-e50cd9704f63/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
-google.golang.org/genproto v0.0.0-20200228133532-8c2c7df3a383/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
-google.golang.org/genproto v0.0.0-20200305110556-506484158171/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
-google.golang.org/genproto v0.0.0-20200312145019-da6875a35672/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
-google.golang.org/genproto v0.0.0-20200331122359-1ee6d9798940/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
-google.golang.org/genproto v0.0.0-20200430143042-b979b6f78d84/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
-google.golang.org/genproto v0.0.0-20200511104702-f5ebc3bea380/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
-google.golang.org/genproto v0.0.0-20200515170657-fc4c6c6a6587/go.mod h1:YsZOwe1myG/8QRHRsmBRE1LrgQY60beZKjly0O1fX9U=
 google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo=
-google.golang.org/genproto v0.0.0-20200618031413-b414f8b61790/go.mod h1:jDfRM7FcilCzHH/e9qn6dsT145K34l5v+OpcnNgKAAA=
-google.golang.org/genproto v0.0.0-20200729003335-053ba62fc06f/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
-google.golang.org/genproto v0.0.0-20200804131852-c06518451d9c/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
-google.golang.org/genproto v0.0.0-20200806141610-86f49bd18e98 h1:LCO0fg4kb6WwkXQXRQQgUYsFeFb5taTX5WAx5O/Vt28=
 google.golang.org/genproto v0.0.0-20200806141610-86f49bd18e98/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
+google.golang.org/genproto v0.0.0-20210226172003-ab064af71705 h1:PYBmACG+YEv8uQPW0r1kJj8tR+gkF0UWq7iFdUezwEw=
+google.golang.org/genproto v0.0.0-20210226172003-ab064af71705/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
 google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c=
-google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38=
-google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM=
 google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg=
 google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY=
-google.golang.org/grpc v1.26.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk=
 google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk=
-google.golang.org/grpc v1.27.1/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk=
-google.golang.org/grpc v1.28.0/go.mod h1:rpkK4SK4GF4Ach/+MFLZUBavHOvF2JJB5uozKKal+60=
-google.golang.org/grpc v1.29.1/go.mod h1:itym6AZVZYACWQqET3MqgPpjcuV5QH3BxFS3IjizoKk=
-google.golang.org/grpc v1.30.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak=
 google.golang.org/grpc v1.31.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak=
-google.golang.org/grpc v1.32.0 h1:zWTV+LMdc3kaiJMSTOFz2UgSBgx8RNQoTGiZu3fR9S0=
-google.golang.org/grpc v1.32.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak=
-google.golang.org/grpc/examples v0.0.0-20200915000551-32e7099cccac h1:oJwcuADYZXzWc+5R6wAvg6Qo86cbqAInuyKepxHGhIo=
-google.golang.org/grpc/examples v0.0.0-20200915000551-32e7099cccac/go.mod h1:Lh55/1hxmVHEkOvSIQ2uj0P12QyOCUNyRwnUlSS13hw=
+google.golang.org/grpc v1.36.0 h1:o1bcQ6imQMIOpdrO3SWf2z5RV72WbDwdXuK0MDlc8As=
+google.golang.org/grpc v1.36.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU=
+google.golang.org/grpc/examples v0.0.0-20210301210255-fc8f38cccf75 h1:IFbcGka/Xo8VIhRtCz9UBaDblKK7/iwLiXwm9D62Jzk=
+google.golang.org/grpc/examples v0.0.0-20210301210255-fc8f38cccf75/go.mod h1:Ly7ZA/ARzg8fnPU9TyZIxoz33sEUuWX7txiqs8lPTgE=
 google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8=
 google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0=
 google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM=
@@ -401,23 +204,16 @@ google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpAD
 google.golang.org/protobuf v1.24.0/go.mod h1:r/3tXBNzIEhYS9I1OUVjXDlt8tc493IdKGjtUeSXeh4=
 google.golang.org/protobuf v1.25.0 h1:Ejskq+SyPohKW+1uil0JJMtmHCgJPJ/qWTxr8qp+R4c=
 google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c=
-gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
-gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY=
-gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
-gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI=
+gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk=
+gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q=
 gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
-gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie2QXMNeKRTHCNY2nXvo=
 gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
+gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b h1:h8qDotaEPuJATrMmW04NCwg7v22aHH28wwpauUhK9Oo=
+gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
 honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
-honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
-honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
 honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
-honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg=
-honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k=
-honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k=
-lab.nexedi.com/kirr/go123 v0.0.0-20200915142026-a281a51cf49b h1:u9EAvkzDKlsTsLj9yuS+dGSli2H1MAHCQy4rFJ0dD7g=
-lab.nexedi.com/kirr/go123 v0.0.0-20200915142026-a281a51cf49b/go.mod h1:Vt3dj/2vggjnIDaDyIWjvf7xmKlyyc4DsQSp7hUf6z4=
-rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8=
-rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0=
-rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA=
+lab.nexedi.com/kirr/go123 v0.0.0-20210128150852-c20e95f0f789 h1:IP+xm2DdYqLn81x8J8PdRkoMrCxLDBsXYe4KZm3FkME=
+lab.nexedi.com/kirr/go123 v0.0.0-20210128150852-c20e95f0f789/go.mod h1:1wkWl3WhmutZiho+wsE7ymOKvRkN7hV3YZtL0f0gXTo=
+lab.nexedi.com/kirr/go123 v0.0.0-20210906140734-c9eb28d9e408 h1:H7YpNUDfTSvvRpKivUMrL9C09tQssQ6brEoX6K/OxOw=
+lab.nexedi.com/kirr/go123 v0.0.0-20210906140734-c9eb28d9e408/go.mod h1:pwDpdCuvtz0QxisDzV/z9eUb9zc/rMQec520h4i8VWQ=

go/internal/xtesting/xtesting.go

-// Copyright (C) 2017-2020  Nexedi SA and Contributors.
+// Copyright (C) 2017-2021  Nexedi SA and Contributors.
 //                          Kirill Smelkov <kirr@nexedi.com>
 //
 // This program is free software: you can Use, Study, Modify and Redistribute
@@ -29,10 +29,13 @@ import (
 	"os"
 	"os/exec"
 	"reflect"
+	"strings"
 	"sync"
 	"testing"
+	"time"
 
 	"lab.nexedi.com/kirr/go123/xerr"
+	"lab.nexedi.com/kirr/go123/xstrings"
 	"lab.nexedi.com/kirr/neo/go/zodb"
 )
 
@@ -92,13 +95,14 @@ func NeedPy(t testing.TB, modules ...string) {
 }
 
 // ZRawObject represents raw ZODB object state.
-type ZRawObject struct {
+type ZRawObject struct { // keep in sync with zodb(test).ZRawObject
 	Oid  zodb.Oid
 	Data []byte // raw serialized zodb data
 }
 
 // ZPyCommitRaw commits new transaction into database @ zurl with raw data specified by objv.
 //
+// Nil data means "delete object".
 // The commit is performed via zodbtools/py.
 func ZPyCommitRaw(zurl string, at zodb.Tid, objv ...ZRawObject) (_ zodb.Tid, err error) {
 	defer xerr.Contextf(&err, "%s: zpycommit @%s", zurl, at)
@@ -109,10 +113,15 @@ func ZPyCommitRaw(zurl string, at zodb.Tid, objv ...ZRawObject) (_ zodb.Tid, err
 	fmt.Fprintf(zin, "description %q\n", fmt.Sprintf("test commit; at=%s", at))
 	fmt.Fprintf(zin, "extension %q\n", "")
 	for _, obj := range objv {
+		// !data -> delete
+		if obj.Data == nil {
+			fmt.Fprintf(zin, "obj %s delete\n", obj.Oid)
+		} else {
 			fmt.Fprintf(zin, "obj %s %d null:00\n", obj.Oid, len(obj.Data))
 			zin.Write(obj.Data)
 			zin.WriteString("\n")
 		}
+	}
 	zin.WriteString("\n")
 
 	// run py `zodb commit`
@@ -135,6 +144,33 @@ func ZPyCommitRaw(zurl string, at zodb.Tid, objv ...ZRawObject) (_ zodb.Tid, err
 
 // XXX + ZPyCommitSrv ?
 
+// ZPyRestore restores transactions specified by zin in zodbdump format.
+//
+// The restore is performed via zodbtools/py.
+func ZPyRestore(zurl string, zin string) (tidv []zodb.Tid, err error) {
+	defer xerr.Contextf(&err, "%s: zpyrestore", zurl)
+
+	// run py `zodb restore`
+	cmd:= exec.Command("python", "-m", "zodbtools.zodb", "restore", zurl)
+	cmd.Stdin  = strings.NewReader(zin)
+	cmd.Stderr = os.Stderr
+	out, err := cmd.Output()
+	if err != nil {
+		return nil, err
+	}
+
+	for _, line := range xstrings.SplitLines(string(out), "\n") {
+		tid, err := zodb.ParseTid(line)
+		if err != nil {
+			return nil, fmt.Errorf("restored, but invalid output: %s", err)
+		}
+
+		tidv = append(tidv, tid)
+	}
+
+	return tidv, nil
+}
+
 
 // ---- tests for storage drivers ----
 
@@ -352,7 +388,7 @@ func DrvTestWatch(t *testing.T, zurl string, zdrvOpen zodb.DriverOpener, bugv ..
 
 	ctx := context.Background()
 	watchq := make(chan zodb.Event)
-	zdrv, at0, err := zdrvOpen(ctx, u, &zodb.DriverOptions{ReadOnly: true, Watchq: watchq})
+	zdrv, at0, err := zdrvOpen(ctx, u, &zodb.DriverOptions{ReadOnly: true, Watchq: watchq}); X(err)
 	if at0 != at {
 		t.Fatalf("opened @ %s  ; want %s", at0, at)
 	}
@@ -438,6 +474,11 @@ func DrvTestWatch(t *testing.T, zurl string, zdrvOpen zodb.DriverOpener, bugv ..
 		}
 	}
 
+	// commit something more and wait a bit to raise chances the driver enqueues to watchq<- .
+	_ = xcommit(at, ZRawObject{0, b("at the end")})
+	time.Sleep(1*time.Second)
+
+	// the driver must handle Close and cancel that watchq<-
 	err = zdrv.Close(); X(err)
 
 	e, ok := <-watchq
@@ -450,6 +491,7 @@ func DrvTestWatch(t *testing.T, zurl string, zdrvOpen zodb.DriverOpener, bugv ..
 // FatalIf(t) returns function f(err), which call t.Fatal if err != nil.
 func FatalIf(t *testing.T) func(error) {
 	return func(err error) {
+		t.Helper()
 		if err != nil {
 			t.Fatal(err)
 		}

go/neo/client.go

@@ -68,6 +68,9 @@ type Client struct {
 	at0Initialized bool                // true after .at0 is initialized
 	at0Ready       chan(struct{})      // ready after .at0 is initialized
 
+	closeOnce sync.Once
+	closed    chan struct{} // ready when Closed
+
 	ownNet bool // true if Client "owns" networker and should release it on Close
 }
 
@@ -81,6 +84,7 @@ func NewClient(clusterName, masterAddr string, net xnet.Networker) *Client {
 	c := &Client{
 		node:     newMasteredNode(proto.CLIENT, clusterName, net, masterAddr),
 		at0Ready: make(chan struct{}),
+		closed:   make(chan struct{}),
 	}
 
 	var runCtx context.Context
@@ -91,6 +95,9 @@ func NewClient(clusterName, masterAddr string, net xnet.Networker) *Client {
 
 // Close implements zodb.IStorageDriver.
 func (c *Client) Close() (err error) {
+	c.closeOnce.Do(func() {
+		close(c.closed)
+	})
 	c.runCancel()
 	err = c.runWG.Wait()
 	if errors.Is(err, context.Canceled) {
@@ -190,7 +197,7 @@ func (c *Client) invalidateObjects(msg *proto.InvalidateObjects) error {
 	defer c.at0Mu.Unlock()
 
 	// queue initial events until .at0 is initialized after register
-	// queued events will be sent to watchq by zeo ctor after initializing .at0
+	// queued events will be sent to watchq by syncMaster after initializing .at0
 	if !c.at0Initialized {
 		c.eventq0 = append(c.eventq0, event)
 		return nil
@@ -198,7 +205,13 @@ func (c *Client) invalidateObjects(msg *proto.InvalidateObjects) error {
 
 	// at0 is initialized - ok to send current event if it goes > at0
 	if tid > c.at0 {
-		c.watchq <- event
+		select {
+		case <-c.closed:
+			// closed - client does not read watchq anymore
+
+		case c.watchq <- event:
+			// ok
+		}
 	}
 	return nil
 }
@@ -250,7 +263,13 @@ func (c *Client) flushEventq0() {
 	if c.watchq != nil {
 		for _, e := range c.eventq0 {
 			if e.Tid > c.at0 {
-				c.watchq <- e
+				select {
+				case <-c.closed:
+					// closed - client does not read watchq anymore
+
+				case c.watchq <- e:
+					// ok
+				}
 			}
 		}
 	}
@@ -410,6 +429,8 @@ func openClientByURL(ctx context.Context, u *url.URL, opt *zodb.DriverOptions) (
 	}
 
 	cred := u.User.String()
+	// ca=ca.crt;cert=my.crt;key=my.key
+	cred = strings.ReplaceAll(cred, ";", "&") // ; is no longer in default separators set https://github.com/golang/go/issues/25192
 	x, err := xurl.ParseQuery(cred)
 	if err != nil {
 		return nil, zodb.InvalidTid, fmt.Errorf("credentials: %s", err)
@@ -442,9 +463,7 @@ func openClientByURL(ctx context.Context, u *url.URL, opt *zodb.DriverOptions) (
 	}
 
 	name := u.Path
-	if strings.HasPrefix(name, "/") {
-		name = name[1:]
-	}
+	name = strings.TrimPrefix(name, "/")
 	if name == "" {
 		return nil, zodb.InvalidTid, fmt.Errorf("cluster name not specified")
 	}
@@ -491,14 +510,18 @@ func openClientByURL(ctx context.Context, u *url.URL, opt *zodb.DriverOptions) (
 		// close .watchq after serve is over
 		c.at0Mu.Lock()
 		defer c.at0Mu.Unlock()
-		if c.at0Initialized {
-			c.flushEventq0()
-		}
 		if c.watchq != nil {
-			if err != nil {
-				c.watchq <- &zodb.EventError{Err: err}
+			if err != nil && /* already flushed .eventq0 */c.at0Initialized {
+				select {
+				case <-c.closed:
+					// closed - client does not read watchq anymore
+
+				case c.watchq <- &zodb.EventError{Err: err}:
+					// ok
+				}
 			}
 			close(c.watchq)
+			c.watchq = nil // prevent flushEventq0 to send to closed chan
 		}
 
 		errq <- err

go/neo/client_test.go

@@ -362,6 +362,7 @@ func (opt NEOSrvOptions) URLPrefix() string {
 // ----------------
 
 // tOptions represents options for testing.
+// TODO -> xtesting
 type tOptions struct {
 	Preload string // preload database with data from this location
 }

go/neo/mastered.go

@@ -76,12 +76,12 @@ type _MasteredNode struct {
 type _MasteredNodeFlags int
 const (
 	// δPartTabPassThrough tells mlink.Recv1 not to filter out messages related
-	// to partition table changes. When mlink.Recv1 receives such messages there
+	// to partition table changes. When mlink.Recv1 receives such messages they
 	// are already processed internally to update .state.PartTab correspondingly.
 	//
 	// Storage uses this mode to receive δPartTab notifications to know
 	// when to persist it.
-	δPartTabPassThrough _MasteredNodeFlags = iota
+	δPartTabPassThrough _MasteredNodeFlags = 1 << iota
 )
 
 // newMasteredNode creates new _MasteredNode that connects to masterAddr/cluster via net.

go/neo/t/neotest

 #!/bin/bash -e
 # neotest: run tests and benchmarks against FileStorage, ZEO and various NEO/py and NEO/go clusters
 
-# Copyright (C) 2017-2020  Nexedi SA and Contributors.
+# Copyright (C) 2017-2021  Nexedi SA and Contributors.
 #                          Kirill Smelkov <kirr@nexedi.com>
 #
 # This program is free software: you can Use, Study, Modify and Redistribute
@@ -141,7 +141,10 @@ $@
 
 # ---- go/py unit tests ----
 cmd_test-go() {
+	(
+		cd $NEOt
 		go test -count=1 lab.nexedi.com/kirr/neo/go/... # -count=1 disables tests caching
+	)
 }
 
 cmd_test-py() {
@@ -1503,18 +1506,22 @@ cpustat)	f=(                     );;
 esac
 
 
+NEOt=$(cd `dirname $0` && pwd)
+
 for flag in ${f[*]}; do
 	case "$flag" in
 	build)
 		# make sure tzodb*, tcpu* and zgenprod are on PATH (because we could be invoked from another dir)
-		X=$(cd `dirname $0` && pwd)
-		export PATH=$X:$PATH
+		export PATH=$NEOt:$PATH
 
 		# rebuild go bits
 		# neo/py, wendelin.core, ... - must be pip install'ed - `neotest deploy` cares about that
+		(
+			cd $NEOt
 			go install -v lab.nexedi.com/kirr/neo/go/...
-		go build -o $X/tzodb_go $X/tzodb.go
-		go build -o $X/tcpu_go $X/tcpu.go
+			go build -o tzodb_go tzodb.go
+			go build -o tcpu_go tcpu.go
+		)
 		;;
 
 	net)

go/neo/t/tzodb.go

@@ -431,7 +431,7 @@ func zwrkPreconnect(ctx context.Context, url string, at zodb.Tid, nwrk int) (_ [
 	if err != nil {
 		for _, stor := range storv {
 			if stor != nil {
-				xio.LClose(stor)
+				xio.LClose(ctx, stor)
 			}
 		}
 		return nil, err

go/zodb/btree/btree.go.in

 // Copyright (c) 2001, 2002 Zope Foundation and Contributors.
 // All Rights Reserved.
 //
-// Copyright (C) 2018-2019  Nexedi SA and Contributors.
+// Copyright (C) 2018-2021  Nexedi SA and Contributors.
 //                          Kirill Smelkov <kirr@nexedi.com>
 //
 // This software is subject to the provisions of the Zope Public License,
@@ -65,7 +65,7 @@ type BTree struct {
 
 // Entry is one BTree node entry.
 //
-// It contains key and child, who is either BTree or Bucket.
+// It contains key and child, which is either BTree or Bucket.
 //
 // Key limits child's keys - see BTree.Entryv for details.
 type Entry struct {
@@ -102,6 +102,15 @@ type BucketEntry struct {
 	value interface{}
 }
 
+// KeyRange represents [lo,hi) key range.
+type KeyRange struct {
+	Lo  KEY
+	Hi_ KEY // NOTE _not_ hi) to avoid overflow at ∞;  hi = hi_ + 1
+}
+
+const _KeyMin KEY = math.Min<Key>
+const _KeyMax KEY = math.Max<Key>
+
 // ---- access []entry ----
 
 // Key returns BTree entry key.
@@ -123,7 +132,7 @@ func (e *Entry) Child() Node { return e.child }
 // Children of all entries are guaranteed to be of the same kind - either all BTree, or all Bucket.
 //
 // The caller must not modify returned array.
-func (t *BTree) Entryv() []Entry {
+func (t *BTree) Entryv() /*readonly*/ []Entry {
 	return t.data
 }
 
@@ -134,7 +143,7 @@ func (e *BucketEntry) Key() KEY { return e.key }
 func (e *BucketEntry) Value() interface{} { return e.value }
 
 // Entryv returns entries of a Bucket node.
-func (b *Bucket) Entryv() []BucketEntry {
+func (b *Bucket) Entryv() /*readonly*/ []BucketEntry {
 	ev := make([]BucketEntry, len(b.keys))
 	for i, k := range b.keys {
 		ev[i] = BucketEntry{k, b.values[i]}
@@ -168,36 +177,54 @@ func (t *BTree) Get(ctx context.Context, key KEY) (_ interface{}, _ bool, err er
 // VGet is like Get but also calls visit while traversing the tree.
 //
 // Visit is called with node being activated.
-func (t *BTree) VGet(ctx context.Context, key KEY, visit func(node Node)) (_ interface{}, _ bool, err error) {
+func (t *BTree) VGet(ctx context.Context, key KEY, visit func(node Node, keycov KeyRange)) (_ interface{}, _ bool, err error) {
 	defer xerr.Contextf(&err, "btree(%s): get %v", t.POid(), key)
 	err = t.PActivate(ctx)
 	if err != nil {
 		return nil, false, err
 	}
 
+	keycov := KeyRange{Lo: _KeyMin, Hi_: _KeyMax}
 	if visit != nil {
-		visit(t)
+		visit(t, keycov)
 	}
 
-	if len(t.data) == 0 {
-		// empty btree
+	for {
+		l := len(t.data)
+		if l == 0 {
+			// empty btree (top, or in leaf)
 			t.PDeactivate()
 			return nil, false, nil
 		}
 
-	for {
 		// search i: K(i) ≤ k < K(i+1)	; K(0) = -∞, K(len) = +∞
-		i := sort.Search(len(t.data), func(i int) bool {
+		i := sort.Search(l, func(i int) bool {
 			j := i + 1
 			if j == len(t.data) {
 				return true // [len].key = +∞
 			}
 			return key < t.data[j].key
 		})
-
-		// FIXME panic index out of range (empty T without children;
-		// logically incorrect, but Restructure generated it once)
+		// i < l
 		child := t.data[i].child
+
+		// shorten global keycov by local [lo,hi) for this child
+		lo := _KeyMin
+		if i > 0 {
+			lo = t.data[i].key
+		}
+		i++
+		hi_ := _KeyMax
+		if i < l {
+			hi_ = t.data[i].key
+		}
+		if hi_ != _KeyMax {
+			hi_--
+		}
+		keycov.Lo  = kmax(keycov.Lo, lo)
+		keycov.Hi_ = kmin(keycov.Hi_, hi_)
+
+
 		t.PDeactivate()
 		err = child.PActivate(ctx)
 		if err != nil {
@@ -207,7 +234,7 @@ func (t *BTree) VGet(ctx context.Context, key KEY, visit func(node Node)) (_ int
 		// XXX verify child keys are in valid range according to parent
 
 		if visit != nil {
-			visit(child)
+			visit(child, keycov)
 		}
 
 		switch child := child.(type) {
@@ -250,26 +277,40 @@ func (t *BTree) MinKey(ctx context.Context) (_ KEY, ok bool, err error) {
 // VMinKey is like MinKey but also calls visit while traversing the tree.
 //
 // Visit is called with node being activated.
-func (t *BTree) VMinKey(ctx context.Context, visit func(node Node)) (_ KEY, ok bool, err error) {
+func (t *BTree) VMinKey(ctx context.Context, visit func(node Node, keycov KeyRange)) (_ KEY, ok bool, err error) {
 	defer xerr.Contextf(&err, "btree(%s): minkey", t.POid())
 	err = t.PActivate(ctx)
 	if err != nil {
 		return 0, false, err
 	}
 
+	keycov := KeyRange{Lo: _KeyMin, Hi_: _KeyMax}
 	if visit != nil {
-		visit(t)
+		visit(t, keycov)
 	}
 
-	if len(t.data) == 0 {
-		// empty btree
+	// NOTE -> can also use t.firstbucket
+	for {
+		l := len(t.data)
+		if l == 0 {
+			// empty btree (top, or in leaf)
 			t.PDeactivate()
 			return 0, false, nil
 		}
 
-	// NOTE -> can also use t.firstbucket
-	for {
 		child := t.data[0].child
+
+		// shorten global keycov by local hi) for this child
+		hi_ := _KeyMax
+		if 1 < l {
+			hi_ = t.data[1].key
+		}
+		if hi_ != _KeyMax {
+			hi_--
+		}
+		// keycov.Lo stays -∞
+		keycov.Hi_ = kmin(keycov.Hi_, hi_)
+
 		t.PDeactivate()
 		err = child.PActivate(ctx)
 		if err != nil {
@@ -277,7 +318,7 @@ func (t *BTree) VMinKey(ctx context.Context, visit func(node Node)) (_ KEY, ok b
 		}
 
 		if visit != nil {
-			visit(child)
+			visit(child, keycov)
 		}
 
 		// XXX verify child keys are in valid range according to parent
@@ -305,26 +346,36 @@ func (t *BTree) MaxKey(ctx context.Context) (_ KEY, _ bool, err error) {
 // VMaxKey is like MaxKey but also calls visit while traversing the tree.
 //
 // Visit is called with node being activated.
-func (t *BTree) VMaxKey(ctx context.Context, visit func(node Node)) (_ KEY, _ bool, err error) {
+func (t *BTree) VMaxKey(ctx context.Context, visit func(node Node, keycov KeyRange)) (_ KEY, _ bool, err error) {
 	defer xerr.Contextf(&err, "btree(%s): maxkey", t.POid())
 	err = t.PActivate(ctx)
 	if err != nil {
 		return 0, false, err
 	}
 
+	keycov := KeyRange{Lo: _KeyMin, Hi_: _KeyMax}
 	if visit != nil {
-		visit(t)
+		visit(t, keycov)
 	}
 
+	for {
 		l := len(t.data)
 		if l == 0 {
-		// empty btree
+			// empty btree (top, or in leaf)
 			t.PDeactivate()
 			return 0, false, nil
 		}
 
-	for {
 		child := t.data[l-1].child
+
+		// shorten global keycov by local [lo for this chile
+		lo := _KeyMin
+		if l-1 > 0 {
+			lo = t.data[l-1].key
+		}
+		keycov.Lo  = kmax(keycov.Lo, lo)
+		// keycov.Hi_ stays ∞
+
 		t.PDeactivate()
 		err = child.PActivate(ctx)
 		if err != nil {
@@ -332,7 +383,7 @@ func (t *BTree) VMaxKey(ctx context.Context, visit func(node Node)) (_ KEY, _ bo
 		}
 
 		if visit != nil {
-			visit(child)
+			visit(child, keycov)
 		}
 
 		// XXX verify child keys are in valid range according to parent
@@ -591,7 +642,7 @@ func (bt *btreeState) PySetState(pystate interface{}) (err error) {
 	var kprev int64
 	var childrenKind int // 1 - BTree, 2 - Bucket
 	for i, idx := 0, 0; i < n; i++ {
-		key := int64(math.Min<Key>) // KEY(-∞)   (qualifies for ≤)
+		key := int64(_KeyMin) // KEY(-∞)   (qualifies for ≤)
 		if i > 0 {
 			// key[0] is unused and not saved
 			key, ok = t[idx].(int64) // XXX Xint
@@ -609,7 +660,7 @@ func (bt *btreeState) PySetState(pystate interface{}) (err error) {
 		}
 
 		if i > 1 && !(key > kprev) {
-			fmt.Errorf("data: [%d]: key not ↑", i)
+			return fmt.Errorf("data: [%d]: key not ↑", i)
 		}
 		kprev = key
 
@@ -629,7 +680,7 @@ func (bt *btreeState) PySetState(pystate interface{}) (err error) {
 			childrenKind = kind
 		}
 		if kind != childrenKind {
-			fmt.Errorf("data: [%d]: children must be of the same type", i)
+			return fmt.Errorf("data: [%d]: children must be of the same type", i)
 		}
 
 		bt.data = append(bt.data, Entry{key: kkey, child: child.(Node)})
@@ -646,3 +697,60 @@ func init() {
 	zodb.RegisterClass("BTrees.BTree.BTree",  t(BTree{}),  t(btreeState{}))
 	zodb.RegisterClass("BTrees.BTree.Bucket", t(Bucket{}), t(bucketState{}))
 }
+
+
+// ---- misc ----
+
+// Has returns whether key k belongs to the range.
+func (r *KeyRange) Has(k KEY) bool {
+	return (r.Lo <= k && k <= r.Hi_)
+}
+
+// Empty returns whether key range is empty.
+func (r *KeyRange) Empty() bool {
+	hi := r.Hi_
+	if hi == _KeyMax {
+		// [x,∞] cannot be empty because max x is ∞ and [∞,∞] has one element: ∞
+		return false
+	}
+	hi++ // no overflow
+	return r.Lo >= hi
+}
+
+func (r KeyRange) String() string {
+	var shi string
+	if r.Hi_ == _KeyMax {
+		shi = kstr(r.Hi_) // ∞
+	} else {
+		shi = fmt.Sprintf("%d", r.Hi_+1)
+	}
+	return fmt.Sprintf("[%s,%s)", kstr(r.Lo), shi)
+}
+
+
+func kmin(a, b KEY) KEY {
+	if a < b {
+		return a
+	} else {
+		return b
+	}
+}
+
+func kmax(a, b KEY) KEY {
+	if a > b {
+		return a
+	} else {
+		return b
+	}
+}
+
+// kstr formats key as string.
+func kstr(k KEY) string {
+	if k == _KeyMin {
+		return "-∞"
+	}
+	if k == _KeyMax {
+		return "∞"
+	}
+	return fmt.Sprintf("%d", k)
+}

go/zodb/btree/btree_test.go

-// Copyright (C) 2018-2019  Nexedi SA and Contributors.
+// Copyright (C) 2018-2021  Nexedi SA and Contributors.
 //                          Kirill Smelkov <kirr@nexedi.com>
 //
 // This program is free software: you can Use, Study, Modify and Redistribute
@@ -23,6 +23,7 @@ package btree
 
 import (
 	"context"
+	"reflect"
 	"testing"
 
 	"lab.nexedi.com/kirr/go123/exc"
@@ -98,6 +99,12 @@ func (b *bucketWrap) MaxKey(ctx context.Context) (k int64, ok bool, err error) {
 	return
 }
 
+// tVisit is information about one visit call.
+type tVisit struct {
+	node   zodb.Oid
+	keycov LKeyRange
+}
+
 func TestBTree(t *testing.T) {
 	X := exc.Raiseif
 	ctx := context.Background()
@@ -105,7 +112,11 @@ func TestBTree(t *testing.T) {
 	if err != nil {
 		t.Fatal(err)
 	}
-	db := zodb.NewDB(stor)
+	defer func() {
+		err := stor.Close(); X(err)
+	}()
+
+	db := zodb.NewDB(stor, &zodb.DBOptions{})
 	defer func() {
 		err := db.Close(); X(err)
 	}()
@@ -118,8 +129,6 @@ func TestBTree(t *testing.T) {
 		t.Fatal(err)
 	}
 
-	// XXX close db/stor
-
 	// go through small test Buckets/BTrees and verify that Get(key) is as expected.
 	for _, tt := range smallTestv {
 		xobj, err := conn.Get(ctx, tt.oid)
@@ -262,4 +271,39 @@ func TestBTree(t *testing.T) {
 	// XXX verify FirstBucket / Next ?
 
 	verifyFirstBucket(B3)
+
+	// verify nodes/keycov visited through VGet/V{Min,Max}Key
+	xBv, err := conn.Get(ctx, Bv_oid);  X(err)
+	Bv, ok := xBv.(*LOBTree)
+	if !ok {
+		t.Fatalf("Bv: %v; got %T;  want LOBTree", Bv_oid, xBv)
+	}
+
+	for k, visitOK := range Bvdict {
+		visit := []tVisit{}
+		_, _, err := Bv.VGet(ctx, k, func(node LONode, keycov LKeyRange) {
+			visit = append(visit, tVisit{node.POid(), keycov})
+		}); X(err)
+		if !reflect.DeepEqual(visit, visitOK) {
+			t.Errorf("VGet(%d): visit:\nhave: %v\nwant: %v", k, visit, visitOK)
+		}
+	}
+
+	visitMinOK := Bvdict[Bv_kmin]
+	visitMaxOK := Bvdict[Bv_kmax]
+	visitMin := []tVisit{}
+	visitMax := []tVisit{}
+	_, _, err = Bv.VMinKey(ctx, func(node LONode, keycov LKeyRange) {
+		visitMin = append(visitMin, tVisit{node.POid(), keycov})
+	}); X(err)
+	_, _, err = Bv.VMaxKey(ctx, func(node LONode, keycov LKeyRange) {
+		visitMax = append(visitMax, tVisit{node.POid(), keycov})
+	}); X(err)
+
+	if !reflect.DeepEqual(visitMin, visitMinOK) {
+		t.Errorf("VMinKey(): visit:\nhave: %v\nwant: %v", visitMin, visitMinOK)
+	}
+	if !reflect.DeepEqual(visitMax, visitMaxOK) {
+		t.Errorf("VMaxKey(): visit:\nhave: %v\nwant: %v", visitMax, visitMaxOK)
+	}
 }

go/zodb/btree/gen-btree

@@ -28,6 +28,9 @@ out=$3
 kind=${KIND,,}	# IO -> io
 Key=${KEY^}
 
+KEYKIND=${KIND:0:1}	# IO -> I
+keykind=${KEYKIND,,}	# I -> i
+
 input=$(dirname $0)/btree.go.in
 
 echo "// Code generated by gen-btree; DO NOT EDIT." >$out
@@ -45,4 +48,10 @@ sed	\
 	-e "s/\bBucketEntry\b/${KIND}BucketEntry/g"	\
 	-e "s/\bbtreeState\b/${kind}btreeState/g"	\
 	-e "s/\bbucketState\b/${kind}bucketState/g"	\
+	-e "s/\b_KeyMin\b/_${KEYKIND}KeyMin/g"		\
+	-e "s/\b_KeyMax\b/_${KEYKIND}KeyMax/g"		\
+	-e "s/\bKeyRange\b/${KEYKIND}KeyRange/g"	\
+	-e "s/\bkmin\b/${keykind}kmin/g"		\
+	-e "s/\bkmax\b/${keykind}kmax/g"		\
+	-e "s/\bkstr\b/${keykind}kstr/g"		\
 	$input >>$out

go/zodb/btree/testdata/gen-testdata

 #!/usr/bin/env python2
 # -*- coding: utf-8 -*-
-# Copyright (C) 2018-2020  Nexedi SA and Contributors.
+# Copyright (C) 2018-2021  Nexedi SA and Contributors.
 #                          Kirill Smelkov <kirr@nexedi.com>
 #
 # This program is free software: you can Use, Study, Modify and Redistribute
@@ -22,8 +22,9 @@
 
 from ZODB.DB import DB
 from BTrees.LOBTree import LOBucket, LOBTree
+from BTrees.check import check as bcheck
 from ZODB.utils import u64
-from zodbtools.test.gen_testdata import run_with_zodb3py2_compat
+from zodbtools.test.gen_testdata import run_with_zodb4py2_compat
 import os, os.path, transaction
 
 from golang.gcompat import qq
@@ -52,14 +53,37 @@ def main2():
 
     root['B3'] = B3 = LOBTree(dict([(_, _) for _ in range(10000)]))
 
+
+    # T4/T2-T/B1-B2-T7,9/B5-B8-B10  (to verify VGet->visit)
+    # TODO use xbtree.py:Restructure after gimport works through modules and
+    #      xbtree.py is moved from wcfs to zodb/go.
+    v1 = LOBucket([(1,"a")])
+    v2 = LOBucket([(2,"b")])
+    v5 = LOBucket([(5,"c")])
+    v8 = LOBucket([(8,"d")])
+    v9 = LOBucket([(9,"e")])
+    T2, T79, T, T4 = LOBTree(), LOBTree(), LOBTree(), LOBTree()
+    T2.__setstate__ (((v1, 2, v2),         v1)) # (child, key, child, ...), firstbucket
+    T79.__setstate__(((v5, 7, v8, 9, v9),  v5))
+    T.__setstate__  (((T79,),              v5))
+    T4.__setstate__ (((T2, 4, T),          v1))
+    root['Bv'] = Bv = T4
+
     transaction.commit()
 
+    bcheck(Bv)
+    assert Bv[1] == "a"
+    assert Bv[2] == "b"
+    assert Bv[5] == "c"
+    assert Bv[8] == "d"
+    assert Bv[9] == "e"
+
+
     with open("ztestdata_expect_test.go", "w") as f:
         def emit(v):
             print >>f, v
         emit("// Code generated by %s; DO NOT EDIT." % __file__)
         emit("package btree\n")
-        #emit("import \"lab.nexedi.com/kirr/neo/go/zodb\"\n")
 
         def emititems(b):
             s = "testEntry{oid: %s, kind: %s, itemv: []kv{" \
@@ -84,12 +108,34 @@ def main2():
         emit("\nconst B3_oid = %s" % u64(B3._p_oid))
         emit("const B3_maxkey = %d" % B3.maxKey())
 
+        emit("\nconst Bv_oid = %s" % u64(Bv._p_oid))
+        emit("const Bv_kmin = %d"  % Bv.minKey())
+        emit("const Bv_kmax = %d"  % Bv.maxKey())
+        emit("var Bvdict = map[int64][]tVisit{")
+        noo = "_LKeyMin"
+        oo  = "_LKeyMax"
+        def emitVisit(key, *visitv): # visitv = [](node, lo,hi)
+            vstr = []
+            for node, lo,hi in visitv:
+                if isinstance(hi, str):
+                    hi_ = hi    # oo or noo
+                else:
+                    hi_ = hi-1
+                vstr.append("{%d, LKeyRange{%s, %s}}" % (u64(node._p_oid), lo, hi_))
+            emit("\t%d: []tVisit{%s}," % (key, ", ".join(vstr)))
+
+        emitVisit(1, (T4, noo,oo), (T2, noo,4), (v1, noo,2))
+        emitVisit(2, (T4, noo,oo), (T2, noo,4), (v2, 2,4))
+        emitVisit(5, (T4, noo,oo), (T,  4,oo),  (T79, 4,oo), (v5, 4,7))
+        emitVisit(8, (T4, noo,oo), (T,  4,oo),  (T79, 4,oo), (v8, 7,9))
+        emitVisit(9, (T4, noo,oo), (T,  4,oo),  (T79, 4,oo), (v9, 9,oo))
+        emit("}")
 
     conn.close()
     db.close()
 
 def main():
-    run_with_zodb3py2_compat(main2)
+    run_with_zodb4py2_compat(main2)
 
 
 if __name__ == '__main__':

go/zodb/btree/ziobtree.go

@@ -3,7 +3,7 @@
 // Copyright (c) 2001, 2002 Zope Foundation and Contributors.
 // All Rights Reserved.
 //
-// Copyright (C) 2018-2019  Nexedi SA and Contributors.
+// Copyright (C) 2018-2021  Nexedi SA and Contributors.
 //                          Kirill Smelkov <kirr@nexedi.com>
 //
 // This software is subject to the provisions of the Zope Public License,
@@ -67,7 +67,7 @@ type IOBTree struct {
 
 // IOEntry is one IOBTree node entry.
 //
-// It contains key and child, who is either IOBTree or IOBucket.
+// It contains key and child, which is either IOBTree or IOBucket.
 //
 // Key limits child's keys - see IOBTree.Entryv for details.
 type IOEntry struct {
@@ -104,6 +104,15 @@ type IOBucketEntry struct {
 	value interface{}
 }
 
+// IKeyRange represents [lo,hi) key range.
+type IKeyRange struct {
+	Lo  int32
+	Hi_ int32 // NOTE _not_ hi) to avoid overflow at ∞;  hi = hi_ + 1
+}
+
+const _IKeyMin int32 = math.MinInt32
+const _IKeyMax int32 = math.MaxInt32
+
 // ---- access []entry ----
 
 // Key returns IOBTree entry key.
@@ -125,7 +134,7 @@ func (e *IOEntry) Child() IONode { return e.child }
 // Children of all entries are guaranteed to be of the same kind - either all IOBTree, or all IOBucket.
 //
 // The caller must not modify returned array.
-func (t *IOBTree) Entryv() []IOEntry {
+func (t *IOBTree) Entryv() /*readonly*/ []IOEntry {
 	return t.data
 }
 
@@ -136,7 +145,7 @@ func (e *IOBucketEntry) Key() int32 { return e.key }
 func (e *IOBucketEntry) Value() interface{} { return e.value }
 
 // Entryv returns entries of a IOBucket node.
-func (b *IOBucket) Entryv() []IOBucketEntry {
+func (b *IOBucket) Entryv() /*readonly*/ []IOBucketEntry {
 	ev := make([]IOBucketEntry, len(b.keys))
 	for i, k := range b.keys {
 		ev[i] = IOBucketEntry{k, b.values[i]}
@@ -170,34 +179,54 @@ func (t *IOBTree) Get(ctx context.Context, key int32) (_ interface{}, _ bool, er
 // VGet is like Get but also calls visit while traversing the tree.
 //
 // Visit is called with node being activated.
-func (t *IOBTree) VGet(ctx context.Context, key int32, visit func(node IONode)) (_ interface{}, _ bool, err error) {
+func (t *IOBTree) VGet(ctx context.Context, key int32, visit func(node IONode, keycov IKeyRange)) (_ interface{}, _ bool, err error) {
 	defer xerr.Contextf(&err, "btree(%s): get %v", t.POid(), key)
 	err = t.PActivate(ctx)
 	if err != nil {
 		return nil, false, err
 	}
 
+	keycov := IKeyRange{Lo: _IKeyMin, Hi_: _IKeyMax}
 	if visit != nil {
-		visit(t)
+		visit(t, keycov)
 	}
 
-	if len(t.data) == 0 {
-		// empty btree
+	for {
+		l := len(t.data)
+		if l == 0 {
+			// empty btree (top, or in leaf)
 			t.PDeactivate()
 			return nil, false, nil
 		}
 
-	for {
 		// search i: K(i) ≤ k < K(i+1)	; K(0) = -∞, K(len) = +∞
-		i := sort.Search(len(t.data), func(i int) bool {
+		i := sort.Search(l, func(i int) bool {
 			j := i + 1
 			if j == len(t.data) {
 				return true // [len].key = +∞
 			}
 			return key < t.data[j].key
 		})
-
+		// i < l
 		child := t.data[i].child
+
+		// shorten global keycov by local [lo,hi) for this child
+		lo := _IKeyMin
+		if i > 0 {
+			lo = t.data[i].key
+		}
+		i++
+		hi_ := _IKeyMax
+		if i < l {
+			hi_ = t.data[i].key
+		}
+		if hi_ != _IKeyMax {
+			hi_--
+		}
+		keycov.Lo  = ikmax(keycov.Lo, lo)
+		keycov.Hi_ = ikmin(keycov.Hi_, hi_)
+
+
 		t.PDeactivate()
 		err = child.PActivate(ctx)
 		if err != nil {
@@ -207,7 +236,7 @@ func (t *IOBTree) VGet(ctx context.Context, key int32, visit func(node IONode))
 		// XXX verify child keys are in valid range according to parent
 
 		if visit != nil {
-			visit(child)
+			visit(child, keycov)
 		}
 
 		switch child := child.(type) {
@@ -250,26 +279,40 @@ func (t *IOBTree) MinKey(ctx context.Context) (_ int32, ok bool, err error) {
 // VMinKey is like MinKey but also calls visit while traversing the tree.
 //
 // Visit is called with node being activated.
-func (t *IOBTree) VMinKey(ctx context.Context, visit func(node IONode)) (_ int32, ok bool, err error) {
+func (t *IOBTree) VMinKey(ctx context.Context, visit func(node IONode, keycov IKeyRange)) (_ int32, ok bool, err error) {
 	defer xerr.Contextf(&err, "btree(%s): minkey", t.POid())
 	err = t.PActivate(ctx)
 	if err != nil {
 		return 0, false, err
 	}
 
+	keycov := IKeyRange{Lo: _IKeyMin, Hi_: _IKeyMax}
 	if visit != nil {
-		visit(t)
+		visit(t, keycov)
 	}
 
-	if len(t.data) == 0 {
-		// empty btree
+	// NOTE -> can also use t.firstbucket
+	for {
+		l := len(t.data)
+		if l == 0 {
+			// empty btree (top, or in leaf)
 			t.PDeactivate()
 			return 0, false, nil
 		}
 
-	// NOTE -> can also use t.firstbucket
-	for {
 		child := t.data[0].child
+
+		// shorten global keycov by local hi) for this child
+		hi_ := _IKeyMax
+		if 1 < l {
+			hi_ = t.data[1].key
+		}
+		if hi_ != _IKeyMax {
+			hi_--
+		}
+		// keycov.Lo stays -∞
+		keycov.Hi_ = ikmin(keycov.Hi_, hi_)
+
 		t.PDeactivate()
 		err = child.PActivate(ctx)
 		if err != nil {
@@ -277,7 +320,7 @@ func (t *IOBTree) VMinKey(ctx context.Context, visit func(node IONode)) (_ int32
 		}
 
 		if visit != nil {
-			visit(child)
+			visit(child, keycov)
 		}
 
 		// XXX verify child keys are in valid range according to parent
@@ -305,26 +348,36 @@ func (t *IOBTree) MaxKey(ctx context.Context) (_ int32, _ bool, err error) {
 // VMaxKey is like MaxKey but also calls visit while traversing the tree.
 //
 // Visit is called with node being activated.
-func (t *IOBTree) VMaxKey(ctx context.Context, visit func(node IONode)) (_ int32, _ bool, err error) {
+func (t *IOBTree) VMaxKey(ctx context.Context, visit func(node IONode, keycov IKeyRange)) (_ int32, _ bool, err error) {
 	defer xerr.Contextf(&err, "btree(%s): maxkey", t.POid())
 	err = t.PActivate(ctx)
 	if err != nil {
 		return 0, false, err
 	}
 
+	keycov := IKeyRange{Lo: _IKeyMin, Hi_: _IKeyMax}
 	if visit != nil {
-		visit(t)
+		visit(t, keycov)
 	}
 
+	for {
 		l := len(t.data)
 		if l == 0 {
-		// empty btree
+			// empty btree (top, or in leaf)
 			t.PDeactivate()
 			return 0, false, nil
 		}
 
-	for {
 		child := t.data[l-1].child
+
+		// shorten global keycov by local [lo for this chile
+		lo := _IKeyMin
+		if l-1 > 0 {
+			lo = t.data[l-1].key
+		}
+		keycov.Lo  = ikmax(keycov.Lo, lo)
+		// keycov.Hi_ stays ∞
+
 		t.PDeactivate()
 		err = child.PActivate(ctx)
 		if err != nil {
@@ -332,7 +385,7 @@ func (t *IOBTree) VMaxKey(ctx context.Context, visit func(node IONode)) (_ int32
 		}
 
 		if visit != nil {
-			visit(child)
+			visit(child, keycov)
 		}
 
 		// XXX verify child keys are in valid range according to parent
@@ -591,7 +644,7 @@ func (bt *iobtreeState) PySetState(pystate interface{}) (err error) {
 	var kprev int64
 	var childrenKind int // 1 - IOBTree, 2 - IOBucket
 	for i, idx := 0, 0; i < n; i++ {
-		key := int64(math.MinInt32) // int32(-∞)   (qualifies for ≤)
+		key := int64(_IKeyMin) // int32(-∞)   (qualifies for ≤)
 		if i > 0 {
 			// key[0] is unused and not saved
 			key, ok = t[idx].(int64) // XXX Xint
@@ -609,7 +662,7 @@ func (bt *iobtreeState) PySetState(pystate interface{}) (err error) {
 		}
 
 		if i > 1 && !(key > kprev) {
-			fmt.Errorf("data: [%d]: key not ↑", i)
+			return fmt.Errorf("data: [%d]: key not ↑", i)
 		}
 		kprev = key
 
@@ -629,7 +682,7 @@ func (bt *iobtreeState) PySetState(pystate interface{}) (err error) {
 			childrenKind = kind
 		}
 		if kind != childrenKind {
-			fmt.Errorf("data: [%d]: children must be of the same type", i)
+			return fmt.Errorf("data: [%d]: children must be of the same type", i)
 		}
 
 		bt.data = append(bt.data, IOEntry{key: kkey, child: child.(IONode)})
@@ -646,3 +699,60 @@ func init() {
 	zodb.RegisterClass("BTrees.IOBTree.IOBTree",  t(IOBTree{}),  t(iobtreeState{}))
 	zodb.RegisterClass("BTrees.IOBTree.IOBucket", t(IOBucket{}), t(iobucketState{}))
 }
+
+
+// ---- misc ----
+
+// Has returns whether key k belongs to the range.
+func (r *IKeyRange) Has(k int32) bool {
+	return (r.Lo <= k && k <= r.Hi_)
+}
+
+// Empty returns whether key range is empty.
+func (r *IKeyRange) Empty() bool {
+	hi := r.Hi_
+	if hi == _IKeyMax {
+		// [x,∞] cannot be empty because max x is ∞ and [∞,∞] has one element: ∞
+		return false
+	}
+	hi++ // no overflow
+	return r.Lo >= hi
+}
+
+func (r IKeyRange) String() string {
+	var shi string
+	if r.Hi_ == _IKeyMax {
+		shi = ikstr(r.Hi_) // ∞
+	} else {
+		shi = fmt.Sprintf("%d", r.Hi_+1)
+	}
+	return fmt.Sprintf("[%s,%s)", ikstr(r.Lo), shi)
+}
+
+
+func ikmin(a, b int32) int32 {
+	if a < b {
+		return a
+	} else {
+		return b
+	}
+}
+
+func ikmax(a, b int32) int32 {
+	if a > b {
+		return a
+	} else {
+		return b
+	}
+}
+
+// ikstr formats key as string.
+func ikstr(k int32) string {
+	if k == _IKeyMin {
+		return "-∞"
+	}
+	if k == _IKeyMax {
+		return "∞"
+	}
+	return fmt.Sprintf("%d", k)
+}

go/zodb/btree/zlobtree.go

@@ -3,7 +3,7 @@
 // Copyright (c) 2001, 2002 Zope Foundation and Contributors.
 // All Rights Reserved.
 //
-// Copyright (C) 2018-2019  Nexedi SA and Contributors.
+// Copyright (C) 2018-2021  Nexedi SA and Contributors.
 //                          Kirill Smelkov <kirr@nexedi.com>
 //
 // This software is subject to the provisions of the Zope Public License,
@@ -67,7 +67,7 @@ type LOBTree struct {
 
 // LOEntry is one LOBTree node entry.
 //
-// It contains key and child, who is either LOBTree or LOBucket.
+// It contains key and child, which is either LOBTree or LOBucket.
 //
 // Key limits child's keys - see LOBTree.Entryv for details.
 type LOEntry struct {
@@ -104,6 +104,15 @@ type LOBucketEntry struct {
 	value interface{}
 }
 
+// LKeyRange represents [lo,hi) key range.
+type LKeyRange struct {
+	Lo  int64
+	Hi_ int64 // NOTE _not_ hi) to avoid overflow at ∞;  hi = hi_ + 1
+}
+
+const _LKeyMin int64 = math.MinInt64
+const _LKeyMax int64 = math.MaxInt64
+
 // ---- access []entry ----
 
 // Key returns LOBTree entry key.
@@ -125,7 +134,7 @@ func (e *LOEntry) Child() LONode { return e.child }
 // Children of all entries are guaranteed to be of the same kind - either all LOBTree, or all LOBucket.
 //
 // The caller must not modify returned array.
-func (t *LOBTree) Entryv() []LOEntry {
+func (t *LOBTree) Entryv() /*readonly*/ []LOEntry {
 	return t.data
 }
 
@@ -136,7 +145,7 @@ func (e *LOBucketEntry) Key() int64 { return e.key }
 func (e *LOBucketEntry) Value() interface{} { return e.value }
 
 // Entryv returns entries of a LOBucket node.
-func (b *LOBucket) Entryv() []LOBucketEntry {
+func (b *LOBucket) Entryv() /*readonly*/ []LOBucketEntry {
 	ev := make([]LOBucketEntry, len(b.keys))
 	for i, k := range b.keys {
 		ev[i] = LOBucketEntry{k, b.values[i]}
@@ -170,36 +179,54 @@ func (t *LOBTree) Get(ctx context.Context, key int64) (_ interface{}, _ bool, er
 // VGet is like Get but also calls visit while traversing the tree.
 //
 // Visit is called with node being activated.
-func (t *LOBTree) VGet(ctx context.Context, key int64, visit func(node LONode)) (_ interface{}, _ bool, err error) {
+func (t *LOBTree) VGet(ctx context.Context, key int64, visit func(node LONode, keycov LKeyRange)) (_ interface{}, _ bool, err error) {
 	defer xerr.Contextf(&err, "btree(%s): get %v", t.POid(), key)
 	err = t.PActivate(ctx)
 	if err != nil {
 		return nil, false, err
 	}
 
+	keycov := LKeyRange{Lo: _LKeyMin, Hi_: _LKeyMax}
 	if visit != nil {
-		visit(t)
+		visit(t, keycov)
 	}
 
-	if len(t.data) == 0 {
-		// empty btree
+	for {
+		l := len(t.data)
+		if l == 0 {
+			// empty btree (top, or in leaf)
 			t.PDeactivate()
 			return nil, false, nil
 		}
 
-	for {
 		// search i: K(i) ≤ k < K(i+1)	; K(0) = -∞, K(len) = +∞
-		i := sort.Search(len(t.data), func(i int) bool {
+		i := sort.Search(l, func(i int) bool {
 			j := i + 1
 			if j == len(t.data) {
 				return true // [len].key = +∞
 			}
 			return key < t.data[j].key
 		})
-
-		// FIXME panic index out of range (empty T without children;
-		// logically incorrect, but Restructure generated it once)
+		// i < l
 		child := t.data[i].child
+
+		// shorten global keycov by local [lo,hi) for this child
+		lo := _LKeyMin
+		if i > 0 {
+			lo = t.data[i].key
+		}
+		i++
+		hi_ := _LKeyMax
+		if i < l {
+			hi_ = t.data[i].key
+		}
+		if hi_ != _LKeyMax {
+			hi_--
+		}
+		keycov.Lo  = lkmax(keycov.Lo, lo)
+		keycov.Hi_ = lkmin(keycov.Hi_, hi_)
+
+
 		t.PDeactivate()
 		err = child.PActivate(ctx)
 		if err != nil {
@@ -209,7 +236,7 @@ func (t *LOBTree) VGet(ctx context.Context, key int64, visit func(node LONode))
 		// XXX verify child keys are in valid range according to parent
 
 		if visit != nil {
-			visit(child)
+			visit(child, keycov)
 		}
 
 		switch child := child.(type) {
@@ -252,26 +279,40 @@ func (t *LOBTree) MinKey(ctx context.Context) (_ int64, ok bool, err error) {
 // VMinKey is like MinKey but also calls visit while traversing the tree.
 //
 // Visit is called with node being activated.
-func (t *LOBTree) VMinKey(ctx context.Context, visit func(node LONode)) (_ int64, ok bool, err error) {
+func (t *LOBTree) VMinKey(ctx context.Context, visit func(node LONode, keycov LKeyRange)) (_ int64, ok bool, err error) {
 	defer xerr.Contextf(&err, "btree(%s): minkey", t.POid())
 	err = t.PActivate(ctx)
 	if err != nil {
 		return 0, false, err
 	}
 
+	keycov := LKeyRange{Lo: _LKeyMin, Hi_: _LKeyMax}
 	if visit != nil {
-		visit(t)
+		visit(t, keycov)
 	}
 
-	if len(t.data) == 0 {
-		// empty btree
+	// NOTE -> can also use t.firstbucket
+	for {
+		l := len(t.data)
+		if l == 0 {
+			// empty btree (top, or in leaf)
 			t.PDeactivate()
 			return 0, false, nil
 		}
 
-	// NOTE -> can also use t.firstbucket
-	for {
 		child := t.data[0].child
+
+		// shorten global keycov by local hi) for this child
+		hi_ := _LKeyMax
+		if 1 < l {
+			hi_ = t.data[1].key
+		}
+		if hi_ != _LKeyMax {
+			hi_--
+		}
+		// keycov.Lo stays -∞
+		keycov.Hi_ = lkmin(keycov.Hi_, hi_)
+
 		t.PDeactivate()
 		err = child.PActivate(ctx)
 		if err != nil {
@@ -279,7 +320,7 @@ func (t *LOBTree) VMinKey(ctx context.Context, visit func(node LONode)) (_ int64
 		}
 
 		if visit != nil {
-			visit(child)
+			visit(child, keycov)
 		}
 
 		// XXX verify child keys are in valid range according to parent
@@ -307,26 +348,36 @@ func (t *LOBTree) MaxKey(ctx context.Context) (_ int64, _ bool, err error) {
 // VMaxKey is like MaxKey but also calls visit while traversing the tree.
 //
 // Visit is called with node being activated.
-func (t *LOBTree) VMaxKey(ctx context.Context, visit func(node LONode)) (_ int64, _ bool, err error) {
+func (t *LOBTree) VMaxKey(ctx context.Context, visit func(node LONode, keycov LKeyRange)) (_ int64, _ bool, err error) {
 	defer xerr.Contextf(&err, "btree(%s): maxkey", t.POid())
 	err = t.PActivate(ctx)
 	if err != nil {
 		return 0, false, err
 	}
 
+	keycov := LKeyRange{Lo: _LKeyMin, Hi_: _LKeyMax}
 	if visit != nil {
-		visit(t)
+		visit(t, keycov)
 	}
 
+	for {
 		l := len(t.data)
 		if l == 0 {
-		// empty btree
+			// empty btree (top, or in leaf)
 			t.PDeactivate()
 			return 0, false, nil
 		}
 
-	for {
 		child := t.data[l-1].child
+
+		// shorten global keycov by local [lo for this chile
+		lo := _LKeyMin
+		if l-1 > 0 {
+			lo = t.data[l-1].key
+		}
+		keycov.Lo  = lkmax(keycov.Lo, lo)
+		// keycov.Hi_ stays ∞
+
 		t.PDeactivate()
 		err = child.PActivate(ctx)
 		if err != nil {
@@ -334,7 +385,7 @@ func (t *LOBTree) VMaxKey(ctx context.Context, visit func(node LONode)) (_ int64
 		}
 
 		if visit != nil {
-			visit(child)
+			visit(child, keycov)
 		}
 
 		// XXX verify child keys are in valid range according to parent
@@ -593,7 +644,7 @@ func (bt *lobtreeState) PySetState(pystate interface{}) (err error) {
 	var kprev int64
 	var childrenKind int // 1 - LOBTree, 2 - LOBucket
 	for i, idx := 0, 0; i < n; i++ {
-		key := int64(math.MinInt64) // int64(-∞)   (qualifies for ≤)
+		key := int64(_LKeyMin) // int64(-∞)   (qualifies for ≤)
 		if i > 0 {
 			// key[0] is unused and not saved
 			key, ok = t[idx].(int64) // XXX Xint
@@ -611,7 +662,7 @@ func (bt *lobtreeState) PySetState(pystate interface{}) (err error) {
 		}
 
 		if i > 1 && !(key > kprev) {
-			fmt.Errorf("data: [%d]: key not ↑", i)
+			return fmt.Errorf("data: [%d]: key not ↑", i)
 		}
 		kprev = key
 
@@ -631,7 +682,7 @@ func (bt *lobtreeState) PySetState(pystate interface{}) (err error) {
 			childrenKind = kind
 		}
 		if kind != childrenKind {
-			fmt.Errorf("data: [%d]: children must be of the same type", i)
+			return fmt.Errorf("data: [%d]: children must be of the same type", i)
 		}
 
 		bt.data = append(bt.data, LOEntry{key: kkey, child: child.(LONode)})
@@ -648,3 +699,60 @@ func init() {
 	zodb.RegisterClass("BTrees.LOBTree.LOBTree",  t(LOBTree{}),  t(lobtreeState{}))
 	zodb.RegisterClass("BTrees.LOBTree.LOBucket", t(LOBucket{}), t(lobucketState{}))
 }
+
+
+// ---- misc ----
+
+// Has returns whether key k belongs to the range.
+func (r *LKeyRange) Has(k int64) bool {
+	return (r.Lo <= k && k <= r.Hi_)
+}
+
+// Empty returns whether key range is empty.
+func (r *LKeyRange) Empty() bool {
+	hi := r.Hi_
+	if hi == _LKeyMax {
+		// [x,∞] cannot be empty because max x is ∞ and [∞,∞] has one element: ∞
+		return false
+	}
+	hi++ // no overflow
+	return r.Lo >= hi
+}
+
+func (r LKeyRange) String() string {
+	var shi string
+	if r.Hi_ == _LKeyMax {
+		shi = lkstr(r.Hi_) // ∞
+	} else {
+		shi = fmt.Sprintf("%d", r.Hi_+1)
+	}
+	return fmt.Sprintf("[%s,%s)", lkstr(r.Lo), shi)
+}
+
+
+func lkmin(a, b int64) int64 {
+	if a < b {
+		return a
+	} else {
+		return b
+	}
+}
+
+func lkmax(a, b int64) int64 {
+	if a > b {
+		return a
+	} else {
+		return b
+	}
+}
+
+// lkstr formats key as string.
+func lkstr(k int64) string {
+	if k == _LKeyMin {
+		return "-∞"
+	}
+	if k == _LKeyMax {
+		return "∞"
+	}
+	return fmt.Sprintf("%d", k)
+}

go/zodb/btree/ztestdata_expect_test.go

@@ -3,13 +3,24 @@ package btree
 
 
 var smallTestv = [...]testEntry{
-	testEntry{oid: 6, kind: kindBucket, itemv: []kv{}},
-	testEntry{oid: 3, kind: kindBucket, itemv: []kv{{10, int64(17)}, }},
+	testEntry{oid: 7, kind: kindBucket, itemv: []kv{}},
+	testEntry{oid: 4, kind: kindBucket, itemv: []kv{{10, int64(17)}, }},
 	testEntry{oid: 1, kind: kindBucket, itemv: []kv{{15, int64(1)}, {23, "hello"}, }},
-	testEntry{oid: 2, kind: kindBTree, itemv: []kv{}},
-	testEntry{oid: 7, kind: kindBTree, itemv: []kv{{5, int64(4)}, }},
-	testEntry{oid: 4, kind: kindBTree, itemv: []kv{{7, int64(3)}, {9, "world"}, }},
+	testEntry{oid: 3, kind: kindBTree, itemv: []kv{}},
+	testEntry{oid: 8, kind: kindBTree, itemv: []kv{{5, int64(4)}, }},
+	testEntry{oid: 5, kind: kindBTree, itemv: []kv{{7, int64(3)}, {9, "world"}, }},
 }
 
-const B3_oid = 5
+const B3_oid = 6
 const B3_maxkey = 9999
+
+const Bv_oid = 2
+const Bv_kmin = 1
+const Bv_kmax = 9
+var Bvdict = map[int64][]tVisit{
+	1: []tVisit{{2, LKeyRange{_LKeyMin, _LKeyMax}}, {342, LKeyRange{_LKeyMin, 3}}, {344, LKeyRange{_LKeyMin, 1}}},
+	2: []tVisit{{2, LKeyRange{_LKeyMin, _LKeyMax}}, {342, LKeyRange{_LKeyMin, 3}}, {349, LKeyRange{2, 3}}},
+	5: []tVisit{{2, LKeyRange{_LKeyMin, _LKeyMax}}, {343, LKeyRange{4, _LKeyMax}}, {345, LKeyRange{4, _LKeyMax}}, {346, LKeyRange{4, 6}}},
+	8: []tVisit{{2, LKeyRange{_LKeyMin, _LKeyMax}}, {343, LKeyRange{4, _LKeyMax}}, {345, LKeyRange{4, _LKeyMax}}, {347, LKeyRange{7, 8}}},
+	9: []tVisit{{2, LKeyRange{_LKeyMin, _LKeyMax}}, {343, LKeyRange{4, _LKeyMax}}, {345, LKeyRange{4, _LKeyMax}}, {348, LKeyRange{9, _LKeyMax}}},
+}

go/zodb/connection.go

-// Copyright (C) 2018-2020  Nexedi SA and Contributors.
+// Copyright (C) 2018-2021  Nexedi SA and Contributors.
 //                          Kirill Smelkov <kirr@nexedi.com>
 //
 // This program is free software: you can Use, Study, Modify and Redistribute
@@ -180,7 +180,7 @@ const (
 
 // newConnection creates new Connection associated with db.
 func newConnection(db *DB, at Tid) *Connection {
-	return &Connection{
+	conn := &Connection{
 		db:    db,
 		at:    at,
 		cache: LiveCache{
@@ -188,6 +188,10 @@ func newConnection(db *DB, at Tid) *Connection {
 			objtab: make(map[Oid]*weak.Ref),
 		},
 	}
+	if cc := db.opt.CacheControl; cc != nil {
+		conn.cache.SetControl(cc)
+	}
+	return conn
 }
 
 // DB returns database handle under which the connection was opened.

go/zodb/db.go

-// Copyright (C) 2018-2020  Nexedi SA and Contributors.
+// Copyright (C) 2018-2021  Nexedi SA and Contributors.
 //                          Kirill Smelkov <kirr@nexedi.com>
 //
 // This program is free software: you can Use, Study, Modify and Redistribute
@@ -48,6 +48,8 @@ type DB struct {
 	stor   IStorage
 	watchq chan Event // we are watching .stor via here
 
+	opt *DBOptions
+
 	down     chan struct{} // ready when DB is no longer operational
 	downOnce sync.Once     // shutdown may be due to both Close and IO error in watcher
 	downErr  error         // reason for shutdown
@@ -121,15 +123,23 @@ type DB struct {
 	// (so it is not duplicated many times for many DB case)
 }
 
+// DBOptions describes options to NewDB.
+type DBOptions struct {
+	// CacheControl, if !nil, is set as default live cache control for
+	// newly created connections.
+	CacheControl LiveCacheControl
+}
 
 // NewDB creates new database handle.
 //
 // Created database handle must be closed when no longer needed.
-func NewDB(stor IStorage) *DB {
-	// XXX db options?
+func NewDB(stor IStorage, opt *DBOptions) *DB {
+	// copy opts in case caller will change them later
+	opt_ := *opt
 	db := &DB{
 		stor:   stor,
 		watchq: make(chan Event),
+		opt:    &opt_,
 		down:   make(chan struct{}),
 		hwait:  make(map[hwaiter]struct{}),
 

go/zodb/impexp_test.go

-// Copyright (C) 2019  Nexedi SA and Contributors.
+// Copyright (C) 2019-2021  Nexedi SA and Contributors.
 //                          Kirill Smelkov <kirr@nexedi.com>
 //
 // This program is free software: you can Use, Study, Modify and Redistribute
@@ -26,9 +26,15 @@ import (
 // imported at runtime via import_x_test due to cyclic dependency:
 
 var ZPyCommit    func(string, Tid, ...IPersistent) (Tid, error)
+var ZPyCommitRaw func(string, Tid, ...ZRawObject)  (Tid, error)
 
 // exported for zodb_test package:
 
+type ZRawObject struct { // keep in sync with xtesting.ZRawObject
+	Oid  Oid
+	Data []byte
+}
+
 func PSerialize(obj IPersistent) *mem.Buf {
 	return obj.persistent().pSerialize()
 }

go/zodb/import_x_test.go

-// Copyright (C) 2019  Nexedi SA and Contributors.
+// Copyright (C) 2019-2021  Nexedi SA and Contributors.
 //                          Kirill Smelkov <kirr@nexedi.com>
 //
 // This program is free software: you can Use, Study, Modify and Redistribute
@@ -34,6 +34,7 @@ import (
 
 func init() {
 	zodb.ZPyCommit    = ZPyCommit
+	zodb.ZPyCommitRaw = ZPyCommitRaw
 }
 
 // ZPyCommit commits new transaction with specified objects.
@@ -41,7 +42,7 @@ func init() {
 // The objects need to be alive, but do not need to be marked as changed.
 // The commit is performed via zodb/py.
 func ZPyCommit(zurl string, at zodb.Tid, objv ...zodb.IPersistent) (zodb.Tid, error) {
-	var rawobjv []xtesting.ZRawObject // raw zodb objects data to commit
+	var rawobjv []zodb.ZRawObject // raw zodb objects data to commit
 	var bufv []*mem.Buf           // buffers to release
 	defer func() {
 		for _, buf := range bufv {
@@ -51,7 +52,7 @@ func ZPyCommit(zurl string, at zodb.Tid, objv ...zodb.IPersistent) (zodb.Tid, er
 
 	for _, obj := range objv {
 		buf := zodb.PSerialize(obj)
-		rawobj := xtesting.ZRawObject{
+		rawobj := zodb.ZRawObject{
 			Oid:  obj.POid(),
 			Data: buf.Data,
 		}
@@ -59,5 +60,13 @@ func ZPyCommit(zurl string, at zodb.Tid, objv ...zodb.IPersistent) (zodb.Tid, er
 		bufv = append(bufv, buf)
 	}
 
-	return xtesting.ZPyCommitRaw(zurl, at, rawobjv...)
+	return ZPyCommitRaw(zurl, at, rawobjv...)
+}
+
+func ZPyCommitRaw(zurl string, at zodb.Tid, rawobjv ...zodb.ZRawObject) (zodb.Tid, error) {
+	var xrawobjv []xtesting.ZRawObject
+	for _, obj := range rawobjv {
+		xrawobjv = append(xrawobjv, xtesting.ZRawObject{Oid: obj.Oid, Data: obj.Data})
+	}
+	return xtesting.ZPyCommitRaw(zurl, at, xrawobjv...)
 }

go/zodb/persistent.go

-// Copyright (C) 2018-2019  Nexedi SA and Contributors.
+// Copyright (C) 2018-2021  Nexedi SA and Contributors.
 //                          Kirill Smelkov <kirr@nexedi.com>
 //
 // This program is free software: you can Use, Study, Modify and Redistribute
@@ -163,7 +163,7 @@ func (obj *Persistent) pSerialize() *mem.Buf {
 func (obj *Persistent) PActivate(ctx context.Context) (err error) {
 	obj.mu.Lock()
 	obj.refcnt++
-	doload := (obj.refcnt == 1 && obj.state == GHOST)
+	doload := (obj.state == GHOST && obj.loading == nil)
 	defer func() {
 		if err != nil {
 			obj.PDeactivate()
@@ -208,10 +208,10 @@ func (obj *Persistent) PActivate(ctx context.Context) (err error) {
 			"%v (want %v); .loading = %p (want %p)", s, GHOST, l, loading))
 	}
 
-	obj.serial = serial
-
 	// try to pass loaded state to object
 	if err == nil {
+		obj.serial = serial
+
 		switch istate := obj.istate().(type) {
 		case Stateful:
 			err = istate.SetState(state)
@@ -229,9 +229,13 @@ func (obj *Persistent) PActivate(ctx context.Context) (err error) {
 		if err == nil {
 			obj.state = UPTODATE
 		}
+
+	} else {
+		obj.serial = InvalidTid
+		// force reload on next activate if it was an error
+		obj.loading = nil
 	}
 
-	// XXX set state to load error? (to avoid panic on second activate after load error)
 	loading.err = err
 
 	obj.mu.Unlock()
@@ -257,7 +261,7 @@ func (obj *Persistent) PDeactivate() {
 	if obj.state >= CHANGED {
 		return
 	}
-	if obj.oid == InvalidOid { // newly created not-yet committed object	// TODO tests
+	if obj.oid == InvalidOid { // newly created not-yet committed object
 		return
 	}
 

go/zodb/persistent_test.go

-// Copyright (C) 2018-2020  Nexedi SA and Contributors.
+// Copyright (C) 2018-2021  Nexedi SA and Contributors.
 //                          Kirill Smelkov <kirr@nexedi.com>
 //
 // This program is free software: you can Use, Study, Modify and Redistribute
@@ -21,6 +21,7 @@ package zodb
 
 import (
 	"context"
+	"errors"
 	"fmt"
 	"io/ioutil"
 	"os"
@@ -276,7 +277,7 @@ func (t *tDB) Reopen() {
 		ReadOnly: true,
 		NoCache: !t.rawcache,
 	}); X(err)
-	db := NewDB(stor)
+	db := NewDB(stor, &DBOptions{})
 	t.stor = stor
 	t.db = db
 }
@@ -317,7 +318,7 @@ func (t *tDB) Add(oid Oid, value string) {
 }
 
 // Commit commits objects queued by Add.
-func (t *tDB) Commit() {
+func (t *tDB) Commit() Tid {
 	t.Helper()
 
 	head, err := ZPyCommit(t.zurl, t.head, t.commitq...)
@@ -326,6 +327,19 @@ func (t *tDB) Commit() {
 	}
 	t.head = head
 	t.commitq = nil
+	return head
+}
+
+// CommitRaw commits raw changes.
+func (t *tDB) CommitRaw(rawobjv ...ZRawObject) Tid {
+	t.Helper()
+
+	head, err := ZPyCommitRaw(t.zurl, t.head, rawobjv...)
+	if err != nil {
+		t.Fatal(err)
+	}
+	t.head = head
+	return head
 }
 
 // Open opens new test transaction/connection.
@@ -443,12 +457,14 @@ func (t *tConnection) Abort() {
 }
 
 func (t *tDB) fatalif(err error) {
+	t.Helper()
 	if err != nil {
 		t.Fatal(err)
 	}
 }
 
 func (t *tConnection) fatalif(err error) {
+	t.Helper()
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -473,13 +489,11 @@ func testPersistentDB(t0 *testing.T, rawcache bool) {
 
 	tdb.Add(101, "bonjour")
 	tdb.Add(102, "monde")
-	tdb.Commit()
-	at0 := tdb.head
+	at0 := tdb.Commit()
 
 	tdb.Add(101, "hello")
 	tdb.Add(102, "world")
-	tdb.Commit()
-	at1 := tdb.head
+	at1 := tdb.Commit()
 
 	tdb.Reopen() // so that at0 is not covered by db.δtail
 	db := tdb.db
@@ -541,8 +555,7 @@ func testPersistentDB(t0 *testing.T, rawcache bool) {
 
 	// commit change to obj2 from external process
 	tdb.Add(102, "kitty")
-	tdb.Commit()
-	at2 := tdb.head
+	at2 := tdb.Commit()
 
 	// new db connection should see the change
 	t2 := tdb.Open(&ConnOptions{})
@@ -608,6 +621,18 @@ func testPersistentDB(t0 *testing.T, rawcache bool) {
 	t.checkObj(obj2, 102, at2, UPTODATE,    0, "kitty")
 
 
+	// newly created object should not go to ghost on deactivate
+	obj3 := NewMyObject(t.conn)
+	obj3.value = "new"
+	checkObj(t0, obj3, t.conn, InvalidOid, InvalidTid, UPTODATE, 0)
+	assert.Equal(obj3.value, "new")
+	t.PActivate(obj3)
+	checkObj(t0, obj3, t.conn, InvalidOid, InvalidTid, UPTODATE, 1)
+	obj3.PDeactivate()
+	checkObj(t0, obj3, t.conn, InvalidOid, InvalidTid, UPTODATE, 0)
+	assert.Equal(obj3.value, "new")
+
+
 	// finish tnx3 and txn2 - conn1 and conn2 go back to db pool
 	t.Abort()
 	t2.Abort()
@@ -719,6 +744,69 @@ func testPersistentDB(t0 *testing.T, rawcache bool) {
 	t.checkObj(robj2, 102, InvalidTid, GHOST, 0)
 }
 
+// Verify that PActivate works correctly after hitting an error from the storage.
+// In this test the error is "object was deleted".
+func TestActivateAfterDelete(t0 *testing.T) {
+	assert := assert.New(t0)
+
+	tdb := testdb(t0, /*rawcache=*/false)
+	defer tdb.Close()
+	db := tdb.db
+
+	tdb.Add(101, "object")
+	at0 := tdb.Commit()
+
+	t := tdb.Open(&ConnOptions{})
+
+	// do not evict the object from live cache.
+	zcc := &zcacheControl{map[Oid]PCachePolicy{
+		101: PCachePinObject | PCacheKeepState,
+	}}
+	zcache := t.conn.Cache()
+	zcache.Lock()
+	zcache.SetControl(zcc)
+	zcache.Unlock()
+
+	// load the object
+	obj := t.Get(101)
+	t.checkObj(obj, 101, InvalidTid, GHOST, 0)
+	t.PActivate(obj)
+	t.checkObj(obj, 101, at0, UPTODATE, 1, "object")
+	obj.PDeactivate()
+	t.checkObj(obj, 101, at0, UPTODATE, 0, "object")
+
+	// delete obj
+	at1 := tdb.CommitRaw(ZRawObject{Oid: 101, Data: nil})
+
+	// conn stays at older view with obj pinned into the cache
+	t.checkObj(obj, 101, at0, UPTODATE, 0, "object")
+
+	// finish transaction and reopen new connection - it should be the same conn
+	t.Abort()
+	assert.Equal(db.pool, []*Connection{t.conn})
+	t_ := tdb.Open(&ConnOptions{})
+	assert.Same(t_.conn, t.conn)
+	t = t_
+	assert.Equal(t.conn.At(), at1)
+
+	// obj should be invalidated but present in the cache
+	t.checkObj(obj, 101, InvalidTid, GHOST, 0)
+
+	// activating obj should give "no data" error
+	// loop because second activate used to panic
+	for i := 0; i < 10; i++ {
+		err := obj.PActivate(t.ctx)
+		eok := &NoDataError{Oid: obj.POid(), DeletedAt: at1}
+		var e *NoDataError
+		errors.As(err, &e)
+		if !reflect.DeepEqual(e, eok) {
+			t.Fatalf("(%d) after delete: err:\nhave: %s\nwant cause: %s", i, err, eok)
+		}
+		// obj should stay in the cache in ghost state
+		t.checkObj(obj, 101, InvalidTid, GHOST, 0)
+	}
+}
+
 // Test details of how LiveCache handles live caching policy.
 func TestLiveCache(t0 *testing.T) {
 	assert := assert.New(t0)
@@ -730,8 +818,7 @@ func TestLiveCache(t0 *testing.T) {
 	tdb.Add(102, "труд")
 	tdb.Add(103, "май")
 	tdb.Add(104, "весна")
-	tdb.Commit()
-	at1 := tdb.head
+	at1 := tdb.Commit()
 
 	zcc := &zcacheControl{map[Oid]PCachePolicy{
 		// obj1 - default (currently: don't pin and don't keep state)

go/zodb/py/pydata-gen-testdata

 #!/usr/bin/env python2
 # -*- coding: utf-8 -*-
-# Copyright (C) 2017-2020  Nexedi SA and Contributors.
+# Copyright (C) 2017-2021  Nexedi SA and Contributors.
 #                          Kirill Smelkov <kirr@nexedi.com>
 #
 # This program is free software: you can Use, Study, Modify and Redistribute
@@ -21,7 +21,7 @@
 """generate reference pickle objects encoding for tests"""
 
 from ZODB import serialize
-from zodbtools.test.gen_testdata import run_with_zodb3py2_compat
+from zodbtools.test.gen_testdata import run_with_zodb4py2_compat
 from golang.gcompat import qq
 
 def main2():
@@ -56,7 +56,7 @@ def main2():
         emit("}")
 
 def main():
-    run_with_zodb3py2_compat(main2)
+    run_with_zodb4py2_compat(main2)
 
 if __name__ == '__main__':
     main()

go/zodb/storage.go

-// Copyright (C) 2017-2019  Nexedi SA and Contributors.
+// Copyright (C) 2017-2021  Nexedi SA and Contributors.
 //                          Kirill Smelkov <kirr@nexedi.com>
 //
 // This program is free software: you can Use, Study, Modify and Redistribute
@@ -46,12 +46,30 @@ type DriverOptions struct {
 	//
 	// Watchq can be nil to ignore such events. However if Watchq != nil, the events
 	// have to be consumed or else the storage driver will misbehave - e.g.
-	// it can get out of sync with the on-disk database file.
+	// it can get out of sync with the on-disk database file, or deadlock
+	// on any user-called operation.
 	//
 	// The storage driver closes !nil Watchq when the driver is closed.
 	//
 	// The storage driver will send only and all events in (at₀, +∞] range,
 	// where at₀ is at returned by driver open.
+	//
+	// The storage driver will stop sending events after call to Close.
+	// In particular the following example is valid and safe from deadlock:
+	//
+	//	watchq := make(chan zodb.Event)
+	//	stor, at0, err := zodb.OpenDriver(..., &DriverOptions{Watchq: watchq})
+	//	defer stor.Close()
+	//
+	//	for {
+	//		select {
+	//		case <-ctx.Done():
+	//			return ctx.Err()
+	//
+	//		case <-watchq:
+	//			...
+	//		}
+	//	}
 	Watchq chan<- Event
 }
 
@@ -73,10 +91,13 @@ func RegisterDriver(scheme string, opener DriverOpener) {
 	driverRegistry[scheme] = opener
 }
 
-// XXX
-func openDriver(ctx context.Context, zurl string, opt *DriverOptions) (_ IStorageDriver, at0 Tid, _ error) {
+// OpenDriver opens ZODB storage driver by URL.
+//
+// It is similar to Open but returns low-level IStorageDriver instead of IStorage.
+// Most users should use Open.
+func OpenDriver(ctx context.Context, zurl string, opt *DriverOptions) (_ IStorageDriver, at0 Tid, _ error) {
 	// no scheme -> file://
-	if !strings.Contains(zurl, "://") {
+	if !strings.Contains(zurl, ":") {
 		zurl = "file://" + zurl
 	}
 
@@ -91,7 +112,7 @@ func openDriver(ctx context.Context, zurl string, opt *DriverOptions) (_ IStorag
 
 	opener, ok := driverRegistry[u.Scheme]
 	if !ok {
-		return nil, InvalidTid, fmt.Errorf("zodb: URL scheme \"%s://\" not supported", u.Scheme)
+		return nil, InvalidTid, fmt.Errorf("zodb: URL scheme \"%s:\" not supported", u.Scheme)
 	}
 
 	storDriver, at0, err := opener(ctx, u, opt)
@@ -108,7 +129,7 @@ func openDriver(ctx context.Context, zurl string, opt *DriverOptions) (_ IStorag
 // Users should import in storage packages they use or zodb/wks package to
 // get support for well-known storages.
 //
-// Storage authors should register their storages with RegisterStorage.
+// Storage authors should register their storages with RegisterDriver.
 func Open(ctx context.Context, zurl string, opt *OpenOptions) (IStorage, error) {
 	drvWatchq := make(chan Event)
 	drvOpt := &DriverOptions{
@@ -116,7 +137,7 @@ func Open(ctx context.Context, zurl string, opt *OpenOptions) (IStorage, error)
 		Watchq:   drvWatchq,
 	}
 
-	storDriver, at0, err := openDriver(ctx, zurl, drvOpt)
+	storDriver, at0, err := OpenDriver(ctx, zurl, drvOpt)
 	if err != nil {
 		return nil, err
 	}
@@ -163,6 +184,7 @@ type storage struct {
 	down        chan struct{} // ready when no longer operational
 	downOnce    sync.Once     // shutdown may be due to both Close and IO error in watcher|Sync
 	downErr     error         // reason for shutdown
+	drvCloseErr error         // err from .driver.Close()
 
 	// watcher
 
@@ -185,6 +207,11 @@ func (s *storage) shutdown(reason error) {
 	s.downOnce.Do(func() {
 		close(s.down)
 		s.downErr = fmt.Errorf("not operational due: %s", reason)
+		// - if called by Close or failed Sync: driver.Close will close
+		//   drvWatchq and cause watcher to stop.
+		// - if called by failed watcher: closing driver will prevent
+		//   drvWatchq<- deadlock in driver because we no longer read from it.
+		s.drvCloseErr = s.driver.Close()
 	})
 }
 
@@ -198,7 +225,7 @@ func (s *storage) Iterate(ctx context.Context, tidMin, tidMax Tid) ITxnIterator
 
 func (s *storage) Close() error {
 	s.shutdown(fmt.Errorf("closed"))
-	return s.driver.Close() // this will close drvWatchq and cause watcher stop
+	return s.drvCloseErr
 }
 
 // loading goes through cache - this way prefetching can work
@@ -369,7 +396,7 @@ func (s *storage) _watcher() error {
 func (s *storage) AddWatch(watchq chan<- Event) (at0 Tid) {
 	ack := make(chan Tid)
 	select {
-	// no longer operational: behave if watchq was registered before that
+	// no longer operational: behave as if watchq was registered before that
 	// and then seen down/close events. Interact with DelWatch directly.
 	case <-s.down:
 		s.headMu.Lock()   // shutdown may be due to Close call and watcher might be
@@ -471,6 +498,14 @@ func (s *storage) Sync(ctx context.Context) (err error) {
 	}
 
 	// wait till .head >= head
+	// XXX instead require from drivers that `drv.Sync() -> head`
+	// guarantees that all EventCommit with .tid <= head were sent to
+	// watchq
+	//
+	// https://lab.nexedi.com/nexedi/ZODB/commit/40116375
+	// https://github.com/zopefoundation/ZODB/commit/4a6b0283#diff-d2a01f71a79ac2b379e218cf72fa1205d3426cad19e7b72d71899f643be4bb73
+	//
+	// ?
 	watchq := make(chan Event)
 	at = s.AddWatch(watchq)
 	defer s.DelWatch(watchq)

go/zodb/storage/demo/demo.go

+// Copyright (C) 2021  Nexedi SA and Contributors.
+//                     Kirill Smelkov <kirr@nexedi.com>
+//
+// This program is free software: you can Use, Study, Modify and Redistribute
+// it under the terms of the GNU General Public License version 3, or (at your
+// option) any later version, as published by the Free Software Foundation.
+//
+// You can also Link and Combine this program with other software covered by
+// the terms of any of the Free Software licenses or any of the Open Source
+// Initiative approved licenses and Convey the resulting work. Corresponding
+// source of such a combination shall include the source code for all other
+// software used.
+//
+// This program is distributed WITHOUT ANY WARRANTY; without even the implied
+// warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
+//
+// See COPYING file for full licensing terms.
+// See https://www.nexedi.com/licensing for rationale and options.
+
+// Package demo provides overlayed storage, similar to DemoStorage in ZODB/py.
+//
+// Storage combines base and δ storages as if δ transactional log is logically
+// appended to base.
+package demo
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"io"
+	"net/url"
+	"regexp"
+	"sync"
+
+	"lab.nexedi.com/kirr/go123/mem"
+	"lab.nexedi.com/kirr/go123/xerr"
+	"lab.nexedi.com/kirr/go123/xsync"
+	"lab.nexedi.com/kirr/neo/go/internal/task"
+	"lab.nexedi.com/kirr/neo/go/zodb"
+)
+
+// Storage combines base and δ storages as if δ transactional log is logically
+// appended to base.
+//
+// Base must remain unmodified, and all δ transactions must be with TIDs
+// strictly after base.Head .
+type Storage struct {
+	base zodb.IStorageDriver
+	δ    zodb.IStorageDriver
+
+	baseAt0    zodb.Tid
+	baseWatchq <-chan zodb.Event
+	δWatchq    <-chan zodb.Event // nil if demo is opened with watchq=nil
+	δWatchq0   <-chan zodb.Event // buffer for δ events queued while δ was initially verified
+	watchq     chan<- zodb.Event // user requested to deliver events here
+
+	watchWG     *xsync.WorkGroup
+	watchCancel func()
+
+	downOnce sync.Once
+	down     chan struct{} // ready when storage is down
+	downErr  error         // reason for shutdown
+
+	closeOnce sync.Once
+	closed    chan struct{} // ready when storage is Closed
+}
+
+// baseMutatedError is reported when Storage.base is detected to change.
+type baseMutatedError struct {
+	baseAt0  zodb.Tid
+	baseHead zodb.Tid
+}
+
+func (e *baseMutatedError) Error() string {
+	return fmt.Sprintf("base.head mutated from @%s to @%s", e.baseAt0, e.baseHead)
+}
+
+// baseErrorEvent is reported on error event detected on Storage.base .
+type baseErrorEvent struct {
+	err error
+}
+
+func (e *baseErrorEvent) Error() string {
+	return fmt.Sprintf("base: error event: %s", e.err)
+}
+
+func (e *baseErrorEvent) Cause() error  { return e.err }
+func (e *baseErrorEvent) Unwrap() error { return e.err }
+
+
+// watcher task detects base mutation and proxies δ events to user watchq.
+func (d *Storage) watcher(ctx context.Context) error {
+	if d.watchq != nil {
+		defer close(d.watchq)
+	}
+
+	for {
+		var δWatchq <-chan zodb.Event
+		if len(d.δWatchq0) != 0 {
+			δWatchq = d.δWatchq0
+		} else {
+			δWatchq = d.δWatchq
+		}
+
+		select {
+		// Close requests to stop watching
+		case <-ctx.Done():
+			return ctx.Err()
+
+		// event on base -> base mutated/error + shutdown
+		case event, ok := <-d.baseWatchq:
+			if !ok {
+				// base closed
+				d.baseWatchq = nil
+				continue
+			}
+
+			var edown error
+			switch event := event.(type) {
+			case *zodb.EventCommit:
+				edown = &baseMutatedError{d.baseAt0, event.Tid}
+
+			case *zodb.EventError:
+				edown = &baseErrorEvent{event.Err}
+
+			default:
+				edown = fmt.Errorf("base: unexpected event %T", event)
+			}
+
+			d.shutdown(edown)
+			ev := &zodb.EventError{&zodb.OpError{URL: d.URL(), Op: "watcher", Err: edown}}
+			if d.watchq != nil {
+				select {
+				case <-d.closed:
+					// wakeup to return edown
+
+				case d.watchq <- ev:
+					// ok
+				}
+			}
+			return edown
+
+		// event on δ -> proxy to user
+		case event, ok := <-δWatchq:
+			if !ok {
+				// δ closed
+				d.δWatchq = nil
+				continue
+			}
+
+			select {
+			case <-d.closed:
+				return nil
+
+			case d.watchq <- event: // !nil because d.δWatchq != nil
+				// ok
+			}
+		}
+	}
+}
+
+// shutdown marks Storage as no longer operational due to reason.
+func (d *Storage) shutdown(reason error) {
+	d.downOnce.Do(func() {
+		d.downErr = reason
+		close(d.down)
+	})
+}
+
+var errClosed = errors.New("storage is closed")
+
+// Close implements zodb.IStorageDriver .
+func (d *Storage) Close() (err error) {
+	defer func() {
+		if err != nil {
+			err = &zodb.OpError{URL: d.URL(), Op: "close", Err: err}
+		}
+	}()
+
+	d.shutdown(errClosed)
+	d.closeOnce.Do(func() {
+		close(d.closed)
+	})
+	errδ    := d.δ.Close()
+	errBase := d.base.Close()
+
+	// cancel watcher; don't propagate its error to Close - watcher error
+	// goes to watchq and op errors.
+	d.watchCancel()
+	_ = d.watchWG.Wait()
+
+	return xerr.Merge(errδ, errBase)
+}
+
+// Sync implements zodb.IStorageDriver .
+func (d *Storage) Sync(ctx context.Context) (_ zodb.Tid, err error) {
+	defer func() {
+		if err != nil {
+			err = &zodb.OpError{URL: d.URL(), Op: "sync", Err: err}
+		}
+	}()
+	if ready(d.down) {
+		return zodb.InvalidTid, d.downErr
+	}
+
+	var head zodb.Tid
+	wg := xsync.NewWorkGroup(ctx)
+	wg.Go(func(ctx context.Context) error {
+		h, err := d.δ.Sync(ctx)
+		head = h
+		return err
+	})
+	wg.Go(func(ctx context.Context) error {
+		baseHead, err := d.base.Sync(ctx)
+		if err != nil {
+			return err
+		}
+		if baseHead != d.baseAt0 {
+			return &baseMutatedError{d.baseAt0, baseHead}
+		}
+		return nil
+	})
+
+	err = wg.Wait()
+	if err != nil {
+		return zodb.InvalidTid, err
+	}
+
+	// δ is just created database
+	if head == 0 {
+		head = d.baseAt0
+	}
+
+	return head, nil
+}
+
+// Load implements zodb.IStorageDriver .
+func (d *Storage) Load(ctx context.Context, xid zodb.Xid) (_ *mem.Buf, _ zodb.Tid, err error) {
+	defer func() {
+		if err != nil {
+			err = &zodb.OpError{URL: d.URL(), Op: "load", Args: xid, Err: err}
+		}
+	}()
+	if ready(d.down) {
+		return nil, zodb.InvalidTid, d.downErr
+	}
+
+	var eNoData   *zodb.NoDataError
+	var eNoObject *zodb.NoObjectError
+
+	inδ := false
+	if xid.At > d.baseAt0 {
+		data, serial, err := d.δ.Load(ctx, xid)
+		if err == nil {
+			// object data is present in δ
+			return data, serial, nil
+		}
+
+		useBase := false
+		switch {
+		case errors.As(err, &eNoData):
+			if eNoData.DeletedAt != 0 {
+				// object deleted in δ -> whiteout
+				return data, serial, eNoData
+			} else {
+				// object present in δ but not yet created as of xid.at
+				useBase = true
+				inδ = true
+			}
+
+		case errors.As(err, &eNoObject):
+			// object not created in δ
+			useBase = true
+		}
+
+		if !useBase {
+			return data, serial, err
+		}
+	}
+
+	// cap xid.At to .baseAt0 (we convert it back on error return, and
+	// it makes Load more robust wrt simultaneous base mutation).
+	xidBase := xid
+	if xid.At > d.baseAt0 {
+		xidBase.At = d.baseAt0
+	}
+
+	data, serial, err := d.base.Load(ctx, xidBase)
+	if err == nil {
+		return data, serial, nil
+	}
+
+	switch {
+	case errors.As(err, &eNoData):
+		err = eNoData
+	case errors.As(err, &eNoObject):
+		if !inδ {
+			err = eNoObject
+		} else {
+			// object is present in δ
+			err = &zodb.NoDataError{Oid: xid.Oid, DeletedAt: 0}
+		}
+	}
+
+	return data, serial, err
+}
+
+// Iterator implements zodb.IStorageDriver .
+func (d *Storage) Iterate(ctx context.Context, tidMin, tidMax zodb.Tid) zodb.ITxnIterator {
+	panic("TODO")
+}
+
+// URL implements zodb.IStorageDriver .
+func (d *Storage) URL() string {
+	return "demo:(" + d.base.URL() + ")/(" + d.δ.URL() + ")"
+}
+
+var demoRe = regexp.MustCompile(`^\((.*)\)/\((.*)\)$`)
+
+func openByURL(ctx context.Context, u *url.URL, opt *zodb.DriverOptions) (_ zodb.IStorageDriver, _ zodb.Tid, err error) {
+	// demo:(base_zurl)/(δ_zurl)
+	defer task.Runningf(&ctx, "demo: open %s", u)(&err)
+
+	if !(u.Opaque != "" && u.RawQuery == "" && u.Fragment == "") {
+		// opaque != "" makes sure user,host,path are empty
+		return nil, zodb.InvalidTid, fmt.Errorf("invalid url")
+	}
+
+	argv := demoRe.FindStringSubmatch(u.Opaque)
+	if argv == nil {
+		return nil, zodb.InvalidTid, fmt.Errorf("invalid url")
+	}
+
+	baseZURL := argv[1]
+	δZURL    := argv[2]
+
+	// open base - always readonly
+	baseWatchq := make(chan zodb.Event)
+	base, baseAt0, err := zodb.OpenDriver(ctx, baseZURL, &zodb.DriverOptions{
+		ReadOnly: true,
+		Watchq:   baseWatchq,
+	})
+	if err != nil {
+		return nil, zodb.InvalidTid, err
+	}
+	defer func() {
+		if err != nil {
+			__ := base.Close()
+			err = xerr.Merge(err, __)
+		}
+	}()
+
+	// open δ    - as requested  but with events going through us
+	δopt := *opt
+	var δWatchq chan zodb.Event
+	if δopt.Watchq != nil {
+		δWatchq = make(chan zodb.Event)
+		δopt.Watchq = δWatchq
+	}
+	δ, δAt0, err := zodb.OpenDriver(ctx, δZURL, &δopt)
+	if err != nil {
+		return nil, zodb.InvalidTid, err
+	}
+	defer func() {
+		if err != nil {
+			__ := δ.Close()
+			err = xerr.Merge(err, __)
+		}
+	}()
+
+
+	// verify that either
+	// - δ is all empty (just created), or
+	// - all δ transactions come strictly after base.Head .
+	at0 := baseAt0
+	var δEventq0 []zodb.Event
+	if δAt0 != 0 {
+		if δAt0 < baseAt0 {
+			return nil, zodb.InvalidTid, fmt.Errorf("base is ahead of δ: base.head=%s  δ.head=%s", baseAt0, δAt0)
+		}
+
+		// read and queue data from δWatchq while we verify δ (not to deadlock δ driver)
+		δq0Stop := make(chan struct{}) // reader <- main  "stop"
+		δq0Done := make(chan struct{}) // reader -> main  "done"
+		go func() {
+			defer close(δq0Done)
+
+			for {
+				select {
+				case <-δq0Stop:
+					return
+
+				case δevent, ok := <-δWatchq:
+					if !ok {
+						return
+					}
+					δEventq0 = append(δEventq0, δevent)
+				}
+			}
+		}()
+
+		// verify δ
+		iδ := δ.Iterate(ctx, 0, baseAt0)
+		δtxni, _, err := iδ.NextTxn(ctx)
+		switch {
+		case err == io.EOF:
+			err = nil // ok - nothing in δ
+
+		case err == nil:
+			// there is a δ transaction ∈ [δAt0, baseAt0)
+			err = fmt.Errorf("base overlaps with δ: base.head=%s  δ.tail=%s", baseAt0, δtxni.Tid)
+		}
+		// TODO iδ.Close()
+
+		close(δq0Stop)
+		<-δq0Done
+		if err != nil {
+			return nil, zodb.InvalidTid, err
+		}
+
+		at0 = δAt0
+	}
+
+	// requeue δWatchq0 <- δEventq0
+	var δWatchq0 chan zodb.Event
+	if l := len(δEventq0); l != 0 {
+		δWatchq0 = make(chan zodb.Event, l)
+		for _, ev := range δEventq0 {
+			δWatchq0 <- ev
+		}
+	}
+
+	d := &Storage{
+		base: base,
+		δ   : δ,
+
+		baseAt0:    baseAt0,
+		baseWatchq: baseWatchq,
+		δWatchq:    δWatchq,
+		δWatchq0:   δWatchq0,
+		watchq:     opt.Watchq,
+
+		down:   make(chan struct{}),
+		closed: make(chan struct{}),
+	}
+
+	// spawn watcher to listen on baseWatchq and shutdown storage if base changes.
+	ctx, cancel := context.WithCancel(context.Background())
+	d.watchWG = xsync.NewWorkGroup(ctx)
+	d.watchCancel = cancel
+	d.watchWG.Go(d.watcher)
+
+	return d, at0, nil
+}
+
+func init() {
+	zodb.RegisterDriver("demo", openByURL)
+}
+
+
+// ---- misc ----
+
+// ready returns whether c is ready.
+func ready(c <-chan struct{}) bool {
+	select {
+	case <-c:
+		return true
+	default:
+		return false
+	}
+}

go/zodb/storage/demo/demo_test.go

+// Copyright (C) 2021  Nexedi SA and Contributors.
+//                     Kirill Smelkov <kirr@nexedi.com>
+//
+// This program is free software: you can Use, Study, Modify and Redistribute
+// it under the terms of the GNU General Public License version 3, or (at your
+// option) any later version, as published by the Free Software Foundation.
+//
+// You can also Link and Combine this program with other software covered by
+// the terms of any of the Free Software licenses or any of the Open Source
+// Initiative approved licenses and Convey the resulting work. Corresponding
+// source of such a combination shall include the source code for all other
+// software used.
+//
+// This program is distributed WITHOUT ANY WARRANTY; without even the implied
+// warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
+//
+// See COPYING file for full licensing terms.
+// See https://www.nexedi.com/licensing for rationale and options.
+
+package demo
+
+import (
+	"bytes"
+	"context"
+	"io/ioutil"
+	"fmt"
+	"net/url"
+	"os"
+	"reflect"
+	"regexp"
+	"testing"
+
+	"lab.nexedi.com/kirr/go123/xerr"
+
+	"lab.nexedi.com/kirr/neo/go/internal/xtesting"
+	"lab.nexedi.com/kirr/neo/go/zodb"
+	"lab.nexedi.com/kirr/neo/go/zodb/zodbtools"
+
+	// for file: scheme support
+	_ "lab.nexedi.com/kirr/neo/go/zodb/storage/fs1"
+)
+
+// DemoData represents data for a demo: storage.
+type DemoData struct {
+	base string // url for   base.fs
+	δ    string // ----/---- δ.fs
+}
+
+func (ddat *DemoData) URL() string {
+	return fmt.Sprintf("demo:(%s)/(%s)", ddat.base, ddat.δ)
+}
+
+
+// tOptions represents options for testing.
+// TODO -> xtesting
+type tOptions struct {
+	Preload string // preload database with data from this location
+}
+
+// withDemoData tests f with all kinds of opt.Preload data split into base + δ.
+func withDemoData(t *testing.T, f func(t *testing.T, ddat *DemoData), optv ...tOptions) {
+	t.Helper()
+	X := xtesting.FatalIf(t)
+
+	opt := tOptions{}
+	if len(optv) > 1 {
+		panic("multiple tOptions not allowed")
+	}
+	if len(optv) == 1 {
+		opt = optv[0]
+	}
+
+	// retrieve zdump of Preload
+	zdump := ""
+	if opt.Preload != "" {
+		ctx := context.Background()
+		buf := &bytes.Buffer{}
+		stor, err := zodb.Open(ctx, opt.Preload, &zodb.OpenOptions{ReadOnly: true}); X(err)
+		err = zodbtools.Dump(ctx, buf, stor, 0, zodb.TidMax, /*hashonly=*/false)
+		stor.Close()
+		X(err)
+		zdump = buf.String()
+	}
+
+	// split zdump into transactions
+	// XXX hacky; TODO -> zodbtools.DumpReader
+	txnRe := regexp.MustCompile(`(?m)^txn (?P<tid>[0-9a-f]{16}) "(?P<status>.)"$`)
+	type zdumpTxn struct {
+		tid zodb.Tid
+		pos int      // where this transaction starts in the dump
+	}
+	var txnv []zdumpTxn
+	for _, m := range txnRe.FindAllStringSubmatchIndex(zdump, -1) {
+		// [m[0]:m[1]] refers to whole txn line
+		__ := zdump[m[2]:m[3]]
+		tid, err := zodb.ParseTid(__); X(err)
+		txnv = append(txnv, zdumpTxn{tid, m[0]})
+	}
+
+	// verify f on all combinations of preload being split into base+δ
+	work := xtempdir(t)
+	defer os.RemoveAll(work)
+	test1 := func(δstart zodb.Tid, zdumpBase, zdumpδ string) {
+		t.Helper()
+		t.Run(fmt.Sprintf("δstart=%s", δstart), func(t *testing.T) {
+			t.Helper()
+			X := xtesting.FatalIf(t)
+			work1 := work + "/δ" + δstart.String()
+			err := os.Mkdir(work1, 0777); X(err)
+
+			base := "file://"+work1+"/base.fs"
+			δ    := "file://"+work1+"/δ.fs"
+			ddat := &DemoData{base, δ}
+
+			_, err = xtesting.ZPyRestore(base, zdumpBase);  X(err)
+			// restore δ part via `demo:(base)/(δ)` - not `file:δ`.
+			// The reason we do this is because restoring δ via
+			// just its file will fail when restoring copy data
+			// record with copy_from transaction being in base.
+			_, err = xtesting.ZPyRestore(ddat.URL(), zdumpδ);  X(err)
+
+			f(t, ddat)
+		})
+	}
+	for i := 0; i < len(txnv); i++ {
+		δtail := txnv[i]
+		test1(δtail.tid, zdump[:δtail.pos], zdump[δtail.pos:])
+	}
+	test1(zodb.TidMax, zdump, "")
+}
+
+// withDemo tests f with demo: client connected to all kind of demo data splits.
+func withDemo(t *testing.T, f func(t *testing.T, ddat *DemoData, ddrv *Storage), optv ...tOptions) {
+	t.Helper()
+	withDemoData(t, func(t *testing.T, ddat *DemoData) {
+		t.Helper()
+		X := xtesting.FatalIf(t)
+		ddrv, _, err := demoOpen(ddat.URL(), &zodb.DriverOptions{ReadOnly: true}); X(err)
+		defer func() {
+			err := ddrv.Close(); X(err)
+		}()
+
+		f(t, ddat, ddrv)
+	}, optv...)
+}
+
+func TestURL(t *testing.T) {
+	withDemo(t, func(t *testing.T, ddat *DemoData, ddrv *Storage) {
+		zurl   := ddrv.URL()
+		zurlOk := ddat.URL()
+		if zurl != zurlOk {
+			t.Fatalf("bad zurl:\nhave: %s\nwant: %s", zurl, zurlOk)
+		}
+	})
+}
+
+func TestEmptyDB(t *testing.T) {
+	withDemo(t, func(t *testing.T, _ *DemoData, ddrv *Storage) {
+		xtesting.DrvTestEmptyDB(t, ddrv)
+	})
+}
+
+func TestLoad(t *testing.T) {
+	X := xtesting.FatalIf(t)
+
+	data := "../fs1/testdata/1.fs"
+	txnvOk, err := xtesting.LoadDBHistory(data); X(err)
+
+	withDemo(t, func(t *testing.T, _ *DemoData, ddrv *Storage) {
+		xtesting.DrvTestLoad(t, ddrv, txnvOk)
+	}, tOptions{
+		Preload: data,
+	})
+}
+
+func TestWatch(t *testing.T) {
+	withDemoData(t, func(t *testing.T, ddat *DemoData) {
+		xtesting.DrvTestWatch(t, ddat.URL(), openByURL)
+	})
+}
+
+// MutateBase mutates ddat.base with new commit.
+func (ddat *DemoData) MutateBase() (zodb.Tid, error) {
+	return xtesting.ZPyCommitRaw(ddat.base, 0, xtesting.ZRawObject{
+		Oid:  1,
+		Data: []byte("ZZZ"),
+	})
+}
+
+// TestSync_vs_BaseMutate verifies Sync wrt base mutation.
+func TestSync_vs_BaseMutate(t *testing.T) {
+	withDemo(t, func(t *testing.T, ddat *DemoData, ddrv *Storage) {
+		X := xtesting.FatalIf(t)
+
+		head, err := ddrv.Sync(context.Background())
+		if !(head == 0 && err == nil) {
+			t.Fatalf("sync0: head=%s  err=%s", head, err)
+		}
+
+		tid, err := ddat.MutateBase(); X(err)
+
+		head, err = ddrv.Sync(context.Background())
+		errOk := &zodb.OpError{URL: ddrv.URL(), Op: "sync", Err: &baseMutatedError{
+			baseAt0:  0,
+			baseHead: tid,
+		}}
+		if !reflect.DeepEqual(err, errOk) {
+			t.Fatalf("after base mutate: sync: unexpected error:\nhave: %s\nwant: %s",
+				err, errOk)
+		}
+	})
+}
+
+// TestWatchLoad_vs_BaseMutate verifies Watch and Load wrt base mutation.
+func TestWatchLoad_vs_BaseMutate(t *testing.T) {
+	withDemoData(t, func(t *testing.T, ddat *DemoData) {
+		X := xtesting.FatalIf(t)
+
+		watchq := make(chan zodb.Event)
+		ddrv, at0, err := demoOpen(ddat.URL(), &zodb.DriverOptions{
+			ReadOnly: true,
+			Watchq:   watchq,
+		}); X(err)
+		defer func() {
+			err := ddrv.Close(); X(err)
+		}()
+
+		tid, err := ddat.MutateBase(); X(err)
+
+		// first wait for error from watchq
+		event, ok := <-watchq
+		if !ok {
+			t.Fatal("after base mutate: premature watchq close")
+		}
+		evErr, ok := event.(*zodb.EventError)
+		if !ok {
+			t.Fatalf("after base mutate: unexpected event: %T", event)
+		}
+
+		errBaseMutated := &baseMutatedError{
+			baseAt0:  0,
+			baseHead: tid,
+		}
+		evErrOk := &zodb.EventError{&zodb.OpError{URL: ddrv.URL(), Op: "watcher", Err: errBaseMutated}}
+		if !reflect.DeepEqual(evErr, evErrOk) {
+			t.Fatalf("after base mutate: unexpected event:\nhave: %s\nwant: %s", evErr, evErrOk)
+		}
+
+		// now make sure Load fails with "base mutated" error
+		xid := zodb.Xid{Oid: 1, At: at0}
+		data, serial, err := ddrv.Load(context.Background(), xid)
+		errOk := &zodb.OpError{URL: ddrv.URL(), Op: "load", Args: xid, Err: errBaseMutated}
+		if !reflect.DeepEqual(err, errOk) {
+			t.Fatalf("after base mutate: load: unexpected error:\nhave: %s\nwant: %s",
+				err, errOk)
+		}
+		if !(data == nil && serial == zodb.InvalidTid) {
+			t.Fatalf("after base mutate: load: unexpected data=%v  serial=%v", data, serial)
+		}
+	})
+}
+
+
+func demoOpen(zurl string, opt *zodb.DriverOptions) (_ *Storage, at0 zodb.Tid, err error) {
+	defer xerr.Contextf(&err, "opendemo %s", zurl)
+	u, err := url.Parse(zurl)
+	if err != nil {
+		return nil, 0, err
+	}
+
+	d, at0, err := openByURL(context.Background(), u, opt)
+	if err != nil {
+		return nil, 0, err
+	}
+
+	return d.(*Storage), at0, nil
+}
+
+func xtempdir(t *testing.T) string {
+	t.Helper()
+	tmpd, err := ioutil.TempDir("", "demo")
+	if err != nil {
+		t.Fatal(err)
+	}
+	return tmpd
+}

go/zodb/storage/fs1/filestorage.go

-// Copyright (C) 2017-2020  Nexedi SA and Contributors.
+// Copyright (C) 2017-2021  Nexedi SA and Contributors.
 //                          Kirill Smelkov <kirr@nexedi.com>
 //
 // This program is free software: you can Use, Study, Modify and Redistribute
@@ -109,6 +109,9 @@ type FileStorage struct {
 	downOnce sync.Once      // shutdown may be due to both Close and IO error in watcher
 	errClose error          // error from .file.Close()
 	watchWg  sync.WaitGroup // to wait for watcher finish
+
+	closed    chan struct{} // ready when storage was Closed
+	closeOnce sync.Once
 }
 
 // IStorageDriver
@@ -132,7 +135,7 @@ func (fs *FileStorage) LastOid(_ context.Context) (zodb.Oid, error) {
 }
 
 func (fs *FileStorage) URL() string {
-	return fs.file.Name()
+	return "file://" + fs.file.Name()
 }
 
 // freelist(DataHeader)
@@ -468,7 +471,7 @@ func (fs *FileStorage) watcher(w *fsnotify.Watcher, errFirstRead chan<- error) {
 		// XXX it can also be internal.poll.ErrFileClosing
 		e.Err.Error() == "use of closed file") {
 		select {
-		case <-fs.down:
+		case <-fs.closed:
 			err = nil
 		default:
 		}
@@ -485,7 +488,13 @@ func (fs *FileStorage) watcher(w *fsnotify.Watcher, errFirstRead chan<- error) {
 
 	if fs.watchq != nil {
 		if err != nil {
-			fs.watchq <- &zodb.EventError{err}
+			select {
+			case <-fs.closed:
+				// closed - skip send to watchq
+
+			case fs.watchq <- &zodb.EventError{err}:
+				// ok
+			}
 		}
 		close(fs.watchq)
 	}
@@ -535,9 +544,9 @@ mainloop:
 		if !first {
 			traceWatch("select ...")
 			select {
-			case <-fs.down:
+			case <-fs.closed:
 				// closed
-				traceWatch("down")
+				traceWatch("closed")
 				return nil
 
 			case err := <-w.Errors:
@@ -694,7 +703,7 @@ mainloop:
 			// notify client
 			if fs.watchq != nil {
 				select {
-				case <-fs.down:
+				case <-fs.closed:
 					return nil
 
 				case fs.watchq <- &zodb.EventCommit{it.Txnh.Tid, δoid}:
@@ -772,6 +781,9 @@ func (fs *FileStorage) shutdown(reason error) {
 }
 
 func (fs *FileStorage) Close() error {
+	fs.closeOnce.Do(func() {
+		close(fs.closed)
+	})
 	fs.shutdown(fmt.Errorf("closed"))
 	fs.watchWg.Wait()
 
@@ -794,6 +806,7 @@ func Open(ctx context.Context, path string, opt *zodb.DriverOptions) (_ *FileSto
 	fs := &FileStorage{
 		watchq: opt.Watchq,
 		down:   make(chan struct{}),
+		closed: make(chan struct{}),
 	}
 
 	f, err := os.Open(path)

go/zodb/storage/fs1/filestorage_test.go

-// Copyright (C) 2017-2020  Nexedi SA and Contributors.
+// Copyright (C) 2017-2021  Nexedi SA and Contributors.
 //                          Kirill Smelkov <kirr@nexedi.com>
 //
 // This program is free software: you can Use, Study, Modify and Redistribute
@@ -370,3 +370,37 @@ func TestOpenRecovery(t *testing.T) {
 		})
 	}
 }
+
+
+// TestLoadWhiteout verifies access to whiteout data record.
+//
+// Whiteout is data record with deletion when object was not previously there.
+// It has both len(data)=0 and backpointer=0.
+//
+// TODO merge into regular tests on testdata/1.fs when
+// FileStorage/py.deleteObject allows to create whiteouts instead of raising
+// POSKeyError.
+func TestLoadWhiteout(t *testing.T) {
+	fs, _ := xfsopen(t, "testdata/whiteout.fs")
+	defer exc.XRun(fs.Close)
+
+	xid := zodb.Xid{At: zodb.Tid(0x17), Oid: zodb.Oid(1)}
+	buf, serial, err := fs.Load(context.Background(), xid)
+
+	errOk := &zodb.OpError{
+		URL:  fs.URL(),
+		Op:   "load",
+		Args: xid,
+		Err:  &zodb.NoDataError{Oid: xid.Oid, DeletedAt: xid.At},
+	}
+
+	if !reflect.DeepEqual(err, errOk) {
+		t.Errorf("load %s: bad err:\nhave: %v\nwant: %v", xid, err, errOk)
+	}
+	if buf != nil {
+		t.Errorf("load %s: buf != nil", xid)
+	}
+	if serial != 0 {
+		t.Errorf("load %s: bad serial %s  ; want 0", xid, serial)
+	}
+}

go/zodb/storage/fs1/format.go

-// Copyright (C) 2017-2020  Nexedi SA and Contributors.
+// Copyright (C) 2017-2021  Nexedi SA and Contributors.
 //                          Kirill Smelkov <kirr@nexedi.com>
 //
 // This program is free software: you can Use, Study, Modify and Redistribute
@@ -207,7 +207,7 @@ func (txnh *TxnHeader) CloneFrom(txnh2 *TxnHeader) {
 type TxnLoadFlags int
 const (
 	LoadAll       TxnLoadFlags = 0x00 // load whole transaction header
-	LoadNoStrings              = 0x01 // do not load user/desc/ext strings
+	LoadNoStrings TxnLoadFlags = 0x01 // do not load user/desc/ext strings
 )
 
 // Load reads and decodes transaction record header @ pos.
@@ -575,7 +575,10 @@ func (dh *DataHeader) LoadBackRef(r io.ReaderAt) (backPos int64, err error) {
 	}
 
 	backPos = int64(binary.BigEndian.Uint64(dh.workMem[0:]))
-	if !(backPos == 0 || backPos >= dataValidFrom) {
+	if backPos == 0 {
+		return 0, nil // deletion
+	}
+	if backPos < dataValidFrom {
 		return 0, checkErr(r, dh, "invalid backpointer: %v", backPos)
 	}
 	if backPos + DataHeaderSize > dh.TxnPos - 8 {

go/zodb/storage/fs1/fs1tools/dump.go

-// Copyright (C) 2017  Nexedi SA and Contributors.
+// Copyright (C) 2017-2021  Nexedi SA and Contributors.
 //                          Kirill Smelkov <kirr@nexedi.com>
 //
 // This program is free software: you can Use, Study, Modify and Redistribute
@@ -51,6 +51,9 @@ type Dumper interface {
 	//
 	// If dumper return io.EOF the whole dumping process finishes.
 	DumpTxn(buf *xfmt.Buffer, it *fs1.Iter) error
+
+	// DumpEndOK is called at the end of successfull dump.
+	DumpEndOK(buf *xfmt.Buffer) error
 }
 
 // Dump dumps content of a FileStorage file @ path.
@@ -101,7 +104,7 @@ func Dump(w io.Writer, path string, dir fs1.IterDir, d Dumper) (err error) {
 		err = it.NextTxn(fs1.LoadAll)
 		if err != nil {
 			if err == io.EOF {
-				err = nil
+				break
 			}
 			return err
 		}
@@ -109,7 +112,7 @@ func Dump(w io.Writer, path string, dir fs1.IterDir, d Dumper) (err error) {
 		err = d.DumpTxn(buf, it)
 		if err != nil {
 			if err == io.EOF {
-				err = nil
+				break
 			}
 			return err
 		}
@@ -119,6 +122,12 @@ func Dump(w io.Writer, path string, dir fs1.IterDir, d Dumper) (err error) {
 			return err
 		}
 	}
+
+	err = d.DumpEndOK(buf)
+	if err != nil {
+		return err
+	}
+	return nil
 }
 
 // ----------------------------------------
@@ -198,6 +207,10 @@ func (d *DumperFsDump) DumpTxn(buf *xfmt.Buffer, it *fs1.Iter) error {
 	}
 }
 
+func (d *DumperFsDump) DumpEndOK(buf *xfmt.Buffer) error {
+	return nil
+}
+
 // DumperFsDumpVerbose implements a very verbose dumper with output identical
 // to fsdump.Dumper in zodb/py originally written by Jeremy Hylton:
 //
@@ -281,6 +294,10 @@ func (d *DumperFsDumpVerbose) dumpData(buf *xfmt.Buffer, it *fs1.Iter) error {
 	return nil
 }
 
+func (d *DumperFsDumpVerbose) DumpEndOK(buf *xfmt.Buffer) error {
+	return nil
+}
+
 const dumpSummary = "dump database transactions"
 
 func dumpUsage(w io.Writer) {
@@ -383,6 +400,10 @@ func (d *DumperFsTail) DumpTxn(buf *xfmt.Buffer, it *fs1.Iter) error {
 	return nil
 }
 
+func (d *DumperFsTail) DumpEndOK(buf *xfmt.Buffer) error {
+	return nil
+}
+
 const tailSummary = "dump last few transactions of a database"
 const ntxnDefault = 10
 

go/zodb/storage/fs1/fs1tools/dump_test.go

-// Copyright (C) 2017  Nexedi SA and Contributors.
+// Copyright (C) 2017-2021  Nexedi SA and Contributors.
 //                          Kirill Smelkov <kirr@nexedi.com>
 //
 // This program is free software: you can Use, Study, Modify and Redistribute
@@ -31,6 +31,10 @@ package fs1tools
 //go:generate sh -c "python2 -c 'from ZODB.FileStorage import fsdump; fsdump.main()' ../testdata/empty.fs >testdata/empty.fsdump.ok"
 //go:generate sh -c "python2 -c 'from ZODB.FileStorage.fsdump import Dumper; import sys; d = Dumper(sys.argv[1]); d.dump()' ../testdata/empty.fs >testdata/empty.fsdumpv.ok"
 
+//go:generate sh -c "python2 -m ZODB.scripts.fstail -n 1000000 ../testdata/whiteout.fs >testdata/whiteout.fstail.ok"
+//go:generate sh -c "python2 -c 'from ZODB.FileStorage import fsdump; fsdump.main()' ../testdata/whiteout.fs >testdata/whiteout.fsdump.ok"
+//go:generate sh -c "python2 -c 'from ZODB.FileStorage.fsdump import Dumper; import sys; d = Dumper(sys.argv[1]); d.dump()' ../testdata/whiteout.fs >testdata/whiteout.fsdumpv.ok"
+
 import (
 	"bytes"
 	"fmt"
@@ -50,10 +54,11 @@ func loadFile(t *testing.T, path string) string {
 	return string(data)
 }
 
-func testDump(t *testing.T, dir fs1.IterDir, d Dumper) {
-	testv := []string{"1", "empty"}
+func testDump(t *testing.T, dir fs1.IterDir, newd func() Dumper) {
+	testv := []string{"1", "empty", "whiteout"}
 	for _, tt := range testv {
 		t.Run("db=" + tt, func(t *testing.T) {
+			d   := newd()
 			buf := bytes.Buffer{}
 
 			err := Dump(&buf, fmt.Sprintf("../testdata/%s.fs", tt), dir, d)
@@ -70,9 +75,13 @@ func testDump(t *testing.T, dir fs1.IterDir, d Dumper) {
 	}
 }
 
-func TestFsDump(t *testing.T)	{ testDump(t, fs1.IterForward,  &DumperFsDump{}) }
-func TestFsDumpv(t *testing.T)	{ testDump(t, fs1.IterForward,  &DumperFsDumpVerbose{}) }
-func TestFsTail(t *testing.T)	{ testDump(t, fs1.IterBackward, &DumperFsTail{Ntxn: 1000000}) }
+func newFsDump()  Dumper { return &DumperFsDump{} }
+func newFsDumpv() Dumper { return &DumperFsDumpVerbose{} }
+func newFsTail()  Dumper { return &DumperFsTail{Ntxn: 1000000} }
+
+func TestFsDump(t *testing.T)	{ testDump(t, fs1.IterForward,  newFsDump) }
+func TestFsDumpv(t *testing.T)	{ testDump(t, fs1.IterForward,  newFsDumpv) }
+func TestFsTail(t *testing.T)	{ testDump(t, fs1.IterBackward, newFsTail) }
 
 func BenchmarkTail(b *testing.B) {
 	// FIXME small testdata/1.fs is not representative for benchmarking

go/zodb/storage/fs1/fs1tools/main.go

-// Copyright (C) 2017  Nexedi SA and Contributors.
+// Copyright (C) 2017-2021  Nexedi SA and Contributors.
 //                          Kirill Smelkov <kirr@nexedi.com>
 //
 // This program is free software: you can Use, Study, Modify and Redistribute
@@ -30,7 +30,7 @@ var commands = prog.CommandRegistry{
 	{"reindex", reindexSummary, reindexUsage, reindexMain},
 	{"verify-index", verifyIdxSummary, verifyIdxUsage, verifyIdxMain},
 	// recover		(fsrecover.py)
-	// verify		(fstest.py)
+	{"verify",  verifySummary,  verifyUsage,  verifyMain},
 
 	// XXX repozo ?
 }

go/zodb/storage/fs1/fs1tools/testdata/whiteout.fsdump.ok

+Trans #00000 tid=0000000000000017 time=1900-01-01 00:00:00.000000 offset=27
+    status=' ' user='' description=''
+  data #00000 oid=0000000000000001 class=undo or abort of object creation

go/zodb/storage/fs1/fs1tools/testdata/whiteout.fsdumpv.ok

+************************************************************
+file identifier: 'FS21'
+============================================================
+offset: 4
+end pos: 77
+transaction id: 0000000000000017
+trec len: 73
+status: ' '
+user: ''
+description: ''
+len(extra): 0
+------------------------------------------------------------
+offset: 27
+oid: 0000000000000001
+revid: 0000000000000017
+previous record offset: 0
+transaction offset: 4
+len(data): 0
+backpointer: 0
+redundant trec len: 73

go/zodb/storage/fs1/fs1tools/testdata/whiteout.fstail.ok

+1900-01-01 00:00:00.000000: hash=e3e0ff6c686f2fa02266e744f6629d592d432061
+user='' description='' length=73 offset=4 (+23)
+

go/zodb/storage/fs1/fs1tools/verify.go

+// Copyright (C) 2021  Nexedi SA and Contributors.
+//                     Kirill Smelkov <kirr@nexedi.com>
+//
+// This program is free software: you can Use, Study, Modify and Redistribute
+// it under the terms of the GNU General Public License version 3, or (at your
+// option) any later version, as published by the Free Software Foundation.
+//
+// You can also Link and Combine this program with other software covered by
+// the terms of any of the Free Software licenses or any of the Open Source
+// Initiative approved licenses and Convey the resulting work. Corresponding
+// source of such a combination shall include the source code for all other
+// software used.
+//
+// This program is distributed WITHOUT ANY WARRANTY; without even the implied
+// warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
+//
+// See COPYING file for full licensing terms.
+// See https://www.nexedi.com/licensing for rationale and options.
+
+package fs1tools
+// verify subcommand
+//
+// verification output mimics fstest from ZODB/py as originally written by Jeremy Hylton:
+// https://github.com/zopefoundation/ZODB/blob/5.6.0-35-g1fb097b41/src/ZODB/scripts/fstest.py
+
+import (
+	"flag"
+	"fmt"
+	"io"
+	"os"
+	"time"
+
+	"lab.nexedi.com/kirr/neo/go/zodb/storage/fs1"
+
+	"lab.nexedi.com/kirr/go123/prog"
+	"lab.nexedi.com/kirr/go123/xflag"
+	"lab.nexedi.com/kirr/go123/xfmt"
+)
+
+// Verify verifies content of a FileStorage file @ path.
+//
+// Only data part of the database is verified (the *.fs file).
+// Use VerifyIndexFor to verify the index part (*.fs.index).
+func Verify(w io.Writer, path string, verbose int, progress bool) (err error) {
+	// just iterate through the file and emit progress.
+	// the FileStorage driver implements all consistency checks by itself.
+	fi, err := os.Stat(path)
+	if err != nil {
+		return fmt.Errorf("verify: %s: %s", path, err)
+	}
+	fsize := fi.Size()
+
+	v := &Verifier{verbose: verbose}
+
+	// display progress updates once per tick
+	if progress {
+		tick := time.NewTicker(time.Second / 4)
+		defer tick.Stop()
+
+		xcr := ""
+		if verbose > 0 {
+			xcr = "\n"
+		}
+		v.progress = func(force bool) error {
+			if !force {
+				select {
+				case <-tick.C:
+				default:
+					return nil
+				}
+			}
+
+			_, err := fmt.Fprintf(w,
+				"\rVerified data bytes: %.1f%% (%d/%d);  #txn: %d%s",
+				100 * float64(v.donePos) / float64(fsize),
+				v.donePos, fsize,
+				v.ntxn,
+				xcr)
+			return err
+		}
+	}
+
+	return Dump(w, path, fs1.IterForward, v)
+}
+
+// Verifier implements Dumper that is used by Verify.
+type Verifier struct {
+	ntxn    int // current transaction record #
+	verbose int // >=1 (print txn)   >=2 (print objects)
+
+	// for loading data
+	dhLoading fs1.DataHeader
+
+	donePos  int64                  // done verifying till this position
+	progress func(force bool) error // called after each transaction if !nil
+}
+
+func (v *Verifier) DumperName() string {
+	return "fsverify"
+}
+
+func (v *Verifier) DumpFileHeader(buf *xfmt.Buffer, fh *fs1.FileHeader) error {
+	return nil
+}
+
+func (v *Verifier) DumpTxn(buf *xfmt.Buffer, it *fs1.Iter) error {
+	txnh := &it.Txnh
+
+	for i := 0; ; i++ {
+		err := it.NextData()
+		if err != nil {
+			if err == io.EOF {
+				break
+			}
+			return err
+		}
+
+		dh := &it.Datah
+
+		// load data
+		v.dhLoading = *dh
+		dbuf, err := v.dhLoading.LoadData(it.R)
+		if err != nil {
+			return err
+		}
+
+		if v.verbose >= 2 {
+			buf .S(fmt.Sprintf("%10d: object oid 0x%s #%d\n", dh.Pos, dh.Oid, i))
+		}
+
+		dbuf.Release()
+	}
+
+	if v.verbose >= 1 {
+		buf .S(fmt.Sprintf("%10d: transaction tid 0x%s #%d \n", txnh.Pos, txnh.Tid, v.ntxn))
+	}
+
+	v.ntxn++
+
+	if v.progress != nil {
+		v.donePos = txnh.Pos + txnh.Len
+		err := v.progress(/*force=*/false)
+		if err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
+func (v *Verifier) DumpEndOK(buf *xfmt.Buffer) error {
+	if v.progress != nil {
+		err := v.progress(/*force=*/true)
+		if err != nil {
+			return err
+		}
+	}
+	if v.verbose >= 1 {
+		buf .S("no errors detected\n")
+	}
+	return nil
+}
+
+
+// ----------------------------------------
+
+const verifySummary = "verify database content"
+
+func verifyUsage(w io.Writer) {
+	fmt.Fprintf(w,
+`Usage: fs1 verify [options] <storage>
+Verify FileStorage records for consistency
+
+<storage> is a path to FileStorage
+
+  options:
+
+	-h --help       this help text.
+	-v		increase verbosity.
+	-p		display progress.
+`)
+}
+
+func verifyMain(argv []string) {
+	verbose := 0
+	var progress bool
+	flags := flag.FlagSet{Usage: func() { verifyUsage(os.Stderr) }}
+	flags.Init("", flag.ExitOnError)
+	flags.Var((*xflag.Count)(&verbose), "v", "verbosity level")
+	flags.BoolVar(&progress, "p", false, "display progress")
+	flags.Parse(argv[1:])
+
+	argv = flags.Args()
+	if len(argv) < 1 {
+		flags.Usage()
+		prog.Exit(2)
+	}
+	storPath := argv[0]
+
+	err := Verify(os.Stdout, storPath, verbose, progress)
+	if err != nil {
+		prog.Fatal(err)
+	}
+}

go/zodb/storage/fs1/py/gen-testdata

 #!/usr/bin/env python2
 # -*- coding: utf-8 -*-
-# Copyright (C) 2017-2020  Nexedi SA and Contributors.
+# Copyright (C) 2017-2021  Nexedi SA and Contributors.
 #                          Kirill Smelkov <kirr@nexedi.com>
 #
 # This program is free software: you can Use, Study, Modify and Redistribute
@@ -21,9 +21,10 @@
 """generate reference fs1 database and index for tests"""
 
 from ZODB.FileStorage import FileStorage
+from ZODB.FileStorage.FileStorage import FILESTORAGE_MAGIC, TxnHeader, DataHeader, TRANS_HDR_LEN
 from ZODB import DB
 from ZODB.Connection import TransactionMetaData
-from zodbtools.test.gen_testdata import gen_testdb, precommit, run_with_zodb3py2_compat
+from zodbtools.test.gen_testdata import gen_testdb, precommit, run_with_zodb4py2_compat
 from os import stat, remove
 from shutil import copyfile
 from golang.gcompat import qq
@@ -156,7 +157,7 @@ def main():
         vstor.store(vroot._p_oid, vroot._p_serial, '000 data 000', '', txn_stormeta)
         vstor.tpc_vote(txn_stormeta)
         # NO tpc_finish here so that status remain 'c' (voted) instead of ' ' (committed)
-    run_with_zodb3py2_compat(_)
+    run_with_zodb4py2_compat(_)
 
     st = stat(outfs)
     l  = st.st_size
@@ -174,5 +175,36 @@ def main():
     remove(voted+".lock")
 
 
+    # prepare file with whiteout (deletion of previously non-existing object)
+    whiteout = "testdata/whiteout.fs"
+    # as of 20210317 FileStorage.deleteObject verifies that object exists
+    # -> prepare magic/transaction/data records manually
+    with open(whiteout, "wb") as f:
+        oid = p64(1)
+        tid = p64(0x17)
+
+        # file header
+        f.write(FILESTORAGE_MAGIC)
+        tpos = f.tell()
+
+        # data record (see FileStorage.deleteObject)
+        dh = DataHeader(oid, tid, 0, tpos, 0, 0)
+        drec = dh.asString() + p64(0)
+
+        # emit txn header (see FileStorage.tpc_vote)
+        tlen = TRANS_HDR_LEN + 0 + 0 + 0 + len(drec) # empty u,d,e
+        th = TxnHeader(tid, tlen, ' ', 0, 0, 0)
+        th.user  = b''
+        th.descr = b''
+        th.ext   = b''
+        f.write(th.asString())
+
+        # emit data record
+        f.write(drec)
+
+        # emit txn tail
+        f.write(p64(tlen))
+
+
 if __name__ == '__main__':
     main()

go/zodb/storage/zeo/proto.go

-// Copyright (C) 2018-2020  Nexedi SA and Contributors.
+// Copyright (C) 2018-2021  Nexedi SA and Contributors.
 //                          Kirill Smelkov <kirr@nexedi.com>
 //
 // This program is free software: you can Use, Study, Modify and Redistribute
@@ -54,7 +54,7 @@ type msg struct {
 type msgFlags int64
 const (
 	msgAsync  msgFlags = 1 // message does not need a reply
-	msgExcept          = 2 // exception was raised on remote side (ZEO5)
+	msgExcept msgFlags = 2 // exception was raised on remote side (ZEO5)
 )
 
 // encoding represents messages encoding.
@@ -104,7 +104,7 @@ func pktEncodeM(m msg) *pktBuf {
 	// arg
 	// it is interface{} - use shamaton/msgpack since msgp does not handle
 	// arbitrary interfaces well.
-	dataArg, err := msgpack.Encode(m.arg)
+	dataArg, err := msgpack.Marshal(m.arg)
 	if err != nil {
 		panic(err) // all our types are expected to be supported by msgpack
 	}
@@ -240,7 +240,7 @@ func pktDecodeM(pkb *pktBuf) (msg, error) {
 	if len(btail) != 0 {
 		return m, derrf(".%d: payload has extra data after message")
 	}
-	err = msgpack.Decode(b, &m.arg)
+	err = msgpack.Unmarshal(b, &m.arg)
 	if err != nil {
 		return m, derrf(".%d: arg: %s", m.msgid, err)
 	}

go/zodb/storage/zeo/zeo.go

@@ -52,6 +52,9 @@ type zeo struct {
 	// becomes ready when serve loop finishes
 	serveWG sync.WaitGroup
 
+	closeOnce sync.Once
+	closed    chan struct{} // ready when Closed
+
 	url string // we were opened via this
 }
 
@@ -174,7 +177,13 @@ func (z *zeo) invalidateTransaction(arg interface{}) (err error) {
 
 	// at0 is initialized - ok to send current event if it goes > at0
 	if tid > z.at0 {
-		z.watchq <- event
+		select {
+		case <-z.closed:
+			// closed - client does not read watchq anymore
+
+		case z.watchq <- event:
+			// ok
+		}
 	}
 	return nil
 }
@@ -188,7 +197,13 @@ func (z *zeo) flushEventq0() {
 	if z.watchq != nil {
 		for _, e := range z.eventq0 {
 			if e.Tid > z.at0 {
-				z.watchq <- e
+				select {
+				case <-z.closed:
+					// closed - client does not read watchq anymore
+
+				case z.watchq <- e:
+					// ok
+				}
 			}
 		}
 	}
@@ -264,7 +279,7 @@ func (r rpc) call(ctx context.Context, argv ...interface{}) (interface{}, error)
 // excError returns error corresponding to an exception.
 //
 // well-known exceptions are mapped to corresponding well-known errors - e.g.
-// POSKeyError -> zodb.NoObjectError, and rest are returned wrapper into rpcExcept.
+// POSKeyError -> zodb.NoObjectError, and rest are returned wrapped into rpcExcept.
 func (r rpc) excError(exc string, argv tuple) error {
 	// translate well-known exceptions
 	switch exc {
@@ -438,7 +453,7 @@ func openByURL(ctx context.Context, u *url.URL, opt *zodb.DriverOptions) (_ zodb
 	}()
 
 
-	z := &zeo{link: zlink, watchq: opt.Watchq, url: url}
+	z := &zeo{link: zlink, watchq: opt.Watchq, closed: make(chan struct{}), url: url}
 
 	// start serve loop on the link
 	z.serveWG.Add(1)
@@ -456,14 +471,18 @@ func openByURL(ctx context.Context, u *url.URL, opt *zodb.DriverOptions) (_ zodb
 		// close .watchq after serve is over
 		z.at0Mu.Lock()
 		defer z.at0Mu.Unlock()
-		if z.at0Initialized {
-			z.flushEventq0()
-		}
 		if z.watchq != nil {
-			if err != nil {
-				z.watchq <- &zodb.EventError{Err: err}
+			if err != nil && /* already flushed .eventq0 */z.at0Initialized {
+				select {
+				case <-z.closed:
+					// closed - client does not read watchq anymore
+
+				case z.watchq <- &zodb.EventError{Err: err}:
+					// ok
+				}
 			}
 			close(z.watchq)
+			z.watchq = nil // prevent flushEventq0 to send to closed chan
 		}
 	}()
 
@@ -496,13 +515,16 @@ func openByURL(ctx context.Context, u *url.URL, opt *zodb.DriverOptions) (_ zodb
 	// "invalidateTransaction" server notification.
 	//
 	// filter-out first < at0 messages for this reason.
+	//
+	// do this in separate task not to deadlock in watchq<- : we did not
+	// yet returned z to caller and so noone might be yet reading from watchq.
+	go func() {
 		z.at0Mu.Lock()
 		z.at0 = lastTid
 		z.at0Initialized = true
 		z.flushEventq0()
 		z.at0Mu.Unlock()
-
-
+	}()
 
 	//call('get_info') -> {}str->str, ex	// XXX can be omitted
 /*
@@ -520,11 +542,15 @@ func openByURL(ctx context.Context, u *url.URL, opt *zodb.DriverOptions) (_ zodb
   'supports_record_iternext': True})
 */
 
-	return z, z.at0, nil
+	return z, lastTid, nil
 }
 
 func (z *zeo) Close() error {
-	err := z.link.Close()
+	var err error
+	z.closeOnce.Do(func() {
+		close(z.closed)
+		err = z.link.Close()
+	})
 	z.serveWG.Wait()
 	return err
 }

go/zodb/storage/zeo/zeo_test.go

@@ -153,7 +153,7 @@ func (z *ZEOPySrv) Encoding() encoding {
 // ----------------
 
 // tOptions represents options for testing.
-// XXX dup in NEO
+// TODO -> xtesting
 type tOptions struct {
 	Preload string // preload database with data from this location
 }

go/zodb/wks/wks.go

-// Copyright (C) 2017  Nexedi SA and Contributors.
+// Copyright (C) 2017-2021  Nexedi SA and Contributors.
 //                          Kirill Smelkov <kirr@nexedi.com>
 //
 // This program is free software: you can Use, Study, Modify and Redistribute
@@ -31,4 +31,5 @@ import (
 	_ "lab.nexedi.com/kirr/neo/go/zodb/storage/fs1"
 	_ "lab.nexedi.com/kirr/neo/go/zodb/storage/zeo"
 	_ "lab.nexedi.com/kirr/neo/go/neo"
+	_ "lab.nexedi.com/kirr/neo/go/zodb/storage/demo"
 )

go/zodb/zodb.go

-// Copyright (C) 2016-2020  Nexedi SA and Contributors.
+// Copyright (C) 2016-2021  Nexedi SA and Contributors.
 //                          Kirill Smelkov <kirr@nexedi.com>
 //
 // This program is free software: you can Use, Study, Modify and Redistribute
@@ -275,7 +275,7 @@ type NoObjectError struct {
 	Oid Oid
 }
 
-func (e NoObjectError) Error() string {
+func (e *NoObjectError) Error() string {
 	return fmt.Sprintf("%s: no such object", e.Oid)
 }
 

go/zodb/zodbtools/help.go

@@ -44,6 +44,7 @@ There are also following simpler ways:
 - neo://<master>/<db>         for a NEO database	XXX + neos:// ?
 - zeo://<host>:<port>         for a ZEO database
 - /path/to/file               for a FileStorage database
+- demo:(zurl_base)/(zurl_δ)   for a DemoStorage overlay
 
 Please see zodburi documentation for full details:
 

go/zodb/δtail.go

-// Copyright (C) 2018-2020  Nexedi SA and Contributors.
+// Copyright (C) 2018-2021  Nexedi SA and Contributors.
 //                          Kirill Smelkov <kirr@nexedi.com>
 //
 // This program is free software: you can Use, Study, Modify and Redistribute
@@ -202,46 +202,35 @@ func (δtail *ΔTail) ForgetPast(revCut Tid) {
 // XXX -> RevAt ?
 // LastRevOf tries to return what was the last revision that changed id as of at database state.
 //
-// Depending on current information in δtail it returns either exact result, or
-// an upper-bound estimate for the last id revision, assuming id was changed ≤ at:
+// it must be called with the following condition:
 //
-// 1) if δtail does not cover at, at is returned:
+//	tail ≤ at ≤ head
 //
-//	# at ∉ [min(rev ∈ δtail), max(rev ∈ δtail)]
-//	LastRevOf(id, at) = at
+// Depending on current information in δtail it returns either exact result, or
+// an upper-bound estimate for the last id revision:
 //
-// 2) if δtail has an entry corresponding to id change, it gives exactly the
+// 1) if δtail has an entry corresponding to id change, it gives exactly the
 // last revision that changed id:
 //
-//	# at ∈ [min(rev ∈ δtail), max(rev ∈ δtail)]
 //	# ∃ rev ∈ δtail: rev changed id && rev ≤ at
-//	LastRevOf(id, at) = max(rev: rev changed id && rev ≤ at)
+//	LastRevOf(id, at) = max(rev: rev changed id && rev ≤ at), true
 //
-// 3) if δtail does not contain appropriate record with id - it returns δtail's
+// 2) if δtail does not contain appropriate record with id - it returns δtail's
 // lower bound as the estimate for the upper bound of the last id revision:
 //
-//	# at ∈ [min(rev ∈ δtail), max(rev ∈ δtail)]
 //	# ∄ rev ∈ δtail: rev changed id && rev ≤ at
-//	LastRevOf(id, at) = min(rev ∈ δtail)
+//	LastRevOf(id, at) = δtail.tail, false
 //
 // On return exact indicates whether returned revision is exactly the last
 // revision of id, or only an upper-bound estimate of it.
 func (δtail *ΔTail) LastRevOf(id Oid, at Tid) (_ Tid, exact bool) {
-	// check if we have no coverage at all
-	l := len(δtail.tailv)
-	if l == 0 {
-		return at, false
-	}
-	revMin := δtail.tailv[0].Rev
-	revMax := δtail.tailv[l-1].Rev
-	if !(revMin <= at && at <= revMax) {
-		return at, false
+	if !(δtail.tail <= at && at <= δtail.head) {
+		panic(fmt.Sprintf("at out of bounds: at: @%s,  (tail, head] = (@%s, @%s]", at, δtail.tail, δtail.head))
         }
 
-	// we have the coverage
 	rev, ok := δtail.lastRevOf[id]
 	if !ok {
-		return δtail.tailv[0].Rev, false
+		return δtail.tail, false
 	}
 
 	if rev <= at {
@@ -250,7 +239,7 @@ func (δtail *ΔTail) LastRevOf(id Oid, at Tid) (_ Tid, exact bool) {
 
 	// what's in index is after at - scan tailv back to find appropriate entry
 	// XXX linear scan - see .lastRevOf comment.
-	for i := l - 1; i >= 0; i-- {
+	for i := len(δtail.tailv) - 1; i >= 0; i-- {
 		δ := δtail.tailv[i]
 		if δ.Rev > at {
 			continue
@@ -264,5 +253,5 @@ func (δtail *ΔTail) LastRevOf(id Oid, at Tid) (_ Tid, exact bool) {
 	}
 
 	// nothing found
-	return δtail.tailv[0].Rev, false
+	return δtail.tail, false
 }

go/zodb/δtail_test.go

-// Copyright (C) 2018-2019  Nexedi SA and Contributors.
+// Copyright (C) 2018-2021  Nexedi SA and Contributors.
 //                          Kirill Smelkov <kirr@nexedi.com>
 //
 // This program is free software: you can Use, Study, Modify and Redistribute
@@ -174,14 +174,13 @@ func TestΔTail(t *testing.T) {
 	δtail = NewΔTail(3)
 
 	δCheck(3,3)
-	δCheckLastUP(4, 12, 12)	// δtail = ø
+	δCheckLastUP(3, 3, 3)	// δtail = ø
 
 	δAppend(R(10, 3,5))
 	δCheck(3,10, R(10, 3,5))
 
-	δCheckLastUP(3,  2,  2)	// at < δtail
-	δCheckLastUP(3, 12, 12)	// at > δtail
-	δCheckLastUP(4, 10, 10)	// id ∉ δtail
+	δCheckLastUP(3,  9, 3)  // id ∈ δtail, but has no entry with rev ≤ at
+	δCheckLastUP(4, 10, 3)	// id ∉ δtail
 
 	δAppend(R(11, 7))
 	δCheck(3,11, R(10, 3,5), R(11, 7))
@@ -192,7 +191,7 @@ func TestΔTail(t *testing.T) {
 	δAppend(R(14, 3,8))
 	δCheck(3,14, R(10, 3,5), R(11, 7), R(12, 7), R(14, 3,8))
 
-	δCheckLastUP(8, 12, 10) // id ∈ δtail, but has no entry with rev ≤ at
+	δCheckLastUP(8, 12, 3)  // id ∈ δtail, but has no entry with rev ≤ at
 
 	δtail.ForgetPast(9)
 	δCheck(9,14, R(10, 3,5), R(11, 7), R(12, 7), R(14, 3,8))

neo/client/zodburi.py

@@ -84,6 +84,7 @@ def _resolve_uri(uri):
         if scheme != "neos":
             raise ValueError("invalid uri: %s : credentials can be specified only with neos:// scheme" % uri)
         # ca=ca.crt;cert=my.crt;key=my.key
+        cred = cred.replace(';', '&') # ; is no longer in default separators set bugs.python.org/issue42967
         for k, v in OrderedDict(parse_qsl(cred)).items():
             if k not in _credopts:
                 raise ValueError("invalid uri: %s : unexpected credential %s" % (uri, k))

neo/tests/ca.crt

 -----BEGIN CERTIFICATE-----
-MIIC7TCCAdWgAwIBAgIJAL8e44sA7PDMMA0GCSqGSIb3DQEBCwUAMA0xCzAJBgNV
-BAMMAkNBMB4XDTE1MDkzMDEzNTQzMFoXDTIxMDMyMjEzNTQzMFowDTELMAkGA1UE
-AwwCQ0EwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQCgT7DVKM4ViQt3
-B0oJC4RGi10dNfpNZZpgA5iC2UJ1u6AqqCf0PCQkrmuIzW3l1TenlOiLNdVASkkT
-wf1lekIgg4tR8/22oGTAnfY6R9r1C6jAMV72v1sffz8D6qfkMPzKchJt55zywdhm
-KscUsMGzXPGIeKrG20m83dSIO4RmCmq/f4BcuWJu6Kkq4n9Wc2IsvpKk+lqEUxI/
-QoqdT6OvMXooGs3t892uvKDu++muBj2Y/yyaXt1tCCjDFsRMLWl3Skks+4PeMCZ4
-wugyXEBk3d5Yzdv5NsFzFBjAuRCGxJXEOEcfHj4Xj9qTCErZ1jKzgnuxJCtgdqRC
-r4beX1U3AgMBAAGjUDBOMB0GA1UdDgQWBBSFY/jKvo0iSTEzzOIcZZUZCT8JfTAf
-BgNVHSMEGDAWgBSFY/jKvo0iSTEzzOIcZZUZCT8JfTAMBgNVHRMEBTADAQH/MA0G
-CSqGSIb3DQEBCwUAA4IBAQAB0LKDAuhodpyNVwEE9Yl+Q/IiPEPCaix6URJnRn1O
-gQnXuZLo1xtJh6wJh1faG1/qNCFMxWEJ+0VkJ7r6v38cNXfYG9OcmD0S6YnNjSuO
-VliAtqVVtj8MppJ4vMatLrNi4cvyYucebtNyBCzSIAi+6bkkHeaVgi1EtxXvq+AS
-iZp3gl84oXv/gV7Bz4SXmVpFJnhsDMoQZG2KAULAgfZ2Am2I+ffG90cD/oEnS/3O
-k3btqTvgIO8MWt8PY3sUOhJEoJYKnC9DppmhOhUTn4zzIIDSluKEOBHZiFb9AcmF
-PvzL+8xiORCdUe1d6ANQQlUd0MM810BXZFYEXFbgKg8o
+MIIC+zCCAeOgAwIBAgIUN2aMGxxsLX4IyuSR2DRfzbTumrAwDQYJKoZIhvcNAQEL
+BQAwDTELMAkGA1UEAwwCQ0EwHhcNMjEwMzIyMTU1MzI5WhcNMjYwOTEyMTU1MzI5
+WjANMQswCQYDVQQDDAJDQTCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEB
+AMVipTeJN6FEpdF6JJkkVw7vPR14pXCMehSWxBMrmICVAHct4fWz4sI/lp3VTiFl
+HnEBOhehLsKfZyosGIMpJ0zI2GsQopkHa5ZPf/cILrQ3ybu7ZsQITAyYKWqhMVdt
+d+YRetfkE874gUHqTo+WSBpCbRvuYK4XhBEmrMiwIHV7nwuIaKMdG6FxNGn8du8R
+Ti96VduJGdjTFIRri2jNP47VKidioPkxecA/48yrDdewn/cxsU/yNCO9Uii/dTyS
+Ro/Jqxql6Leb7xaZlODS5QQAGWjAI6w0BCobwYaUuJOZ1HSUL6A42rnMqI6gGjCY
+Dju313RlE5jOilWQhodmawECAwEAAaNTMFEwHQYDVR0OBBYEFBfzeqZl9fHZR2bH
+rb+T6+O6mu1aMB8GA1UdIwQYMBaAFBfzeqZl9fHZR2bHrb+T6+O6mu1aMA8GA1Ud
+EwEB/wQFMAMBAf8wDQYJKoZIhvcNAQELBQADggEBAK6+04BOOP/PF6BR6FaZjLHN
+qhcSBfG4Y8RyLtERwGTQW68PwCU2TloJgpoY8updT0V9EsxnkWjsU/Wgnl8pdHjp
+GMAivelTcDoYH4qn8E6mDEKE47Be4fE0FdGvU9QU7UBWG+Uf8pxgnFJcGou5grLG
+RXTSNCSrBnJG9tmSPA/aw/Q9MVfXFuB7K8aPJ/d2Z/8v/QFdBK25JH8I+ErFU3ys
+ufn58Jqsy4dp8R4qYVP1TKPfGsZ+RZeIIJ4cDIik/vbv9+Ymi+m8LZ66kUNq1d4X
+tGbPjcNt/qFQ/FG7T6ZUPe9vRpYAhIHGBYZFs59LJcQsH1DzSvei3141Gs7TYFg=
 -----END CERTIFICATE-----

neo/tests/node.crt

 -----BEGIN CERTIFICATE-----
-MIICkDCCAXgCAQEwDQYJKoZIhvcNAQELBQAwDTELMAkGA1UEAwwCQ0EwHhcNMTUw
-OTMwMTM1NDMwWhcNMjEwMzIyMTM1NDMwWjAPMQ0wCwYDVQQDDARub2RlMIIBIjAN
+MIICkDCCAXgCAQEwDQYJKoZIhvcNAQELBQAwDTELMAkGA1UEAwwCQ0EwHhcNMjEw
+MzIyMTU1MzI5WhcNMjYwOTEyMTU1MzI5WjAPMQ0wCwYDVQQDDARub2RlMIIBIjAN
 BgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAuJ+ClJyjhJOJdGUyqHn79opMLP3m
 1g27uBWKT+OGd4FcreVoRDxPVuxZxMtDCZcBfUHVvOoSlS06khwSxViEe1hxwHRa
 n2qMWlwvaWeNY0CFH5V+DI4XSNojgny85Lb5jB69FuPcrHnwxLk2OFntrXEeNbEa
 d7QSoNbPajbJIp5BS/WR9iu5Z5JYdumLWjTvOU+eZc4iA6Wa2kdDtbqkGi4wOJ1L
 /ggATL+p+QFcubVptztPT8vq7gvDdGJgXLJ2lPHV0V/sdJB1FB4mSJEDDjSm2Hpp
 qPVJSO1GrAy5Ld+0SnXIZZejhIUJIumocY08r+vzDSQ/8NnqXR4Odz1TWwIDAQAB
-MA0GCSqGSIb3DQEBCwUAA4IBAQBlYkkInDDDcgnNRdUmzxwejs1PmEehZ3H5FkMp
-TsmpoVC+oqM+QywMu8UJRtCjXnnJdAUbVYuZ1Tjm7qvFIhN+5OlIVxJ+8WcmZPSe
-lj0N7Dv2nE1diTDS+qPZVPZ0demo1LafRmPomPWiM/CQRlMPxXnimuiYOROhWGn6
-jsyoOwquMkAc6Ub++l4OCxLAP0eTgJFkivmqpaYZXG4o7zFvcQ3rQ66rQrMl69sR
-8/MVqbT5Sq1CEJbepP4GaFfa5l3CVy7WH2MhCV1/9mNwcXafkTgx3q2HsPon4Dze
-kNwiguNAM4L/j4dbIwz+CIVWcgpBCrfv2JYu+jGlRpxIDeWR
+MA0GCSqGSIb3DQEBCwUAA4IBAQAvtJztgZ8TuY5Q52Lv4hE2WX3/IZQPYc9cGQqv
+sI0xKQEWD4hVHAmxL7bLFOUovzFqNMY0pYBGVJ53nypsKNorKbiepSMPosfcbk9o
+U8xF9YDJO+yS6V2lWFi6iKe4WX2t+L2j5x+q4dptJoIvsM777xGatri59q8UK3ne
+YTINui8+tsOIGEpcXMHMI/k3RvBUS3Mwy0pX6rsi8/YsUVUShpPDjuAqQIYxU4gP
+qAIWrhXNYwOGuMQ1V+j83R5lqDoHrVauWznc0n0qg/EJArhwkLGfjQoqrogCQOVp
+iMEV6YDLC8EMqZfYVUT0i2+/bS737Z2lcdgz2essW45nk7+H
 -----END CERTIFICATE-----