Skip to content

N
neo
Commit 95fbc76f authored by Kirill Smelkov's avatar Kirill Smelkov
Browse files
Options

Merge branch 't' into t2 

* t: (129 commits)
  X go.mod: v↑ *
  go/zodb/btree: Change V<op> family to also provide visited node key coverage on visit callback
  go/zodb/btree: Add KeyRange type
  go/zodb/btree: Introduce constants for min/max key value
  go/zodb/btree: tests: Don't forget to close storage
  go/zodb/btree: Cosmetics
  .
  .
  .
  .
  .
  .
  .
  X Start reworking BTree to provide keycov on visit callback
  X go/neo: Fix credentials parsing with go1.17
  fixup! Y client: Fix URI scheme to move credentials out of query
  go/internal/xtesting: Add missing X
  go/zodb/{fs1,zeo}: ~staticcheck
  go/zodb/btree: Fix missing return on data-consistency error
  go/zodb, go/zodb/btree: Fix go generate after rename on zodbtools side
  ...
parents ad5940f3 ea538368
Hide whitespace changes
Inline Side-by-side
Showing with 2218 additions and 630 deletions
go/go.mod
View file @ 95fbc76f
......@@ -3,24 +3,36 @@ module lab.nexedi.com/kirr/neo/go
go 1.14
require (
github.com/DataDog/czlib v0.0.0-20160811164712-4bc9a24e37f2
github.com/DataDog/czlib v0.0.0-20210322182103-8087f4e14ae7
github.com/cznic/mathutil v0.0.0-20181122101859-297441e03548 // indirect
github.com/cznic/strutil v0.0.0-20181122101858-275e90344537
github.com/fsnotify/fsnotify v1.4.10-0.20200417215612-7f4cf4dd2b52
github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b
github.com/gwenn/gosqlite v0.0.0-20200521090053-24878be1a237
github.com/kisielk/og-rek v1.0.1-0.20180928202415-8b25c4cefd6c
github.com/davecgh/go-spew v1.1.1 // indirect
github.com/fsnotify/fsnotify v1.5.1
github.com/golang/glog v1.0.0
github.com/golang/protobuf v1.4.3 // indirect
github.com/google/go-cmp v0.5.4 // indirect
github.com/gwenn/gosqlite v0.0.0-20211101095637-b18efb2e44c8
github.com/gwenn/yacr v0.0.0-20211101095056-492fb0c571bc // indirect
github.com/kisielk/og-rek v1.2.0
github.com/kr/text v0.2.0 // indirect
github.com/kylelemons/godebug v1.1.0
github.com/philhofer/fwd v1.0.0 // indirect
github.com/philhofer/fwd v1.1.1
github.com/pkg/errors v0.9.1
github.com/remyoudompheng/bigfft v0.0.0-20200410134404-eec4a21b6bb0 // indirect
github.com/shamaton/msgpack v1.1.1
github.com/soheilhy/cmux v0.1.4
github.com/shamaton/msgpack v1.2.1
github.com/soheilhy/cmux v0.1.5
github.com/someonegg/gocontainer v1.0.0
github.com/stretchr/testify v1.6.1
github.com/tinylib/msgp v1.1.3-0.20200327023543-e88e92c0ccca
golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208
google.golang.org/grpc v1.32.0 // indirect
google.golang.org/grpc/examples v0.0.0-20200915000551-32e7099cccac // indirect
lab.nexedi.com/kirr/go123 v0.0.0-20200915142026-a281a51cf49b
github.com/stretchr/testify v1.7.0
github.com/tinylib/msgp v1.1.6
github.com/ttacon/chalk v0.0.0-20160626202418-22c06c80ed31 // indirect
golang.org/x/net v0.0.0-20211111160137-58aab5ef257a // indirect
golang.org/x/sync v0.0.0-20210220032951-036812b2e83c // indirect
golang.org/x/sys v0.0.0-20211111213525-f221eed1c01e // indirect
golang.org/x/text v0.3.7 // indirect
google.golang.org/genproto v0.0.0-20210226172003-ab064af71705 // indirect
google.golang.org/grpc v1.36.0 // indirect
google.golang.org/grpc/examples v0.0.0-20210301210255-fc8f38cccf75 // indirect
gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c // indirect
gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b // indirect
lab.nexedi.com/kirr/go123 v0.0.0-20210906140734-c9eb28d9e408
)
go/go.sum
View file @ 95fbc76f
cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
cloud.google.com/go v0.38.0/go.mod h1:990N+gfupTy94rShfmMCWGDn0LpTmnzTp2qbd1dvSRU=
cloud.google.com/go v0.44.1/go.mod h1:iSa0KzasP4Uvy3f1mN/7PiObzGgflwredwwASm/v6AU=
cloud.google.com/go v0.44.2/go.mod h1:60680Gw3Yr4ikxnPRS/oxxkBccT6SA1yMk63TGekxKY=
cloud.google.com/go v0.45.1/go.mod h1:RpBamKRgapWJb87xiFSdk4g1CME7QZg3uwTez+TSTjc=
cloud.google.com/go v0.46.3/go.mod h1:a6bKKbmY7er1mI7TEI4lsAkts/mkhTSZK8w33B4RAg0=
cloud.google.com/go v0.50.0/go.mod h1:r9sluTvynVuxRIOHXQEHMFffphuXHOMZMycpNR5e6To=
cloud.google.com/go v0.52.0/go.mod h1:pXajvRH/6o3+F9jDHZWQ5PbGhn+o8w9qiu/CffaVdO4=
cloud.google.com/go v0.53.0/go.mod h1:fp/UouUEsRkN6ryDKNW/Upv/JBKnv6WDthjR6+vze6M=
cloud.google.com/go v0.54.0/go.mod h1:1rq2OEkV3YMf6n/9ZvGWI3GWw0VoqH/1x2nd8Is/bPc=
cloud.google.com/go v0.56.0/go.mod h1:jr7tqZxxKOVYizybht9+26Z/gUq7tiRzu+ACVAMbKVk=
cloud.google.com/go v0.57.0/go.mod h1:oXiQ6Rzq3RAkkY7N6t3TcE6jE+CIBBbA36lwQ1JyzZs=
cloud.google.com/go v0.62.0/go.mod h1:jmCYTdRCQuc1PHIIJ/maLInMho30T/Y0M4hTdTShOYc=
cloud.google.com/go v0.63.0/go.mod h1:GmezbQc7T2snqkEXWfZ0sy0VfkB/ivI2DdtJL2DEmlg=
cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o=
cloud.google.com/go/bigquery v1.3.0/go.mod h1:PjpwJnslEMmckchkHFfq+HTD2DmtT67aNFKH1/VBDHE=
cloud.google.com/go/bigquery v1.4.0/go.mod h1:S8dzgnTigyfTmLBfrtrhyYhwRxG72rYxvftPBK2Dvzc=
cloud.google.com/go/bigquery v1.5.0/go.mod h1:snEHRnqQbz117VIFhE8bmtwIDY80NLUZUMb4Nv6dBIg=
cloud.google.com/go/bigquery v1.7.0/go.mod h1://okPTzCYNXSlb24MZs83e2Do+h+VXtc4gLoIoXIAPc=
cloud.google.com/go/bigquery v1.8.0/go.mod h1:J5hqkt3O0uAFnINi6JXValWIb1v0goeZM77hZzJN/fQ=
cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE=
cloud.google.com/go/datastore v1.1.0/go.mod h1:umbIZjpQpHh4hmRpGhH4tLFup+FVzqBi1b3c64qFpCk=
cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I=
cloud.google.com/go/pubsub v1.1.0/go.mod h1:EwwdRX2sKPjnvnqCa270oGRyludottCI76h+R3AArQw=
cloud.google.com/go/pubsub v1.2.0/go.mod h1:jhfEVHT8odbXTkndysNHCcx0awwzvfOlguIAii9o8iA=
cloud.google.com/go/pubsub v1.3.1/go.mod h1:i+ucay31+CNRpDW4Lu78I4xXG+O1r/MAHgjpRVR+TSU=
cloud.google.com/go/storage v1.0.0/go.mod h1:IhtSnM/ZTZV8YYJWCY8RULGVqBDmpoyjwiyrjsg+URw=
cloud.google.com/go/storage v1.5.0/go.mod h1:tpKbwo567HUNpVclU5sGELwQWBDZ8gh0ZeosJ0Rtdos=
cloud.google.com/go/storage v1.6.0/go.mod h1:N7U0C8pVQ/+NIKOBQyamJIeKQKkZ+mxpohlUTyfDhBk=
cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RXyy7KQOVs=
cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0=
crawshaw.io/iox v0.0.0-20181124134642-c51c3df30797/go.mod h1:sXBiorCo8c46JlQV3oXPKINnZ8mcqnye1EkVkqsectk=
crawshaw.io/sqlite v0.3.2 h1:N6IzTjkiw9FItHAa0jp+ZKC6tuLzXqAYIv+ccIWos1I=
crawshaw.io/sqlite v0.3.2/go.mod h1:igAO5JulrQ1DbdZdtVq48mnZUBAPOeFzer7VhDWNtW4=
dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU=
github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo=
github.com/DataDog/czlib v0.0.0-20160811164712-4bc9a24e37f2 h1:/Xf0UwdF1dlv6cNu0q1TWTQgfSsALgXrdHNtBMj7ybU=
github.com/DataDog/czlib v0.0.0-20160811164712-4bc9a24e37f2/go.mod h1:xK7es18l1aQtR1g0i6RhEQ78ckvRAWiwlG0vBFud/Jk=
github.com/DataDog/czlib v0.0.0-20210322182103-8087f4e14ae7 h1:6ZJZdzkbvKb6HRXmZ12ICZ0IbqfR+0Cd2C0IutWHHIA=
github.com/DataDog/czlib v0.0.0-20210322182103-8087f4e14ae7/go.mod h1:ROY4muaTWpoeQAx/oUkvxe9zKCmgU5xDGXsfEbA+omc=
github.com/bmizerany/assert v0.0.0-20160611221934-b7ed37b82869 h1:DDGfHa7BWjL4YnC6+E63dPcxHo2sUxDIu8g3QgEJdRY=
github.com/bmizerany/assert v0.0.0-20160611221934-b7ed37b82869/go.mod h1:Ekp36dRnpXw/yCqJaO+ZrUyxD+3VXMFFr56k5XYrpB4=
github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI=
github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI=
github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU=
github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc=
github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk=
github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
github.com/cznic/mathutil v0.0.0-20181122101859-297441e03548 h1:iwZdTE0PVqJCos1vaoKsclOGD3ADKpshg3SRtYBbwso=
github.com/cznic/mathutil v0.0.0-20181122101859-297441e03548/go.mod h1:e6NPNENfs9mPDVNRekM7lKScauxd5kXTr1Mfyig6TDM=
github.com/cznic/strutil v0.0.0-20181122101858-275e90344537 h1:MZRmHqDBd0vxNwenEbKSQqRVT24d3C05ft8kduSwlqM=
github.com/cznic/strutil v0.0.0-20181122101858-275e90344537/go.mod h1:AHHPPPXTw0h6pVabbcbyGRK1DckRn7r/STdZEeIDzZc=
github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8=
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98=
github.com/envoyproxy/go-control-plane v0.9.9-0.20201210154907-fd9021fe5dad/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk=
github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c=
github.com/fsnotify/fsnotify v1.4.10-0.20200417215612-7f4cf4dd2b52 h1:0NmERxogGTU8hgzOhRKNoKivtBZkDW29GeuJtK9e0sc=
github.com/fsnotify/fsnotify v1.4.10-0.20200417215612-7f4cf4dd2b52/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ=
github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU=
github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8=
github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8=
github.com/fsnotify/fsnotify v1.5.1 h1:mZcQUHVQUQWoPXXtuf9yuEXKudkV2sx1E06UadKWpgI=
github.com/fsnotify/fsnotify v1.5.1/go.mod h1:T3375wBYaZdLLcVNkcVbzGHY7f1l/uK5T5Ai1i3InKU=
github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b h1:VKtxabqXZkF25pY9ekfRL6a582T4P37/31XEstQ5p58=
github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
github.com/golang/glog v1.0.0 h1:nfP3RFugxnNRyKgeWd4oI1nYvXpxrx8ck8ZrcizshdQ=
github.com/golang/glog v1.0.0/go.mod h1:EWib/APOK0SL3dFbYqvxE3UYd8E6s1ouQ7iEp/0LWV4=
github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFUx0Y=
github.com/golang/mock v1.4.0/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw=
github.com/golang/mock v1.4.1/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw=
github.com/golang/mock v1.4.3/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw=
github.com/golang/mock v1.4.4/go.mod h1:l3mdAwkq5BuhzHwde/uurv3sEJeZMXNpwsxVWU71h+4=
github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw=
github.com/golang/protobuf v1.3.4/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw=
github.com/golang/protobuf v1.3.5/go.mod h1:6O5/vntMXwX2lRkT1hjjk0nAC1IDOTvTlVgjlRvqsdk=
github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8=
github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA=
github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs=
github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w=
github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0=
github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8=
github.com/golang/protobuf v1.4.2 h1:+Z5KGCizgyZCbGh1KZqA0fcLLkwbsjIzS4aV2v7wJX0=
github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI=
github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
github.com/golang/protobuf v1.4.3 h1:JjCZWpVbqXDqFVmTfYWEVTMIYrL/NPdPSCHPJ0T/raM=
github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI=
github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M=
github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
github.com/google/go-cmp v0.4.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
github.com/google/go-cmp v0.5.1 h1:JFrFEBb2xKufg6XkJsJr+WbKb4FQlURi5RUcBveYu9k=
github.com/google/go-cmp v0.5.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs=
github.com/google/martian/v3 v3.0.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0=
github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc=
github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc=
github.com/google/pprof v0.0.0-20191218002539-d4f498aebedc/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM=
github.com/google/pprof v0.0.0-20200212024743-f11f1df84d12/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM=
github.com/google/pprof v0.0.0-20200229191704-1ebb73c60ed3/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM=
github.com/google/pprof v0.0.0-20200430221834-fc25d7d30c6d/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM=
github.com/google/pprof v0.0.0-20200708004538-1a94d8640e99/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM=
github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI=
github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg=
github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk=
github.com/gwenn/gosqlite v0.0.0-20200521090053-24878be1a237 h1:RY/HYWiI6K/HL8ePZYpgvMGf4wX3/FgGxgE8kPh8vDM=
github.com/gwenn/gosqlite v0.0.0-20200521090053-24878be1a237/go.mod h1:WBYs9HfQGOYDCz7rFwMk7aHkbTTB0cUkQe3pZQARvIg=
github.com/gwenn/yacr v0.0.0-20200110180258-a66d8c42d0ff h1:xv97Y/sC4qXwoZQ9HEhgoeDN/7j4JfA5qBWLNxihZQo=
github.com/google/go-cmp v0.5.4 h1:L8R9j+yAqZuZjsqh/z+F1NCffTKKLShY6zXTItVIZ8M=
github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
github.com/gwenn/gosqlite v0.0.0-20201008200117-82e079acf5b6 h1:GUdDtRtiCm0bipN/Hc03rqJl++aX2wdWs+F3pGCaPTU=
github.com/gwenn/gosqlite v0.0.0-20201008200117-82e079acf5b6/go.mod h1:WBYs9HfQGOYDCz7rFwMk7aHkbTTB0cUkQe3pZQARvIg=
github.com/gwenn/gosqlite v0.0.0-20211101095637-b18efb2e44c8 h1:sWkgaGez8CNa2KHGBTTop16/mC03VP6MDqPKfvhEmCU=
github.com/gwenn/gosqlite v0.0.0-20211101095637-b18efb2e44c8/go.mod h1:WBYs9HfQGOYDCz7rFwMk7aHkbTTB0cUkQe3pZQARvIg=
github.com/gwenn/yacr v0.0.0-20200110180258-a66d8c42d0ff/go.mod h1:5SNcBGxZ5OaJAMJCSI/x3V7SGsvXqbwnwP/sHZLgYsw=
github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc=
github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU=
github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk=
github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
github.com/kisielk/og-rek v1.0.1-0.20180928202415-8b25c4cefd6c h1:JKB9XSrj+dsCcfOZANbOTgDf3XLZVmX+rgFlKmuIZ+8=
github.com/kisielk/og-rek v1.0.1-0.20180928202415-8b25c4cefd6c/go.mod h1:Fo0gkB8QTO7j5Ha+ObtDMW5GtL5UoY66JUU8ZoLJeLQ=
github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
github.com/kr/pretty v0.2.0 h1:s5hAObm+yFO5uHYt5dYjxi2rXrsnmRpJx4OYvIWUaQs=
github.com/gwenn/yacr v0.0.0-20200112083327-bbe82c1f4d60 h1:JX4Yy6S9U/f3Jix82M58NLIAFeW/UjBFVrnYn5GS4X8=
github.com/gwenn/yacr v0.0.0-20200112083327-bbe82c1f4d60/go.mod h1:Ps/gikIXcn2rRmeP0HQ9EvUYJrfrjAi51Wg8acsrkP0=
github.com/gwenn/yacr v0.0.0-20211101095056-492fb0c571bc h1:AUv494HF3D9ht26o89DuJjqM9QDHwZeYCNU/JSU5jqI=
github.com/gwenn/yacr v0.0.0-20211101095056-492fb0c571bc/go.mod h1:Ps/gikIXcn2rRmeP0HQ9EvUYJrfrjAi51Wg8acsrkP0=
github.com/kisielk/og-rek v1.1.0 h1:u10TvQbPtrlY/6H4+BiFsBywwSVTGFsx0YOVtpx3IbI=
github.com/kisielk/og-rek v1.1.0/go.mod h1:6ihsOSzSAxR/65S3Bn9zNihoEqRquhDQZ2c6I2+MG3c=
github.com/kisielk/og-rek v1.2.0 h1:CTvDIin+YnetsSQAYbe+QNAxXU3B50C5hseEz8xEoJw=
github.com/kisielk/og-rek v1.2.0/go.mod h1:6ihsOSzSAxR/65S3Bn9zNihoEqRquhDQZ2c6I2+MG3c=
github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI=
github.com/kr/pretty v0.2.1 h1:Fmg33tUaq4/8ym9TJN1x7sLJnHVwhP33CNkpYV/7rwI=
github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI=
github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE=
github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc=
github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw=
github.com/philhofer/fwd v1.0.0 h1:UbZqGr5Y38ApvM/V/jEljVxwocdweyH+vmYvRPBnbqQ=
github.com/philhofer/fwd v1.0.0/go.mod h1:gk3iGcWd9+svBvR0sR+KPcfE+RNWozjowpeBVG3ZVNU=
github.com/philhofer/fwd v1.1.1 h1:GdGcTjf5RNAxwS4QLsiMzJYj5KEvPJD3Abr261yRQXQ=
github.com/philhofer/fwd v1.1.1/go.mod h1:gk3iGcWd9+svBvR0sR+KPcfE+RNWozjowpeBVG3ZVNU=
github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
......@@ -137,259 +87,112 @@ github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZN
github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
github.com/remyoudompheng/bigfft v0.0.0-20200410134404-eec4a21b6bb0 h1:OdAsTTz6OkFY5QxjkYwrChwuRruF69c169dPK26NUlk=
github.com/remyoudompheng/bigfft v0.0.0-20200410134404-eec4a21b6bb0/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo=
github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
github.com/shamaton/msgpack v1.1.1 h1:wZcK/aIifWvsDl6jXo+c/tRYo9Vnb8abWCossDbj2Ww=
github.com/shamaton/msgpack v1.1.1/go.mod h1:ibiaNQRTCUISAYkkyOpaSCEBiCAxXe6u6Mu1sQ6945U=
github.com/shamaton/msgpack v1.2.1 h1:40cwW7YAEdOIxcxIsUkAxSMUyYWZUyNiazI5AyiBntI=
github.com/shamaton/msgpack v1.2.1/go.mod h1:ibiaNQRTCUISAYkkyOpaSCEBiCAxXe6u6Mu1sQ6945U=
github.com/soheilhy/cmux v0.1.4 h1:0HKaf1o97UwFjHH9o5XsHUOF+tqmdA7KEzXLpiyaw0E=
github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM=
github.com/soheilhy/cmux v0.1.5-0.20210205191134-5ec6847320e5 h1:GJTW+uNMIV1RKwox+T4aN0/sQlYRg78uHZf2H0aBcDw=
github.com/soheilhy/cmux v0.1.5-0.20210205191134-5ec6847320e5/go.mod h1:T7TcVDs9LWfQgPlPsdngu6I6QIoyIFZDDC6sNE1GqG0=
github.com/soheilhy/cmux v0.1.5 h1:jjzc5WVemNEDTLwv9tlmemhC73tI08BNOIGwBOo10Js=
github.com/soheilhy/cmux v0.1.5/go.mod h1:T7TcVDs9LWfQgPlPsdngu6I6QIoyIFZDDC6sNE1GqG0=
github.com/someonegg/gocontainer v1.0.0 h1:9MMUFbQf7g+g9sMG4ggBHPDS1+Iz+wd9Ee/O4BNRdw0=
github.com/someonegg/gocontainer v1.0.0/go.mod h1:zGJcXRK0ikzEYPFKTaFXi6UU/ulNuJypfADX4UQGtMw=
github.com/someonegg/gox v1.0.0/go.mod h1:pngAcWxBFnyYM4oY+h9Rgv0WaikLkSfY5dBYuabUnBE=
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
github.com/stretchr/testify v1.6.1 h1:hDPOHmpOpP40lSULcqw7IrRb/u7w6RpDC9399XyoNd0=
github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
github.com/tinylib/msgp v1.1.3-0.20200327023543-e88e92c0ccca h1:RhRT3kgpvYL7aEUQoWwmhRTvObmqRJPV3rEMkr1qTfM=
github.com/tinylib/msgp v1.1.3-0.20200327023543-e88e92c0ccca/go.mod h1:+d+yLhGm8mzTaHzB+wgMYrodPfmZrzkirds8fDWklFE=
github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA=
github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY=
github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
github.com/tinylib/msgp v1.1.5 h1:2gXmtWueD2HefZHQe1QOy9HVzmFrLOVvsXwXBQ0ayy0=
github.com/tinylib/msgp v1.1.5/go.mod h1:eQsjooMTnV42mHu917E26IogZ2930nFyBQdofk10Udg=
github.com/tinylib/msgp v1.1.6 h1:i+SbKraHhnrf9M5MYmvQhFnbLhAXSDWF8WWsuyRdocw=
github.com/tinylib/msgp v1.1.6/go.mod h1:75BAfg2hauQhs3qedfdDZmWAPcFMAvJE5b9rGOMufyw=
github.com/ttacon/chalk v0.0.0-20160626202418-22c06c80ed31/go.mod h1:onvgF043R+lC5RZ8IT9rBXDaEDnpnw/Cl+HFiw+v/7Q=
github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU=
go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8=
go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=
go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=
go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8=
golang.org/x/exp v0.0.0-20190829153037-c13cbed26979/go.mod h1:86+5VVa7VpoJ4kLfm080zCjGlMRFzhUhsZKEZO7MGek=
golang.org/x/exp v0.0.0-20191030013958-a1ab85dbe136/go.mod h1:JXzH8nQsPlswgeRAPE3MuO9GYsAcnJvJ4vnMwN/5qkY=
golang.org/x/exp v0.0.0-20191129062945-2f5052295587/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4=
golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4=
golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4=
golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM=
golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU=
golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js=
golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0=
golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU=
golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
golang.org/x/lint v0.0.0-20190409202823-959b441ac422/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
golang.org/x/lint v0.0.0-20190909230951-414d861bb4ac/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
golang.org/x/lint v0.0.0-20191125180803-fdd1cda4f05f/go.mod h1:5qLYkcX4OjUUV8bRuDixDT3tpyyb+LUpUlRWLxfhWrs=
golang.org/x/lint v0.0.0-20200130185559-910be7a94367/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY=
golang.org/x/lint v0.0.0-20200302205851-738671d3881b/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY=
golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE=
golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o=
golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc=
golang.org/x/mod v0.1.0/go.mod h1:0QHyrYULN0/3qlju5TqG8bIK38QM8yzMo5ekMj3DlcY=
golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg=
golang.org/x/mod v0.1.1-0.20191107180719-034126e5016b/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg=
golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
golang.org/x/mod v0.3.0 h1:RM4zey1++hCTbCVQfnWeKs9/IEsaBLA8vTkd0WVtmH4=
golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
golang.org/x/mod v0.4.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
golang.org/x/mod v0.4.1/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
golang.org/x/net v0.0.0-20190501004415-9ce7a6920f09/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks=
golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
golang.org/x/net v0.0.0-20190628185345-da137c7871d7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
golang.org/x/net v0.0.0-20190724013045-ca1201d0de80/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
golang.org/x/net v0.0.0-20200114155413-6afb5195e5aa/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
golang.org/x/net v0.0.0-20200222125558-5a598a2470a0/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
golang.org/x/net v0.0.0-20200301022130-244492dfa37a/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
golang.org/x/net v0.0.0-20200501053045-e0ff5e5a1de5/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
golang.org/x/net v0.0.0-20200506145744-7e3656a0809f/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
golang.org/x/net v0.0.0-20200513185701-a91f0712d120/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
golang.org/x/net v0.0.0-20200520182314-0ba52f642ac2/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
golang.org/x/net v0.0.0-20200822124328-c89045814202 h1:VvcQYSHwXgi7W+TpUR6A9g6Up98WAHf3f/ulnJ62IyA=
golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
golang.org/x/net v0.0.0-20201202161906-c7110b5ffcbb/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
golang.org/x/net v0.0.0-20210226172049-e18ecbb05110 h1:qWPm9rbaAMKs8Bq/9LRpbMqxWRVUAQwMI9fVrssnTfw=
golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
golang.org/x/net v0.0.0-20211111160137-58aab5ef257a h1:c83jeVQW0KGKNaKBRfelNYNHaev+qawl9yaA825s8XE=
golang.org/x/net v0.0.0-20211111160137-58aab5ef257a/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
golang.org/x/oauth2 v0.0.0-20191202225959-858c2ad4c8b6/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208 h1:qwRHBd0NqMbJxfbotnDhm2ByMI1Shq4Y6oRJo21SGJA=
golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20210220032951-036812b2e83c h1:5KslGYwFpkhGh+Q16bwMP3cOontH8FOep7tGV86Y7SQ=
golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20191001151750-bb3f8db39f24/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20191228213918-04cbcbbfeed8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20200113162924-86b910548bc1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20200122134326-e047566fdf82/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20200212091648-12a6c2dcc1e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20200302150141-5c8b2ff67527/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd h1:xhmwyvizuTgC2qz7ZlMluP20uW+C3Rm0FD/WLDX8884=
golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20200331124033-c3d80250170d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20200501052902-10377860bb8e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20200511232937-7e40ca221e25/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20200515095857-1151b9dac4a9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20200523222454-059865788121/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20200803210538-64077c9b5642 h1:B6caxRw+hozq68X2MY7jEpZh/cr4/aHLv9xU8Kkadrw=
golang.org/x/sys v0.0.0-20200803210538-64077c9b5642/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg=
golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20210301091718-77cc2087c03b h1:kHlr0tATeLRMEiZJu5CknOw/E8V6h69sXXQFGoPtjcc=
golang.org/x/sys v0.0.0-20210301091718-77cc2087c03b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20211111213525-f221eed1c01e h1:zeJt6jBtVDK23XK9QXcmG0FvO0elikp0dYZQZOeL1y0=
golang.org/x/sys v0.0.0-20211111213525-f221eed1c01e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
golang.org/x/text v0.3.3 h1:cokOdA+Jmi5PJGXLlLllQSgYigAEfHXJAERHVMaCc2k=
golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
golang.org/x/text v0.3.5 h1:i6eZZ+zk0SOf0xgBpEpPD18qWcJda6q1sxt3S0kzyUQ=
golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
golang.org/x/text v0.3.7 h1:olpwvP2KacW1ZWvsR7uQhoyTYvKAupfQrRGBFM352Gk=
golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY=
golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
golang.org/x/tools v0.0.0-20190312151545-0bb0c0a6e846/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
golang.org/x/tools v0.0.0-20190425150028-36563e24a262/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
golang.org/x/tools v0.0.0-20190506145303-2d16b83fe98c/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
golang.org/x/tools v0.0.0-20190606124116-d0a3d012864b/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
golang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
golang.org/x/tools v0.0.0-20190816200558-6889da9d5479/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
golang.org/x/tools v0.0.0-20190911174233-4f2ddba30aff/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
golang.org/x/tools v0.0.0-20191113191852-77e3bb0ad9e7/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
golang.org/x/tools v0.0.0-20191115202509-3a792d9c32b2/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
golang.org/x/tools v0.0.0-20191125144606-a911d9008d1f/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
golang.org/x/tools v0.0.0-20191130070609-6e064ea0cf2d/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
golang.org/x/tools v0.0.0-20191216173652-a0e659d51361/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
golang.org/x/tools v0.0.0-20191227053925-7b8e75db28f4/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
golang.org/x/tools v0.0.0-20200117161641-43d50277825c/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
golang.org/x/tools v0.0.0-20200122220014-bf1340f18c4a/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
golang.org/x/tools v0.0.0-20200130002326-2f3ba24bd6e7/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
golang.org/x/tools v0.0.0-20200204074204-1cc6d1ef6c74/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
golang.org/x/tools v0.0.0-20200207183749-b753a1ba74fa/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
golang.org/x/tools v0.0.0-20200212150539-ea181f53ac56/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
golang.org/x/tools v0.0.0-20200224181240-023911ca70b2/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
golang.org/x/tools v0.0.0-20200227222343-706bc42d1f0d/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
golang.org/x/tools v0.0.0-20200304193943-95d2e580d8eb/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw=
golang.org/x/tools v0.0.0-20200312045724-11d5b4c81c7d/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw=
golang.org/x/tools v0.0.0-20200331025713-a30bf2db82d4/go.mod h1:Sl4aGygMT6LrqrWclx+PTx3U+LnKx/seiNR+3G19Ar8=
golang.org/x/tools v0.0.0-20200501065659-ab2804fb9c9d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
golang.org/x/tools v0.0.0-20200512131952-2bc93b1c0c88/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
golang.org/x/tools v0.0.0-20200515010526-7d3b6ebf133d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
golang.org/x/tools v0.0.0-20200618134242-20370b0cb4b2/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
golang.org/x/tools v0.0.0-20200729194436-6467de6f59a7/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA=
golang.org/x/tools v0.0.0-20200804011535-6c149bb5ef0d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA=
golang.org/x/tools v0.0.0-20200806022845-90696ccdc692/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA=
golang.org/x/tools v0.0.0-20200915031644-64986481280e h1:tfSNPIxC48Azhz4nLSPskz/yE9R6ftFRK8pfgfqWUAc=
golang.org/x/tools v0.0.0-20200915031644-64986481280e/go.mod h1:z6u4i615ZeAfBE4XtMziQW1fSVJXACjjbWkB/mvPzlU=
golang.org/x/tools v0.0.0-20201022035929-9cf592e881e9/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
golang.org/x/tools v0.0.0-20210108195828-e2f9c7f1fc8e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0=
golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE=
golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE=
google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M=
google.golang.org/api v0.8.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg=
google.golang.org/api v0.9.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg=
google.golang.org/api v0.13.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI=
google.golang.org/api v0.14.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI=
google.golang.org/api v0.15.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI=
google.golang.org/api v0.17.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE=
google.golang.org/api v0.18.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE=
google.golang.org/api v0.19.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE=
google.golang.org/api v0.20.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE=
google.golang.org/api v0.22.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE=
google.golang.org/api v0.24.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE=
google.golang.org/api v0.28.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE=
google.golang.org/api v0.29.0/go.mod h1:Lcubydp8VUV7KeIHD9z2Bys/sm/vGKnG1UHuDBSrHWM=
google.golang.org/api v0.30.0/go.mod h1:QGmEvQ87FHZNiUVJkT14jQNYJ4ZJjdRF23ZXz5138Fc=
google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=
google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0=
google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc=
google.golang.org/appengine v1.6.6/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc=
google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc=
google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
google.golang.org/genproto v0.0.0-20190425155659-357c62f0e4bb/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
google.golang.org/genproto v0.0.0-20190502173448-54afdca5d873/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
google.golang.org/genproto v0.0.0-20190801165951-fa694d86fc64/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc=
google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc=
google.golang.org/genproto v0.0.0-20190911173649-1774047e7e51/go.mod h1:IbNlFCBrqXvoKpeg0TB2l7cyZUmoaFKYIwrEpbDKLA8=
google.golang.org/genproto v0.0.0-20191108220845-16a3f7862a1a/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc=
google.golang.org/genproto v0.0.0-20191115194625-c23dd37a84c9/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc=
google.golang.org/genproto v0.0.0-20191216164720-4f79533eabd1/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc=
google.golang.org/genproto v0.0.0-20191230161307-f3c370f40bfb/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc=
google.golang.org/genproto v0.0.0-20200115191322-ca5a22157cba/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc=
google.golang.org/genproto v0.0.0-20200122232147-0452cf42e150/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc=
google.golang.org/genproto v0.0.0-20200204135345-fa8e72b47b90/go.mod h1:GmwEX6Z4W5gMy59cAlVYjN9JhxgbQH6Gn+gFDQe2lzA=
google.golang.org/genproto v0.0.0-20200212174721-66ed5ce911ce/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
google.golang.org/genproto v0.0.0-20200224152610-e50cd9704f63/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
google.golang.org/genproto v0.0.0-20200228133532-8c2c7df3a383/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
google.golang.org/genproto v0.0.0-20200305110556-506484158171/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
google.golang.org/genproto v0.0.0-20200312145019-da6875a35672/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
google.golang.org/genproto v0.0.0-20200331122359-1ee6d9798940/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
google.golang.org/genproto v0.0.0-20200430143042-b979b6f78d84/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
google.golang.org/genproto v0.0.0-20200511104702-f5ebc3bea380/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
google.golang.org/genproto v0.0.0-20200515170657-fc4c6c6a6587/go.mod h1:YsZOwe1myG/8QRHRsmBRE1LrgQY60beZKjly0O1fX9U=
google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo=
google.golang.org/genproto v0.0.0-20200618031413-b414f8b61790/go.mod h1:jDfRM7FcilCzHH/e9qn6dsT145K34l5v+OpcnNgKAAA=
google.golang.org/genproto v0.0.0-20200729003335-053ba62fc06f/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
google.golang.org/genproto v0.0.0-20200804131852-c06518451d9c/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
google.golang.org/genproto v0.0.0-20200806141610-86f49bd18e98 h1:LCO0fg4kb6WwkXQXRQQgUYsFeFb5taTX5WAx5O/Vt28=
google.golang.org/genproto v0.0.0-20200806141610-86f49bd18e98/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
google.golang.org/genproto v0.0.0-20210226172003-ab064af71705 h1:PYBmACG+YEv8uQPW0r1kJj8tR+gkF0UWq7iFdUezwEw=
google.golang.org/genproto v0.0.0-20210226172003-ab064af71705/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c=
google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38=
google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM=
google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg=
google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY=
google.golang.org/grpc v1.26.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk=
google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk=
google.golang.org/grpc v1.27.1/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk=
google.golang.org/grpc v1.28.0/go.mod h1:rpkK4SK4GF4Ach/+MFLZUBavHOvF2JJB5uozKKal+60=
google.golang.org/grpc v1.29.1/go.mod h1:itym6AZVZYACWQqET3MqgPpjcuV5QH3BxFS3IjizoKk=
google.golang.org/grpc v1.30.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak=
google.golang.org/grpc v1.31.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak=
google.golang.org/grpc v1.32.0 h1:zWTV+LMdc3kaiJMSTOFz2UgSBgx8RNQoTGiZu3fR9S0=
google.golang.org/grpc v1.32.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak=
google.golang.org/grpc/examples v0.0.0-20200915000551-32e7099cccac h1:oJwcuADYZXzWc+5R6wAvg6Qo86cbqAInuyKepxHGhIo=
google.golang.org/grpc/examples v0.0.0-20200915000551-32e7099cccac/go.mod h1:Lh55/1hxmVHEkOvSIQ2uj0P12QyOCUNyRwnUlSS13hw=
google.golang.org/grpc v1.36.0 h1:o1bcQ6imQMIOpdrO3SWf2z5RV72WbDwdXuK0MDlc8As=
google.golang.org/grpc v1.36.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU=
google.golang.org/grpc/examples v0.0.0-20210301210255-fc8f38cccf75 h1:IFbcGka/Xo8VIhRtCz9UBaDblKK7/iwLiXwm9D62Jzk=
google.golang.org/grpc/examples v0.0.0-20210301210255-fc8f38cccf75/go.mod h1:Ly7ZA/ARzg8fnPU9TyZIxoz33sEUuWX7txiqs8lPTgE=
google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8=
google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0=
google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM=
......@@ -401,23 +204,16 @@ google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpAD
google.golang.org/protobuf v1.24.0/go.mod h1:r/3tXBNzIEhYS9I1OUVjXDlt8tc493IdKGjtUeSXeh4=
google.golang.org/protobuf v1.25.0 h1:Ejskq+SyPohKW+1uil0JJMtmHCgJPJ/qWTxr8qp+R4c=
google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY=
gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI=
gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk=
gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q=
gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie2QXMNeKRTHCNY2nXvo=
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b h1:h8qDotaEPuJATrMmW04NCwg7v22aHH28wwpauUhK9Oo=
gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg=
honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k=
honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k=
lab.nexedi.com/kirr/go123 v0.0.0-20200915142026-a281a51cf49b h1:u9EAvkzDKlsTsLj9yuS+dGSli2H1MAHCQy4rFJ0dD7g=
lab.nexedi.com/kirr/go123 v0.0.0-20200915142026-a281a51cf49b/go.mod h1:Vt3dj/2vggjnIDaDyIWjvf7xmKlyyc4DsQSp7hUf6z4=
rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8=
rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0=
rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA=
lab.nexedi.com/kirr/go123 v0.0.0-20210128150852-c20e95f0f789 h1:IP+xm2DdYqLn81x8J8PdRkoMrCxLDBsXYe4KZm3FkME=
lab.nexedi.com/kirr/go123 v0.0.0-20210128150852-c20e95f0f789/go.mod h1:1wkWl3WhmutZiho+wsE7ymOKvRkN7hV3YZtL0f0gXTo=
lab.nexedi.com/kirr/go123 v0.0.0-20210906140734-c9eb28d9e408 h1:H7YpNUDfTSvvRpKivUMrL9C09tQssQ6brEoX6K/OxOw=
lab.nexedi.com/kirr/go123 v0.0.0-20210906140734-c9eb28d9e408/go.mod h1:pwDpdCuvtz0QxisDzV/z9eUb9zc/rMQec520h4i8VWQ=
go/internal/xtesting/xtesting.go
View file @ 95fbc76f
// Copyright (C) 2017-2020 Nexedi SA and Contributors.
// Copyright (C) 2017-2021 Nexedi SA and Contributors.
// Kirill Smelkov <kirr@nexedi.com>
//
// This program is free software: you can Use, Study, Modify and Redistribute
......@@ -29,10 +29,13 @@ import (
"os"
"os/exec"
"reflect"
"strings"
"sync"
"testing"
"time"
"lab.nexedi.com/kirr/go123/xerr"
"lab.nexedi.com/kirr/go123/xstrings"
"lab.nexedi.com/kirr/neo/go/zodb"
)
......@@ -92,13 +95,14 @@ func NeedPy(t testing.TB, modules ...string) {
}
// ZRawObject represents raw ZODB object state.
type ZRawObject struct {
type ZRawObject struct { // keep in sync with zodb(test).ZRawObject
Oid zodb.Oid
Data []byte // raw serialized zodb data
}
// ZPyCommitRaw commits new transaction into database @ zurl with raw data specified by objv.
//
// Nil data means "delete object".
// The commit is performed via zodbtools/py.
func ZPyCommitRaw(zurl string, at zodb.Tid, objv ...ZRawObject) (_ zodb.Tid, err error) {
defer xerr.Contextf(&err, "%s: zpycommit @%s", zurl, at)
......@@ -109,9 +113,14 @@ func ZPyCommitRaw(zurl string, at zodb.Tid, objv ...ZRawObject) (_ zodb.Tid, err
fmt.Fprintf(zin, "description %q\n", fmt.Sprintf("test commit; at=%s", at))
fmt.Fprintf(zin, "extension %q\n", "")
for _, obj := range objv {
fmt.Fprintf(zin, "obj %s %d null:00\n", obj.Oid, len(obj.Data))
zin.Write(obj.Data)
zin.WriteString("\n")
// !data -> delete
if obj.Data == nil {
fmt.Fprintf(zin, "obj %s delete\n", obj.Oid)
} else {
fmt.Fprintf(zin, "obj %s %d null:00\n", obj.Oid, len(obj.Data))
zin.Write(obj.Data)
zin.WriteString("\n")
}
}
zin.WriteString("\n")
......@@ -135,6 +144,33 @@ func ZPyCommitRaw(zurl string, at zodb.Tid, objv ...ZRawObject) (_ zodb.Tid, err
// XXX + ZPyCommitSrv ?
// ZPyRestore restores transactions specified by zin in zodbdump format.
//
// The restore is performed via zodbtools/py.
func ZPyRestore(zurl string, zin string) (tidv []zodb.Tid, err error) {
defer xerr.Contextf(&err, "%s: zpyrestore", zurl)
// run py `zodb restore`
cmd:= exec.Command("python", "-m", "zodbtools.zodb", "restore", zurl)
cmd.Stdin = strings.NewReader(zin)
cmd.Stderr = os.Stderr
out, err := cmd.Output()
if err != nil {
return nil, err
}
for _, line := range xstrings.SplitLines(string(out), "\n") {
tid, err := zodb.ParseTid(line)
if err != nil {
return nil, fmt.Errorf("restored, but invalid output: %s", err)
}
tidv = append(tidv, tid)
}
return tidv, nil
}
// ---- tests for storage drivers ----
......@@ -352,7 +388,7 @@ func DrvTestWatch(t *testing.T, zurl string, zdrvOpen zodb.DriverOpener, bugv ..
ctx := context.Background()
watchq := make(chan zodb.Event)
zdrv, at0, err := zdrvOpen(ctx, u, &zodb.DriverOptions{ReadOnly: true, Watchq: watchq})
zdrv, at0, err := zdrvOpen(ctx, u, &zodb.DriverOptions{ReadOnly: true, Watchq: watchq}); X(err)
if at0 != at {
t.Fatalf("opened @ %s ; want %s", at0, at)
}
......@@ -438,6 +474,11 @@ func DrvTestWatch(t *testing.T, zurl string, zdrvOpen zodb.DriverOpener, bugv ..
}
}
// commit something more and wait a bit to raise chances the driver enqueues to watchq<- .
_ = xcommit(at, ZRawObject{0, b("at the end")})
time.Sleep(1*time.Second)
// the driver must handle Close and cancel that watchq<-
err = zdrv.Close(); X(err)
e, ok := <-watchq
......@@ -450,6 +491,7 @@ func DrvTestWatch(t *testing.T, zurl string, zdrvOpen zodb.DriverOpener, bugv ..
// FatalIf(t) returns function f(err), which call t.Fatal if err != nil.
func FatalIf(t *testing.T) func(error) {
return func(err error) {
t.Helper()
if err != nil {
t.Fatal(err)
}
......
go/neo/client.go
View file @ 95fbc76f
......@@ -68,6 +68,9 @@ type Client struct {
at0Initialized bool // true after .at0 is initialized
at0Ready chan(struct{}) // ready after .at0 is initialized
closeOnce sync.Once
closed chan struct{} // ready when Closed
ownNet bool // true if Client "owns" networker and should release it on Close
}
......@@ -79,8 +82,9 @@ var _ zodb.IStorageDriver = (*Client)(nil)
// Use Run to actually start running the node.
func NewClient(clusterName, masterAddr string, net xnet.Networker) *Client {
c := &Client{
node: newMasteredNode(proto.CLIENT, clusterName, net, masterAddr),
node: newMasteredNode(proto.CLIENT, clusterName, net, masterAddr),
at0Ready: make(chan struct{}),
closed: make(chan struct{}),
}
var runCtx context.Context
......@@ -91,6 +95,9 @@ func NewClient(clusterName, masterAddr string, net xnet.Networker) *Client {
// Close implements zodb.IStorageDriver.
func (c *Client) Close() (err error) {
c.closeOnce.Do(func() {
close(c.closed)
})
c.runCancel()
err = c.runWG.Wait()
if errors.Is(err, context.Canceled) {
......@@ -190,7 +197,7 @@ func (c *Client) invalidateObjects(msg *proto.InvalidateObjects) error {
defer c.at0Mu.Unlock()
// queue initial events until .at0 is initialized after register
// queued events will be sent to watchq by zeo ctor after initializing .at0
// queued events will be sent to watchq by syncMaster after initializing .at0
if !c.at0Initialized {
c.eventq0 = append(c.eventq0, event)
return nil
......@@ -198,7 +205,13 @@ func (c *Client) invalidateObjects(msg *proto.InvalidateObjects) error {
// at0 is initialized - ok to send current event if it goes > at0
if tid > c.at0 {
c.watchq <- event
select {
case <-c.closed:
// closed - client does not read watchq anymore
case c.watchq <- event:
// ok
}
}
return nil
}
......@@ -250,7 +263,13 @@ func (c *Client) flushEventq0() {
if c.watchq != nil {
for _, e := range c.eventq0 {
if e.Tid > c.at0 {
c.watchq <- e
select {
case <-c.closed:
// closed - client does not read watchq anymore
case c.watchq <- e:
// ok
}
}
}
}
......@@ -410,6 +429,8 @@ func openClientByURL(ctx context.Context, u *url.URL, opt *zodb.DriverOptions) (
}
cred := u.User.String()
// ca=ca.crt;cert=my.crt;key=my.key
cred = strings.ReplaceAll(cred, ";", "&") // ; is no longer in default separators set https://github.com/golang/go/issues/25192
x, err := xurl.ParseQuery(cred)
if err != nil {
return nil, zodb.InvalidTid, fmt.Errorf("credentials: %s", err)
......@@ -442,9 +463,7 @@ func openClientByURL(ctx context.Context, u *url.URL, opt *zodb.DriverOptions) (
}
name := u.Path
if strings.HasPrefix(name, "/") {
name = name[1:]
}
name = strings.TrimPrefix(name, "/")
if name == "" {
return nil, zodb.InvalidTid, fmt.Errorf("cluster name not specified")
}
......@@ -491,14 +510,18 @@ func openClientByURL(ctx context.Context, u *url.URL, opt *zodb.DriverOptions) (
// close .watchq after serve is over
c.at0Mu.Lock()
defer c.at0Mu.Unlock()
if c.at0Initialized {
c.flushEventq0()
}
if c.watchq != nil {
if err != nil {
c.watchq <- &zodb.EventError{Err: err}
if err != nil && /* already flushed .eventq0 */c.at0Initialized {
select {
case <-c.closed:
// closed - client does not read watchq anymore
case c.watchq <- &zodb.EventError{Err: err}:
// ok
}
}
close(c.watchq)
c.watchq = nil // prevent flushEventq0 to send to closed chan
}
errq <- err
......
go/neo/client_test.go
View file @ 95fbc76f
......@@ -362,6 +362,7 @@ func (opt NEOSrvOptions) URLPrefix() string {
// ----------------
// tOptions represents options for testing.
// TODO -> xtesting
type tOptions struct {
Preload string // preload database with data from this location
}
......
go/neo/mastered.go
View file @ 95fbc76f
......@@ -76,12 +76,12 @@ type _MasteredNode struct {
type _MasteredNodeFlags int
const (
// δPartTabPassThrough tells mlink.Recv1 not to filter out messages related
// to partition table changes. When mlink.Recv1 receives such messages there
// to partition table changes. When mlink.Recv1 receives such messages they
// are already processed internally to update .state.PartTab correspondingly.
//
// Storage uses this mode to receive δPartTab notifications to know
// when to persist it.
δPartTabPassThrough _MasteredNodeFlags = iota
δPartTabPassThrough _MasteredNodeFlags = 1 << iota
)
// newMasteredNode creates new _MasteredNode that connects to masterAddr/cluster via net.
......
go/neo/t/neotest
View file @ 95fbc76f
#!/bin/bash -e
# neotest: run tests and benchmarks against FileStorage, ZEO and various NEO/py and NEO/go clusters
# Copyright (C) 2017-2020 Nexedi SA and Contributors.
# Copyright (C) 2017-2021 Nexedi SA and Contributors.
# Kirill Smelkov <kirr@nexedi.com>
#
# This program is free software: you can Use, Study, Modify and Redistribute
......@@ -141,7 +141,10 @@ $@
# ---- go/py unit tests ----
cmd_test-go() {
go test -count=1 lab.nexedi.com/kirr/neo/go/... # -count=1 disables tests caching
(
cd $NEOt
go test -count=1 lab.nexedi.com/kirr/neo/go/... # -count=1 disables tests caching
)
}
cmd_test-py() {
......@@ -1503,18 +1506,22 @@ cpustat) f=( );;
esac
NEOt=$(cd `dirname $0` && pwd)
for flag in ${f[*]}; do
case "$flag" in
build)
# make sure tzodb*, tcpu* and zgenprod are on PATH (because we could be invoked from another dir)
X=$(cd `dirname $0` && pwd)
export PATH=$X:$PATH
export PATH=$NEOt:$PATH
# rebuild go bits
# neo/py, wendelin.core, ... - must be pip install'ed - `neotest deploy` cares about that
go install -v lab.nexedi.com/kirr/neo/go/...
go build -o $X/tzodb_go $X/tzodb.go
go build -o $X/tcpu_go $X/tcpu.go
(
cd $NEOt
go install -v lab.nexedi.com/kirr/neo/go/...
go build -o tzodb_go tzodb.go
go build -o tcpu_go tcpu.go
)
;;
net)
......
go/neo/t/tzodb.go
View file @ 95fbc76f
......@@ -431,7 +431,7 @@ func zwrkPreconnect(ctx context.Context, url string, at zodb.Tid, nwrk int) (_ [
if err != nil {
for _, stor := range storv {
if stor != nil {
xio.LClose(stor)
xio.LClose(ctx, stor)
}
}
return nil, err
......
go/zodb/btree/btree.go.in
View file @ 95fbc76f
// Copyright (c) 2001, 2002 Zope Foundation and Contributors.
// All Rights Reserved.
//
// Copyright (C) 2018-2019 Nexedi SA and Contributors.
// Copyright (C) 2018-2021 Nexedi SA and Contributors.
// Kirill Smelkov <kirr@nexedi.com>
//
// This software is subject to the provisions of the Zope Public License,
......@@ -65,7 +65,7 @@ type BTree struct {
// Entry is one BTree node entry.
//
// It contains key and child, who is either BTree or Bucket.
// It contains key and child, which is either BTree or Bucket.
//
// Key limits child's keys - see BTree.Entryv for details.
type Entry struct {
......@@ -102,6 +102,15 @@ type BucketEntry struct {
value interface{}
}
// KeyRange represents [lo,hi) key range.
type KeyRange struct {
Lo KEY
Hi_ KEY // NOTE _not_ hi) to avoid overflow at ∞; hi = hi_ + 1
}
const _KeyMin KEY = math.Min<Key>
const _KeyMax KEY = math.Max<Key>
// ---- access []entry ----
// Key returns BTree entry key.
......@@ -123,7 +132,7 @@ func (e *Entry) Child() Node { return e.child }
// Children of all entries are guaranteed to be of the same kind - either all BTree, or all Bucket.
//
// The caller must not modify returned array.
func (t *BTree) Entryv() []Entry {
func (t *BTree) Entryv() /*readonly*/ []Entry {
return t.data
}
......@@ -134,7 +143,7 @@ func (e *BucketEntry) Key() KEY { return e.key }
func (e *BucketEntry) Value() interface{} { return e.value }
// Entryv returns entries of a Bucket node.
func (b *Bucket) Entryv() []BucketEntry {
func (b *Bucket) Entryv() /*readonly*/ []BucketEntry {
ev := make([]BucketEntry, len(b.keys))
for i, k := range b.keys {
ev[i] = BucketEntry{k, b.values[i]}
......@@ -168,36 +177,54 @@ func (t *BTree) Get(ctx context.Context, key KEY) (_ interface{}, _ bool, err er
// VGet is like Get but also calls visit while traversing the tree.
//
// Visit is called with node being activated.
func (t *BTree) VGet(ctx context.Context, key KEY, visit func(node Node)) (_ interface{}, _ bool, err error) {
func (t *BTree) VGet(ctx context.Context, key KEY, visit func(node Node, keycov KeyRange)) (_ interface{}, _ bool, err error) {
defer xerr.Contextf(&err, "btree(%s): get %v", t.POid(), key)
err = t.PActivate(ctx)
if err != nil {
return nil, false, err
}
keycov := KeyRange{Lo: _KeyMin, Hi_: _KeyMax}
if visit != nil {
visit(t)
}
if len(t.data) == 0 {
// empty btree
t.PDeactivate()
return nil, false, nil
visit(t, keycov)
}
for {
l := len(t.data)
if l == 0 {
// empty btree (top, or in leaf)
t.PDeactivate()
return nil, false, nil
}
// search i: K(i) ≤ k < K(i+1) ; K(0) = -∞, K(len) = +∞
i := sort.Search(len(t.data), func(i int) bool {
i := sort.Search(l, func(i int) bool {
j := i + 1
if j == len(t.data) {
return true // [len].key = +∞
}
return key < t.data[j].key
})
// FIXME panic index out of range (empty T without children;
// logically incorrect, but Restructure generated it once)
// i < l
child := t.data[i].child
// shorten global keycov by local [lo,hi) for this child
lo := _KeyMin
if i > 0 {
lo = t.data[i].key
}
i++
hi_ := _KeyMax
if i < l {
hi_ = t.data[i].key
}
if hi_ != _KeyMax {
hi_--
}
keycov.Lo = kmax(keycov.Lo, lo)
keycov.Hi_ = kmin(keycov.Hi_, hi_)
t.PDeactivate()
err = child.PActivate(ctx)
if err != nil {
......@@ -207,7 +234,7 @@ func (t *BTree) VGet(ctx context.Context, key KEY, visit func(node Node)) (_ int
// XXX verify child keys are in valid range according to parent
if visit != nil {
visit(child)
visit(child, keycov)
}
switch child := child.(type) {
......@@ -250,26 +277,40 @@ func (t *BTree) MinKey(ctx context.Context) (_ KEY, ok bool, err error) {
// VMinKey is like MinKey but also calls visit while traversing the tree.
//
// Visit is called with node being activated.
func (t *BTree) VMinKey(ctx context.Context, visit func(node Node)) (_ KEY, ok bool, err error) {
func (t *BTree) VMinKey(ctx context.Context, visit func(node Node, keycov KeyRange)) (_ KEY, ok bool, err error) {
defer xerr.Contextf(&err, "btree(%s): minkey", t.POid())
err = t.PActivate(ctx)
if err != nil {
return 0, false, err
}
keycov := KeyRange{Lo: _KeyMin, Hi_: _KeyMax}
if visit != nil {
visit(t)
}
if len(t.data) == 0 {
// empty btree
t.PDeactivate()
return 0, false, nil
visit(t, keycov)
}
// NOTE -> can also use t.firstbucket
for {
l := len(t.data)
if l == 0 {
// empty btree (top, or in leaf)
t.PDeactivate()
return 0, false, nil
}
child := t.data[0].child
// shorten global keycov by local hi) for this child
hi_ := _KeyMax
if 1 < l {
hi_ = t.data[1].key
}
if hi_ != _KeyMax {
hi_--
}
// keycov.Lo stays -∞
keycov.Hi_ = kmin(keycov.Hi_, hi_)
t.PDeactivate()
err = child.PActivate(ctx)
if err != nil {
......@@ -277,7 +318,7 @@ func (t *BTree) VMinKey(ctx context.Context, visit func(node Node)) (_ KEY, ok b
}
if visit != nil {
visit(child)
visit(child, keycov)
}
// XXX verify child keys are in valid range according to parent
......@@ -305,26 +346,36 @@ func (t *BTree) MaxKey(ctx context.Context) (_ KEY, _ bool, err error) {
// VMaxKey is like MaxKey but also calls visit while traversing the tree.
//
// Visit is called with node being activated.
func (t *BTree) VMaxKey(ctx context.Context, visit func(node Node)) (_ KEY, _ bool, err error) {
func (t *BTree) VMaxKey(ctx context.Context, visit func(node Node, keycov KeyRange)) (_ KEY, _ bool, err error) {
defer xerr.Contextf(&err, "btree(%s): maxkey", t.POid())
err = t.PActivate(ctx)
if err != nil {
return 0, false, err
}
keycov := KeyRange{Lo: _KeyMin, Hi_: _KeyMax}
if visit != nil {
visit(t)
}
l := len(t.data)
if l == 0 {
// empty btree
t.PDeactivate()
return 0, false, nil
visit(t, keycov)
}
for {
l := len(t.data)
if l == 0 {
// empty btree (top, or in leaf)
t.PDeactivate()
return 0, false, nil
}
child := t.data[l-1].child
// shorten global keycov by local [lo for this chile
lo := _KeyMin
if l-1 > 0 {
lo = t.data[l-1].key
}
keycov.Lo = kmax(keycov.Lo, lo)
// keycov.Hi_ stays ∞
t.PDeactivate()
err = child.PActivate(ctx)
if err != nil {
......@@ -332,7 +383,7 @@ func (t *BTree) VMaxKey(ctx context.Context, visit func(node Node)) (_ KEY, _ bo
}
if visit != nil {
visit(child)
visit(child, keycov)
}
// XXX verify child keys are in valid range according to parent
......@@ -591,7 +642,7 @@ func (bt *btreeState) PySetState(pystate interface{}) (err error) {
var kprev int64
var childrenKind int // 1 - BTree, 2 - Bucket
for i, idx := 0, 0; i < n; i++ {
key := int64(math.Min<Key>) // KEY(-∞) (qualifies for ≤)
key := int64(_KeyMin) // KEY(-∞) (qualifies for ≤)
if i > 0 {
// key[0] is unused and not saved
key, ok = t[idx].(int64) // XXX Xint
......@@ -609,7 +660,7 @@ func (bt *btreeState) PySetState(pystate interface{}) (err error) {
}
if i > 1 && !(key > kprev) {
fmt.Errorf("data: [%d]: key not ↑", i)
return fmt.Errorf("data: [%d]: key not ↑", i)
}
kprev = key
......@@ -629,7 +680,7 @@ func (bt *btreeState) PySetState(pystate interface{}) (err error) {
childrenKind = kind
}
if kind != childrenKind {
fmt.Errorf("data: [%d]: children must be of the same type", i)
return fmt.Errorf("data: [%d]: children must be of the same type", i)
}
bt.data = append(bt.data, Entry{key: kkey, child: child.(Node)})
......@@ -646,3 +697,60 @@ func init() {
zodb.RegisterClass("BTrees.BTree.BTree", t(BTree{}), t(btreeState{}))
zodb.RegisterClass("BTrees.BTree.Bucket", t(Bucket{}), t(bucketState{}))
}
// ---- misc ----
// Has returns whether key k belongs to the range.
func (r *KeyRange) Has(k KEY) bool {
return (r.Lo <= k && k <= r.Hi_)
}
// Empty returns whether key range is empty.
func (r *KeyRange) Empty() bool {
hi := r.Hi_
if hi == _KeyMax {
// [x,∞] cannot be empty because max x is ∞ and [∞,∞] has one element: ∞
return false
}
hi++ // no overflow
return r.Lo >= hi
}
func (r KeyRange) String() string {
var shi string
if r.Hi_ == _KeyMax {
shi = kstr(r.Hi_) // ∞
} else {
shi = fmt.Sprintf("%d", r.Hi_+1)
}
return fmt.Sprintf("[%s,%s)", kstr(r.Lo), shi)
}
func kmin(a, b KEY) KEY {
if a < b {
return a
} else {
return b
}
}
func kmax(a, b KEY) KEY {
if a > b {
return a
} else {
return b
}
}
// kstr formats key as string.
func kstr(k KEY) string {
if k == _KeyMin {
return "-∞"
}
if k == _KeyMax {
return "∞"
}
return fmt.Sprintf("%d", k)
}
go/zodb/btree/btree_test.go
View file @ 95fbc76f
// Copyright (C) 2018-2019 Nexedi SA and Contributors.
// Copyright (C) 2018-2021 Nexedi SA and Contributors.
// Kirill Smelkov <kirr@nexedi.com>
//
// This program is free software: you can Use, Study, Modify and Redistribute
......@@ -23,6 +23,7 @@ package btree
import (
"context"
"reflect"
"testing"
"lab.nexedi.com/kirr/go123/exc"
......@@ -98,6 +99,12 @@ func (b *bucketWrap) MaxKey(ctx context.Context) (k int64, ok bool, err error) {
return
}
// tVisit is information about one visit call.
type tVisit struct {
node zodb.Oid
keycov LKeyRange
}
func TestBTree(t *testing.T) {
X := exc.Raiseif
ctx := context.Background()
......@@ -105,7 +112,11 @@ func TestBTree(t *testing.T) {
if err != nil {
t.Fatal(err)
}
db := zodb.NewDB(stor)
defer func() {
err := stor.Close(); X(err)
}()
db := zodb.NewDB(stor, &zodb.DBOptions{})
defer func() {
err := db.Close(); X(err)
}()
......@@ -118,8 +129,6 @@ func TestBTree(t *testing.T) {
t.Fatal(err)
}
// XXX close db/stor
// go through small test Buckets/BTrees and verify that Get(key) is as expected.
for _, tt := range smallTestv {
xobj, err := conn.Get(ctx, tt.oid)
......@@ -262,4 +271,39 @@ func TestBTree(t *testing.T) {
// XXX verify FirstBucket / Next ?
verifyFirstBucket(B3)
// verify nodes/keycov visited through VGet/V{Min,Max}Key
xBv, err := conn.Get(ctx, Bv_oid); X(err)
Bv, ok := xBv.(*LOBTree)
if !ok {
t.Fatalf("Bv: %v; got %T; want LOBTree", Bv_oid, xBv)
}
for k, visitOK := range Bvdict {
visit := []tVisit{}
_, _, err := Bv.VGet(ctx, k, func(node LONode, keycov LKeyRange) {
visit = append(visit, tVisit{node.POid(), keycov})
}); X(err)
if !reflect.DeepEqual(visit, visitOK) {
t.Errorf("VGet(%d): visit:\nhave: %v\nwant: %v", k, visit, visitOK)
}
}
visitMinOK := Bvdict[Bv_kmin]
visitMaxOK := Bvdict[Bv_kmax]
visitMin := []tVisit{}
visitMax := []tVisit{}
_, _, err = Bv.VMinKey(ctx, func(node LONode, keycov LKeyRange) {
visitMin = append(visitMin, tVisit{node.POid(), keycov})
}); X(err)
_, _, err = Bv.VMaxKey(ctx, func(node LONode, keycov LKeyRange) {
visitMax = append(visitMax, tVisit{node.POid(), keycov})
}); X(err)
if !reflect.DeepEqual(visitMin, visitMinOK) {
t.Errorf("VMinKey(): visit:\nhave: %v\nwant: %v", visitMin, visitMinOK)
}
if !reflect.DeepEqual(visitMax, visitMaxOK) {
t.Errorf("VMaxKey(): visit:\nhave: %v\nwant: %v", visitMax, visitMaxOK)
}
}
go/zodb/btree/gen-btree
View file @ 95fbc76f
......@@ -28,6 +28,9 @@ out=$3
kind=${KIND,,} # IO -> io
Key=${KEY^}
KEYKIND=${KIND:0:1} # IO -> I
keykind=${KEYKIND,,} # I -> i
input=$(dirname $0)/btree.go.in
echo "// Code generated by gen-btree; DO NOT EDIT." >$out
......@@ -45,4 +48,10 @@ sed \
-e "s/\bBucketEntry\b/${KIND}BucketEntry/g" \
-e "s/\bbtreeState\b/${kind}btreeState/g" \
-e "s/\bbucketState\b/${kind}bucketState/g" \
-e "s/\b_KeyMin\b/_${KEYKIND}KeyMin/g" \
-e "s/\b_KeyMax\b/_${KEYKIND}KeyMax/g" \
-e "s/\bKeyRange\b/${KEYKIND}KeyRange/g" \
-e "s/\bkmin\b/${keykind}kmin/g" \
-e "s/\bkmax\b/${keykind}kmax/g" \
-e "s/\bkstr\b/${keykind}kstr/g" \
$input >>$out
go/zodb/btree/testdata/gen-testdata
View file @ 95fbc76f
#!/usr/bin/env python2
# -*- coding: utf-8 -*-
# Copyright (C) 2018-2020 Nexedi SA and Contributors.
# Copyright (C) 2018-2021 Nexedi SA and Contributors.
# Kirill Smelkov <kirr@nexedi.com>
#
# This program is free software: you can Use, Study, Modify and Redistribute
......@@ -22,8 +22,9 @@
from ZODB.DB import DB
from BTrees.LOBTree import LOBucket, LOBTree
from BTrees.check import check as bcheck
from ZODB.utils import u64
from zodbtools.test.gen_testdata import run_with_zodb3py2_compat
from zodbtools.test.gen_testdata import run_with_zodb4py2_compat
import os, os.path, transaction
from golang.gcompat import qq
......@@ -52,14 +53,37 @@ def main2():
root['B3'] = B3 = LOBTree(dict([(_, _) for _ in range(10000)]))
# T4/T2-T/B1-B2-T7,9/B5-B8-B10 (to verify VGet->visit)
# TODO use xbtree.py:Restructure after gimport works through modules and
# xbtree.py is moved from wcfs to zodb/go.
v1 = LOBucket([(1,"a")])
v2 = LOBucket([(2,"b")])
v5 = LOBucket([(5,"c")])
v8 = LOBucket([(8,"d")])
v9 = LOBucket([(9,"e")])
T2, T79, T, T4 = LOBTree(), LOBTree(), LOBTree(), LOBTree()
T2.__setstate__ (((v1, 2, v2), v1)) # (child, key, child, ...), firstbucket
T79.__setstate__(((v5, 7, v8, 9, v9), v5))
T.__setstate__ (((T79,), v5))
T4.__setstate__ (((T2, 4, T), v1))
root['Bv'] = Bv = T4
transaction.commit()
bcheck(Bv)
assert Bv[1] == "a"
assert Bv[2] == "b"
assert Bv[5] == "c"
assert Bv[8] == "d"
assert Bv[9] == "e"
with open("ztestdata_expect_test.go", "w") as f:
def emit(v):
print >>f, v
emit("// Code generated by %s; DO NOT EDIT." % __file__)
emit("package btree\n")
#emit("import \"lab.nexedi.com/kirr/neo/go/zodb\"\n")
def emititems(b):
s = "testEntry{oid: %s, kind: %s, itemv: []kv{" \
......@@ -84,12 +108,34 @@ def main2():
emit("\nconst B3_oid = %s" % u64(B3._p_oid))
emit("const B3_maxkey = %d" % B3.maxKey())
emit("\nconst Bv_oid = %s" % u64(Bv._p_oid))
emit("const Bv_kmin = %d" % Bv.minKey())
emit("const Bv_kmax = %d" % Bv.maxKey())
emit("var Bvdict = map[int64][]tVisit{")
noo = "_LKeyMin"
oo = "_LKeyMax"
def emitVisit(key, *visitv): # visitv = [](node, lo,hi)
vstr = []
for node, lo,hi in visitv:
if isinstance(hi, str):
hi_ = hi # oo or noo
else:
hi_ = hi-1
vstr.append("{%d, LKeyRange{%s, %s}}" % (u64(node._p_oid), lo, hi_))
emit("\t%d: []tVisit{%s}," % (key, ", ".join(vstr)))
emitVisit(1, (T4, noo,oo), (T2, noo,4), (v1, noo,2))
emitVisit(2, (T4, noo,oo), (T2, noo,4), (v2, 2,4))
emitVisit(5, (T4, noo,oo), (T, 4,oo), (T79, 4,oo), (v5, 4,7))
emitVisit(8, (T4, noo,oo), (T, 4,oo), (T79, 4,oo), (v8, 7,9))
emitVisit(9, (T4, noo,oo), (T, 4,oo), (T79, 4,oo), (v9, 9,oo))
emit("}")
conn.close()
db.close()
def main():
run_with_zodb3py2_compat(main2)
run_with_zodb4py2_compat(main2)
if __name__ == '__main__':
......
go/zodb/btree/ziobtree.go
View file @ 95fbc76f
......@@ -3,7 +3,7 @@
// Copyright (c) 2001, 2002 Zope Foundation and Contributors.
// All Rights Reserved.
//
// Copyright (C) 2018-2019 Nexedi SA and Contributors.
// Copyright (C) 2018-2021 Nexedi SA and Contributors.
// Kirill Smelkov <kirr@nexedi.com>
//
// This software is subject to the provisions of the Zope Public License,
......@@ -67,7 +67,7 @@ type IOBTree struct {
// IOEntry is one IOBTree node entry.
//
// It contains key and child, who is either IOBTree or IOBucket.
// It contains key and child, which is either IOBTree or IOBucket.
//
// Key limits child's keys - see IOBTree.Entryv for details.
type IOEntry struct {
......@@ -104,6 +104,15 @@ type IOBucketEntry struct {
value interface{}
}
// IKeyRange represents [lo,hi) key range.
type IKeyRange struct {
Lo int32
Hi_ int32 // NOTE _not_ hi) to avoid overflow at ∞; hi = hi_ + 1
}
const _IKeyMin int32 = math.MinInt32
const _IKeyMax int32 = math.MaxInt32
// ---- access []entry ----
// Key returns IOBTree entry key.
......@@ -125,7 +134,7 @@ func (e *IOEntry) Child() IONode { return e.child }
// Children of all entries are guaranteed to be of the same kind - either all IOBTree, or all IOBucket.
//
// The caller must not modify returned array.
func (t *IOBTree) Entryv() []IOEntry {
func (t *IOBTree) Entryv() /*readonly*/ []IOEntry {
return t.data
}
......@@ -136,7 +145,7 @@ func (e *IOBucketEntry) Key() int32 { return e.key }
func (e *IOBucketEntry) Value() interface{} { return e.value }
// Entryv returns entries of a IOBucket node.
func (b *IOBucket) Entryv() []IOBucketEntry {
func (b *IOBucket) Entryv() /*readonly*/ []IOBucketEntry {
ev := make([]IOBucketEntry, len(b.keys))
for i, k := range b.keys {
ev[i] = IOBucketEntry{k, b.values[i]}
......@@ -170,34 +179,54 @@ func (t *IOBTree) Get(ctx context.Context, key int32) (_ interface{}, _ bool, er
// VGet is like Get but also calls visit while traversing the tree.
//
// Visit is called with node being activated.
func (t *IOBTree) VGet(ctx context.Context, key int32, visit func(node IONode)) (_ interface{}, _ bool, err error) {
func (t *IOBTree) VGet(ctx context.Context, key int32, visit func(node IONode, keycov IKeyRange)) (_ interface{}, _ bool, err error) {
defer xerr.Contextf(&err, "btree(%s): get %v", t.POid(), key)
err = t.PActivate(ctx)
if err != nil {
return nil, false, err
}
keycov := IKeyRange{Lo: _IKeyMin, Hi_: _IKeyMax}
if visit != nil {
visit(t)
}
if len(t.data) == 0 {
// empty btree
t.PDeactivate()
return nil, false, nil
visit(t, keycov)
}
for {
l := len(t.data)
if l == 0 {
// empty btree (top, or in leaf)
t.PDeactivate()
return nil, false, nil
}
// search i: K(i) ≤ k < K(i+1) ; K(0) = -∞, K(len) = +∞
i := sort.Search(len(t.data), func(i int) bool {
i := sort.Search(l, func(i int) bool {
j := i + 1
if j == len(t.data) {
return true // [len].key = +∞
}
return key < t.data[j].key
})
// i < l
child := t.data[i].child
// shorten global keycov by local [lo,hi) for this child
lo := _IKeyMin
if i > 0 {
lo = t.data[i].key
}
i++
hi_ := _IKeyMax
if i < l {
hi_ = t.data[i].key
}
if hi_ != _IKeyMax {
hi_--
}
keycov.Lo = ikmax(keycov.Lo, lo)
keycov.Hi_ = ikmin(keycov.Hi_, hi_)
t.PDeactivate()
err = child.PActivate(ctx)
if err != nil {
......@@ -207,7 +236,7 @@ func (t *IOBTree) VGet(ctx context.Context, key int32, visit func(node IONode))
// XXX verify child keys are in valid range according to parent
if visit != nil {
visit(child)
visit(child, keycov)
}
switch child := child.(type) {
......@@ -250,26 +279,40 @@ func (t *IOBTree) MinKey(ctx context.Context) (_ int32, ok bool, err error) {
// VMinKey is like MinKey but also calls visit while traversing the tree.
//
// Visit is called with node being activated.
func (t *IOBTree) VMinKey(ctx context.Context, visit func(node IONode)) (_ int32, ok bool, err error) {
func (t *IOBTree) VMinKey(ctx context.Context, visit func(node IONode, keycov IKeyRange)) (_ int32, ok bool, err error) {
defer xerr.Contextf(&err, "btree(%s): minkey", t.POid())
err = t.PActivate(ctx)
if err != nil {
return 0, false, err
}
keycov := IKeyRange{Lo: _IKeyMin, Hi_: _IKeyMax}
if visit != nil {
visit(t)
}
if len(t.data) == 0 {
// empty btree
t.PDeactivate()
return 0, false, nil
visit(t, keycov)
}
// NOTE -> can also use t.firstbucket
for {
l := len(t.data)
if l == 0 {
// empty btree (top, or in leaf)
t.PDeactivate()
return 0, false, nil
}
child := t.data[0].child
// shorten global keycov by local hi) for this child
hi_ := _IKeyMax
if 1 < l {
hi_ = t.data[1].key
}
if hi_ != _IKeyMax {
hi_--
}
// keycov.Lo stays -∞
keycov.Hi_ = ikmin(keycov.Hi_, hi_)
t.PDeactivate()
err = child.PActivate(ctx)
if err != nil {
......@@ -277,7 +320,7 @@ func (t *IOBTree) VMinKey(ctx context.Context, visit func(node IONode)) (_ int32
}
if visit != nil {
visit(child)
visit(child, keycov)
}
// XXX verify child keys are in valid range according to parent
......@@ -305,26 +348,36 @@ func (t *IOBTree) MaxKey(ctx context.Context) (_ int32, _ bool, err error) {
// VMaxKey is like MaxKey but also calls visit while traversing the tree.
//
// Visit is called with node being activated.
func (t *IOBTree) VMaxKey(ctx context.Context, visit func(node IONode)) (_ int32, _ bool, err error) {
func (t *IOBTree) VMaxKey(ctx context.Context, visit func(node IONode, keycov IKeyRange)) (_ int32, _ bool, err error) {
defer xerr.Contextf(&err, "btree(%s): maxkey", t.POid())
err = t.PActivate(ctx)
if err != nil {
return 0, false, err
}
keycov := IKeyRange{Lo: _IKeyMin, Hi_: _IKeyMax}
if visit != nil {
visit(t)
}
l := len(t.data)
if l == 0 {
// empty btree
t.PDeactivate()
return 0, false, nil
visit(t, keycov)
}
for {
l := len(t.data)
if l == 0 {
// empty btree (top, or in leaf)
t.PDeactivate()
return 0, false, nil
}
child := t.data[l-1].child
// shorten global keycov by local [lo for this chile
lo := _IKeyMin
if l-1 > 0 {
lo = t.data[l-1].key
}
keycov.Lo = ikmax(keycov.Lo, lo)
// keycov.Hi_ stays ∞
t.PDeactivate()
err = child.PActivate(ctx)
if err != nil {
......@@ -332,7 +385,7 @@ func (t *IOBTree) VMaxKey(ctx context.Context, visit func(node IONode)) (_ int32
}
if visit != nil {
visit(child)
visit(child, keycov)
}
// XXX verify child keys are in valid range according to parent
......@@ -591,7 +644,7 @@ func (bt *iobtreeState) PySetState(pystate interface{}) (err error) {
var kprev int64
var childrenKind int // 1 - IOBTree, 2 - IOBucket
for i, idx := 0, 0; i < n; i++ {
key := int64(math.MinInt32) // int32(-∞) (qualifies for ≤)
key := int64(_IKeyMin) // int32(-∞) (qualifies for ≤)
if i > 0 {
// key[0] is unused and not saved
key, ok = t[idx].(int64) // XXX Xint
......@@ -609,7 +662,7 @@ func (bt *iobtreeState) PySetState(pystate interface{}) (err error) {
}
if i > 1 && !(key > kprev) {
fmt.Errorf("data: [%d]: key not ↑", i)
return fmt.Errorf("data: [%d]: key not ↑", i)
}
kprev = key
......@@ -629,7 +682,7 @@ func (bt *iobtreeState) PySetState(pystate interface{}) (err error) {
childrenKind = kind
}
if kind != childrenKind {
fmt.Errorf("data: [%d]: children must be of the same type", i)
return fmt.Errorf("data: [%d]: children must be of the same type", i)
}
bt.data = append(bt.data, IOEntry{key: kkey, child: child.(IONode)})
......@@ -646,3 +699,60 @@ func init() {
zodb.RegisterClass("BTrees.IOBTree.IOBTree", t(IOBTree{}), t(iobtreeState{}))
zodb.RegisterClass("BTrees.IOBTree.IOBucket", t(IOBucket{}), t(iobucketState{}))
}
// ---- misc ----
// Has returns whether key k belongs to the range.
func (r *IKeyRange) Has(k int32) bool {
return (r.Lo <= k && k <= r.Hi_)
}
// Empty returns whether key range is empty.
func (r *IKeyRange) Empty() bool {
hi := r.Hi_
if hi == _IKeyMax {
// [x,∞] cannot be empty because max x is ∞ and [∞,∞] has one element: ∞
return false
}
hi++ // no overflow
return r.Lo >= hi
}
func (r IKeyRange) String() string {
var shi string
if r.Hi_ == _IKeyMax {
shi = ikstr(r.Hi_) // ∞
} else {
shi = fmt.Sprintf("%d", r.Hi_+1)
}
return fmt.Sprintf("[%s,%s)", ikstr(r.Lo), shi)
}
func ikmin(a, b int32) int32 {
if a < b {
return a
} else {
return b
}
}
func ikmax(a, b int32) int32 {
if a > b {
return a
} else {
return b
}
}
// ikstr formats key as string.
func ikstr(k int32) string {
if k == _IKeyMin {
return "-∞"
}
if k == _IKeyMax {
return "∞"
}
return fmt.Sprintf("%d", k)
}
go/zodb/btree/zlobtree.go
View file @ 95fbc76f
......@@ -3,7 +3,7 @@
// Copyright (c) 2001, 2002 Zope Foundation and Contributors.
// All Rights Reserved.
//
// Copyright (C) 2018-2019 Nexedi SA and Contributors.
// Copyright (C) 2018-2021 Nexedi SA and Contributors.
// Kirill Smelkov <kirr@nexedi.com>
//
// This software is subject to the provisions of the Zope Public License,
......@@ -67,7 +67,7 @@ type LOBTree struct {
// LOEntry is one LOBTree node entry.
//
// It contains key and child, who is either LOBTree or LOBucket.
// It contains key and child, which is either LOBTree or LOBucket.
//
// Key limits child's keys - see LOBTree.Entryv for details.
type LOEntry struct {
......@@ -104,6 +104,15 @@ type LOBucketEntry struct {
value interface{}
}
// LKeyRange represents [lo,hi) key range.
type LKeyRange struct {
Lo int64
Hi_ int64 // NOTE _not_ hi) to avoid overflow at ∞; hi = hi_ + 1
}
const _LKeyMin int64 = math.MinInt64
const _LKeyMax int64 = math.MaxInt64
// ---- access []entry ----
// Key returns LOBTree entry key.
......@@ -125,7 +134,7 @@ func (e *LOEntry) Child() LONode { return e.child }
// Children of all entries are guaranteed to be of the same kind - either all LOBTree, or all LOBucket.
//
// The caller must not modify returned array.
func (t *LOBTree) Entryv() []LOEntry {
func (t *LOBTree) Entryv() /*readonly*/ []LOEntry {
return t.data
}
......@@ -136,7 +145,7 @@ func (e *LOBucketEntry) Key() int64 { return e.key }
func (e *LOBucketEntry) Value() interface{} { return e.value }
// Entryv returns entries of a LOBucket node.
func (b *LOBucket) Entryv() []LOBucketEntry {
func (b *LOBucket) Entryv() /*readonly*/ []LOBucketEntry {
ev := make([]LOBucketEntry, len(b.keys))
for i, k := range b.keys {
ev[i] = LOBucketEntry{k, b.values[i]}
......@@ -170,36 +179,54 @@ func (t *LOBTree) Get(ctx context.Context, key int64) (_ interface{}, _ bool, er
// VGet is like Get but also calls visit while traversing the tree.
//
// Visit is called with node being activated.
func (t *LOBTree) VGet(ctx context.Context, key int64, visit func(node LONode)) (_ interface{}, _ bool, err error) {
func (t *LOBTree) VGet(ctx context.Context, key int64, visit func(node LONode, keycov LKeyRange)) (_ interface{}, _ bool, err error) {
defer xerr.Contextf(&err, "btree(%s): get %v", t.POid(), key)
err = t.PActivate(ctx)
if err != nil {
return nil, false, err
}
keycov := LKeyRange{Lo: _LKeyMin, Hi_: _LKeyMax}
if visit != nil {
visit(t)
}
if len(t.data) == 0 {
// empty btree
t.PDeactivate()
return nil, false, nil
visit(t, keycov)
}
for {
l := len(t.data)
if l == 0 {
// empty btree (top, or in leaf)
t.PDeactivate()
return nil, false, nil
}
// search i: K(i) ≤ k < K(i+1) ; K(0) = -∞, K(len) = +∞
i := sort.Search(len(t.data), func(i int) bool {
i := sort.Search(l, func(i int) bool {
j := i + 1
if j == len(t.data) {
return true // [len].key = +∞
}
return key < t.data[j].key
})
// FIXME panic index out of range (empty T without children;
// logically incorrect, but Restructure generated it once)
// i < l
child := t.data[i].child
// shorten global keycov by local [lo,hi) for this child
lo := _LKeyMin
if i > 0 {
lo = t.data[i].key
}
i++
hi_ := _LKeyMax
if i < l {
hi_ = t.data[i].key
}
if hi_ != _LKeyMax {
hi_--
}
keycov.Lo = lkmax(keycov.Lo, lo)
keycov.Hi_ = lkmin(keycov.Hi_, hi_)
t.PDeactivate()
err = child.PActivate(ctx)
if err != nil {
......@@ -209,7 +236,7 @@ func (t *LOBTree) VGet(ctx context.Context, key int64, visit func(node LONode))
// XXX verify child keys are in valid range according to parent
if visit != nil {
visit(child)
visit(child, keycov)
}
switch child := child.(type) {
......@@ -252,26 +279,40 @@ func (t *LOBTree) MinKey(ctx context.Context) (_ int64, ok bool, err error) {
// VMinKey is like MinKey but also calls visit while traversing the tree.
//
// Visit is called with node being activated.
func (t *LOBTree) VMinKey(ctx context.Context, visit func(node LONode)) (_ int64, ok bool, err error) {
func (t *LOBTree) VMinKey(ctx context.Context, visit func(node LONode, keycov LKeyRange)) (_ int64, ok bool, err error) {
defer xerr.Contextf(&err, "btree(%s): minkey", t.POid())
err = t.PActivate(ctx)
if err != nil {
return 0, false, err
}
keycov := LKeyRange{Lo: _LKeyMin, Hi_: _LKeyMax}
if visit != nil {
visit(t)
}
if len(t.data) == 0 {
// empty btree
t.PDeactivate()
return 0, false, nil
visit(t, keycov)
}
// NOTE -> can also use t.firstbucket
for {
l := len(t.data)
if l == 0 {
// empty btree (top, or in leaf)
t.PDeactivate()
return 0, false, nil
}
child := t.data[0].child
// shorten global keycov by local hi) for this child
hi_ := _LKeyMax
if 1 < l {
hi_ = t.data[1].key
}
if hi_ != _LKeyMax {
hi_--
}
// keycov.Lo stays -∞
keycov.Hi_ = lkmin(keycov.Hi_, hi_)
t.PDeactivate()
err = child.PActivate(ctx)
if err != nil {
......@@ -279,7 +320,7 @@ func (t *LOBTree) VMinKey(ctx context.Context, visit func(node LONode)) (_ int64
}
if visit != nil {
visit(child)
visit(child, keycov)
}
// XXX verify child keys are in valid range according to parent
......@@ -307,26 +348,36 @@ func (t *LOBTree) MaxKey(ctx context.Context) (_ int64, _ bool, err error) {
// VMaxKey is like MaxKey but also calls visit while traversing the tree.
//
// Visit is called with node being activated.
func (t *LOBTree) VMaxKey(ctx context.Context, visit func(node LONode)) (_ int64, _ bool, err error) {
func (t *LOBTree) VMaxKey(ctx context.Context, visit func(node LONode, keycov LKeyRange)) (_ int64, _ bool, err error) {
defer xerr.Contextf(&err, "btree(%s): maxkey", t.POid())
err = t.PActivate(ctx)
if err != nil {
return 0, false, err
}
keycov := LKeyRange{Lo: _LKeyMin, Hi_: _LKeyMax}
if visit != nil {
visit(t)
}
l := len(t.data)
if l == 0 {
// empty btree
t.PDeactivate()
return 0, false, nil
visit(t, keycov)
}
for {
l := len(t.data)
if l == 0 {
// empty btree (top, or in leaf)
t.PDeactivate()
return 0, false, nil
}
child := t.data[l-1].child
// shorten global keycov by local [lo for this chile
lo := _LKeyMin
if l-1 > 0 {
lo = t.data[l-1].key
}
keycov.Lo = lkmax(keycov.Lo, lo)
// keycov.Hi_ stays ∞
t.PDeactivate()
err = child.PActivate(ctx)
if err != nil {
......@@ -334,7 +385,7 @@ func (t *LOBTree) VMaxKey(ctx context.Context, visit func(node LONode)) (_ int64
}
if visit != nil {
visit(child)
visit(child, keycov)
}
// XXX verify child keys are in valid range according to parent
......@@ -593,7 +644,7 @@ func (bt *lobtreeState) PySetState(pystate interface{}) (err error) {
var kprev int64
var childrenKind int // 1 - LOBTree, 2 - LOBucket
for i, idx := 0, 0; i < n; i++ {
key := int64(math.MinInt64) // int64(-∞) (qualifies for ≤)
key := int64(_LKeyMin) // int64(-∞) (qualifies for ≤)
if i > 0 {
// key[0] is unused and not saved
key, ok = t[idx].(int64) // XXX Xint
......@@ -611,7 +662,7 @@ func (bt *lobtreeState) PySetState(pystate interface{}) (err error) {
}
if i > 1 && !(key > kprev) {
fmt.Errorf("data: [%d]: key not ↑", i)
return fmt.Errorf("data: [%d]: key not ↑", i)
}
kprev = key
......@@ -631,7 +682,7 @@ func (bt *lobtreeState) PySetState(pystate interface{}) (err error) {
childrenKind = kind
}
if kind != childrenKind {
fmt.Errorf("data: [%d]: children must be of the same type", i)
return fmt.Errorf("data: [%d]: children must be of the same type", i)
}
bt.data = append(bt.data, LOEntry{key: kkey, child: child.(LONode)})
......@@ -648,3 +699,60 @@ func init() {
zodb.RegisterClass("BTrees.LOBTree.LOBTree", t(LOBTree{}), t(lobtreeState{}))
zodb.RegisterClass("BTrees.LOBTree.LOBucket", t(LOBucket{}), t(lobucketState{}))
}
// ---- misc ----
// Has returns whether key k belongs to the range.
func (r *LKeyRange) Has(k int64) bool {
return (r.Lo <= k && k <= r.Hi_)
}
// Empty returns whether key range is empty.
func (r *LKeyRange) Empty() bool {
hi := r.Hi_
if hi == _LKeyMax {
// [x,∞] cannot be empty because max x is ∞ and [∞,∞] has one element: ∞
return false
}
hi++ // no overflow
return r.Lo >= hi
}
func (r LKeyRange) String() string {
var shi string
if r.Hi_ == _LKeyMax {
shi = lkstr(r.Hi_) // ∞
} else {
shi = fmt.Sprintf("%d", r.Hi_+1)
}
return fmt.Sprintf("[%s,%s)", lkstr(r.Lo), shi)
}
func lkmin(a, b int64) int64 {
if a < b {
return a
} else {
return b
}
}
func lkmax(a, b int64) int64 {
if a > b {
return a
} else {
return b
}
}
// lkstr formats key as string.
func lkstr(k int64) string {
if k == _LKeyMin {
return "-∞"
}
if k == _LKeyMax {
return "∞"
}
return fmt.Sprintf("%d", k)
}
go/zodb/btree/ztestdata_expect_test.go
View file @ 95fbc76f
......@@ -3,13 +3,24 @@ package btree
var smallTestv = [...]testEntry{
testEntry{oid: 6, kind: kindBucket, itemv: []kv{}},
testEntry{oid: 3, kind: kindBucket, itemv: []kv{{10, int64(17)}, }},
testEntry{oid: 7, kind: kindBucket, itemv: []kv{}},
testEntry{oid: 4, kind: kindBucket, itemv: []kv{{10, int64(17)}, }},
testEntry{oid: 1, kind: kindBucket, itemv: []kv{{15, int64(1)}, {23, "hello"}, }},
testEntry{oid: 2, kind: kindBTree, itemv: []kv{}},
testEntry{oid: 7, kind: kindBTree, itemv: []kv{{5, int64(4)}, }},
testEntry{oid: 4, kind: kindBTree, itemv: []kv{{7, int64(3)}, {9, "world"}, }},
testEntry{oid: 3, kind: kindBTree, itemv: []kv{}},
testEntry{oid: 8, kind: kindBTree, itemv: []kv{{5, int64(4)}, }},
testEntry{oid: 5, kind: kindBTree, itemv: []kv{{7, int64(3)}, {9, "world"}, }},
}
const B3_oid = 5
const B3_oid = 6
const B3_maxkey = 9999
const Bv_oid = 2
const Bv_kmin = 1
const Bv_kmax = 9
var Bvdict = map[int64][]tVisit{
1: []tVisit{{2, LKeyRange{_LKeyMin, _LKeyMax}}, {342, LKeyRange{_LKeyMin, 3}}, {344, LKeyRange{_LKeyMin, 1}}},
2: []tVisit{{2, LKeyRange{_LKeyMin, _LKeyMax}}, {342, LKeyRange{_LKeyMin, 3}}, {349, LKeyRange{2, 3}}},
5: []tVisit{{2, LKeyRange{_LKeyMin, _LKeyMax}}, {343, LKeyRange{4, _LKeyMax}}, {345, LKeyRange{4, _LKeyMax}}, {346, LKeyRange{4, 6}}},
8: []tVisit{{2, LKeyRange{_LKeyMin, _LKeyMax}}, {343, LKeyRange{4, _LKeyMax}}, {345, LKeyRange{4, _LKeyMax}}, {347, LKeyRange{7, 8}}},
9: []tVisit{{2, LKeyRange{_LKeyMin, _LKeyMax}}, {343, LKeyRange{4, _LKeyMax}}, {345, LKeyRange{4, _LKeyMax}}, {348, LKeyRange{9, _LKeyMax}}},
}
go/zodb/connection.go
View file @ 95fbc76f
// Copyright (C) 2018-2020 Nexedi SA and Contributors.
// Copyright (C) 2018-2021 Nexedi SA and Contributors.
// Kirill Smelkov <kirr@nexedi.com>
//
// This program is free software: you can Use, Study, Modify and Redistribute
......@@ -180,7 +180,7 @@ const (
// newConnection creates new Connection associated with db.
func newConnection(db *DB, at Tid) *Connection {
return &Connection{
conn := &Connection{
db: db,
at: at,
cache: LiveCache{
......@@ -188,6 +188,10 @@ func newConnection(db *DB, at Tid) *Connection {
objtab: make(map[Oid]*weak.Ref),
},
}
if cc := db.opt.CacheControl; cc != nil {
conn.cache.SetControl(cc)
}
return conn
}
// DB returns database handle under which the connection was opened.
......
go/zodb/db.go
View file @ 95fbc76f
// Copyright (C) 2018-2020 Nexedi SA and Contributors.
// Copyright (C) 2018-2021 Nexedi SA and Contributors.
// Kirill Smelkov <kirr@nexedi.com>
//
// This program is free software: you can Use, Study, Modify and Redistribute
......@@ -48,6 +48,8 @@ type DB struct {
stor IStorage
watchq chan Event // we are watching .stor via here
opt *DBOptions
down chan struct{} // ready when DB is no longer operational
downOnce sync.Once // shutdown may be due to both Close and IO error in watcher
downErr error // reason for shutdown
......@@ -121,15 +123,23 @@ type DB struct {
// (so it is not duplicated many times for many DB case)
}
// DBOptions describes options to NewDB.
type DBOptions struct {
// CacheControl, if !nil, is set as default live cache control for
// newly created connections.
CacheControl LiveCacheControl
}
// NewDB creates new database handle.
//
// Created database handle must be closed when no longer needed.
func NewDB(stor IStorage) *DB {
// XXX db options?
func NewDB(stor IStorage, opt *DBOptions) *DB {
// copy opts in case caller will change them later
opt_ := *opt
db := &DB{
stor: stor,
watchq: make(chan Event),
opt: &opt_,
down: make(chan struct{}),
hwait: make(map[hwaiter]struct{}),
......
go/zodb/impexp_test.go
View file @ 95fbc76f
// Copyright (C) 2019 Nexedi SA and Contributors.
// Kirill Smelkov <kirr@nexedi.com>
// Copyright (C) 2019-2021 Nexedi SA and Contributors.
// Kirill Smelkov <kirr@nexedi.com>
//
// This program is free software: you can Use, Study, Modify and Redistribute
// it under the terms of the GNU General Public License version 3, or (at your
......@@ -25,10 +25,16 @@ import (
// imported at runtime via import_x_test due to cyclic dependency:
var ZPyCommit func(string, Tid, ...IPersistent) (Tid, error)
var ZPyCommit func(string, Tid, ...IPersistent) (Tid, error)
var ZPyCommitRaw func(string, Tid, ...ZRawObject) (Tid, error)
// exported for zodb_test package:
type ZRawObject struct { // keep in sync with xtesting.ZRawObject
Oid Oid
Data []byte
}
func PSerialize(obj IPersistent) *mem.Buf {
return obj.persistent().pSerialize()
}
go/zodb/import_x_test.go
View file @ 95fbc76f
// Copyright (C) 2019 Nexedi SA and Contributors.
// Kirill Smelkov <kirr@nexedi.com>
// Copyright (C) 2019-2021 Nexedi SA and Contributors.
// Kirill Smelkov <kirr@nexedi.com>
//
// This program is free software: you can Use, Study, Modify and Redistribute
// it under the terms of the GNU General Public License version 3, or (at your
......@@ -33,7 +33,8 @@ import (
// due to cyclic dependency.
func init() {
zodb.ZPyCommit = ZPyCommit
zodb.ZPyCommit = ZPyCommit
zodb.ZPyCommitRaw = ZPyCommitRaw
}
// ZPyCommit commits new transaction with specified objects.
......@@ -41,8 +42,8 @@ func init() {
// The objects need to be alive, but do not need to be marked as changed.
// The commit is performed via zodb/py.
func ZPyCommit(zurl string, at zodb.Tid, objv ...zodb.IPersistent) (zodb.Tid, error) {
var rawobjv []xtesting.ZRawObject // raw zodb objects data to commit
var bufv []*mem.Buf // buffers to release
var rawobjv []zodb.ZRawObject // raw zodb objects data to commit
var bufv []*mem.Buf // buffers to release
defer func() {
for _, buf := range bufv {
buf.Release()
......@@ -51,7 +52,7 @@ func ZPyCommit(zurl string, at zodb.Tid, objv ...zodb.IPersistent) (zodb.Tid, er
for _, obj := range objv {
buf := zodb.PSerialize(obj)
rawobj := xtesting.ZRawObject{
rawobj := zodb.ZRawObject{
Oid: obj.POid(),
Data: buf.Data,
}
......@@ -59,5 +60,13 @@ func ZPyCommit(zurl string, at zodb.Tid, objv ...zodb.IPersistent) (zodb.Tid, er
bufv = append(bufv, buf)
}
return xtesting.ZPyCommitRaw(zurl, at, rawobjv...)
return ZPyCommitRaw(zurl, at, rawobjv...)
}
func ZPyCommitRaw(zurl string, at zodb.Tid, rawobjv ...zodb.ZRawObject) (zodb.Tid, error) {
var xrawobjv []xtesting.ZRawObject
for _, obj := range rawobjv {
xrawobjv = append(xrawobjv, xtesting.ZRawObject{Oid: obj.Oid, Data: obj.Data})
}
return xtesting.ZPyCommitRaw(zurl, at, xrawobjv...)
}
go/zodb/persistent.go
View file @ 95fbc76f
// Copyright (C) 2018-2019 Nexedi SA and Contributors.
// Copyright (C) 2018-2021 Nexedi SA and Contributors.
// Kirill Smelkov <kirr@nexedi.com>
//
// This program is free software: you can Use, Study, Modify and Redistribute
......@@ -163,7 +163,7 @@ func (obj *Persistent) pSerialize() *mem.Buf {
func (obj *Persistent) PActivate(ctx context.Context) (err error) {
obj.mu.Lock()
obj.refcnt++
doload := (obj.refcnt == 1 && obj.state == GHOST)
doload := (obj.state == GHOST && obj.loading == nil)
defer func() {
if err != nil {
obj.PDeactivate()
......@@ -208,10 +208,10 @@ func (obj *Persistent) PActivate(ctx context.Context) (err error) {
"%v (want %v); .loading = %p (want %p)", s, GHOST, l, loading))
}
obj.serial = serial
// try to pass loaded state to object
if err == nil {
obj.serial = serial
switch istate := obj.istate().(type) {
case Stateful:
err = istate.SetState(state)
......@@ -229,9 +229,13 @@ func (obj *Persistent) PActivate(ctx context.Context) (err error) {
if err == nil {
obj.state = UPTODATE
}
} else {
obj.serial = InvalidTid
// force reload on next activate if it was an error
obj.loading = nil
}
// XXX set state to load error? (to avoid panic on second activate after load error)
loading.err = err
obj.mu.Unlock()
......@@ -257,7 +261,7 @@ func (obj *Persistent) PDeactivate() {
if obj.state >= CHANGED {
return
}
if obj.oid == InvalidOid { // newly created not-yet committed object // TODO tests
if obj.oid == InvalidOid { // newly created not-yet committed object
return
}
......
go/zodb/persistent_test.go
View file @ 95fbc76f
// Copyright (C) 2018-2020 Nexedi SA and Contributors.
// Copyright (C) 2018-2021 Nexedi SA and Contributors.
// Kirill Smelkov <kirr@nexedi.com>
//
// This program is free software: you can Use, Study, Modify and Redistribute
......@@ -21,6 +21,7 @@ package zodb
import (
"context"
"errors"
"fmt"
"io/ioutil"
"os"
......@@ -276,7 +277,7 @@ func (t *tDB) Reopen() {
ReadOnly: true,
NoCache: !t.rawcache,
}); X(err)
db := NewDB(stor)
db := NewDB(stor, &DBOptions{})
t.stor = stor
t.db = db
}
......@@ -317,7 +318,7 @@ func (t *tDB) Add(oid Oid, value string) {
}
// Commit commits objects queued by Add.
func (t *tDB) Commit() {
func (t *tDB) Commit() Tid {
t.Helper()
head, err := ZPyCommit(t.zurl, t.head, t.commitq...)
......@@ -326,6 +327,19 @@ func (t *tDB) Commit() {
}
t.head = head
t.commitq = nil
return head
}
// CommitRaw commits raw changes.
func (t *tDB) CommitRaw(rawobjv ...ZRawObject) Tid {
t.Helper()
head, err := ZPyCommitRaw(t.zurl, t.head, rawobjv...)
if err != nil {
t.Fatal(err)
}
t.head = head
return head
}
// Open opens new test transaction/connection.
......@@ -443,12 +457,14 @@ func (t *tConnection) Abort() {
}
func (t *tDB) fatalif(err error) {
t.Helper()
if err != nil {
t.Fatal(err)
}
}
func (t *tConnection) fatalif(err error) {
t.Helper()
if err != nil {
t.Fatal(err)
}
......@@ -473,13 +489,11 @@ func testPersistentDB(t0 *testing.T, rawcache bool) {
tdb.Add(101, "bonjour")
tdb.Add(102, "monde")
tdb.Commit()
at0 := tdb.head
at0 := tdb.Commit()
tdb.Add(101, "hello")
tdb.Add(102, "world")
tdb.Commit()
at1 := tdb.head
at1 := tdb.Commit()
tdb.Reopen() // so that at0 is not covered by db.δtail
db := tdb.db
......@@ -541,8 +555,7 @@ func testPersistentDB(t0 *testing.T, rawcache bool) {
// commit change to obj2 from external process
tdb.Add(102, "kitty")
tdb.Commit()
at2 := tdb.head
at2 := tdb.Commit()
// new db connection should see the change
t2 := tdb.Open(&ConnOptions{})
......@@ -608,6 +621,18 @@ func testPersistentDB(t0 *testing.T, rawcache bool) {
t.checkObj(obj2, 102, at2, UPTODATE, 0, "kitty")
// newly created object should not go to ghost on deactivate
obj3 := NewMyObject(t.conn)
obj3.value = "new"
checkObj(t0, obj3, t.conn, InvalidOid, InvalidTid, UPTODATE, 0)
assert.Equal(obj3.value, "new")
t.PActivate(obj3)
checkObj(t0, obj3, t.conn, InvalidOid, InvalidTid, UPTODATE, 1)
obj3.PDeactivate()
checkObj(t0, obj3, t.conn, InvalidOid, InvalidTid, UPTODATE, 0)
assert.Equal(obj3.value, "new")
// finish tnx3 and txn2 - conn1 and conn2 go back to db pool
t.Abort()
t2.Abort()
......@@ -719,6 +744,69 @@ func testPersistentDB(t0 *testing.T, rawcache bool) {
t.checkObj(robj2, 102, InvalidTid, GHOST, 0)
}
// Verify that PActivate works correctly after hitting an error from the storage.
// In this test the error is "object was deleted".
func TestActivateAfterDelete(t0 *testing.T) {
assert := assert.New(t0)
tdb := testdb(t0, /*rawcache=*/false)
defer tdb.Close()
db := tdb.db
tdb.Add(101, "object")
at0 := tdb.Commit()
t := tdb.Open(&ConnOptions{})
// do not evict the object from live cache.
zcc := &zcacheControl{map[Oid]PCachePolicy{
101: PCachePinObject | PCacheKeepState,
}}
zcache := t.conn.Cache()
zcache.Lock()
zcache.SetControl(zcc)
zcache.Unlock()
// load the object
obj := t.Get(101)
t.checkObj(obj, 101, InvalidTid, GHOST, 0)
t.PActivate(obj)
t.checkObj(obj, 101, at0, UPTODATE, 1, "object")
obj.PDeactivate()
t.checkObj(obj, 101, at0, UPTODATE, 0, "object")
// delete obj
at1 := tdb.CommitRaw(ZRawObject{Oid: 101, Data: nil})
// conn stays at older view with obj pinned into the cache
t.checkObj(obj, 101, at0, UPTODATE, 0, "object")
// finish transaction and reopen new connection - it should be the same conn
t.Abort()
assert.Equal(db.pool, []*Connection{t.conn})
t_ := tdb.Open(&ConnOptions{})
assert.Same(t_.conn, t.conn)
t = t_
assert.Equal(t.conn.At(), at1)
// obj should be invalidated but present in the cache
t.checkObj(obj, 101, InvalidTid, GHOST, 0)
// activating obj should give "no data" error
// loop because second activate used to panic
for i := 0; i < 10; i++ {
err := obj.PActivate(t.ctx)
eok := &NoDataError{Oid: obj.POid(), DeletedAt: at1}
var e *NoDataError
errors.As(err, &e)
if !reflect.DeepEqual(e, eok) {
t.Fatalf("(%d) after delete: err:\nhave: %s\nwant cause: %s", i, err, eok)
}
// obj should stay in the cache in ghost state
t.checkObj(obj, 101, InvalidTid, GHOST, 0)
}
}
// Test details of how LiveCache handles live caching policy.
func TestLiveCache(t0 *testing.T) {
assert := assert.New(t0)
......@@ -730,8 +818,7 @@ func TestLiveCache(t0 *testing.T) {
tdb.Add(102, "труд")
tdb.Add(103, "май")
tdb.Add(104, "весна")
tdb.Commit()
at1 := tdb.head
at1 := tdb.Commit()
zcc := &zcacheControl{map[Oid]PCachePolicy{
// obj1 - default (currently: don't pin and don't keep state)
......
go/zodb/py/pydata-gen-testdata
View file @ 95fbc76f
#!/usr/bin/env python2
# -*- coding: utf-8 -*-
# Copyright (C) 2017-2020 Nexedi SA and Contributors.
# Copyright (C) 2017-2021 Nexedi SA and Contributors.
# Kirill Smelkov <kirr@nexedi.com>
#
# This program is free software: you can Use, Study, Modify and Redistribute
......@@ -21,7 +21,7 @@
"""generate reference pickle objects encoding for tests"""
from ZODB import serialize
from zodbtools.test.gen_testdata import run_with_zodb3py2_compat
from zodbtools.test.gen_testdata import run_with_zodb4py2_compat
from golang.gcompat import qq
def main2():
......@@ -56,7 +56,7 @@ def main2():
emit("}")
def main():
run_with_zodb3py2_compat(main2)
run_with_zodb4py2_compat(main2)
if __name__ == '__main__':
main()
go/zodb/storage.go
View file @ 95fbc76f
// Copyright (C) 2017-2019 Nexedi SA and Contributors.
// Copyright (C) 2017-2021 Nexedi SA and Contributors.
// Kirill Smelkov <kirr@nexedi.com>
//
// This program is free software: you can Use, Study, Modify and Redistribute
......@@ -46,12 +46,30 @@ type DriverOptions struct {
//
// Watchq can be nil to ignore such events. However if Watchq != nil, the events
// have to be consumed or else the storage driver will misbehave - e.g.
// it can get out of sync with the on-disk database file.
// it can get out of sync with the on-disk database file, or deadlock
// on any user-called operation.
//
// The storage driver closes !nil Watchq when the driver is closed.
//
// The storage driver will send only and all events in (at₀, +∞] range,
// where at₀ is at returned by driver open.
//
// The storage driver will stop sending events after call to Close.
// In particular the following example is valid and safe from deadlock:
//
// watchq := make(chan zodb.Event)
// stor, at0, err := zodb.OpenDriver(..., &DriverOptions{Watchq: watchq})
// defer stor.Close()
//
// for {
// select {
// case <-ctx.Done():
// return ctx.Err()
//
// case <-watchq:
// ...
// }
// }
Watchq chan<- Event
}
......@@ -73,10 +91,13 @@ func RegisterDriver(scheme string, opener DriverOpener) {
driverRegistry[scheme] = opener
}
// XXX
func openDriver(ctx context.Context, zurl string, opt *DriverOptions) (_ IStorageDriver, at0 Tid, _ error) {
// OpenDriver opens ZODB storage driver by URL.
//
// It is similar to Open but returns low-level IStorageDriver instead of IStorage.
// Most users should use Open.
func OpenDriver(ctx context.Context, zurl string, opt *DriverOptions) (_ IStorageDriver, at0 Tid, _ error) {
// no scheme -> file://
if !strings.Contains(zurl, "://") {
if !strings.Contains(zurl, ":") {
zurl = "file://" + zurl
}
......@@ -91,7 +112,7 @@ func openDriver(ctx context.Context, zurl string, opt *DriverOptions) (_ IStorag
opener, ok := driverRegistry[u.Scheme]
if !ok {
return nil, InvalidTid, fmt.Errorf("zodb: URL scheme \"%s://\" not supported", u.Scheme)
return nil, InvalidTid, fmt.Errorf("zodb: URL scheme \"%s:\" not supported", u.Scheme)
}
storDriver, at0, err := opener(ctx, u, opt)
......@@ -108,7 +129,7 @@ func openDriver(ctx context.Context, zurl string, opt *DriverOptions) (_ IStorag
// Users should import in storage packages they use or zodb/wks package to
// get support for well-known storages.
//
// Storage authors should register their storages with RegisterStorage.
// Storage authors should register their storages with RegisterDriver.
func Open(ctx context.Context, zurl string, opt *OpenOptions) (IStorage, error) {
drvWatchq := make(chan Event)
drvOpt := &DriverOptions{
......@@ -116,7 +137,7 @@ func Open(ctx context.Context, zurl string, opt *OpenOptions) (IStorage, error)
Watchq: drvWatchq,
}
storDriver, at0, err := openDriver(ctx, zurl, drvOpt)
storDriver, at0, err := OpenDriver(ctx, zurl, drvOpt)
if err != nil {
return nil, err
}
......@@ -160,9 +181,10 @@ type storage struct {
driver IStorageDriver
l1cache *Cache // can be =nil, if opened with NoCache
down chan struct{} // ready when no longer operational
downOnce sync.Once // shutdown may be due to both Close and IO error in watcher|Sync
downErr error // reason for shutdown
down chan struct{} // ready when no longer operational
downOnce sync.Once // shutdown may be due to both Close and IO error in watcher|Sync
downErr error // reason for shutdown
drvCloseErr error // err from .driver.Close()
// watcher
......@@ -185,6 +207,11 @@ func (s *storage) shutdown(reason error) {
s.downOnce.Do(func() {
close(s.down)
s.downErr = fmt.Errorf("not operational due: %s", reason)
// - if called by Close or failed Sync: driver.Close will close
// drvWatchq and cause watcher to stop.
// - if called by failed watcher: closing driver will prevent
// drvWatchq<- deadlock in driver because we no longer read from it.
s.drvCloseErr = s.driver.Close()
})
}
......@@ -198,7 +225,7 @@ func (s *storage) Iterate(ctx context.Context, tidMin, tidMax Tid) ITxnIterator
func (s *storage) Close() error {
s.shutdown(fmt.Errorf("closed"))
return s.driver.Close() // this will close drvWatchq and cause watcher stop
return s.drvCloseErr
}
// loading goes through cache - this way prefetching can work
......@@ -369,7 +396,7 @@ func (s *storage) _watcher() error {
func (s *storage) AddWatch(watchq chan<- Event) (at0 Tid) {
ack := make(chan Tid)
select {
// no longer operational: behave if watchq was registered before that
// no longer operational: behave as if watchq was registered before that
// and then seen down/close events. Interact with DelWatch directly.
case <-s.down:
s.headMu.Lock() // shutdown may be due to Close call and watcher might be
......@@ -471,6 +498,14 @@ func (s *storage) Sync(ctx context.Context) (err error) {
}
// wait till .head >= head
// XXX instead require from drivers that `drv.Sync() -> head`
// guarantees that all EventCommit with .tid <= head were sent to
// watchq
//
// https://lab.nexedi.com/nexedi/ZODB/commit/40116375
// https://github.com/zopefoundation/ZODB/commit/4a6b0283#diff-d2a01f71a79ac2b379e218cf72fa1205d3426cad19e7b72d71899f643be4bb73
//
// ?
watchq := make(chan Event)
at = s.AddWatch(watchq)
defer s.DelWatch(watchq)
......
go/zodb/storage/demo/demo.go 0 → 100644
View file @ 95fbc76f
// Copyright (C) 2021 Nexedi SA and Contributors.
// Kirill Smelkov <kirr@nexedi.com>
//
// This program is free software: you can Use, Study, Modify and Redistribute
// it under the terms of the GNU General Public License version 3, or (at your
// option) any later version, as published by the Free Software Foundation.
//
// You can also Link and Combine this program with other software covered by
// the terms of any of the Free Software licenses or any of the Open Source
// Initiative approved licenses and Convey the resulting work. Corresponding
// source of such a combination shall include the source code for all other
// software used.
//
// This program is distributed WITHOUT ANY WARRANTY; without even the implied
// warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
//
// See COPYING file for full licensing terms.
// See https://www.nexedi.com/licensing for rationale and options.
// Package demo provides overlayed storage, similar to DemoStorage in ZODB/py.
//
// Storage combines base and δ storages as if δ transactional log is logically
// appended to base.
package demo
import (
"context"
"errors"
"fmt"
"io"
"net/url"
"regexp"
"sync"
"lab.nexedi.com/kirr/go123/mem"
"lab.nexedi.com/kirr/go123/xerr"
"lab.nexedi.com/kirr/go123/xsync"
"lab.nexedi.com/kirr/neo/go/internal/task"
"lab.nexedi.com/kirr/neo/go/zodb"
)
// Storage combines base and δ storages as if δ transactional log is logically
// appended to base.
//
// Base must remain unmodified, and all δ transactions must be with TIDs
// strictly after base.Head .
type Storage struct {
base zodb.IStorageDriver
δ zodb.IStorageDriver
baseAt0 zodb.Tid
baseWatchq <-chan zodb.Event
δWatchq <-chan zodb.Event // nil if demo is opened with watchq=nil
δWatchq0 <-chan zodb.Event // buffer for δ events queued while δ was initially verified
watchq chan<- zodb.Event // user requested to deliver events here
watchWG *xsync.WorkGroup
watchCancel func()
downOnce sync.Once
down chan struct{} // ready when storage is down
downErr error // reason for shutdown
closeOnce sync.Once
closed chan struct{} // ready when storage is Closed
}
// baseMutatedError is reported when Storage.base is detected to change.
type baseMutatedError struct {
baseAt0 zodb.Tid
baseHead zodb.Tid
}
func (e *baseMutatedError) Error() string {
return fmt.Sprintf("base.head mutated from @%s to @%s", e.baseAt0, e.baseHead)
}
// baseErrorEvent is reported on error event detected on Storage.base .
type baseErrorEvent struct {
err error
}
func (e *baseErrorEvent) Error() string {
return fmt.Sprintf("base: error event: %s", e.err)
}
func (e *baseErrorEvent) Cause() error { return e.err }
func (e *baseErrorEvent) Unwrap() error { return e.err }
// watcher task detects base mutation and proxies δ events to user watchq.
func (d *Storage) watcher(ctx context.Context) error {
if d.watchq != nil {
defer close(d.watchq)
}
for {
var δWatchq <-chan zodb.Event
if len(d.δWatchq0) != 0 {
δWatchq = d.δWatchq0
} else {
δWatchq = d.δWatchq
}
select {
// Close requests to stop watching
case <-ctx.Done():
return ctx.Err()
// event on base -> base mutated/error + shutdown
case event, ok := <-d.baseWatchq:
if !ok {
// base closed
d.baseWatchq = nil
continue
}
var edown error
switch event := event.(type) {
case *zodb.EventCommit:
edown = &baseMutatedError{d.baseAt0, event.Tid}
case *zodb.EventError:
edown = &baseErrorEvent{event.Err}
default:
edown = fmt.Errorf("base: unexpected event %T", event)
}
d.shutdown(edown)
ev := &zodb.EventError{&zodb.OpError{URL: d.URL(), Op: "watcher", Err: edown}}
if d.watchq != nil {
select {
case <-d.closed:
// wakeup to return edown
case d.watchq <- ev:
// ok
}
}
return edown
// event on δ -> proxy to user
case event, ok := <-δWatchq:
if !ok {
// δ closed
d.δWatchq = nil
continue
}
select {
case <-d.closed:
return nil
case d.watchq <- event: // !nil because d.δWatchq != nil
// ok
}
}
}
}
// shutdown marks Storage as no longer operational due to reason.
func (d *Storage) shutdown(reason error) {
d.downOnce.Do(func() {
d.downErr = reason
close(d.down)
})
}
var errClosed = errors.New("storage is closed")
// Close implements zodb.IStorageDriver .
func (d *Storage) Close() (err error) {
defer func() {
if err != nil {
err = &zodb.OpError{URL: d.URL(), Op: "close", Err: err}
}
}()
d.shutdown(errClosed)
d.closeOnce.Do(func() {
close(d.closed)
})
errδ := d.δ.Close()
errBase := d.base.Close()
// cancel watcher; don't propagate its error to Close - watcher error
// goes to watchq and op errors.
d.watchCancel()
_ = d.watchWG.Wait()
return xerr.Merge(errδ, errBase)
}
// Sync implements zodb.IStorageDriver .
func (d *Storage) Sync(ctx context.Context) (_ zodb.Tid, err error) {
defer func() {
if err != nil {
err = &zodb.OpError{URL: d.URL(), Op: "sync", Err: err}
}
}()
if ready(d.down) {
return zodb.InvalidTid, d.downErr
}
var head zodb.Tid
wg := xsync.NewWorkGroup(ctx)
wg.Go(func(ctx context.Context) error {
h, err := d.δ.Sync(ctx)
head = h
return err
})
wg.Go(func(ctx context.Context) error {
baseHead, err := d.base.Sync(ctx)
if err != nil {
return err
}
if baseHead != d.baseAt0 {
return &baseMutatedError{d.baseAt0, baseHead}
}
return nil
})
err = wg.Wait()
if err != nil {
return zodb.InvalidTid, err
}
// δ is just created database
if head == 0 {
head = d.baseAt0
}
return head, nil
}
// Load implements zodb.IStorageDriver .
func (d *Storage) Load(ctx context.Context, xid zodb.Xid) (_ *mem.Buf, _ zodb.Tid, err error) {
defer func() {
if err != nil {
err = &zodb.OpError{URL: d.URL(), Op: "load", Args: xid, Err: err}
}
}()
if ready(d.down) {
return nil, zodb.InvalidTid, d.downErr
}
var eNoData *zodb.NoDataError
var eNoObject *zodb.NoObjectError
inδ := false
if xid.At > d.baseAt0 {
data, serial, err := d.δ.Load(ctx, xid)
if err == nil {
// object data is present in δ
return data, serial, nil
}
useBase := false
switch {
case errors.As(err, &eNoData):
if eNoData.DeletedAt != 0 {
// object deleted in δ -> whiteout
return data, serial, eNoData
} else {
// object present in δ but not yet created as of xid.at
useBase = true
inδ = true
}
case errors.As(err, &eNoObject):
// object not created in δ
useBase = true
}
if !useBase {
return data, serial, err
}
}
// cap xid.At to .baseAt0 (we convert it back on error return, and
// it makes Load more robust wrt simultaneous base mutation).
xidBase := xid
if xid.At > d.baseAt0 {
xidBase.At = d.baseAt0
}
data, serial, err := d.base.Load(ctx, xidBase)
if err == nil {
return data, serial, nil
}
switch {
case errors.As(err, &eNoData):
err = eNoData
case errors.As(err, &eNoObject):
if !inδ {
err = eNoObject
} else {
// object is present in δ
err = &zodb.NoDataError{Oid: xid.Oid, DeletedAt: 0}
}
}
return data, serial, err
}
// Iterator implements zodb.IStorageDriver .
func (d *Storage) Iterate(ctx context.Context, tidMin, tidMax zodb.Tid) zodb.ITxnIterator {
panic("TODO")
}
// URL implements zodb.IStorageDriver .
func (d *Storage) URL() string {
return "demo:(" + d.base.URL() + ")/(" + d.δ.URL() + ")"
}
var demoRe = regexp.MustCompile(`^\((.*)\)/\((.*)\)$`)
func openByURL(ctx context.Context, u *url.URL, opt *zodb.DriverOptions) (_ zodb.IStorageDriver, _ zodb.Tid, err error) {
// demo:(base_zurl)/(δ_zurl)
defer task.Runningf(&ctx, "demo: open %s", u)(&err)
if !(u.Opaque != "" && u.RawQuery == "" && u.Fragment == "") {
// opaque != "" makes sure user,host,path are empty
return nil, zodb.InvalidTid, fmt.Errorf("invalid url")
}
argv := demoRe.FindStringSubmatch(u.Opaque)
if argv == nil {
return nil, zodb.InvalidTid, fmt.Errorf("invalid url")
}
baseZURL := argv[1]
δZURL := argv[2]
// open base - always readonly
baseWatchq := make(chan zodb.Event)
base, baseAt0, err := zodb.OpenDriver(ctx, baseZURL, &zodb.DriverOptions{
ReadOnly: true,
Watchq: baseWatchq,
})
if err != nil {
return nil, zodb.InvalidTid, err
}
defer func() {
if err != nil {
__ := base.Close()
err = xerr.Merge(err, __)
}
}()
// open δ - as requested but with events going through us
δopt := *opt
var δWatchq chan zodb.Event
if δopt.Watchq != nil {
δWatchq = make(chan zodb.Event)
δopt.Watchq = δWatchq
}
δ, δAt0, err := zodb.OpenDriver(ctx, δZURL, &δopt)
if err != nil {
return nil, zodb.InvalidTid, err
}
defer func() {
if err != nil {
__ := δ.Close()
err = xerr.Merge(err, __)
}
}()
// verify that either
// - δ is all empty (just created), or
// - all δ transactions come strictly after base.Head .
at0 := baseAt0
var δEventq0 []zodb.Event
if δAt0 != 0 {
if δAt0 < baseAt0 {
return nil, zodb.InvalidTid, fmt.Errorf("base is ahead of δ: base.head=%s δ.head=%s", baseAt0, δAt0)
}
// read and queue data from δWatchq while we verify δ (not to deadlock δ driver)
δq0Stop := make(chan struct{}) // reader <- main "stop"
δq0Done := make(chan struct{}) // reader -> main "done"
go func() {
defer close(δq0Done)
for {
select {
case <-δq0Stop:
return
case δevent, ok := <-δWatchq:
if !ok {
return
}
δEventq0 = append(δEventq0, δevent)
}
}
}()
// verify δ
iδ := δ.Iterate(ctx, 0, baseAt0)
δtxni, _, err := iδ.NextTxn(ctx)
switch {
case err == io.EOF:
err = nil // ok - nothing in δ
case err == nil:
// there is a δ transaction ∈ [δAt0, baseAt0)
err = fmt.Errorf("base overlaps with δ: base.head=%s δ.tail=%s", baseAt0, δtxni.Tid)
}
// TODO iδ.Close()
close(δq0Stop)
<-δq0Done
if err != nil {
return nil, zodb.InvalidTid, err
}
at0 = δAt0
}
// requeue δWatchq0 <- δEventq0
var δWatchq0 chan zodb.Event
if l := len(δEventq0); l != 0 {
δWatchq0 = make(chan zodb.Event, l)
for _, ev := range δEventq0 {
δWatchq0 <- ev
}
}
d := &Storage{
base: base,
δ : δ,
baseAt0: baseAt0,
baseWatchq: baseWatchq,
δWatchq: δWatchq,
δWatchq0: δWatchq0,
watchq: opt.Watchq,
down: make(chan struct{}),
closed: make(chan struct{}),
}
// spawn watcher to listen on baseWatchq and shutdown storage if base changes.
ctx, cancel := context.WithCancel(context.Background())
d.watchWG = xsync.NewWorkGroup(ctx)
d.watchCancel = cancel
d.watchWG.Go(d.watcher)
return d, at0, nil
}
func init() {
zodb.RegisterDriver("demo", openByURL)
}
// ---- misc ----
// ready returns whether c is ready.
func ready(c <-chan struct{}) bool {
select {
case <-c:
return true
default:
return false
}
}
go/zodb/storage/demo/demo_test.go 0 → 100644
View file @ 95fbc76f
// Copyright (C) 2021 Nexedi SA and Contributors.
// Kirill Smelkov <kirr@nexedi.com>
//
// This program is free software: you can Use, Study, Modify and Redistribute
// it under the terms of the GNU General Public License version 3, or (at your
// option) any later version, as published by the Free Software Foundation.
//
// You can also Link and Combine this program with other software covered by
// the terms of any of the Free Software licenses or any of the Open Source
// Initiative approved licenses and Convey the resulting work. Corresponding
// source of such a combination shall include the source code for all other
// software used.
//
// This program is distributed WITHOUT ANY WARRANTY; without even the implied
// warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
//
// See COPYING file for full licensing terms.
// See https://www.nexedi.com/licensing for rationale and options.
package demo
import (
"bytes"
"context"
"io/ioutil"
"fmt"
"net/url"
"os"
"reflect"
"regexp"
"testing"
"lab.nexedi.com/kirr/go123/xerr"
"lab.nexedi.com/kirr/neo/go/internal/xtesting"
"lab.nexedi.com/kirr/neo/go/zodb"
"lab.nexedi.com/kirr/neo/go/zodb/zodbtools"
// for file: scheme support
_ "lab.nexedi.com/kirr/neo/go/zodb/storage/fs1"
)
// DemoData represents data for a demo: storage.
type DemoData struct {
base string // url for base.fs
δ string // ----/---- δ.fs
}
func (ddat *DemoData) URL() string {
return fmt.Sprintf("demo:(%s)/(%s)", ddat.base, ddat.δ)
}
// tOptions represents options for testing.
// TODO -> xtesting
type tOptions struct {
Preload string // preload database with data from this location
}
// withDemoData tests f with all kinds of opt.Preload data split into base + δ.
func withDemoData(t *testing.T, f func(t *testing.T, ddat *DemoData), optv ...tOptions) {
t.Helper()
X := xtesting.FatalIf(t)
opt := tOptions{}
if len(optv) > 1 {
panic("multiple tOptions not allowed")
}
if len(optv) == 1 {
opt = optv[0]
}
// retrieve zdump of Preload
zdump := ""
if opt.Preload != "" {
ctx := context.Background()
buf := &bytes.Buffer{}
stor, err := zodb.Open(ctx, opt.Preload, &zodb.OpenOptions{ReadOnly: true}); X(err)
err = zodbtools.Dump(ctx, buf, stor, 0, zodb.TidMax, /*hashonly=*/false)
stor.Close()
X(err)
zdump = buf.String()
}
// split zdump into transactions
// XXX hacky; TODO -> zodbtools.DumpReader
txnRe := regexp.MustCompile(`(?m)^txn (?P<tid>[0-9a-f]{16}) "(?P<status>.)"$`)
type zdumpTxn struct {
tid zodb.Tid
pos int // where this transaction starts in the dump
}
var txnv []zdumpTxn
for _, m := range txnRe.FindAllStringSubmatchIndex(zdump, -1) {
// [m[0]:m[1]] refers to whole txn line
__ := zdump[m[2]:m[3]]
tid, err := zodb.ParseTid(__); X(err)
txnv = append(txnv, zdumpTxn{tid, m[0]})
}
// verify f on all combinations of preload being split into base+δ
work := xtempdir(t)
defer os.RemoveAll(work)
test1 := func(δstart zodb.Tid, zdumpBase, zdumpδ string) {
t.Helper()
t.Run(fmt.Sprintf("δstart=%s", δstart), func(t *testing.T) {
t.Helper()
X := xtesting.FatalIf(t)
work1 := work + "/δ" + δstart.String()
err := os.Mkdir(work1, 0777); X(err)
base := "file://"+work1+"/base.fs"
δ := "file://"+work1+"/δ.fs"
ddat := &DemoData{base, δ}
_, err = xtesting.ZPyRestore(base, zdumpBase); X(err)
// restore δ part via `demo:(base)/(δ)` - not `file:δ`.
// The reason we do this is because restoring δ via
// just its file will fail when restoring copy data
// record with copy_from transaction being in base.
_, err = xtesting.ZPyRestore(ddat.URL(), zdumpδ); X(err)
f(t, ddat)
})
}
for i := 0; i < len(txnv); i++ {
δtail := txnv[i]
test1(δtail.tid, zdump[:δtail.pos], zdump[δtail.pos:])
}
test1(zodb.TidMax, zdump, "")
}
// withDemo tests f with demo: client connected to all kind of demo data splits.
func withDemo(t *testing.T, f func(t *testing.T, ddat *DemoData, ddrv *Storage), optv ...tOptions) {
t.Helper()
withDemoData(t, func(t *testing.T, ddat *DemoData) {
t.Helper()
X := xtesting.FatalIf(t)
ddrv, _, err := demoOpen(ddat.URL(), &zodb.DriverOptions{ReadOnly: true}); X(err)
defer func() {
err := ddrv.Close(); X(err)
}()
f(t, ddat, ddrv)
}, optv...)
}
func TestURL(t *testing.T) {
withDemo(t, func(t *testing.T, ddat *DemoData, ddrv *Storage) {
zurl := ddrv.URL()
zurlOk := ddat.URL()
if zurl != zurlOk {
t.Fatalf("bad zurl:\nhave: %s\nwant: %s", zurl, zurlOk)
}
})
}
func TestEmptyDB(t *testing.T) {
withDemo(t, func(t *testing.T, _ *DemoData, ddrv *Storage) {
xtesting.DrvTestEmptyDB(t, ddrv)
})
}
func TestLoad(t *testing.T) {
X := xtesting.FatalIf(t)
data := "../fs1/testdata/1.fs"
txnvOk, err := xtesting.LoadDBHistory(data); X(err)
withDemo(t, func(t *testing.T, _ *DemoData, ddrv *Storage) {
xtesting.DrvTestLoad(t, ddrv, txnvOk)
}, tOptions{
Preload: data,
})
}
func TestWatch(t *testing.T) {
withDemoData(t, func(t *testing.T, ddat *DemoData) {
xtesting.DrvTestWatch(t, ddat.URL(), openByURL)
})
}
// MutateBase mutates ddat.base with new commit.
func (ddat *DemoData) MutateBase() (zodb.Tid, error) {
return xtesting.ZPyCommitRaw(ddat.base, 0, xtesting.ZRawObject{
Oid: 1,
Data: []byte("ZZZ"),
})
}
// TestSync_vs_BaseMutate verifies Sync wrt base mutation.
func TestSync_vs_BaseMutate(t *testing.T) {
withDemo(t, func(t *testing.T, ddat *DemoData, ddrv *Storage) {
X := xtesting.FatalIf(t)
head, err := ddrv.Sync(context.Background())
if !(head == 0 && err == nil) {
t.Fatalf("sync0: head=%s err=%s", head, err)
}
tid, err := ddat.MutateBase(); X(err)
head, err = ddrv.Sync(context.Background())
errOk := &zodb.OpError{URL: ddrv.URL(), Op: "sync", Err: &baseMutatedError{
baseAt0: 0,
baseHead: tid,
}}
if !reflect.DeepEqual(err, errOk) {
t.Fatalf("after base mutate: sync: unexpected error:\nhave: %s\nwant: %s",
err, errOk)
}
})
}
// TestWatchLoad_vs_BaseMutate verifies Watch and Load wrt base mutation.
func TestWatchLoad_vs_BaseMutate(t *testing.T) {
withDemoData(t, func(t *testing.T, ddat *DemoData) {
X := xtesting.FatalIf(t)
watchq := make(chan zodb.Event)
ddrv, at0, err := demoOpen(ddat.URL(), &zodb.DriverOptions{
ReadOnly: true,
Watchq: watchq,
}); X(err)
defer func() {
err := ddrv.Close(); X(err)
}()
tid, err := ddat.MutateBase(); X(err)
// first wait for error from watchq
event, ok := <-watchq
if !ok {
t.Fatal("after base mutate: premature watchq close")
}
evErr, ok := event.(*zodb.EventError)
if !ok {
t.Fatalf("after base mutate: unexpected event: %T", event)
}
errBaseMutated := &baseMutatedError{
baseAt0: 0,
baseHead: tid,
}
evErrOk := &zodb.EventError{&zodb.OpError{URL: ddrv.URL(), Op: "watcher", Err: errBaseMutated}}
if !reflect.DeepEqual(evErr, evErrOk) {
t.Fatalf("after base mutate: unexpected event:\nhave: %s\nwant: %s", evErr, evErrOk)
}
// now make sure Load fails with "base mutated" error
xid := zodb.Xid{Oid: 1, At: at0}
data, serial, err := ddrv.Load(context.Background(), xid)
errOk := &zodb.OpError{URL: ddrv.URL(), Op: "load", Args: xid, Err: errBaseMutated}
if !reflect.DeepEqual(err, errOk) {
t.Fatalf("after base mutate: load: unexpected error:\nhave: %s\nwant: %s",
err, errOk)
}
if !(data == nil && serial == zodb.InvalidTid) {
t.Fatalf("after base mutate: load: unexpected data=%v serial=%v", data, serial)
}
})
}
func demoOpen(zurl string, opt *zodb.DriverOptions) (_ *Storage, at0 zodb.Tid, err error) {
defer xerr.Contextf(&err, "opendemo %s", zurl)
u, err := url.Parse(zurl)
if err != nil {
return nil, 0, err
}
d, at0, err := openByURL(context.Background(), u, opt)
if err != nil {
return nil, 0, err
}
return d.(*Storage), at0, nil
}
func xtempdir(t *testing.T) string {
t.Helper()
tmpd, err := ioutil.TempDir("", "demo")
if err != nil {
t.Fatal(err)
}
return tmpd
}
go/zodb/storage/fs1/filestorage.go
View file @ 95fbc76f
// Copyright (C) 2017-2020 Nexedi SA and Contributors.
// Copyright (C) 2017-2021 Nexedi SA and Contributors.
// Kirill Smelkov <kirr@nexedi.com>
//
// This program is free software: you can Use, Study, Modify and Redistribute
......@@ -109,6 +109,9 @@ type FileStorage struct {
downOnce sync.Once // shutdown may be due to both Close and IO error in watcher
errClose error // error from .file.Close()
watchWg sync.WaitGroup // to wait for watcher finish
closed chan struct{} // ready when storage was Closed
closeOnce sync.Once
}
// IStorageDriver
......@@ -132,7 +135,7 @@ func (fs *FileStorage) LastOid(_ context.Context) (zodb.Oid, error) {
}
func (fs *FileStorage) URL() string {
return fs.file.Name()
return "file://" + fs.file.Name()
}
// freelist(DataHeader)
......@@ -468,7 +471,7 @@ func (fs *FileStorage) watcher(w *fsnotify.Watcher, errFirstRead chan<- error) {
// XXX it can also be internal.poll.ErrFileClosing
e.Err.Error() == "use of closed file") {
select {
case <-fs.down:
case <-fs.closed:
err = nil
default:
}
......@@ -485,7 +488,13 @@ func (fs *FileStorage) watcher(w *fsnotify.Watcher, errFirstRead chan<- error) {
if fs.watchq != nil {
if err != nil {
fs.watchq <- &zodb.EventError{err}
select {
case <-fs.closed:
// closed - skip send to watchq
case fs.watchq <- &zodb.EventError{err}:
// ok
}
}
close(fs.watchq)
}
......@@ -535,9 +544,9 @@ mainloop:
if !first {
traceWatch("select ...")
select {
case <-fs.down:
case <-fs.closed:
// closed
traceWatch("down")
traceWatch("closed")
return nil
case err := <-w.Errors:
......@@ -694,7 +703,7 @@ mainloop:
// notify client
if fs.watchq != nil {
select {
case <-fs.down:
case <-fs.closed:
return nil
case fs.watchq <- &zodb.EventCommit{it.Txnh.Tid, δoid}:
......@@ -772,6 +781,9 @@ func (fs *FileStorage) shutdown(reason error) {
}
func (fs *FileStorage) Close() error {
fs.closeOnce.Do(func() {
close(fs.closed)
})
fs.shutdown(fmt.Errorf("closed"))
fs.watchWg.Wait()
......@@ -794,6 +806,7 @@ func Open(ctx context.Context, path string, opt *zodb.DriverOptions) (_ *FileSto
fs := &FileStorage{
watchq: opt.Watchq,
down: make(chan struct{}),
closed: make(chan struct{}),
}
f, err := os.Open(path)
......
go/zodb/storage/fs1/filestorage_test.go
View file @ 95fbc76f
// Copyright (C) 2017-2020 Nexedi SA and Contributors.
// Copyright (C) 2017-2021 Nexedi SA and Contributors.
// Kirill Smelkov <kirr@nexedi.com>
//
// This program is free software: you can Use, Study, Modify and Redistribute
......@@ -370,3 +370,37 @@ func TestOpenRecovery(t *testing.T) {
})
}
}
// TestLoadWhiteout verifies access to whiteout data record.
//
// Whiteout is data record with deletion when object was not previously there.
// It has both len(data)=0 and backpointer=0.
//
// TODO merge into regular tests on testdata/1.fs when
// FileStorage/py.deleteObject allows to create whiteouts instead of raising
// POSKeyError.
func TestLoadWhiteout(t *testing.T) {
fs, _ := xfsopen(t, "testdata/whiteout.fs")
defer exc.XRun(fs.Close)
xid := zodb.Xid{At: zodb.Tid(0x17), Oid: zodb.Oid(1)}
buf, serial, err := fs.Load(context.Background(), xid)
errOk := &zodb.OpError{
URL: fs.URL(),
Op: "load",
Args: xid,
Err: &zodb.NoDataError{Oid: xid.Oid, DeletedAt: xid.At},
}
if !reflect.DeepEqual(err, errOk) {
t.Errorf("load %s: bad err:\nhave: %v\nwant: %v", xid, err, errOk)
}
if buf != nil {
t.Errorf("load %s: buf != nil", xid)
}
if serial != 0 {
t.Errorf("load %s: bad serial %s ; want 0", xid, serial)
}
}
go/zodb/storage/fs1/format.go
View file @ 95fbc76f
// Copyright (C) 2017-2020 Nexedi SA and Contributors.
// Copyright (C) 2017-2021 Nexedi SA and Contributors.
// Kirill Smelkov <kirr@nexedi.com>
//
// This program is free software: you can Use, Study, Modify and Redistribute
......@@ -207,7 +207,7 @@ func (txnh *TxnHeader) CloneFrom(txnh2 *TxnHeader) {
type TxnLoadFlags int
const (
LoadAll TxnLoadFlags = 0x00 // load whole transaction header
LoadNoStrings = 0x01 // do not load user/desc/ext strings
LoadNoStrings TxnLoadFlags = 0x01 // do not load user/desc/ext strings
)
// Load reads and decodes transaction record header @ pos.
......@@ -575,7 +575,10 @@ func (dh *DataHeader) LoadBackRef(r io.ReaderAt) (backPos int64, err error) {
}
backPos = int64(binary.BigEndian.Uint64(dh.workMem[0:]))
if !(backPos == 0 || backPos >= dataValidFrom) {
if backPos == 0 {
return 0, nil // deletion
}
if backPos < dataValidFrom {
return 0, checkErr(r, dh, "invalid backpointer: %v", backPos)
}
if backPos + DataHeaderSize > dh.TxnPos - 8 {
......
go/zodb/storage/fs1/fs1tools/dump.go
View file @ 95fbc76f
// Copyright (C) 2017 Nexedi SA and Contributors.
// Kirill Smelkov <kirr@nexedi.com>
// Copyright (C) 2017-2021 Nexedi SA and Contributors.
// Kirill Smelkov <kirr@nexedi.com>
//
// This program is free software: you can Use, Study, Modify and Redistribute
// it under the terms of the GNU General Public License version 3, or (at your
......@@ -51,6 +51,9 @@ type Dumper interface {
//
// If dumper return io.EOF the whole dumping process finishes.
DumpTxn(buf *xfmt.Buffer, it *fs1.Iter) error
// DumpEndOK is called at the end of successfull dump.
DumpEndOK(buf *xfmt.Buffer) error
}
// Dump dumps content of a FileStorage file @ path.
......@@ -101,7 +104,7 @@ func Dump(w io.Writer, path string, dir fs1.IterDir, d Dumper) (err error) {
err = it.NextTxn(fs1.LoadAll)
if err != nil {
if err == io.EOF {
err = nil
break
}
return err
}
......@@ -109,7 +112,7 @@ func Dump(w io.Writer, path string, dir fs1.IterDir, d Dumper) (err error) {
err = d.DumpTxn(buf, it)
if err != nil {
if err == io.EOF {
err = nil
break
}
return err
}
......@@ -119,6 +122,12 @@ func Dump(w io.Writer, path string, dir fs1.IterDir, d Dumper) (err error) {
return err
}
}
err = d.DumpEndOK(buf)
if err != nil {
return err
}
return nil
}
// ----------------------------------------
......@@ -198,6 +207,10 @@ func (d *DumperFsDump) DumpTxn(buf *xfmt.Buffer, it *fs1.Iter) error {
}
}
func (d *DumperFsDump) DumpEndOK(buf *xfmt.Buffer) error {
return nil
}
// DumperFsDumpVerbose implements a very verbose dumper with output identical
// to fsdump.Dumper in zodb/py originally written by Jeremy Hylton:
//
......@@ -281,6 +294,10 @@ func (d *DumperFsDumpVerbose) dumpData(buf *xfmt.Buffer, it *fs1.Iter) error {
return nil
}
func (d *DumperFsDumpVerbose) DumpEndOK(buf *xfmt.Buffer) error {
return nil
}
const dumpSummary = "dump database transactions"
func dumpUsage(w io.Writer) {
......@@ -383,6 +400,10 @@ func (d *DumperFsTail) DumpTxn(buf *xfmt.Buffer, it *fs1.Iter) error {
return nil
}
func (d *DumperFsTail) DumpEndOK(buf *xfmt.Buffer) error {
return nil
}
const tailSummary = "dump last few transactions of a database"
const ntxnDefault = 10
......
go/zodb/storage/fs1/fs1tools/dump_test.go
View file @ 95fbc76f
// Copyright (C) 2017 Nexedi SA and Contributors.
// Kirill Smelkov <kirr@nexedi.com>
// Copyright (C) 2017-2021 Nexedi SA and Contributors.
// Kirill Smelkov <kirr@nexedi.com>
//
// This program is free software: you can Use, Study, Modify and Redistribute
// it under the terms of the GNU General Public License version 3, or (at your
......@@ -31,6 +31,10 @@ package fs1tools
//go:generate sh -c "python2 -c 'from ZODB.FileStorage import fsdump; fsdump.main()' ../testdata/empty.fs >testdata/empty.fsdump.ok"
//go:generate sh -c "python2 -c 'from ZODB.FileStorage.fsdump import Dumper; import sys; d = Dumper(sys.argv[1]); d.dump()' ../testdata/empty.fs >testdata/empty.fsdumpv.ok"
//go:generate sh -c "python2 -m ZODB.scripts.fstail -n 1000000 ../testdata/whiteout.fs >testdata/whiteout.fstail.ok"
//go:generate sh -c "python2 -c 'from ZODB.FileStorage import fsdump; fsdump.main()' ../testdata/whiteout.fs >testdata/whiteout.fsdump.ok"
//go:generate sh -c "python2 -c 'from ZODB.FileStorage.fsdump import Dumper; import sys; d = Dumper(sys.argv[1]); d.dump()' ../testdata/whiteout.fs >testdata/whiteout.fsdumpv.ok"
import (
"bytes"
"fmt"
......@@ -50,10 +54,11 @@ func loadFile(t *testing.T, path string) string {
return string(data)
}
func testDump(t *testing.T, dir fs1.IterDir, d Dumper) {
testv := []string{"1", "empty"}
func testDump(t *testing.T, dir fs1.IterDir, newd func() Dumper) {
testv := []string{"1", "empty", "whiteout"}
for _, tt := range testv {
t.Run("db=" + tt, func(t *testing.T) {
d := newd()
buf := bytes.Buffer{}
err := Dump(&buf, fmt.Sprintf("../testdata/%s.fs", tt), dir, d)
......@@ -70,9 +75,13 @@ func testDump(t *testing.T, dir fs1.IterDir, d Dumper) {
}
}
func TestFsDump(t *testing.T) { testDump(t, fs1.IterForward, &DumperFsDump{}) }
func TestFsDumpv(t *testing.T) { testDump(t, fs1.IterForward, &DumperFsDumpVerbose{}) }
func TestFsTail(t *testing.T) { testDump(t, fs1.IterBackward, &DumperFsTail{Ntxn: 1000000}) }
func newFsDump() Dumper { return &DumperFsDump{} }
func newFsDumpv() Dumper { return &DumperFsDumpVerbose{} }
func newFsTail() Dumper { return &DumperFsTail{Ntxn: 1000000} }
func TestFsDump(t *testing.T) { testDump(t, fs1.IterForward, newFsDump) }
func TestFsDumpv(t *testing.T) { testDump(t, fs1.IterForward, newFsDumpv) }
func TestFsTail(t *testing.T) { testDump(t, fs1.IterBackward, newFsTail) }
func BenchmarkTail(b *testing.B) {
// FIXME small testdata/1.fs is not representative for benchmarking
......
go/zodb/storage/fs1/fs1tools/main.go
View file @ 95fbc76f
// Copyright (C) 2017 Nexedi SA and Contributors.
// Kirill Smelkov <kirr@nexedi.com>
// Copyright (C) 2017-2021 Nexedi SA and Contributors.
// Kirill Smelkov <kirr@nexedi.com>
//
// This program is free software: you can Use, Study, Modify and Redistribute
// it under the terms of the GNU General Public License version 3, or (at your
......@@ -30,7 +30,7 @@ var commands = prog.CommandRegistry{
{"reindex", reindexSummary, reindexUsage, reindexMain},
{"verify-index", verifyIdxSummary, verifyIdxUsage, verifyIdxMain},
// recover (fsrecover.py)
// verify (fstest.py)
{"verify", verifySummary, verifyUsage, verifyMain},
// XXX repozo ?
}
......
go/zodb/storage/fs1/fs1tools/testdata/whiteout.fsdump.ok 0 → 100644
View file @ 95fbc76f
Trans #00000 tid=0000000000000017 time=1900-01-01 00:00:00.000000 offset=27
status=' ' user='' description=''
data #00000 oid=0000000000000001 class=undo or abort of object creation
go/zodb/storage/fs1/fs1tools/testdata/whiteout.fsdumpv.ok 0 → 100644
View file @ 95fbc76f
************************************************************
file identifier: 'FS21'
============================================================
offset: 4
end pos: 77
transaction id: 0000000000000017
trec len: 73
status: ' '
user: ''
description: ''
len(extra): 0
------------------------------------------------------------
offset: 27
oid: 0000000000000001
revid: 0000000000000017
previous record offset: 0
transaction offset: 4
len(data): 0
backpointer: 0
redundant trec len: 73
go/zodb/storage/fs1/fs1tools/testdata/whiteout.fstail.ok 0 → 100644
View file @ 95fbc76f
1900-01-01 00:00:00.000000: hash=e3e0ff6c686f2fa02266e744f6629d592d432061
user='' description='' length=73 offset=4 (+23)
go/zodb/storage/fs1/fs1tools/verify.go 0 → 100644
View file @ 95fbc76f
// Copyright (C) 2021 Nexedi SA and Contributors.
// Kirill Smelkov <kirr@nexedi.com>
//
// This program is free software: you can Use, Study, Modify and Redistribute
// it under the terms of the GNU General Public License version 3, or (at your
// option) any later version, as published by the Free Software Foundation.
//
// You can also Link and Combine this program with other software covered by
// the terms of any of the Free Software licenses or any of the Open Source
// Initiative approved licenses and Convey the resulting work. Corresponding
// source of such a combination shall include the source code for all other
// software used.
//
// This program is distributed WITHOUT ANY WARRANTY; without even the implied
// warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
//
// See COPYING file for full licensing terms.
// See https://www.nexedi.com/licensing for rationale and options.
package fs1tools
// verify subcommand
//
// verification output mimics fstest from ZODB/py as originally written by Jeremy Hylton:
// https://github.com/zopefoundation/ZODB/blob/5.6.0-35-g1fb097b41/src/ZODB/scripts/fstest.py
import (
"flag"
"fmt"
"io"
"os"
"time"
"lab.nexedi.com/kirr/neo/go/zodb/storage/fs1"
"lab.nexedi.com/kirr/go123/prog"
"lab.nexedi.com/kirr/go123/xflag"
"lab.nexedi.com/kirr/go123/xfmt"
)
// Verify verifies content of a FileStorage file @ path.
//
// Only data part of the database is verified (the *.fs file).
// Use VerifyIndexFor to verify the index part (*.fs.index).
func Verify(w io.Writer, path string, verbose int, progress bool) (err error) {
// just iterate through the file and emit progress.
// the FileStorage driver implements all consistency checks by itself.
fi, err := os.Stat(path)
if err != nil {
return fmt.Errorf("verify: %s: %s", path, err)
}
fsize := fi.Size()
v := &Verifier{verbose: verbose}
// display progress updates once per tick
if progress {
tick := time.NewTicker(time.Second / 4)
defer tick.Stop()
xcr := ""
if verbose > 0 {
xcr = "\n"
}
v.progress = func(force bool) error {
if !force {
select {
case <-tick.C:
default:
return nil
}
}
_, err := fmt.Fprintf(w,
"\rVerified data bytes: %.1f%% (%d/%d); #txn: %d%s",
100 * float64(v.donePos) / float64(fsize),
v.donePos, fsize,
v.ntxn,
xcr)
return err
}
}
return Dump(w, path, fs1.IterForward, v)
}
// Verifier implements Dumper that is used by Verify.
type Verifier struct {
ntxn int // current transaction record #
verbose int // >=1 (print txn) >=2 (print objects)
// for loading data
dhLoading fs1.DataHeader
donePos int64 // done verifying till this position
progress func(force bool) error // called after each transaction if !nil
}
func (v *Verifier) DumperName() string {
return "fsverify"
}
func (v *Verifier) DumpFileHeader(buf *xfmt.Buffer, fh *fs1.FileHeader) error {
return nil
}
func (v *Verifier) DumpTxn(buf *xfmt.Buffer, it *fs1.Iter) error {
txnh := &it.Txnh
for i := 0; ; i++ {
err := it.NextData()
if err != nil {
if err == io.EOF {
break
}
return err
}
dh := &it.Datah
// load data
v.dhLoading = *dh
dbuf, err := v.dhLoading.LoadData(it.R)
if err != nil {
return err
}
if v.verbose >= 2 {
buf .S(fmt.Sprintf("%10d: object oid 0x%s #%d\n", dh.Pos, dh.Oid, i))
}
dbuf.Release()
}
if v.verbose >= 1 {
buf .S(fmt.Sprintf("%10d: transaction tid 0x%s #%d \n", txnh.Pos, txnh.Tid, v.ntxn))
}
v.ntxn++
if v.progress != nil {
v.donePos = txnh.Pos + txnh.Len
err := v.progress(/*force=*/false)
if err != nil {
return err
}
}
return nil
}
func (v *Verifier) DumpEndOK(buf *xfmt.Buffer) error {
if v.progress != nil {
err := v.progress(/*force=*/true)
if err != nil {
return err
}
}
if v.verbose >= 1 {
buf .S("no errors detected\n")
}
return nil
}
// ----------------------------------------
const verifySummary = "verify database content"
func verifyUsage(w io.Writer) {
fmt.Fprintf(w,
`Usage: fs1 verify [options] <storage>
Verify FileStorage records for consistency
<storage> is a path to FileStorage
options:
-h --help this help text.
-v increase verbosity.
-p display progress.
`)
}
func verifyMain(argv []string) {
verbose := 0
var progress bool
flags := flag.FlagSet{Usage: func() { verifyUsage(os.Stderr) }}
flags.Init("", flag.ExitOnError)
flags.Var((*xflag.Count)(&verbose), "v", "verbosity level")
flags.BoolVar(&progress, "p", false, "display progress")
flags.Parse(argv[1:])
argv = flags.Args()
if len(argv) < 1 {
flags.Usage()
prog.Exit(2)
}
storPath := argv[0]
err := Verify(os.Stdout, storPath, verbose, progress)
if err != nil {
prog.Fatal(err)
}
}
go/zodb/storage/fs1/py/gen-testdata
View file @ 95fbc76f
#!/usr/bin/env python2
# -*- coding: utf-8 -*-
# Copyright (C) 2017-2020 Nexedi SA and Contributors.
# Copyright (C) 2017-2021 Nexedi SA and Contributors.
# Kirill Smelkov <kirr@nexedi.com>
#
# This program is free software: you can Use, Study, Modify and Redistribute
......@@ -21,9 +21,10 @@
"""generate reference fs1 database and index for tests"""
from ZODB.FileStorage import FileStorage
from ZODB.FileStorage.FileStorage import FILESTORAGE_MAGIC, TxnHeader, DataHeader, TRANS_HDR_LEN
from ZODB import DB
from ZODB.Connection import TransactionMetaData
from zodbtools.test.gen_testdata import gen_testdb, precommit, run_with_zodb3py2_compat
from zodbtools.test.gen_testdata import gen_testdb, precommit, run_with_zodb4py2_compat
from os import stat, remove
from shutil import copyfile
from golang.gcompat import qq
......@@ -156,7 +157,7 @@ def main():
vstor.store(vroot._p_oid, vroot._p_serial, '000 data 000', '', txn_stormeta)
vstor.tpc_vote(txn_stormeta)
# NO tpc_finish here so that status remain 'c' (voted) instead of ' ' (committed)
run_with_zodb3py2_compat(_)
run_with_zodb4py2_compat(_)
st = stat(outfs)
l = st.st_size
......@@ -174,5 +175,36 @@ def main():
remove(voted+".lock")
# prepare file with whiteout (deletion of previously non-existing object)
whiteout = "testdata/whiteout.fs"
# as of 20210317 FileStorage.deleteObject verifies that object exists
# -> prepare magic/transaction/data records manually
with open(whiteout, "wb") as f:
oid = p64(1)
tid = p64(0x17)
# file header
f.write(FILESTORAGE_MAGIC)
tpos = f.tell()
# data record (see FileStorage.deleteObject)
dh = DataHeader(oid, tid, 0, tpos, 0, 0)
drec = dh.asString() + p64(0)
# emit txn header (see FileStorage.tpc_vote)
tlen = TRANS_HDR_LEN + 0 + 0 + 0 + len(drec) # empty u,d,e
th = TxnHeader(tid, tlen, ' ', 0, 0, 0)
th.user = b''
th.descr = b''
th.ext = b''
f.write(th.asString())
# emit data record
f.write(drec)
# emit txn tail
f.write(p64(tlen))
if __name__ == '__main__':
main()
go/zodb/storage/zeo/proto.go
View file @ 95fbc76f
// Copyright (C) 2018-2020 Nexedi SA and Contributors.
// Copyright (C) 2018-2021 Nexedi SA and Contributors.
// Kirill Smelkov <kirr@nexedi.com>
//
// This program is free software: you can Use, Study, Modify and Redistribute
......@@ -54,7 +54,7 @@ type msg struct {
type msgFlags int64
const (
msgAsync msgFlags = 1 // message does not need a reply
msgExcept = 2 // exception was raised on remote side (ZEO5)
msgExcept msgFlags = 2 // exception was raised on remote side (ZEO5)
)
// encoding represents messages encoding.
......@@ -104,7 +104,7 @@ func pktEncodeM(m msg) *pktBuf {
// arg
// it is interface{} - use shamaton/msgpack since msgp does not handle
// arbitrary interfaces well.
dataArg, err := msgpack.Encode(m.arg)
dataArg, err := msgpack.Marshal(m.arg)
if err != nil {
panic(err) // all our types are expected to be supported by msgpack
}
......@@ -240,7 +240,7 @@ func pktDecodeM(pkb *pktBuf) (msg, error) {
if len(btail) != 0 {
return m, derrf(".%d: payload has extra data after message")
}
err = msgpack.Decode(b, &m.arg)
err = msgpack.Unmarshal(b, &m.arg)
if err != nil {
return m, derrf(".%d: arg: %s", m.msgid, err)
}
......
go/zodb/storage/zeo/zeo.go
View file @ 95fbc76f
......@@ -52,6 +52,9 @@ type zeo struct {
// becomes ready when serve loop finishes
serveWG sync.WaitGroup
closeOnce sync.Once
closed chan struct{} // ready when Closed
url string // we were opened via this
}
......@@ -174,7 +177,13 @@ func (z *zeo) invalidateTransaction(arg interface{}) (err error) {
// at0 is initialized - ok to send current event if it goes > at0
if tid > z.at0 {
z.watchq <- event
select {
case <-z.closed:
// closed - client does not read watchq anymore
case z.watchq <- event:
// ok
}
}
return nil
}
......@@ -188,7 +197,13 @@ func (z *zeo) flushEventq0() {
if z.watchq != nil {
for _, e := range z.eventq0 {
if e.Tid > z.at0 {
z.watchq <- e
select {
case <-z.closed:
// closed - client does not read watchq anymore
case z.watchq <- e:
// ok
}
}
}
}
......@@ -264,7 +279,7 @@ func (r rpc) call(ctx context.Context, argv ...interface{}) (interface{}, error)
// excError returns error corresponding to an exception.
//
// well-known exceptions are mapped to corresponding well-known errors - e.g.
// POSKeyError -> zodb.NoObjectError, and rest are returned wrapper into rpcExcept.
// POSKeyError -> zodb.NoObjectError, and rest are returned wrapped into rpcExcept.
func (r rpc) excError(exc string, argv tuple) error {
// translate well-known exceptions
switch exc {
......@@ -438,7 +453,7 @@ func openByURL(ctx context.Context, u *url.URL, opt *zodb.DriverOptions) (_ zodb
}()
z := &zeo{link: zlink, watchq: opt.Watchq, url: url}
z := &zeo{link: zlink, watchq: opt.Watchq, closed: make(chan struct{}), url: url}
// start serve loop on the link
z.serveWG.Add(1)
......@@ -456,14 +471,18 @@ func openByURL(ctx context.Context, u *url.URL, opt *zodb.DriverOptions) (_ zodb
// close .watchq after serve is over
z.at0Mu.Lock()
defer z.at0Mu.Unlock()
if z.at0Initialized {
z.flushEventq0()
}
if z.watchq != nil {
if err != nil {
z.watchq <- &zodb.EventError{Err: err}
if err != nil && /* already flushed .eventq0 */z.at0Initialized {
select {
case <-z.closed:
// closed - client does not read watchq anymore
case z.watchq <- &zodb.EventError{Err: err}:
// ok
}
}
close(z.watchq)
z.watchq = nil // prevent flushEventq0 to send to closed chan
}
}()
......@@ -496,13 +515,16 @@ func openByURL(ctx context.Context, u *url.URL, opt *zodb.DriverOptions) (_ zodb
// "invalidateTransaction" server notification.
//
// filter-out first < at0 messages for this reason.
z.at0Mu.Lock()
z.at0 = lastTid
z.at0Initialized = true
z.flushEventq0()
z.at0Mu.Unlock()
//
// do this in separate task not to deadlock in watchq<- : we did not
// yet returned z to caller and so noone might be yet reading from watchq.
go func() {
z.at0Mu.Lock()
z.at0 = lastTid
z.at0Initialized = true
z.flushEventq0()
z.at0Mu.Unlock()
}()
//call('get_info') -> {}str->str, ex // XXX can be omitted
/*
......@@ -520,11 +542,15 @@ func openByURL(ctx context.Context, u *url.URL, opt *zodb.DriverOptions) (_ zodb
'supports_record_iternext': True})
*/
return z, z.at0, nil
return z, lastTid, nil
}
func (z *zeo) Close() error {
err := z.link.Close()
var err error
z.closeOnce.Do(func() {
close(z.closed)
err = z.link.Close()
})
z.serveWG.Wait()
return err
}
......
go/zodb/storage/zeo/zeo_test.go
View file @ 95fbc76f
......@@ -153,7 +153,7 @@ func (z *ZEOPySrv) Encoding() encoding {
// ----------------
// tOptions represents options for testing.
// XXX dup in NEO
// TODO -> xtesting
type tOptions struct {
Preload string // preload database with data from this location
}
......
go/zodb/wks/wks.go
View file @ 95fbc76f
// Copyright (C) 2017 Nexedi SA and Contributors.
// Kirill Smelkov <kirr@nexedi.com>
// Copyright (C) 2017-2021 Nexedi SA and Contributors.
// Kirill Smelkov <kirr@nexedi.com>
//
// This program is free software: you can Use, Study, Modify and Redistribute
// it under the terms of the GNU General Public License version 3, or (at your
......@@ -31,4 +31,5 @@ import (
_ "lab.nexedi.com/kirr/neo/go/zodb/storage/fs1"
_ "lab.nexedi.com/kirr/neo/go/zodb/storage/zeo"
_ "lab.nexedi.com/kirr/neo/go/neo"
_ "lab.nexedi.com/kirr/neo/go/zodb/storage/demo"
)
go/zodb/zodb.go
View file @ 95fbc76f
// Copyright (C) 2016-2020 Nexedi SA and Contributors.
// Copyright (C) 2016-2021 Nexedi SA and Contributors.
// Kirill Smelkov <kirr@nexedi.com>
//
// This program is free software: you can Use, Study, Modify and Redistribute
......@@ -275,7 +275,7 @@ type NoObjectError struct {
Oid Oid
}
func (e NoObjectError) Error() string {
func (e *NoObjectError) Error() string {
return fmt.Sprintf("%s: no such object", e.Oid)
}
......
go/zodb/zodbtools/help.go
View file @ 95fbc76f
......@@ -41,9 +41,10 @@ and using path to that file with zconfig:// schema:
There are also following simpler ways:
- neo://<master>/<db> for a NEO database XXX + neos:// ?
- zeo://<host>:<port> for a ZEO database
- /path/to/file for a FileStorage database
- neo://<master>/<db> for a NEO database XXX + neos:// ?
- zeo://<host>:<port> for a ZEO database
- /path/to/file for a FileStorage database
- demo:(zurl_base)/(zurl_δ) for a DemoStorage overlay
Please see zodburi documentation for full details:
......
go/zodb/δtail.go
View file @ 95fbc76f
// Copyright (C) 2018-2020 Nexedi SA and Contributors.
// Copyright (C) 2018-2021 Nexedi SA and Contributors.
// Kirill Smelkov <kirr@nexedi.com>
//
// This program is free software: you can Use, Study, Modify and Redistribute
......@@ -202,46 +202,35 @@ func (δtail *ΔTail) ForgetPast(revCut Tid) {
// XXX -> RevAt ?
// LastRevOf tries to return what was the last revision that changed id as of at database state.
//
// Depending on current information in δtail it returns either exact result, or
// an upper-bound estimate for the last id revision, assuming id was changed ≤ at:
// it must be called with the following condition:
//
// 1) if δtail does not cover at, at is returned:
// tail ≤ at ≤ head
//
// # at ∉ [min(rev ∈ δtail), max(rev ∈ δtail)]
// LastRevOf(id, at) = at
// Depending on current information in δtail it returns either exact result, or
// an upper-bound estimate for the last id revision:
//
// 2) if δtail has an entry corresponding to id change, it gives exactly the
// 1) if δtail has an entry corresponding to id change, it gives exactly the
// last revision that changed id:
//
// # at ∈ [min(rev ∈ δtail), max(rev ∈ δtail)]
// # ∃ rev ∈ δtail: rev changed id && rev ≤ at
// LastRevOf(id, at) = max(rev: rev changed id && rev ≤ at)
// LastRevOf(id, at) = max(rev: rev changed id && rev ≤ at), true
//
// 3) if δtail does not contain appropriate record with id - it returns δtail's
// 2) if δtail does not contain appropriate record with id - it returns δtail's
// lower bound as the estimate for the upper bound of the last id revision:
//
// # at ∈ [min(rev ∈ δtail), max(rev ∈ δtail)]
// # ∄ rev ∈ δtail: rev changed id && rev ≤ at
// LastRevOf(id, at) = min(rev ∈ δtail)
// LastRevOf(id, at) = δtail.tail, false
//
// On return exact indicates whether returned revision is exactly the last
// revision of id, or only an upper-bound estimate of it.
func (δtail *ΔTail) LastRevOf(id Oid, at Tid) (_ Tid, exact bool) {
// check if we have no coverage at all
l := len(δtail.tailv)
if l == 0 {
return at, false
}
revMin := δtail.tailv[0].Rev
revMax := δtail.tailv[l-1].Rev
if !(revMin <= at && at <= revMax) {
return at, false
}
if !(δtail.tail <= at && at <= δtail.head) {
panic(fmt.Sprintf("at out of bounds: at: @%s, (tail, head] = (@%s, @%s]", at, δtail.tail, δtail.head))
}
// we have the coverage
rev, ok := δtail.lastRevOf[id]
if !ok {
return δtail.tailv[0].Rev, false
return δtail.tail, false
}
if rev <= at {
......@@ -250,7 +239,7 @@ func (δtail *ΔTail) LastRevOf(id Oid, at Tid) (_ Tid, exact bool) {
// what's in index is after at - scan tailv back to find appropriate entry
// XXX linear scan - see .lastRevOf comment.
for i := l - 1; i >= 0; i-- {
for i := len(δtail.tailv) - 1; i >= 0; i-- {
δ := δtail.tailv[i]
if δ.Rev > at {
continue
......@@ -264,5 +253,5 @@ func (δtail *ΔTail) LastRevOf(id Oid, at Tid) (_ Tid, exact bool) {
}
// nothing found
return δtail.tailv[0].Rev, false
return δtail.tail, false
}
go/zodb/δtail_test.go
View file @ 95fbc76f
// Copyright (C) 2018-2019 Nexedi SA and Contributors.
// Copyright (C) 2018-2021 Nexedi SA and Contributors.
// Kirill Smelkov <kirr@nexedi.com>
//
// This program is free software: you can Use, Study, Modify and Redistribute
......@@ -174,14 +174,13 @@ func TestΔTail(t *testing.T) {
δtail = NewΔTail(3)
δCheck(3,3)
δCheckLastUP(4, 12, 12) // δtail = ø
δCheckLastUP(3, 3, 3) // δtail = ø
δAppend(R(10, 3,5))
δCheck(3,10, R(10, 3,5))
δCheckLastUP(3, 2, 2) // at < δtail
δCheckLastUP(3, 12, 12) // at > δtail
δCheckLastUP(4, 10, 10) // id ∉ δtail
δCheckLastUP(3, 9, 3) // id ∈ δtail, but has no entry with rev ≤ at
δCheckLastUP(4, 10, 3) // id ∉ δtail
δAppend(R(11, 7))
δCheck(3,11, R(10, 3,5), R(11, 7))
......@@ -192,7 +191,7 @@ func TestΔTail(t *testing.T) {
δAppend(R(14, 3,8))
δCheck(3,14, R(10, 3,5), R(11, 7), R(12, 7), R(14, 3,8))
δCheckLastUP(8, 12, 10) // id ∈ δtail, but has no entry with rev ≤ at
δCheckLastUP(8, 12, 3) // id ∈ δtail, but has no entry with rev ≤ at
δtail.ForgetPast(9)
δCheck(9,14, R(10, 3,5), R(11, 7), R(12, 7), R(14, 3,8))
......
neo/client/zodburi.py
View file @ 95fbc76f
......@@ -84,6 +84,7 @@ def _resolve_uri(uri):
if scheme != "neos":
raise ValueError("invalid uri: %s : credentials can be specified only with neos:// scheme" % uri)
# ca=ca.crt;cert=my.crt;key=my.key
cred = cred.replace(';', '&') # ; is no longer in default separators set bugs.python.org/issue42967
for k, v in OrderedDict(parse_qsl(cred)).items():
if k not in _credopts:
raise ValueError("invalid uri: %s : unexpected credential %s" % (uri, k))
......
neo/tests/ca.crt
View file @ 95fbc76f
-----BEGIN CERTIFICATE-----
MIIC7TCCAdWgAwIBAgIJAL8e44sA7PDMMA0GCSqGSIb3DQEBCwUAMA0xCzAJBgNV
BAMMAkNBMB4XDTE1MDkzMDEzNTQzMFoXDTIxMDMyMjEzNTQzMFowDTELMAkGA1UE
AwwCQ0EwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQCgT7DVKM4ViQt3
B0oJC4RGi10dNfpNZZpgA5iC2UJ1u6AqqCf0PCQkrmuIzW3l1TenlOiLNdVASkkT
wf1lekIgg4tR8/22oGTAnfY6R9r1C6jAMV72v1sffz8D6qfkMPzKchJt55zywdhm
KscUsMGzXPGIeKrG20m83dSIO4RmCmq/f4BcuWJu6Kkq4n9Wc2IsvpKk+lqEUxI/
QoqdT6OvMXooGs3t892uvKDu++muBj2Y/yyaXt1tCCjDFsRMLWl3Skks+4PeMCZ4
wugyXEBk3d5Yzdv5NsFzFBjAuRCGxJXEOEcfHj4Xj9qTCErZ1jKzgnuxJCtgdqRC
r4beX1U3AgMBAAGjUDBOMB0GA1UdDgQWBBSFY/jKvo0iSTEzzOIcZZUZCT8JfTAf
BgNVHSMEGDAWgBSFY/jKvo0iSTEzzOIcZZUZCT8JfTAMBgNVHRMEBTADAQH/MA0G
CSqGSIb3DQEBCwUAA4IBAQAB0LKDAuhodpyNVwEE9Yl+Q/IiPEPCaix6URJnRn1O
gQnXuZLo1xtJh6wJh1faG1/qNCFMxWEJ+0VkJ7r6v38cNXfYG9OcmD0S6YnNjSuO
VliAtqVVtj8MppJ4vMatLrNi4cvyYucebtNyBCzSIAi+6bkkHeaVgi1EtxXvq+AS
iZp3gl84oXv/gV7Bz4SXmVpFJnhsDMoQZG2KAULAgfZ2Am2I+ffG90cD/oEnS/3O
k3btqTvgIO8MWt8PY3sUOhJEoJYKnC9DppmhOhUTn4zzIIDSluKEOBHZiFb9AcmF
PvzL+8xiORCdUe1d6ANQQlUd0MM810BXZFYEXFbgKg8o
MIIC+zCCAeOgAwIBAgIUN2aMGxxsLX4IyuSR2DRfzbTumrAwDQYJKoZIhvcNAQEL
BQAwDTELMAkGA1UEAwwCQ0EwHhcNMjEwMzIyMTU1MzI5WhcNMjYwOTEyMTU1MzI5
WjANMQswCQYDVQQDDAJDQTCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEB
AMVipTeJN6FEpdF6JJkkVw7vPR14pXCMehSWxBMrmICVAHct4fWz4sI/lp3VTiFl
HnEBOhehLsKfZyosGIMpJ0zI2GsQopkHa5ZPf/cILrQ3ybu7ZsQITAyYKWqhMVdt
d+YRetfkE874gUHqTo+WSBpCbRvuYK4XhBEmrMiwIHV7nwuIaKMdG6FxNGn8du8R
Ti96VduJGdjTFIRri2jNP47VKidioPkxecA/48yrDdewn/cxsU/yNCO9Uii/dTyS
Ro/Jqxql6Leb7xaZlODS5QQAGWjAI6w0BCobwYaUuJOZ1HSUL6A42rnMqI6gGjCY
Dju313RlE5jOilWQhodmawECAwEAAaNTMFEwHQYDVR0OBBYEFBfzeqZl9fHZR2bH
rb+T6+O6mu1aMB8GA1UdIwQYMBaAFBfzeqZl9fHZR2bHrb+T6+O6mu1aMA8GA1Ud
EwEB/wQFMAMBAf8wDQYJKoZIhvcNAQELBQADggEBAK6+04BOOP/PF6BR6FaZjLHN
qhcSBfG4Y8RyLtERwGTQW68PwCU2TloJgpoY8updT0V9EsxnkWjsU/Wgnl8pdHjp
GMAivelTcDoYH4qn8E6mDEKE47Be4fE0FdGvU9QU7UBWG+Uf8pxgnFJcGou5grLG
RXTSNCSrBnJG9tmSPA/aw/Q9MVfXFuB7K8aPJ/d2Z/8v/QFdBK25JH8I+ErFU3ys
ufn58Jqsy4dp8R4qYVP1TKPfGsZ+RZeIIJ4cDIik/vbv9+Ymi+m8LZ66kUNq1d4X
tGbPjcNt/qFQ/FG7T6ZUPe9vRpYAhIHGBYZFs59LJcQsH1DzSvei3141Gs7TYFg=
-----END CERTIFICATE-----
neo/tests/node.crt
View file @ 95fbc76f
-----BEGIN CERTIFICATE-----
MIICkDCCAXgCAQEwDQYJKoZIhvcNAQELBQAwDTELMAkGA1UEAwwCQ0EwHhcNMTUw
OTMwMTM1NDMwWhcNMjEwMzIyMTM1NDMwWjAPMQ0wCwYDVQQDDARub2RlMIIBIjAN
MIICkDCCAXgCAQEwDQYJKoZIhvcNAQELBQAwDTELMAkGA1UEAwwCQ0EwHhcNMjEw
MzIyMTU1MzI5WhcNMjYwOTEyMTU1MzI5WjAPMQ0wCwYDVQQDDARub2RlMIIBIjAN
BgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAuJ+ClJyjhJOJdGUyqHn79opMLP3m
1g27uBWKT+OGd4FcreVoRDxPVuxZxMtDCZcBfUHVvOoSlS06khwSxViEe1hxwHRa
n2qMWlwvaWeNY0CFH5V+DI4XSNojgny85Lb5jB69FuPcrHnwxLk2OFntrXEeNbEa
d7QSoNbPajbJIp5BS/WR9iu5Z5JYdumLWjTvOU+eZc4iA6Wa2kdDtbqkGi4wOJ1L
/ggATL+p+QFcubVptztPT8vq7gvDdGJgXLJ2lPHV0V/sdJB1FB4mSJEDDjSm2Hpp
qPVJSO1GrAy5Ld+0SnXIZZejhIUJIumocY08r+vzDSQ/8NnqXR4Odz1TWwIDAQAB
MA0GCSqGSIb3DQEBCwUAA4IBAQBlYkkInDDDcgnNRdUmzxwejs1PmEehZ3H5FkMp
TsmpoVC+oqM+QywMu8UJRtCjXnnJdAUbVYuZ1Tjm7qvFIhN+5OlIVxJ+8WcmZPSe
lj0N7Dv2nE1diTDS+qPZVPZ0demo1LafRmPomPWiM/CQRlMPxXnimuiYOROhWGn6
jsyoOwquMkAc6Ub++l4OCxLAP0eTgJFkivmqpaYZXG4o7zFvcQ3rQ66rQrMl69sR
8/MVqbT5Sq1CEJbepP4GaFfa5l3CVy7WH2MhCV1/9mNwcXafkTgx3q2HsPon4Dze
kNwiguNAM4L/j4dbIwz+CIVWcgpBCrfv2JYu+jGlRpxIDeWR
MA0GCSqGSIb3DQEBCwUAA4IBAQAvtJztgZ8TuY5Q52Lv4hE2WX3/IZQPYc9cGQqv
sI0xKQEWD4hVHAmxL7bLFOUovzFqNMY0pYBGVJ53nypsKNorKbiepSMPosfcbk9o
U8xF9YDJO+yS6V2lWFi6iKe4WX2t+L2j5x+q4dptJoIvsM777xGatri59q8UK3ne
YTINui8+tsOIGEpcXMHMI/k3RvBUS3Mwy0pX6rsi8/YsUVUShpPDjuAqQIYxU4gP
qAIWrhXNYwOGuMQ1V+j83R5lqDoHrVauWznc0n0qg/EJArhwkLGfjQoqrogCQOVp
iMEV6YDLC8EMqZfYVUT0i2+/bS737Z2lcdgz2essW45nk7+H
-----END CERTIFICATE-----
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment
GitLab Nexedi Edition | About GitLab | About Nexedi | 沪ICP备2021021310号-2 | 沪ICP备2021021310号-7