Commit 2c0968e4 authored by Levin Zimmermann's avatar Levin Zimmermann Committed by Kirill Smelkov

lib/zodb: Update NEO URI format to be in sync with upstream NEO

NEO/go and NEO/py URI format diverged over time:

- kirr/neo@8c974485

However with nexedi/neoppod!21 a
common solution was found. With kirr/neo!7 NEO/go and NEO/py
URI formats are in sync again. We therefore now need to update
'wendelin.core' to support the finally agreed on URI format.

/reviewed-by @kirr
/reviewed-on nexedi/wendelin.core!28
parent 921ad362
...@@ -513,7 +513,7 @@ def test_zstor_2zurl(tmpdir, neo_ssl_dict): ...@@ -513,7 +513,7 @@ def test_zstor_2zurl(tmpdir, neo_ssl_dict):
# sslp is the ssl encryption uri part of an encrypted NEO node # sslp is the ssl encryption uri part of an encrypted NEO node
q = quote_plus q = quote_plus
sslp = ";".join(("%s=%s" % (q(k), q(v)) for k, v in sorted(neo_ssl_dict.items()))) sslp = "&".join(("%s=%s" % (q(k), q(v)) for k, v in sorted(neo_ssl_dict.items())))
_ = assert_zurl_is_correct _ = assert_zurl_is_correct
_(fs1("test.fs"), "file://%s/test.fs" % tmpdir) # FileStorage _(fs1("test.fs"), "file://%s/test.fs" % tmpdir) # FileStorage
...@@ -523,13 +523,13 @@ def test_zstor_2zurl(tmpdir, neo_ssl_dict): ...@@ -523,13 +523,13 @@ def test_zstor_2zurl(tmpdir, neo_ssl_dict):
_(zeo("test", ("127.0.0.1", 1234)), "zeo://127.0.0.1:1234?storage=test") # + non-default storage name _(zeo("test", ("127.0.0.1", 1234)), "zeo://127.0.0.1:1234?storage=test") # + non-default storage name
_(zeo("1", ("::1", 1234)), "zeo://[::1]:1234") # ZEO/ip6 _(zeo("1", ("::1", 1234)), "zeo://[::1]:1234") # ZEO/ip6
_(zeo("test", ("::1", 1234)), "zeo://[::1]:1234?storage=test") # + non-default storage name _(zeo("test", ("::1", 1234)), "zeo://[::1]:1234?storage=test") # + non-default storage name
_(neo("test", "127.0.0.1:1234"), "neo://127.0.0.1:1234/test") # NEO/ip4 _(neo("test", "127.0.0.1:1234"), "neo://test@127.0.0.1:1234") # NEO/ip4
_(neo("test", "127.0.0.1:1234", 1), "neos://%s@127.0.0.1:1234/test" % sslp) # + ssl _(neo("test", "127.0.0.1:1234", 1), "neos://test@127.0.0.1:1234?%s" % sslp) # + ssl
_(neo("test", "[::1]:1234"), "neo://[::1]:1234/test") # NEO/ip6 _(neo("test", "[::1]:1234"), "neo://test@[::1]:1234") # NEO/ip6
_(neo("test", "[::1]:1234", 1), "neos://%s@[::1]:1234/test" % sslp) # + ssl _(neo("test", "[::1]:1234", 1), "neos://test@[::1]:1234?%s" % sslp) # + ssl
_(neo("test", "[::1]:1234\n[::2]:1234"), # + 2 master nodes _(neo("test", "[::1]:1234\n[::2]:1234"), # + 2 master nodes
# Master order is not specified, so we have 2 possible/acceptable zurl # Master order is not specified, so we have 2 possible/acceptable zurl
"neo://[::1]:1234,[::2]:1234/test", "neo://[::2]:1234,[::1]:1234/test") "neo://test@[::1]:1234,[::2]:1234", "neo://test@[::2]:1234,[::1]:1234")
_(demo(zeo("base", ("1.2.3.4", 5)), # DemoStorage _(demo(zeo("base", ("1.2.3.4", 5)), # DemoStorage
fs1("delta.fs")), "demo:(zeo://1.2.3.4:5?storage=base)/(file://%s/delta.fs)" % tmpdir) fs1("delta.fs")), "demo:(zeo://1.2.3.4:5?storage=base)/(file://%s/delta.fs)" % tmpdir)
...@@ -551,13 +551,18 @@ def test_zstor_2zurl(tmpdir, neo_ssl_dict): ...@@ -551,13 +551,18 @@ def test_zstor_2zurl(tmpdir, neo_ssl_dict):
# ZEO # ZEO
("zeo://localhost:9001", "zeo://localhost:9001"), ("zeo://localhost:9001", "zeo://localhost:9001"),
# NEO # NEO
("neo://127.0.0.1:1234/cluster", "neo://127.0.0.1:1234/cluster"), ("neo://cluster@127.0.0.1:1234", "neo://cluster@127.0.0.1:1234"),
# > 1 master nodes \w different order # > 1 master nodes \w different order
("neo://abc:1,def:2/cluster", "neo://abc:1,def:2/cluster"), ("neo://cluster@abc:1,def:2", "neo://cluster@abc:1,def:2"),
("neo://def:2,abc:1/cluster", "neo://abc:1,def:2/cluster"), ("neo://cluster@def:2,abc:1", "neo://cluster@abc:1,def:2"),
# Different SSL paths # Different SSL paths
("neos://ca=a&key=b&cert=c@xyz:1/cluster", "neos://xyz:1/cluster"), ("neos://cluster@xyz:1?ca=a&key=b&cert=c", "neos://cluster@xyz:1"),
("neos://ca=α&key=β&cert=γ@xyz:1/cluster", "neos://xyz:1/cluster"), ("neos://cluster@xyz:1?ca=α&key=β&cert=γ", "neos://cluster@xyz:1"),
# neo:// with anything SSL-related in query -> neos://
("neo://cluster@xyz:1?cert=c", "neos://cluster@xyz:1"),
# any order of options should result in the same normalized URI
("neo://cluster@xyz:1?a=1&c=10&b=2", "neo://cluster@xyz:1?a=1&b=2&c=10"),
("neo://cluster@xyz:1?b=2&a=1&c=10", "neo://cluster@xyz:1?a=1&b=2&c=10"),
], ],
) )
def test_zurl_normalize_main(zurl, zurl_norm_ok): def test_zurl_normalize_main(zurl, zurl_norm_ok):
......
...@@ -389,15 +389,14 @@ def zstor_2zurl(zstor): ...@@ -389,15 +389,14 @@ def zstor_2zurl(zstor):
# NEO # NEO
if ztype == "neo.client.Storage.Storage": if ztype == "neo.client.Storage.Storage":
# neo(s)://[<credentials>@]<master>/<cluster> # neo(s)://<cluster>@<master>?[<credentials>]
app = zstor.app app = zstor.app
if not app.ssl: if not app.ssl:
u = "neo://" u = "neo://"
else: else:
q = urlparse.quote_plus
u = "neos://" u = "neos://"
ca, cert, key = app.ssl_credentials # .ssl_credentials depend on kirr's patch
u += "ca=%s;cert=%s;key=%s@" % (q(ca), q(cert), q(key)) u += "%s@" % app.name
masterv = app.nm.getMasterList() masterv = app.nm.getMasterList()
if len(masterv) == 0: if len(masterv) == 0:
...@@ -413,7 +412,11 @@ def zstor_2zurl(zstor): ...@@ -413,7 +412,11 @@ def zstor_2zurl(zstor):
master_list.append("%s:%s" % (host, port)) master_list.append("%s:%s" % (host, port))
u += ",".join(master_list) u += ",".join(master_list)
u += "/%s" % app.name
if app.ssl:
q = urlparse.quote_plus
ca, cert, key = app.ssl_credentials # .ssl_credentials depend on kirr's patch
u += "?ca=%s&cert=%s&key=%s" % (q(ca), q(cert), q(key))
return u return u
...@@ -453,7 +456,7 @@ def _is_ipv6(host): ...@@ -453,7 +456,7 @@ def _is_ipv6(host):
# #
# Example: # Example:
# #
# neos://ca=zzz@def:2,abc:1/cluster -> neos://abc:1,def:2/cluster # neos://cluster@def:2,abc:1?ca=zzz -> neos://cluster@abc:1,def:2
def zurl_normalize_main(zurl): def zurl_normalize_main(zurl):
scheme, netloc, path, query, frag = urlsplit(zurl) scheme, netloc, path, query, frag = urlsplit(zurl)
try: try:
...@@ -481,14 +484,31 @@ def _znormalize_neo(scheme, netloc, path, query, frag): ...@@ -481,14 +484,31 @@ def _znormalize_neo(scheme, netloc, path, query, frag):
# The same database can be accessed from different clients with different # The same database can be accessed from different clients with different
# credentials, but we want to map them all to the same single WCFS # credentials, but we want to map them all to the same single WCFS
# instance. # instance.
if "@" in netloc: q = urlparse.parse_qs(query)
netloc = netloc[netloc.index("@") + 1 :] for k in ("ca", "cert", "key"):
# NOTE: We only enforce SSL in case ca/cert/key are a non-empty
# string - this means "?ca=&cert=&key=" is interpreted as !SSL.
if q.pop(k, 0):
# Ensure we use 'neos' scheme (instead of 'neo' scheme),
# otherwise a normalized URL with 'neo' scheme but provided
# ca/cert/key is different than a URL with 'neos' scheme,
# although both point to the same database.
scheme = "neos"
# Explicitly sort query before reassembling into string to insure
# parameter order of input URI doesn't impact normalized URI.
query = urlparse.urlencode(tuple((k, q[k]) for k in sorted(q.keys())), doseq=True)
# Sort multiple master nodes: if a NEO cluster has multiple master # Sort multiple master nodes: if a NEO cluster has multiple master
# nodes, there is no agreed on order in which the master node # nodes, there is no agreed on order in which the master node
# addresses appear in the netloc. In order to insure we always # addresses appear in the netloc. In order to insure we always
# get the same mountpoint among different clients we explicitly # get the same mountpoint among different clients we explicitly
# sort the master node addr order. # sort the master node addr order.
if "@" in netloc:
i = netloc.index("@") + 1
name, netloc = netloc[:i], netloc[i:]
else:
name = ""
netloc = ",".join(sorted(netloc.split(','))) netloc = ",".join(sorted(netloc.split(',')))
netloc = "%s%s" % (name, netloc)
return (scheme, netloc, path, query, frag) return (scheme, netloc, path, query, frag)
_znormalizer('neo', _znormalize_neo) _znormalizer('neo', _znormalize_neo)
_znormalizer('neos', _znormalize_neo) _znormalizer('neos', _znormalize_neo)
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment