client: Add support for zodburi

zodburi[1] provides a way to open ZODB storages by URL/URI. It already
has support for file:// zeo:// zconfig:// memory:// etc schemes out of
the box and third-party-to-ZODB storages can add support for their
schemes via providing zodburi.resolvers entrypoint.

For example relstorage and newtdb do this.

Let's also teach NEO to open itself via neo:// URI schema.

[1] http://docs.pylonsproject.org/projects/zodburi
[2] https://github.com/zodb/relstorage/blob/2.1a1-15-g68c8cf1/relstorage/zodburi_resolver.py
[3] https://github.com/newtdb/db/blob/0.5.2-1-gbd36e90/src/newt/db/zodburi.py

neo/client/zodburi.py

+#
+# Copyright (C) 2017  Nexedi SA
+#
+# This program is free software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License
+# as published by the Free Software Foundation; either version 2
+# of the License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+"""NEO URI resolver for zodburi
+
+URI format:
+
+    neo://name@master1,master2,...,masterN?options
+"""
+
+import ZODB.config
+import ZConfig
+
+from cStringIO import StringIO
+from collections import OrderedDict
+from urlparse import urlsplit, parse_qsl
+
+# neo_zconf_options returns set of zconfig options supported by NEO storage
+def neo_zconf_options():
+    neo_schema = """<schema>
+      <import package="ZODB" />
+      <import package="neo.client" />
+    </schema>"""
+
+    neo_schema = StringIO(neo_schema)
+    neo_schema = ZConfig.loadSchemaFile(neo_schema)
+    neo_storage_zconf = neo_schema.gettype('NeoStorage')
+
+    options = {k for k, _ in neo_storage_zconf}
+    assert 'master_nodes' in options
+    assert 'name' in options
+
+    return options
+
+# canonical_opt_name returns "oPtion_nAme" as "option-name"
+def canonical_opt_name(name):
+    return name.lower().replace('_', '-')
+
+# worker entrypoint for resolve_uri and tests
+def _resolve_uri(uri):
+    scheme, netloc, path, query, frag = urlsplit(uri)
+
+    if scheme != "neo":
+        raise ValueError("invalid uri: %s : expected neo:// scheme" % uri)
+    if path != "":
+        raise ValueError("invalid uri: %s : non-empty path" % uri)
+    if frag != "":
+        raise ValueError("invalid uri: %s : non-empty fragment" % uri)
+
+    # extract master list and name from netloc
+    name, masterloc = netloc.split('@', 1)
+    master_list = masterloc.split(',')
+
+    neokw = OrderedDict()
+    neokw['master_nodes'] = ' '.join(master_list)
+    neokw['name'] = name
+
+    # get options from query: only those that are defined by NEO schema go to
+    # storage - rest are returned as database options
+    dbkw = {}
+    neo_options = neo_zconf_options()
+    for k, v in OrderedDict(parse_qsl(query)).items():
+        if k in neo_options:
+            neokw[k] = v
+        else:
+            # it might be option for storage, but not in canonical form e.g.
+            # read_only -> read-only  (zodburi world settled on using "_" and
+            # ZConfig world on "-" as separators)
+            k2 = canonical_opt_name(k)
+            if k2 in neo_options:
+                neokw[k2] = v
+
+            # else keep this kv as db option
+            else:
+                dbkw[k] = v
+
+
+    # now we have everything. Let ZConfig do actual work for validation options
+    # and borning the storage
+    neozconf = """%import neo.client
+<NEOStorage>
+"""
+    for k, v in neokw.items():
+        neozconf += "  %s\t%s\n" % (k, v)
+
+    neozconf += "</NEOStorage>\n"
+
+    return neozconf, dbkw
+
+# resolve_uri resolves uri according to neo:// schema.
+# see module docstring for uri format.
+def resolve_uri(uri):
+    neozconf, dbkw = _resolve_uri(uri)
+
+    def factory():
+        return ZODB.config.storageFromString(neozconf)
+
+    return factory, dbkw

neo/scripts/runner.py

@@ -66,6 +66,7 @@ UNIT_TEST_MODULES = [
     # client application
     'neo.tests.client.testClientApp',
     'neo.tests.client.testMasterHandler',
+    'neo.tests.client.testZODBURI',
     # light functional tests
     'neo.tests.threaded.test',
     'neo.tests.threaded.testImporter',

neo/tests/client/testZODBURI.py

+#
+# Copyright (C) 2017  Nexedi SA
+#
+# This program is free software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License
+# as published by the Free Software Foundation; either version 2
+# of the License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+import unittest
+from neo.client.zodburi import _resolve_uri
+
+
+testv = [
+    # [] of (uri, zconf_ok, dbkw_ok)
+    ("neo://dbname@master",
+     """\
+  master_nodes\tmaster
+  name\tdbname
+""",
+     {}),
+
+    ("neo://db2@master1:port1,master2:port2,master3:port3",
+     """\
+  master_nodes\tmaster1:port1 master2:port2 master3:port3
+  name\tdb2
+""",
+     {}),
+
+    ("neo://db3@master1,master2:port2?read_only=true",
+     """\
+  master_nodes\tmaster1 master2:port2
+  name\tdb3
+  read-only\ttrue
+""",
+     {}),
+
+    ("neo://db4@[2001:67c:1254:2a::1]:1234,master2:port2?read_only=false"
+     "&compress=true&logfile=xxx&alpha=111&dynamic_master_list=zzz&ca=qqq"
+     "&cert=rrr&key=sss&beta=222",
+     """\
+  master_nodes\t[2001:67c:1254:2a::1]:1234 master2:port2
+  name\tdb4
+  read-only\tfalse
+  compress\ttrue
+  logfile\txxx
+  dynamic_master_list\tzzz
+  ca\tqqq
+  cert\trrr
+  key\tsss
+""",
+     {"alpha": "111", "beta": "222"}),
+]
+
+
+
+class ZODBURITests(unittest.TestCase):
+
+    def test_zodburi(self):
+        # invalid schema / path / fragment
+        self.assertRaises(ValueError, _resolve_uri, "http://db@master")
+        self.assertRaises(ValueError, _resolve_uri, "neo://db@master/path")
+        self.assertRaises(ValueError, _resolve_uri, "neo://db@master#frag")
+
+        # db @ master not fully specified
+        self.assertRaises(ValueError, _resolve_uri, "neo://master")
+
+        # verify zodburi resolver produces expected zconfig
+        for uri, zconf_ok, dbkw_ok in testv:
+            zconf_ok = "%import neo.client\n<NEOStorage>\n" + zconf_ok + \
+                       "</NEOStorage>\n"
+
+            zconf, dbkw = _resolve_uri(uri)
+            self.assertMultiLineEqual(zconf, zconf_ok)
+            self.assertEqual(dbkw, dbkw_ok)
+
+
+if __name__ == '__main__':
+    unittest.main()

setup.py

@@ -84,6 +84,9 @@ setup(
             'neosimple=neo.scripts.simple:main',
             'stat_zodb=neo.tests.stat_zodb:main',
         ],
+        'zodburi.resolvers': [
+            'neo = neo.client.zodburi:resolve_uri [client]',
+        ],
     },
     extras_require = extras_require,
     package_data = {