lib/zodb/zstor_2zurl: Add comprehensive tests

This patch adds comprehensive tests for 'wendelin.lib.zodb.zstor_2zurl'.
Before this patch only one related test existed ('test_zurlstable').
This test only lightly checked correct functionality of 'zstor_2zurl'.
Therefore we added the new tests 'test_zstor_2zurl' and
'test_zurlsamedb'.

The new tests only cover existing functionality.

---

Co-authored-by: kirr

/reviewed-by @kirr
/reviewed-on !13

lib/tests/test_zodb.py

 # Wendelin.core.bigfile | Tests for ZODB utilities and critical properties of ZODB itself
-# Copyright (C) 2014-2021  Nexedi SA and Contributors.
+# Copyright (C) 2014-2022  Nexedi SA and Contributors.
 #                          Kirill Smelkov <kirr@nexedi.com>
 #
 # This program is free software: you can Use, Study, Modify and Redistribute
@@ -17,13 +17,14 @@
 #
 # See COPYING file for full licensing terms.
 # See https://www.nexedi.com/licensing for rationale and options.
-from wendelin.lib.zodb import LivePersistent, deactivate_btree, dbclose, zconn_at, zstor_2zurl, zmajor, _zhasNXDPatch
+from wendelin.lib.zodb import LivePersistent, deactivate_btree, dbclose, zconn_at, zstor_2zurl, zmajor, _zhasNXDPatch, dbstoropen
 from wendelin.lib.testing import getTestDB
 from wendelin.lib import testing
 from persistent import Persistent, UPTODATE, GHOST, CHANGED
 from ZODB import DB, POSException
 from ZODB.FileStorage import FileStorage
 from ZODB.MappingStorage import MappingStorage
+from ZODB.DemoStorage import DemoStorage
 from BTrees.IOBTree import IOBTree
 import transaction
 from transaction import TransactionManager
@@ -31,6 +32,9 @@ from golang import defer, func
 from pytest import raises
 import pytest; xfail = pytest.mark.xfail
 from ZEO.ClientStorage import ClientStorage as ZEOStorage
+from neo.client.Storage import Storage as NEOStorage
+import os
+from six.moves.urllib.parse import quote_plus
 
 from wendelin.lib.tests.testprog import zopenrace, zloadrace
 
@@ -407,6 +411,142 @@ def test_zurlstable():
             assert zurl == zurl0
 
 
+# test that ZODB database opened via storage's zurl, provides access to the same data.
+@func
+def test_zurlsamedb():
+    stor1 = testdb.getZODBStorage()
+    defer(stor1.close)
+
+    # skip on FileStorage - ZODB/py fails with LockError on attempt to create
+    # two FileStorage's connected to the same data.
+    if isinstance(stor1, FileStorage):
+        pytest.skip("skipping on FileStorage")
+
+    zurl = zstor_2zurl(stor1)
+    stor2 = dbstoropen(zurl)
+    defer(stor2.close)
+
+    db1 = DB(stor1)
+    db2 = DB(stor2)
+
+    # get/set retrieves or sets root['X'] = x.
+    @func
+    def set(db, x):
+        conn = db.open();   defer(conn.close)
+        root = conn.root()
+        root['X'] = x
+        transaction.commit()
+    @func
+    def get(db):
+        zsync(db.storage)
+        conn = db.open();   defer(conn.close)
+        root = conn.root()
+        return root['X']
+
+    # stor1/stor2 should have the same data
+    set(db1, 1)
+    assert get(db2) == 1
+    set(db1, 'abc')
+    assert get(db2) == 'abc'
+
+
+# ensure zstor_2zurl returns expected zurl.
+def test_zstor_2zurl(tmpdir, neo_ssl_dict):
+    # fs1 returns new FileStorage located in tmpdir.
+    def fs1(name):
+        return FileStorage("%s/%s" % (tmpdir, name))
+
+    # zeo returns new ZEO client for specified storage name and server address.
+    #
+    # server_addr can be either:
+    # - str (specifying address of UNIX socket), or
+    # - (host, addr) pair - specifying TCP address.
+    #
+    # NOTE the client is returned without waiting until server is connected.
+    def zeo(storage_name, server_addr):
+        if testing.TestDB_ZEO('').z5:
+            return ZEOStorage(server_addr, storage=storage_name, wait=False)
+
+        # It's better to use a mock storage for zeo == 4, because
+        # we would have to wait for a long time. See here the
+        # respective part in ZEO4 source code:
+        #
+        #   https://github.com/zopefoundation/ZEO/blob/4/src/ZEO/ClientStorage.py#L423-L430
+        #
+        # ..compared to ZEO5 which omits the else clause:
+        #
+        #   https://github.com/zopefoundation/ZEO/blob/5.3.0/src/ZEO/ClientStorage.py#L279-L286
+        zeo_storage = type(
+            "ClientStorage",
+            (object,),
+            {
+                "_addr": server_addr,
+                "_storage": storage_name,
+                "close": lambda self: None,
+                "getName": lambda self: self._storage
+            }
+        )()
+        type(zeo_storage).__module__ = "ZEO.ClientStorage"
+        type(zeo_storage).__name__ = "ClientStorage"
+        return zeo_storage
+
+    # neo returns new NEO client for specified cluster name and master address.
+    # NOTE, similarly to ZEO, the client is returned without waiting until server nodes are connected.
+    def neo(cluster_name, master_addr, ssl=0):
+        kwargs = dict(master_nodes=master_addr, name=cluster_name)
+        if ssl:
+            kwargs.update(neo_ssl_dict)
+        return NEOStorage(**kwargs)
+
+    # demo returns new DemoStorage with specified base and delta.
+    def demo(base, delta):
+        return DemoStorage(base=base, changes=delta)
+
+    # assert_zurl_is_correct verifies that zstor_2zurl(zstor) returns zurl_ok.
+    # zstor is closed after this test.
+    @func
+    def assert_zurl_is_correct(zstor, zurl_ok):
+        defer(zstor.close)
+        assert zstor_2zurl(zstor) == zurl_ok
+
+    # sslp is the ssl encryption uri part of an encrypted NEO node
+    q = quote_plus
+    sslp = ";".join(("%s=%s" % (q(k), q(v)) for k, v in sorted(neo_ssl_dict.items())))
+
+    _ = assert_zurl_is_correct
+    _(fs1("test.fs"),                       "file://%s/test.fs" % tmpdir)           # FileStorage
+    _(zeo("1",     "/path/to/zeo.sock"),    "zeo:///path/to/zeo.sock")              # ZEO/unix
+    _(zeo("test",  "/path/to/zeo.sock"),    "zeo:///path/to/zeo.sock?storage=test") #   + non-default storage name
+    _(zeo("1",     ("127.0.0.1", 1234)),    "zeo://127.0.0.1:1234")                 # ZEO/ip4
+    _(zeo("test",  ("127.0.0.1", 1234)),    "zeo://127.0.0.1:1234?storage=test")    #   + non-default storage name
+    _(neo("test",  "127.0.0.1:1234"),       "neo://127.0.0.1:1234/test")            # NEO/ip4
+    _(neo("test",  "127.0.0.1:1234", 1),    "neos://%s@127.0.0.1:1234/test" % sslp) #   + ssl
+    _(demo(zeo("base", ("1.2.3.4",  5)),                                            # DemoStorage
+           fs1("delta.fs")),                "demo:(zeo://1.2.3.4:5?storage=base)/(file://%s/delta.fs)" % tmpdir)
+
+    # Test exceptions
+    #   invalid storage
+    with raises(ValueError, match="in-RAM storages are not supported"):
+        zstor_2zurl(MappingStorage())
+
+    #   invalid object
+    with raises(NotImplementedError):
+        zstor_2zurl("I am not a storage.")
+
+
+# neo_ssl_dict returns the path of precomputed static ssl certificate
+# files.
+@pytest.fixture
+def neo_ssl_dict():
+    ssl_files_base_path = "%s%s%s%s" % (
+      os.path.dirname(__file__), os.sep, "testdata", os.sep
+    )
+    return {
+        k: "%s%s" % (ssl_files_base_path, v)
+        for k, v in dict(ca="ca.crt", key="node.key", cert="node.crt").items()
+    }
+
+
 # ---- tests for critical properties of ZODB ----
 
 # verify race in between Connection.open and invalidations.

lib/tests/testdata/ca.crt

+-----BEGIN CERTIFICATE-----
+MIIC/TCCAeWgAwIBAgIUOkZ7N15eWCwcDK3XGsGrFZFXNT8wDQYJKoZIhvcNAQEL
+BQAwDTELMAkGA1UEAwwCQ0EwIBcNMjIxMTE2MTM0OTQ4WhgPMjI5NjA4MzExMzQ5
+NDhaMA0xCzAJBgNVBAMMAkNBMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKC
+AQEAofbtYCkTLsptpaol4DqjNEz7er3INVp255T7j6pth1Z12255KlLUojTJBYqD
+doeek3XVGY+mZ4Hxt00pyzUIbpKw7HEltkvINdBPBIxO8q2fM5fkSOUkfFQkkpxo
+O7rQLMFCFZ3qiDsZ7sZp16nW3wfwlR9lsHPwH5buMxRnJ9KvltdEHmNcsfPH3/x6
+gbxIpKdpOWQXtWb4CoFpgYIPmn6SE/hpHWV0LBkXEv/l4Pp7LnYyts1F2uO32SLT
+zJ8U1Q7JQe6C0BejoO2g9y5xtHLKjmwE9mWu8iD9qH4MYjU9+O/Z1Djnb1E+Jvjb
+zahvdNmTwxfFF62FaR66f8EBawIDAQABo1MwUTAdBgNVHQ4EFgQUQCW5f7bArbvZ
+SQsNeYPl89xKeWMwHwYDVR0jBBgwFoAUQCW5f7bArbvZSQsNeYPl89xKeWMwDwYD
+VR0TAQH/BAUwAwEB/zANBgkqhkiG9w0BAQsFAAOCAQEAn9Ei+DAOc92Iau3XtEcn
+wLvVB4+KYG75TzwBWFDn0o+ycBJ1Ie8IyK/pEe+Pw6SvsSKEMpzH+R4h3vcbDMTP
+bJaaL4GPosfZCpMkCD3RprvHCW+wJYTWjjEKBswOTLabSAyMNvyT+51qfp6Hlr3M
+Dbl/qwHvSmGehVH/EASQwOnHAO037hbUYilra24TW+qTQSK9A0x3395QZUIHVUVG
+6YrSUdeYzZRUjYyEOjxMkkr0WDF5O9+QDETYhZjtmD/OE6O2Tuqayf3y15yApynq
+KMi6Eti47GUXmvfZDTyIosRNtVp8SUq1yCFV+641iEb9GakUkmc9G8d6/WhI6kmw
+nQ==
+-----END CERTIFICATE-----

lib/tests/testdata/create-certs

+#!/bin/sh -e
+# Same as the one used in NEO/py tests:
+# 	https://lab.nexedi.com/nexedi/neoppod/blob/bff5c82f/neo/tests/create-certs
+DAYS=100000  # = ~270 years
+
+at_exit() { rm -f "$CAkey"; }
+trap at_exit 0
+CAkey=`mktemp`
+openssl req -new -x509 -nodes -keyout "$CAkey" -out ca.crt -subj /CN=CA -days $DAYS
+openssl req -new -nodes -keyout node.key -subj /CN=node |
+openssl x509 -CA ca.crt -CAkey "$CAkey" -req -out node.crt -set_serial 1 -days $DAYS

lib/tests/testdata/node.crt

+-----BEGIN CERTIFICATE-----
+MIICkjCCAXoCAQEwDQYJKoZIhvcNAQELBQAwDTELMAkGA1UEAwwCQ0EwIBcNMjIx
+MTE2MTM0OTQ4WhgPMjI5NjA4MzExMzQ5NDhaMA8xDTALBgNVBAMMBG5vZGUwggEi
+MA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQCrewmDGrWefg+4GOjqZhkrQ0ht
+Kxhh0wZhuJmpcjayc8TE+BbJJ64vOhdmGO5dZ3Ssj4EJkxaX0zG573xn6iJAmRJv
+WCo/ufFzCqMeOXc0JejpZD47sNEd5A9pOe8MBeT8HEt08st7Mh+HNE9SPUePo1y8
+31FRX2DTj0xLxtK+KLSWsrhEu7jotR0XY0K1YYj47Xu8EJmOdmdK7bRosVnoVPa3
+naaKlUt1k2wgzEMOzGsHDPIXLo3Xpe9o1tHSM7phG7prZLWv6r1FUwbHSQVzMQXi
+HsBpHkgMkoH7YN2MtHrjulzYjjtzYOjqvWP+MIUUXTo0PqpiPoeVlxCjVHFxAgMB
+AAEwDQYJKoZIhvcNAQELBQADggEBABdzqOXg0L+dUmFklg6zQlbq8Ca7q6OBpe50
+aV0b6lQQ1euVcPWoBmxC0giTPsBrWCcn0jpCmBEZ3xUerM7Qih1POyKrM8uQ5m3T
+3eifgDE7krwzRDNqsMPh1AC7PpABbRCHXRxdqe4tLjMmI8jXNtsDTuNwjnXSZD5W
+WJjLQ19WXy8V6ffvwjVSwmnu83qY4fQ9vmgG+0liOyXx4tlaH6NrZUMGuPUWJ5Qz
+n6UBeegOi1HhovjBdXhTZgVRCB2TCwNaGA7EiDuAKFglKlw4SBoLSCnV1PJdc/u6
+SBdbRDKBm4zAOfYHs5h86+3+Okj/VIhZlQ0oPofCRxi2fWG0El8=
+-----END CERTIFICATE-----

lib/tests/testdata/node.key

+-----BEGIN PRIVATE KEY-----
+MIIEvAIBADANBgkqhkiG9w0BAQEFAASCBKYwggSiAgEAAoIBAQCrewmDGrWefg+4
+GOjqZhkrQ0htKxhh0wZhuJmpcjayc8TE+BbJJ64vOhdmGO5dZ3Ssj4EJkxaX0zG5
+73xn6iJAmRJvWCo/ufFzCqMeOXc0JejpZD47sNEd5A9pOe8MBeT8HEt08st7Mh+H
+NE9SPUePo1y831FRX2DTj0xLxtK+KLSWsrhEu7jotR0XY0K1YYj47Xu8EJmOdmdK
+7bRosVnoVPa3naaKlUt1k2wgzEMOzGsHDPIXLo3Xpe9o1tHSM7phG7prZLWv6r1F
+UwbHSQVzMQXiHsBpHkgMkoH7YN2MtHrjulzYjjtzYOjqvWP+MIUUXTo0PqpiPoeV
+lxCjVHFxAgMBAAECggEAINhzTPpruUz6I9AZpPQjdAbyM37ir7Ik8cjTHgKzeWD1
+WqkObD8+fOtKmhLCbX+oeHkQRU44MKKSTsQRdzIAl5D6V/tyNDygg+GQtvRY4dM1
+FqdNyE8o7unFDem74PisEKI2zdLHqPRD17jsTJtSPrASzlaUE9sKB9vVyFnv+d4v
+xaTXQzPm0SOc+gNDiyUyEo8yYqESf1Ypz8Lq0iHBQzXd2gPBWwKjjm69xeA7RjLV
+H7WoYOPdXI1OcIDr/FmSkD3Di1vo9Ar5FwB7t99Y9rwkdJvgAIqqttUIO0qJ35RP
+0IiFSGrGfU9Gk8/bSNRrdWF1keAAstI9Vy8LtWyDlQKBgQDbzQ4br+PK/wzGsgt5
+niJ2K7mD2JAuNk3QidAewhxCjG1IxvE4NaqPLWG7A9Bya+1wzPGEgV9q7eu8j8s3
+UsFyaYd5EkFj8uejdhR8HLAfiem66lVSYFFMDSO3w7SeEXognvQzkBlON32q9HRs
+eg+kW8jWzriv77kdV0SVSWxX4wKBgQDHuMPbP/ZV8HXSAjm2z1LfAcrcdoJf3/dL
+cU0CD80f55PhlDehMq0irS1DI4Jt++L3zzqmFaF9kvrjF7UQU3ypxz1iGQLtkPat
+rv2Nnb98afBpmqSQ9cKJpB2fUsKYjtKFZ3Tlk+lgK46m58bAj9xZhBafU4uDftcR
+QgFrdH7JmwKBgCde7IcDE395/fTdUXbgX3XhUSKPuQu+CGQxO0k3Jc65zp15nR2f
+Z0SsMVRJU54bPPW2kBBXCBtkMJSfARyQwzvWMUD7hyD7EojSDstIbDfKAUbM0Kmn
+vQgtVleFQ6YIpWKQdleCGTZMdEMZ9Hy1xHWBR1FKyAQjoXni9tLyyuhXAoGAYuV2
+9mZh0IRVkzxO39shqhP1I937CQaaKd1cyJySci316EnAki3J1dXzB7/ZoiSMazFO
+VVd4s/nIChN/MSpmHJvwJVdFP+X8gFyaaTmnzC+ZQHG286D+R7HgiTgt6+UtEan9
+FIF9moV3CebTYB7ceecSXKq99sskLAOHOHJjJekCgYBM+rvFRWF+pfXwySnP97Me
++377ViGsKf3tGy1tXpuNhc1AC/iyIUelrhOcWrwkMK2kYnT6VWqqpg8rVJMolHLE
+UJRDGrmEW31HUjd+YjFIa9HrWmUP3JToct0YE+E4wHyRVaL2LDnjHO8lKHUS0nQT
+1cNYtzNncApsNw4rdOg6Sg==
+-----END PRIVATE KEY-----

setup.py

@@ -397,6 +397,7 @@ setup(
                    'test': [
                      'pytest',
                      'scipy',
+                     'neoppod',     # lib/tests/test_zodb.py
                      'ZEO',         # lib/tests/test_zodb.py
                     ],
     },