Initial support for NEO storage in unit tests (--neostorage option)

This only implements an equivalent of DemoStorage:
- use 'BTree' as backend storage (entirely in RAM)
- not yet compatible with --load/--save

git-svn-id: https://svn.erp5.org/repos/public/erp5/trunk@45695 20353a03-c40f-0410-a6d1-a30d3c3de9de

product/ERP5Type/tests/custom_zodb.py

@@ -20,6 +20,7 @@ instance_home = os.environ['INSTANCE_HOME']
 zserver_list = os.environ.get('zserver', '').split(',')
 os.environ['zserver'] = zserver_list[0]
 
+neo_storage = 'neo_storage' in os.environ
 zeo_client = os.environ.get('zeo_client')
 if zeo_client:
   zeo_client = zeo_client.rsplit(':', 1)
@@ -28,7 +29,8 @@ if zeo_client:
 try:
   activity_node = int(os.environ['activity_node'])
 except KeyError:
-  activity_node = (zeo_client or 'zeo_server' in os.environ) and 1 or None
+  activity_node = (zeo_client or neo_storage or 'zeo_server' in os.environ
+                  ) and 1 or None
 
 data_fs_path = os.environ.get('erp5_tests_data_fs_path',
                               os.path.join(instance_home, 'var', 'Data.fs'))
@@ -88,7 +90,7 @@ for static_dir in static_dir_list:
     os.mkdir(static_dir)
 
 zeo_server_pid = None
-zeo_client_pid_list = []
+node_pid_list = []
 
 ZEvent = sys.modules.get('ZServer.PubCore.ZEvent')
 def fork():
@@ -103,43 +105,61 @@ def fork():
     instance_random.seed(instance_random.random())
   return pid
 
-while not zeo_client:
-  if activity_node:
-    r, zeo_client = os.pipe()
-    zeo_server_pid = fork()
-    if zeo_server_pid:
-      save_mysql = None
-      os.close(zeo_client)
-      zeo_client = eval(os.fdopen(r).read())
-      continue
-    else:
-      zeo_client_pid_list = activity_node = None
-      os.close(r)
-      signal.signal(signal.SIGINT, signal.SIG_IGN)
-  elif activity_node is not None:
-    # run ZEO server but no need to fork
-    zeo_server_pid = 0
-
-  if save:
-    Storage = FileStorage(data_fs_path)
-  elif load:
-    Storage = FileStorage(data_fs_path, read_only=True)
-    Storage._is_read_only = False # XXX for Zope 2.8
-    Storage = DemoStorage(base=Storage)
-  else:
-    Storage = DemoStorage()
-  break
-else:
+def forkNodes():
+  global node_pid_list
   for i in xrange(1, activity_node):
     pid = fork()
     if not pid:
-      zeo_client_pid_list = None
+      node_pid_list = None
       os.environ['zserver'] = i < len(zserver_list) and zserver_list[i] or ''
       break
-    zeo_client_pid_list.append(pid)
-  # Zope 2.12: do not import ClientStorage before forking due to client trigger
-  from ZEO.ClientStorage import ClientStorage
-  Storage = ClientStorage(zeo_client)
+    node_pid_list.append(pid)
+
+if neo_storage:
+  if load or save or zeo_client:
+    raise Exception("--neo_storage conflicts with --load/save/zeo_client")
+  from neo.tests.functional import NEOCluster
+  neo_cluster = NEOCluster(range(2), partitions=4, adapter='BTree',
+                           temp_dir=os.getcwd(), verbose=False)
+  sigint = signal.signal(signal.SIGINT, signal.SIG_IGN)
+  neo_cluster.start()
+  signal.signal(signal.SIGINT, sigint)
+  forkNodes()
+  Storage = neo_cluster.getZODBStorage()
+else:
+  neo_cluster = None
+  while not zeo_client:
+    if activity_node:
+      r, zeo_client = os.pipe()
+      zeo_server_pid = fork()
+      if zeo_server_pid:
+        save_mysql = None
+        os.close(zeo_client)
+        zeo_client = eval(os.fdopen(r).read())
+        continue
+      else:
+        node_pid_list = activity_node = None
+        os.close(r)
+        signal.signal(signal.SIGINT, signal.SIG_IGN)
+    elif activity_node is not None:
+      # run ZEO server but no need to fork
+      zeo_server_pid = 0
+
+    if save:
+      Storage = FileStorage(data_fs_path)
+    elif load:
+      Storage = FileStorage(data_fs_path, read_only=True)
+      Storage._is_read_only = False # XXX for Zope 2.8
+      Storage = DemoStorage(base=Storage)
+    else:
+      Storage = DemoStorage()
+    break
+  else:
+    forkNodes()
+    # Zope 2.12: do not import ClientStorage
+    # before forking due to client trigger
+    from ZEO.ClientStorage import ClientStorage
+    Storage = ClientStorage(zeo_client)
 
-if zeo_client_pid_list is not None:
+if node_pid_list is not None:
   _print("Instance at %r loaded ... " % instance_home)

product/ERP5Type/tests/runUnitTest.py

@@ -128,6 +128,8 @@ Options:
                              Make ZServer listen on given host:port
                              If used with --activity_node=, this can be a
                              comma-separated list of addresses.
+  --neo_storage              Use a volatile NEO storage instead of a DemoStorage
+                             (not compatible with --save or --load).
   --products_path=path,path  Comma-separated list of products paths locations
                              which shall be used in test environment.
   --sys_path=path,path       Comma-separated list of paths which will be used to
@@ -486,12 +488,12 @@ def runUnitTestList(test_list, verbosity=1, debug=0, run_only=None):
 
   import Lifetime
   from ZEO.ClientStorage import ClientStorage
-  from Zope2.custom_zodb import \
-      save_mysql, zeo_server_pid, zeo_client_pid_list, Storage
+  from Zope2.custom_zodb import Storage, save_mysql, \
+      node_pid_list, neo_cluster, zeo_server_pid
   def shutdown(signum, frame, signum_set=set()):
     Lifetime.shutdown(0)
     signum_set.add(signum)
-    if zeo_client_pid_list is None and len(signum_set) > 1:
+    if node_pid_list is None and len(signum_set) > 1:
       # in case of ^C, a child should also receive a SIGHUP from the parent,
       # so we merge the first 2 different signals in a single exception
       signum_set.remove(signal.SIGHUP)
@@ -508,7 +510,7 @@ def runUnitTestList(test_list, verbosity=1, debug=0, run_only=None):
                       or not load)
     if zeo_server_pid == 0:
       suite = ZEOServerTestCase('asyncore_loop')
-    elif zeo_client_pid_list is None or not test_list:
+    elif node_pid_list is None or not test_list:
       suite = ProcessingNodeTestCase('processing_node')
       if not (dummy or load):
         _print('WARNING: either --save or --load should be used because static'
@@ -538,7 +540,7 @@ def runUnitTestList(test_list, verbosity=1, debug=0, run_only=None):
       if run_only:
         ERP5TypeTestLoader.filter_test_list = None
 
-    if zeo_client_pid_list is None:
+    if node_pid_list is None:
       result = suite()
     else:
       if not test_list:
@@ -548,16 +550,18 @@ def runUnitTestList(test_list, verbosity=1, debug=0, run_only=None):
   finally:
     ProcessingNodeTestCase.unregisterNode()
     Storage.close()
-    if zeo_client_pid_list is not None:
+    if node_pid_list is not None:
       # Wait that child processes exit. Stop ZEO storage (if any) after all
       # other nodes disconnected.
-      for pid in zeo_client_pid_list:
+      for pid in node_pid_list:
         os.kill(pid, signal.SIGHUP)
-      for pid in zeo_client_pid_list:
+      for pid in node_pid_list:
         os.waitpid(pid, 0)
       if zeo_server_pid:
         os.kill(zeo_server_pid, signal.SIGHUP)
         os.waitpid(zeo_server_pid, 0)
+      if neo_cluster:
+        neo_cluster.stop()
 
   if save:
     os.chdir(instance_home)
@@ -584,7 +588,7 @@ def runUnitTestList(test_list, verbosity=1, debug=0, run_only=None):
           if e.errno != errno.ENOENT:
             raise
         os.rename(static_dir, backup_path)
-    elif zeo_client_pid_list is not None:
+    elif node_pid_list is not None:
       _print('WARNING: No static files saved. You will have to do it manually.')
 
   return result
@@ -633,6 +637,7 @@ def main(argument_list=None):
         "zeo_client=",
         "zeo_server=",
         "zserver=",
+        "neo_storage",
         "products_path=",
         "sys_path=",
         "instance_home=",
@@ -732,6 +737,8 @@ def main(argument_list=None):
       os.environ["zeo_server"] = arg
     elif opt == "--zserver":
       os.environ["zserver"] = arg
+    elif opt == "--neo_storage":
+      os.environ["neo_storage"] = ""
     elif opt == "--products_path":
       os.environ["PRODUCTS_PATH"] = arg
     elif opt == "--sys_path":