qa: fix random RuntimeError when starting cluster in testClusterStartup

neo/master/handlers/administration.py

@@ -64,6 +64,9 @@ class AdministrationHandler(MasterHandler):
             for node in storage_list:
                 assert node.isPending(), node
                 if node.getConnection().isPending():
+                    # XXX: It's wrong to use ProtocolError here. We must reply
+                    #      less aggressively because the admin has no way to
+                    #      know that there's still pending activity.
                     raise ProtocolError('Cannot exit recovery now: node %r is '
                         'entering cluster' % (node, ))
             app._startup_allowed = True

neo/tests/functional/__init__.py

@@ -416,6 +416,18 @@ class NEOCluster(object):
         if not pdb.wait(test, MAX_START_TIME):
             raise AssertionError('Timeout when starting cluster')
 
+    def startCluster(self):
+        # Even if the storage nodes are in the expected state, there may still
+        # be activity between them and the master, preventing the cluster to
+        # start.
+        def start(last_try):
+            try:
+                self.neoctl.startCluster()
+            except (NotReadyException, RuntimeError):
+                return False, e
+            return True, None
+        self.expectCondition(start)
+
     def stop(self, clients=True):
         # Suspend all processes to kill before actually killing them, so that
         # nodes don't log errors because they get disconnected from other nodes:

neo/tests/functional/testCluster.py

@@ -31,7 +31,6 @@ class ClusterTests(NEOFunctionalTest):
     def testClusterStartup(self):
         neo = self.neo = NEOCluster(['test_neo1', 'test_neo2'], replicas=1,
                          temp_dir=self.getTempDirectory())
-        neoctl = neo.neoctl
         neo.run()
         # Runing a new cluster doesn't exit Recovery state.
         s1, s2 = neo.getStorageProcessList()
@@ -40,7 +39,7 @@ class ClusterTests(NEOFunctionalTest):
         neo.expectClusterRecovering()
         # When allowing cluster to exit Recovery, it reaches Running state and
         # all present storage nodes reach running state.
-        neoctl.startCluster()
+        neo.startCluster()
         neo.expectRunning(s1)
         neo.expectRunning(s2)
         neo.expectClusterRunning()
@@ -64,7 +63,7 @@ class ClusterTests(NEOFunctionalTest):
         neo.expectPending(s1)
         neo.expectUnknown(s2)
         neo.expectClusterRecovering()
-        neoctl.startCluster()
+        neo.startCluster()
         neo.expectRunning(s1)
         neo.expectUnknown(s2)
         neo.expectClusterRunning()