WIP: Make admin node a web-app

The goal is to get rid off the neoctl command-line tool, and to manage the
cluster via a web browser, or tools like 'wget'. Then, it will be possible to
provide an web user interface to connect to the underlying DB of any storage
node, usually a SQL client.

The design of admin app is finished:
- it's threaded like clients
- it's a WSGI app

I also hacked a HTTP API as quickly as possible to make all tests pass.

TODO:
- SSL
- define a better HTTP API
- there's no UI at all yet
- remove all unused packets from the protocol (those that were only used
  between neoctl and admin node)

There are a few dead files, not deleted yet, in case that they contain a few
pieces of useful code:
 neo/neoctl/app.py
 neo/neoctl/handler.py
 neo/scripts/neoctl.py

neo/admin/handler.py

@@ -14,102 +14,25 @@
 # You should have received a copy of the GNU General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
-from neo.lib import logging, protocol
-from neo.lib.handler import EventHandler
-from neo.lib.protocol import uuid_str, Packets
+from bottle import HTTPError
+from neo.lib.handler import AnswerBaseHandler, EventHandler, MTEventHandler
 from neo.lib.pt import PartitionTable
 from neo.lib.exception import PrimaryFailure
 
-def check_primary_master(func):
-    def wrapper(self, *args, **kw):
-        if self.app.master_conn is not None:
-            return func(self, *args, **kw)
-        raise protocol.NotReadyError('Not connected to a primary master.')
-    return wrapper
-
-def forward_ask(klass):
-    return check_primary_master(lambda self, conn, *args, **kw:
-        self.app.master_conn.ask(klass(*args, **kw),
-                                 conn=conn, msg_id=conn.getPeerId()))
-
-class AdminEventHandler(EventHandler):
-    """This class deals with events for administrating cluster."""
-
-    @check_primary_master
-    def askPartitionList(self, conn, min_offset, max_offset, uuid):
-        logging.info("ask partition list from %s to %s for %s",
-                     min_offset, max_offset, uuid_str(uuid))
-        self.app.sendPartitionTable(conn, min_offset, max_offset, uuid)
-
-    @check_primary_master
-    def askNodeList(self, conn, node_type):
-        if node_type is None:
-            node_type = 'all'
-            node_filter = None
-        else:
-            node_filter = lambda n: n.getType() is node_type
-        logging.info("ask list of %s nodes", node_type)
-        node_list = self.app.nm.getList(node_filter)
-        node_information_list = [node.asTuple() for node in node_list ]
-        p = Packets.AnswerNodeList(node_information_list)
-        conn.answer(p)
-
-    @check_primary_master
-    def askClusterState(self, conn):
-        conn.answer(Packets.AnswerClusterState(self.app.cluster_state))
-
-    @check_primary_master
-    def askPrimary(self, conn):
-        master_node = self.app.master_node
-        conn.answer(Packets.AnswerPrimary(master_node.getUUID()))
-
-    @check_primary_master
-    def flushLog(self, conn):
-        self.app.master_conn.send(Packets.FlushLog())
-        super(AdminEventHandler, self).flushLog(conn)
-
-    askLastIDs = forward_ask(Packets.AskLastIDs)
-    askLastTransaction = forward_ask(Packets.AskLastTransaction)
-    addPendingNodes = forward_ask(Packets.AddPendingNodes)
-    askRecovery = forward_ask(Packets.AskRecovery)
-    tweakPartitionTable = forward_ask(Packets.TweakPartitionTable)
-    setClusterState = forward_ask(Packets.SetClusterState)
-    setNodeState = forward_ask(Packets.SetNodeState)
-    setNumReplicas = forward_ask(Packets.SetNumReplicas)
-    checkReplicas = forward_ask(Packets.CheckReplicas)
-    truncate = forward_ask(Packets.Truncate)
-    repair = forward_ask(Packets.Repair)
-
-
-class MasterEventHandler(EventHandler):
-    """ This class is just used to dispatch message to right handler"""
-
-    def _connectionLost(self, conn):
-        app = self.app
-        if app.listening_conn: # if running
-            assert app.master_conn in (conn, None)
-            conn.cancelRequests("connection to master lost")
-            app.reset()
-            app.uuid = None
-            raise PrimaryFailure
+class MasterBootstrapHandler(EventHandler):
 
     def connectionFailed(self, conn):
-        self._connectionLost(conn)
+        raise AssertionError
 
     def connectionClosed(self, conn):
-        self._connectionLost(conn)
-
-    def dispatch(self, conn, packet, kw={}):
-        if 'conn' in kw:
-            # expected answer
-            if packet.isResponse():
-                packet.setId(kw['msg_id'])
-                kw['conn'].answer(packet)
-            else:
-                self.app.request_handler.dispatch(conn, packet, kw)
-        else:
-            # unexpected answers and notifications
-            super(MasterEventHandler, self).dispatch(conn, packet, kw)
+        app = self.app
+        try:
+            app.__dict__.pop('pt').clear()
+        except KeyError:
+            pass
+        if app.master_conn is not None:
+            assert app.master_conn is conn
+            raise PrimaryFailure
 
     def answerClusterState(self, conn, state):
         self.app.cluster_state = state
@@ -124,7 +47,26 @@ class MasterEventHandler(EventHandler):
     def notifyClusterInformation(self, conn, cluster_state):
         self.app.cluster_state = cluster_state
 
+class MasterEventHandler(MasterBootstrapHandler, MTEventHandler):
+
+    pass
 
-class MasterRequestEventHandler(EventHandler):
+class PrimaryAnswersHandler(AnswerBaseHandler):
     """ This class handle all answer from primary master node"""
-    # XXX: to be deleted ?
+
+    def ack(self, conn, message):
+        super(PrimaryAnswersHandler, self).ack(conn, message)
+        self.app.setHandlerData(message)
+
+    def denied(self, conn, message):
+        raise HTTPError(405, message)
+
+    def protocolError(self, conn, message):
+        raise HTTPError(500, message)
+
+    def answerClusterState(self, conn, state):
+        self.app.cluster_state = state
+
+    answerRecovery = \
+    answerTweakPartitionTable = \
+        lambda self, conn, *args: self.app.setHandlerData(args)

neo/lib/connection.py

@@ -607,6 +607,14 @@ class ClientConnection(Connection):
         handler.connectionStarted(self)
         self._connect()
 
+    def convertToMT(self, dispatcher):
+        # XXX: The bootstrap code of the client should at least been moved to
+        #      threaded_app so that the admin can reuse it. Ideally, we'd like
+        #      to also merge with BootstrapManager.
+        assert self.__class__ is ClientConnection, self
+        self.__class__ = MTClientConnection
+        self._initMT(dispatcher)
+
     def _connect(self):
         try:
             connected = self.connector.makeClientConnection()
@@ -701,11 +709,14 @@ class MTClientConnection(ClientConnection):
                 return func(*args, **kw)
         return wrapper
 
-    def __init__(self, *args, **kwargs):
-        self.lock = lock = RLock()
-        self.dispatcher = kwargs.pop('dispatcher')
-        with lock:
-            super(MTClientConnection, self).__init__(*args, **kwargs)
+    def __init__(self, *args, **kw):
+        self._initMT(kw.pop('dispatcher'))
+        with self.lock:
+            super(MTClientConnection, self).__init__(*args, **kw)
+
+    def _initMT(self, dispatcher):
+        self.lock = RLock()
+        self.dispatcher = dispatcher
 
     # Alias without lock (cheaper than super())
     _ask = ClientConnection.ask.__func__

neo/lib/threaded_app.py

@@ -72,7 +72,9 @@ class ThreadedApplication(BaseApplication):
             logging.debug('Stopping %s', self.poll_thread)
             self.em.wakeup(thread.exit)
         else:
-            super(ThreadedApplication, self).close()
+            self._close()
+
+    _close = BaseApplication.close.__func__
 
     def start(self):
         self.poll_thread.is_alive() or self.poll_thread.start()
@@ -82,7 +84,7 @@ class ThreadedApplication(BaseApplication):
         try:
             self._run()
         finally:
-            super(ThreadedApplication, self).close()
+            self._close()
             logging.debug("Poll thread stopped")
 
     def _run(self):
@@ -135,8 +137,15 @@ class ThreadedApplication(BaseApplication):
             handler.dispatch(conn, packet, kw)
 
     def _ask(self, conn, packet, handler=None, **kw):
-        self.setHandlerData(None)
-        queue = self._thread_container.queue
+        # The following line is more than optimization. If an admin node sends
+        # a packet that causes the master to disconnect (e.g. stop a cluster),
+        # we want at least to return the answer for this request, even if the
+        # polling thread already exited and cleared self.__dict__: returning
+        # the result of getHandlerData() would raise an AttributeError.
+        # This is tested by testShutdown (neo.tests.threaded.test.Test).
+        thread_container = self._thread_container
+        thread_container.answer = None
+        queue = thread_container.queue
         msg_id = conn.ask(packet, queue=queue, **kw)
         get = queue.get
         _handlePacket = self._handlePacket
@@ -144,6 +153,5 @@ class ThreadedApplication(BaseApplication):
             qconn, qpacket, kw = get(True)
             if conn is qconn and msg_id == qpacket.getId():
                 _handlePacket(qconn, qpacket, kw, handler)
-                break
+                return thread_container.answer # see above comment
             _handlePacket(qconn, qpacket, kw)
-        return self.getHandlerData()

neo/scripts/neoadmin.py

@@ -19,6 +19,7 @@
 
 from neo.lib import logging
 
+
 def main(args=None):
     from neo.admin.app import Application
     config = Application.option_parser.parse(args)
@@ -28,5 +29,5 @@ def main(args=None):
 
     # and then, load and run the application
     app = Application(config)
-    app.run()
+    app.serve()
 

neo/tests/functional/__init__.py

@@ -702,14 +702,16 @@ class NEOCluster(object):
     def expectStorageUnknown(self, process, *args, **kw):
         process_uuid = process.getUUID()
         def expected_storage_not_known(last_try):
+            try:
                 for storage in self.getStorageList():
                     if storage[2] == process_uuid:
                         return False, storage
+            except NotReadyException:
+                return False, None
             return True, None
         self.expectCondition(expected_storage_not_known, *args, **kw)
 
     def __del__(self):
-        self.neoctl.close()
         if self.cleanup_on_delete:
             os.removedirs(self.temp_dir)
 

neo/tests/threaded/test.py

@@ -852,9 +852,8 @@ class Test(NEOThreadedTest):
             t1.commit()
         self.assertRaises(ConnectionClosed, t2.join)
         # all nodes except clients should exit
-        cluster.join(cluster.master_list
-                   + cluster.storage_list
-                   + cluster.admin_list)
+        cluster.join(cluster.master_list + cluster.storage_list,
+                     (cluster.admin,))
         cluster.stop() # stop and reopen DB to check partition tables
         cluster.start()
         pt = cluster.admin.pt
@@ -2753,7 +2752,7 @@ class Test(NEOThreadedTest):
     @with_cluster(start_cluster=0, master_count=2)
     def testIdentifyUnknownMaster(self, cluster):
         m0, m1 = cluster.master_list
-        cluster.master_nodes = ()
+        with Patch(cluster, master_nodes=()):
             m0.resetNode()
         cluster.start(master_list=(m0,))
         m1.start()

setup.py

@@ -47,7 +47,7 @@ get3rdParty(x, '3rdparty/' + x, 'https://lab.nexedi.com/nexedi/erp5'
 zodb_require = ['ZODB3>=3.10dev']
 
 extras_require = {
-    'admin': [],
+    'admin': ['bottle'],
     'client': zodb_require,
     'ctl': [],
     'master': [],