Add a configuration manager that load options from :

* Default values
* Command line
* Configuration file
Restore the sample configuration file, updated with new options.
Tests updated.


git-svn-id: https://svn.erp5.org/repos/neo/trunk@1364 71dcc9de-d417-0410-9af5-da40c76e7ee4

neo.conf

+# Note: Unless otherwise noted, all parameters in this configuration file
+# must be identical for all nodes in a given cluster.
+
+# Default parameters.
+[DEFAULT]
+# The cluster name
+# This must be set.
+# It must be a name unique to a given cluster, to prevent foreign
+# misconfigured nodes from interfering.
+cluster:
+
+# The list of master nodes
+# Master nodes not not in this list will be rejected by the cluster.
+# This list should be identical for all nodes in a given cluster for
+# maximum availability.
+masters: 127.0.0.1:10000
+
+# Partition table configuration
+# Data in the cluster is distributed among nodes using a partition table, which
+# has the following parameters.
+# Replicas: How many copies of a partition should exist at a time.
+#   0 means no redundancy
+#   1 means there is a spare copy of all partitions
+replicas: 1
+# Partitions: How data spreads among storage nodes. This number must be at
+# least equal to the number of storage nodes the cluster contains.
+# IMPORTANT: This must not be changed once the cluster contains data.
+partitions: 20
+
+# Individual nodes parameters
+# Some parameters makes no sense to be defined in [DEFAULT] section.
+# They are:
+# bind: The ip:port the node will listen on.
+# database: Storage nodes only. The MySQL database credentials to use
+#           (username:password@database). 
+#           Those database must be created manualy.
+
+# Admin node
+[admin]
+bind: 127.0.0.1:9999
+
+# Master nodes
+[master]
+bind: 127.0.0.1:10000
+
+# Storage nodes
+[storage]
+database: neo:neo@neo1
+bind: 127.0.0.1:20000
+

neo/admin/app.py

@@ -51,32 +51,26 @@ class Dispatcher:
 class Application(object):
     """The storage node application."""
 
-    def __init__(self, cluster, bind, masters, uuid=None):
+    def __init__(self, config):
 
         # always use default connector for now
         self.connector_handler = getConnectorHandler()
 
-        # set the cluster name
-        if cluster is None:
-            raise RuntimeError, 'cluster name must be non-empty'
-        self.name = cluster
-        
-        # set the bind address
-        ip_address, port = bind.split(':')
-        self.server = (ip_address, int(port))
-        logging.debug('IP address is %s, port is %d', *(self.server))
+        self.name = config.getCluster()
+        self.server = config.getBind()
+        self.master_node_list = config.getMasters()
 
-        # load master node list
-        self.master_node_list = parseMasterList(masters)
+        logging.debug('IP address is %s, port is %d', *(self.server))
         logging.debug('master nodes are %s', self.master_node_list)
 
         # Internal attributes.
         self.em = EventManager()
         self.nm = NodeManager()
+
         # The partition table is initialized after getting the number of
         # partitions.
         self.pt = None
-        self.uuid = uuid
+        self.uuid = config.getUUID()
         self.primary_master_node = None
         self.ptid = None
         self.request_handler = MasterRequestEventHandler(self)

neo/config.py

+#
+# Copyright (C) 2006-2009  Nexedi SA
+# 
+# This program is free software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License
+# as published by the Free Software Foundation; either version 2
+# of the License, or (at your option) any later version.
+# 
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+
+from ConfigParser import SafeConfigParser
+from neo.util import bin, parseMasterList
+
+
+class ConfigurationManager(object):
+    """ 
+    Configuration manager that load options from a configuration file and
+    command line arguments
+    """
+
+    def __init__(self, defaults, config_file, section, argument_list):
+        self.defaults = defaults
+        self.argument_list = argument_list
+        self.parser = None
+        if config_file is not None:
+            self.parser = SafeConfigParser(defaults)
+            self.parser.read(config_file)
+        self.section = section
+
+    def __get(self, key, optional=False):
+        value = self.argument_list.get(key)
+        if value is None:
+            if self.parser is None:
+                value = self.defaults.get(key)
+            else:
+                value = self.parser.get(self.section, key)
+        if value is None and not optional:
+            raise RuntimeError("Option '%s' is undefined'" % (key, ))
+        return value
+
+    def getMasters(self):
+        """ Get the master node list except itself """
+        masters = self.__get('masters')
+        if not masters:
+            return []
+        # load master node list except itself
+        return parseMasterList(masters, except_node=self.getBind())
+        
+    def getBind(self):
+        """ Get the address to bind to """
+        bind = self.__get('bind')
+        if ':' in bind:
+            (ip, port) = bind.split(':')
+        else:
+            ip = bind
+            # took port from default bind address
+            port = self.defaults['bind'].split(':')[1]
+        return (ip, int(port))
+
+    def getDatabase(self):
+        """ Get the database credentials (username, password, database) """
+        # expected pattern : [user[:password]@]database
+        username = None
+        password = None
+        database = self.__get('database')
+        if '@' in database:
+            (username, database) = database.split('@')
+            if ':' in username:
+                (username, password) = username.split(':')
+        return (username, password, database)
+
+    def getCluster(self):
+        cluster = self.__get('cluster')
+        assert cluster != '', "Cluster name must be non-empty"
+        return cluster
+
+    def getName(self):
+        return self.__get('name')
+
+    def getReplicas(self):
+        return int(self.__get('replicas'))
+
+    def getPartitions(self):
+        return int(self.__get('partitions'))
+
+    def getReset(self):
+        # only from command line
+        return self.argument_list.get('reset', False)
+
+    def getUUID(self):
+        # only from command line
+        return bin(self.argument_list.get('uuid', None))
+

neo/master/app.py

@@ -40,23 +40,16 @@ REQUIRED_NODE_NUMBER = 1
 class Application(object):
     """The master node application."""
 
-    def __init__(self, cluster, bind, masters, replicas, partitions, uuid):
+    def __init__(self, config):
 
         # always use default connector for now
         self.connector_handler = getConnectorHandler()
 
-        # set the cluster name
-        if cluster is None:
-            raise RuntimeError, 'cluster name must be non-empty'
-        self.name = cluster
+        self.name = config.getCluster()
+        self.server = config.getBind()
+        self.master_node_list = config.getMasters()
 
-        # set the bind address
-        ip_address, port = bind.split(':')
-        self.server = (ip_address, int(port))
         logging.debug('IP address is %s, port is %d', *(self.server))
-
-        # load master node list
-        self.master_node_list = parseMasterList(masters, self.server)
         logging.debug('master nodes are %s', self.master_node_list)
 
         # Internal attributes.
@@ -64,6 +57,7 @@ class Application(object):
         self.nm = NodeManager()
 
         # Partition table
+        replicas, partitions = config.getReplicas(), config.getPartitions()
         if replicas < 0:
             raise RuntimeError, 'replicas must be a positive integer'
         if partitions <= 0:
@@ -78,7 +72,8 @@ class Application(object):
         self.cluster_state = None
 
         # Generate an UUID for self
-        if uuid is None:
+        uuid = config.getUUID() 
+        if uuid is None or uuid == '':
             uuid = self.getNewUUID(NodeTypes.MASTER)
         self.uuid = uuid
 

neo/storage/app.py

@@ -36,38 +36,28 @@ from neo.bootstrap import BootstrapManager
 class Application(object):
     """The storage node application."""
 
-    def __init__(self, cluster, bind, masters, database, uuid, reset):
-
+    def __init__(self, config):
         # always use default connector for now
         self.connector_handler = getConnectorHandler()
 
         # set the cluster name
-        if cluster is None:
-            raise RuntimeError, 'cluster name must be non-empty'
-        self.name = cluster
+        self.name = config.getCluster()
         
         # set the bind address
-        ip_address, port = bind.split(':')
-        self.server = (ip_address, int(port))
+        self.server = config.getBind()
         logging.debug('IP address is %s, port is %d', *(self.server))
 
         # load master node list
-        self.master_node_list = parseMasterList(masters)
+        self.master_node_list = config.getMasters()
         logging.debug('master nodes are %s', self.master_node_list)
 
         # load database connection credentials, from user:password@database
-        if database is None:
-            raise RuntimeError, 'database connection required'
-        (ident, dbname) = database.split('@')
-        if ':' in ident:
-            (username, password) = ident.split(':')
-        else:
-            (username, password) = (ident, None)
+        (username, password, database) = config.getDatabase()
             
         # Internal attributes.
         self.em = EventManager()
         self.nm = NodeManager()
-        self.dm = MySQLDatabaseManager(database=dbname, user=username,
+        self.dm = MySQLDatabaseManager(database=database, user=username,
                 password=password)
 
         # The partition table is initialized after getting the number of
@@ -92,16 +82,17 @@ class Application(object):
         self.has_node_information = False
         self.has_partition_table = False
 
-        self.dm.setup(reset)
+        self.dm.setup(reset=config.getReset())
         self.loadConfiguration()
 
         # force node uuid from command line argument, for testing purpose only
-        if uuid is not None:
-            self.uuid = uuid
+        if config.getUUID() is not None:
+            self.uuid = config.getUUID()
 
     def loadConfiguration(self):
         """Load persistent configuration data from the database.
         If data is not present, generate it."""
+
         dm = self.dm
 
         self.uuid = dm.getUUID()

neo/tests/__init__.py

@@ -61,33 +61,31 @@ class NeoTestBase(unittest.TestCase):
     def getMasterConfiguration(self, cluster='main', master_number=2, 
             replicas=2, partitions=1009, uuid=None):
         assert master_number >= 1 and master_number <= 10
-        masters = ['127.0.0.1:1001%d' % i for i in xrange(master_number)]
-        return {
-                'cluster': cluster,
-                'bind': masters[0],
-                'masters': '/'.join(masters),
-                'replicas': replicas,
-                'partitions': partitions,
-                'uuid': uuid,
-        }
+        masters = [('127.0.0.1', 10010 + i) for i in xrange(master_number)]
+        return Mock({
+                'getCluster': cluster,
+                'getBind': masters[0],
+                'getMasters': masters,
+                'getReplicas': replicas,
+                'getPartitions': partitions,
+                'getUUID': uuid,
+        })
 
     def getStorageConfiguration(self, cluster='main', master_number=2, 
             index=0, prefix=DB_PREFIX, uuid=None):
         assert master_number >= 1 and master_number <= 10
         assert index >= 0 and index <= 9
-        masters = ['127.0.0.1:1001%d' % i for i in xrange(master_number)]
-        if DB_PASSWD is None:
-            database = '%s:@%s%d' % (DB_USER, prefix, index)
-        else:
-            database = '%s:%s@%s%d' % (DB_USER, DB_PASSWD, prefix, index)
-        return {
-                'cluster': cluster,
-                'bind': '127.0.0.1:1002%d' % (index, ),
-                'masters': '/'.join(masters),
-                'database': database,
-                'uuid': uuid,
-                'reset': False,
-        }
+        masters = [('127.0.0.1', 10010 + i) for i in xrange(master_number)]
+        database = (DB_USER, DB_PASSWD, '%s%d' % (prefix, index))
+        return Mock({
+                'getCluster': cluster,
+                'getName': 'storage',
+                'getBind': ('127.0.0.1', 10020 + index),
+                'getMasters': masters,
+                'getDatabase': database,
+                'getUUID': uuid,
+                'getReset': False,
+        })
         
     # XXX: according to changes with namespaced UUIDs, it would be better to 
     # implement get<NodeType>UUID() methods 

neo/tests/master/testClientHandler.py

@@ -30,7 +30,7 @@ class MasterClientHandlerTests(NeoTestBase):
     def setUp(self):
         # create an application object
         config = self.getMasterConfiguration(master_number=1, replicas=1)
-        self.app = Application(**config)
+        self.app = Application(config)
         self.app.pt.clear()
         self.app.pt.setID(pack('!Q', 1))
         self.app.em = Mock({"getConnectionList" : []})

neo/tests/master/testElectionHandler.py

@@ -40,7 +40,7 @@ class MasterClientElectionTests(NeoTestBase):
     def setUp(self):
         # create an application object
         config = self.getMasterConfiguration()
-        self.app = Application(**config)
+        self.app = Application(config)
         self.app.pt.clear()
         self.app.em = Mock({"getConnectionList" : []})
         self.app.finishing_transaction_dict = {}

neo/tests/master/testMasterApp.py

@@ -26,7 +26,7 @@ class MasterAppTests(NeoTestBase):
     def setUp(self):
         # create an application object
         config = self.getMasterConfiguration()
-        self.app = Application(**config)
+        self.app = Application(config)
         self.app.pt.clear()
 
     def tearDown(self):

neo/tests/master/testRecoveryHandler.py

@@ -28,7 +28,7 @@ class MasterRecoveryTests(NeoTestBase):
     def setUp(self):
         # create an application object
         config = self.getMasterConfiguration()
-        self.app = Application(**config)
+        self.app = Application(config)
         self.app.pt.clear()
         self.app.finishing_transaction_dict = {}
         for address in self.app.master_node_list:

neo/tests/master/testStorageHandler.py

@@ -32,7 +32,7 @@ class MasterStorageHandlerTests(NeoTestBase):
     def setUp(self):
         # create an application object
         config = self.getMasterConfiguration(master_number=1, replicas=1)
-        self.app = Application(**config)
+        self.app = Application(config)
         self.app.pt.clear()
         self.app.pt.setID(pack('!Q', 1))
         self.app.em = Mock({"getConnectionList" : []})

neo/tests/master/testVerificationHandler.py

@@ -31,7 +31,7 @@ class MasterVerificationTests(NeoTestBase):
     def setUp(self):
         # create an application object
         config = self.getMasterConfiguration()
-        self.app = Application(**config)
+        self.app = Application(config)
         self.app.pt.clear()
         self.app.finishing_transaction_dict = {}
         for address in self.app.master_node_list:

neo/tests/storage/testClientHandler.py

@@ -46,7 +46,7 @@ class StorageClientHandlerTests(NeoTestBase):
         self.prepareDatabase(number=1)
         # create an application object
         config = self.getStorageConfiguration(master_number=1)
-        self.app = Application(**config)
+        self.app = Application(config)
         self.app.transaction_dict = {}
         self.app.store_lock_dict = {}
         self.app.load_lock_dict = {}

neo/tests/storage/testInitializationHandler.py

@@ -30,7 +30,7 @@ class StorageInitializationHandlerTests(NeoTestBase):
         self.prepareDatabase(number=1)
         # create an application object
         config = self.getStorageConfiguration(master_number=1)
-        self.app = Application(**config)
+        self.app = Application(config)
         self.verification = InitializationHandler(self.app)
         # define some variable to simulate client and storage node
         self.master_port = 10010

neo/tests/storage/testMasterHandler.py

@@ -45,7 +45,7 @@ class StorageMasterHandlerTests(NeoTestBase):
         self.prepareDatabase(number=1)
         # create an application object
         config = self.getStorageConfiguration(master_number=1)
-        self.app = Application(**config)
+        self.app = Application(config)
         self.app.transaction_dict = {}
         self.app.store_lock_dict = {}
         self.app.load_lock_dict = {}

neo/tests/storage/testStorageApp.py

@@ -30,7 +30,7 @@ class StorageAppTests(NeoTestBase):
         self.prepareDatabase(number=1)
         # create an application object
         config = self.getStorageConfiguration(master_number=1)
-        self.app = Application(**config)
+        self.app = Application(config)
         self.app.event_queue = deque()
         
     def tearDown(self):

neo/tests/storage/testStorageHandler.py

@@ -43,7 +43,7 @@ class StorageStorageHandlerTests(NeoTestBase):
         self.prepareDatabase(number=1)
         # create an application object
         config = self.getStorageConfiguration(master_number=1)
-        self.app = Application(**config)
+        self.app = Application(config)
         self.app.transaction_dict = {}
         self.app.store_lock_dict = {}
         self.app.load_lock_dict = {}

neo/tests/storage/testVerificationHandler.py

@@ -32,7 +32,7 @@ class StorageVerificationHandlerTests(NeoTestBase):
         self.prepareDatabase(number=1)
         # create an application object
         config = self.getStorageConfiguration(master_number=1)
-        self.app = Application(**config)
+        self.app = Application(config)
         self.verification = VerificationHandler(self.app)
         # define some variable to simulate client and storage node
         self.master_port = 10010

neo/tests/testBootstrap.py

@@ -28,7 +28,7 @@ class BootstrapManagerTests(NeoTestBase):
         self.prepareDatabase(number=1)
         # create an application object
         config = self.getStorageConfiguration()
-        self.app = Application(**config)
+        self.app = Application(config)
         for address in self.app.master_node_list:
             self.app.nm.createMaster(address=address)
         self.bootstrap = BootstrapManager(self.app, 'main', NodeTypes.STORAGE)

neo/util.py

@@ -89,7 +89,7 @@ def parseMasterList(masters, except_node=None):
         address = (ip_address, int(port))
         if (address != except_node):
             master_node_list.append(address)
-    return master_node_list
+    return tuple(master_node_list)
 
 
 class Enum(dict):

neoadmin

@@ -20,13 +20,15 @@
 
 from optparse import OptionParser
 from neo import setupLog
-from neo.util import bin
+from neo.config import ConfigurationManager
 
 parser = OptionParser()
 parser.add_option('-u', '--uuid', help='specify an UUID to use for this ' \
                   'process')
 parser.add_option('-v', '--verbose', action = 'store_true', 
                   help = 'print verbose messages')
+parser.add_option('-f', '--file', help = 'specify a configuration file') 
+parser.add_option('-s', '--section', help = 'specify a configuration section') 
 parser.add_option('-l', '--logfile', help = 'specify a logging file')
 parser.add_option('-c', '--cluster', help = 'the cluster name')
 parser.add_option('-m', '--masters', help = 'master node list')
@@ -35,18 +37,28 @@ parser.add_option('-n', '--name', help = 'the node name (impove logging)')
 
 # build configuration dict from command line options
 (options, args) = parser.parse_args()
-config = {
-    'uuid': options.uuid,
-    'cluster': options.cluster,
-    'masters': options.masters or '127.0.0.1:10000',
-    'bind': options.bind or '127.0.0.1:9999',
-}
-config['uuid'] = bin(config['uuid'])
+arguments = dict(
+    uuid = options.uuid,
+    cluster = options.cluster,
+    masters = options.masters,
+    bind = options.bind,
+)
+defaults = dict(
+    name = 'admin',
+    bind = '127.0.0.1:9999',
+    masters = '127.0.0.1:10000',
+)
+config = ConfigurationManager(
+        defaults, 
+        options.file, 
+        options.section or 'admin', 
+        arguments,
+)
 
 # setup custom logging
-setupLog(options.name or 'admin', options.logfile or None, options.verbose)
+setupLog(config.getName(), options.logfile or None, options.verbose)
 
 # and then, load and run the application
 from neo.admin.app import Application
-app = Application(**config)
+app = Application(config)
 app.run()

neomaster

@@ -20,11 +20,13 @@
 
 from optparse import OptionParser
 from neo import setupLog
-from neo.util import bin
+from neo.config import ConfigurationManager
 
 parser = OptionParser()
 parser.add_option('-v', '--verbose', action = 'store_true', 
                   help = 'print verbose messages')
+parser.add_option('-f', '--file', help = 'specify a configuration file') 
+parser.add_option('-s', '--section', help = 'specify a configuration section') 
 parser.add_option('-u', '--uuid', help='the node UUID (testing purpose)')
 parser.add_option('-n', '--name', help = 'the node name (impove logging)')
 parser.add_option('-b', '--bind', help = 'the local address to bind to')
@@ -36,22 +38,32 @@ parser.add_option('-l', '--logfile', help = 'specify a logging file')
 
 # build configuration dict from command line options
 (options, args) = parser.parse_args()
-config = {
-    'uuid': options.uuid,
-    'bind': options.bind or '127.0.0.1:10000',
-    'cluster': options.cluster,
-    'masters': options.masters or '',
-    'replicas': options.replicas or 0,
-    'partitions': options.partitions or 100,
-}
-config['uuid'] = bin(config['uuid'])
-config['replicas'] = int(config['replicas'])
-config['partitions'] = int(config['partitions'])
+arguments = dict(
+    uuid = options.uuid or None,
+    bind = options.bind,
+    cluster = options.cluster,
+    masters = options.masters,
+    replicas = options.replicas,
+    partitions = options.partitions,
+)
+defaults = dict(
+    name = 'master',
+    bind = '127.0.0.1:10000',
+    masters = '',
+    replicas = 0,
+    partitions = 100,
+)
+config = ConfigurationManager(
+        defaults, 
+        options.file, 
+        options.section or 'master', 
+        arguments,
+)
 
 # setup custom logging
-setupLog(options.name or 'master', options.logfile or None, options.verbose)
+setupLog(config.getName(), options.logfile or None, options.verbose)
 
 # and then, load and run the application
 from neo.master.app import Application
-app = Application(**config)
+app = Application(config)
 app.run()

neostorage

@@ -20,7 +20,7 @@
 
 from optparse import OptionParser
 from neo import setupLog
-from neo.util import bin
+from neo.config import ConfigurationManager
 
 
 parser = OptionParser()
@@ -29,10 +29,11 @@ parser.add_option('-v', '--verbose', action = 'store_true',
 parser.add_option('-u', '--uuid', help='specify an UUID to use for this ' \
                   'process. Previously assigned UUID takes precedence (ie ' \
                   'you should always use -R with this switch)')
+parser.add_option('-f', '--file', help = 'specify a configuration file') 
+parser.add_option('-s', '--section', help = 'specify a configuration section') 
 parser.add_option('-l', '--logfile', help = 'specify a logging file')
 parser.add_option('-R', '--reset', action = 'store_true',
                   help = 'remove an existing database if any')
-
 parser.add_option('-n', '--name', help = 'the node name (impove logging)')
 parser.add_option('-b', '--bind', help = 'the local address to bind to')
 parser.add_option('-c', '--cluster', help = 'the cluster name')
@@ -40,20 +41,30 @@ parser.add_option('-m', '--masters', help = 'master node list')
 parser.add_option('-d', '--database', help = 'database connections string')
 
 (options, args) = parser.parse_args()
-config = {
-    'uuid': options.uuid,
-    'bind': options.bind or '127.0.0.1:20000',
-    'cluster': options.cluster,
-    'masters': options.masters or '127.0.0.1:10000',
-    'database': options.database,
-    'reset': options.reset,
-}
-config['uuid'] = bin(config['uuid'])
+arguments = dict(
+    uuid = options.uuid,
+    bind = options.bind,
+    cluster = options.cluster,
+    masters = options.masters,
+    database = options.database,
+    reset = options.reset,
+)
+defaults = dict(
+    name = 'storage',
+    bind = '127.0.0.1:20000',
+    masters = '127.0.0.1:10000',
+)
+config = ConfigurationManager(
+        defaults, 
+        options.file, 
+        options.section or 'storage', 
+        arguments,
+)
 
 # setup custom logging
-setupLog(options.name or 'storage', options.logfile or None, options.verbose)
+setupLog(config.getName(), options.logfile or None, options.verbose)
 
 # and then, load and run the application
 from neo.storage.app import Application
-app = Application(**config)
+app = Application(config)
 app.run()