Merge remote-tracking branch 'origin/nosqltestbed'

component/erlang/buildout.cfg

+[buildout]
+parts =
+  erlang
+
+[erlang]
+recipe = hexagonit.recipe.cmmi
+url = http://www.erlang.org/download/otp_src_R14B03.tar.gz
+md5sum = 7979e662d11476b97c462feb7c132fb7

component/libmemcached/buildout.cfg

+[buildout]
+
+parts =
+  libmemcached
+
+[libmemcached]
+<= libmemcached-0.50
+
+[libmemcached-0.50]
+<= libmemcached-common
+url = http://launchpad.net/libmemcached/1.0/0.50/+download/libmemcached-0.50.tar.gz
+md5sum = c8627014a37cd821cf93317b8de6f9f8
+
+[libmemcached-0.44]
+<= libmemcached-common
+url = http://launchpad.net/libmemcached/1.0/0.44/+download/libmemcached-0.44.tar.gz
+md5sum = e6bd825c46fa080b550f90f9001cba8c
+
+[libmemcached-common]
+recipe = hexagonit.recipe.cmmi
+configure-options =
+  --without-docs
+  --without-memcached
+  --without-libgtest-prefix
+  --without-libevent-prefix
+  --without-libinnodb-prefix
+  --without-libsasl-prefix
+  --without-libsasl2-prefix
+
+

component/membase/buildout.cfg

+[buildout]
+extends =
+  ../curl/buildout.cfg
+  ../libevent/buildout.cfg
+  ../erlang/buildout.cfg
+
+parts =
+  membase-source
+  membase-memcached
+  membase-libmemcached
+  membase-libconflate
+  membase-libvbucket
+  membase-moxi
+  membase-bucket_engine
+  membase-ep-engine
+  membase-membase-cli
+  membase-memcachetest
+  membase-ns_server
+  membase-vbucketmigrator
+
+[membase]
+location = ${buildout:parts-directory}/${:_buildout_section_name_}
+
+[membase-source]
+<= membase-1.7.0-source
+
+[membase-1.7.0-source]
+<= membase-source-common
+url = http://files.couchbase.com/source/membase-server_src-1.7.0.tar.gz
+md5sum = c933fffea299d00e43b002cb65738663
+
+[membase-source-common]
+recipe = hexagonit.recipe.download
+strip-top-level-dir = true
+
+[membase-component-common]
+recipe = hexagonit.recipe.cmmi
+path = ${membase-source:location}/${:component}
+prefix = ${membase:location}
+configure-options =
+  --prefix=${:prefix}
+  ${:component-configure-options}
+
+[membase-libmemcached]
+<= membase-component-common
+component = libmemcached
+component-configure-options =
+  --disable-static
+  --enable-shared
+  --disable-dtrace
+  --without-docs
+  --disable-sasl
+  --with-memcached=${membase:location}/bin/memcached
+
+[membase-memcached]
+<= membase-component-common
+component = memcached
+patches =
+configure-command =
+  ./configure
+component-configure-options =
+  --enable-isasl
+
+[membase-libconflate]
+<= membase-component-common
+component = libconflate
+component-configure-options =
+  --disable-static
+  --enable-shared
+  --without-check
+  --with-libcurl-prefix=${curl:location}
+
+[membase-libvbucket]
+<= membase-component-common
+component = libvbucket
+component-configure-options =
+  --disable-static
+  --enable-shared
+  --without-docs
+  --with-libhashkit-prefix=${membase:location}
+
+[membase-moxi]
+<= membase-component-common
+component = moxi
+component-configure-options =
+  --enable-moxi-libvbucket
+  --enable-moxi-libmemcached
+  --without-check
+  --with-libevent-prefix=${libevent:location}
+  --with-libmemcached-prefix=${membase:location}
+  --with-memcached=${membase:location}/bin/memcached
+  --with-libhashkit-prefix=${membase:location}
+  --with-libconflate-prefix=${membase:location}
+  --with-libvbucket-prefix=${membase:location}
+
+[membase-bucket_engine]
+<= membase-component-common
+component = bucket_engine
+component-configure-options =
+  --with-memcached=${membase-source:location}/memcached
+
+[membase-ep-engine]
+<= membase-component-common
+component = ep-engine
+component-configure-options =
+  --with-memcached=${membase-source:location}/memcached
+
+[membase-membase-cli]
+<= membase-component-common
+component = membase-cli
+patches =
+configure-command =
+  ./configure
+component-configure-options =
+
+[membase-memcachetest]
+<= membase-component-common
+component = memcachetest
+component-configure-options =
+  --with-memcached=${membase:location}/bin/memcached
+
+[membase-ns_server]
+<= membase-component-common
+component = ns_server
+patches =
+configure-command =
+  ./configure
+component-configure-options =
+environment =
+  PATH=${erlang:location}/bin:%(PATH)s
+
+[membase-vbucketmigrator]
+<= membase-component-common
+component = vbucketmigrator
+component-configure-options =
+  --without-sasl
+  --with-isasl
+

component/memstrike/buildout.cfg

+[buildout]
+extends =
+  ../libmemcached/buildout.cfg
+  ../git/buildout.cfg
+  ../autoconf/buildout.cfg
+  ../automake/buildout.cfg
+  ../libtool/buildout.cfg
+
+parts =
+  memstrike
+
+[memstrike]
+recipe = hexagonit.recipe.cmmi
+path = ${memstrikesource:location}
+configure-command =
+  ./bootstrap
+  ./configure
+configure-options =
+  --prefix=${buildout:parts-directory}/${:_buildout_section_name_}
+  --with-libmemcached=${libmemcached:location}
+environment =
+  PATH =${autoconf:location}/bin:${automake-1.11:location}/bin:${libtool:location}/bin:%(PATH)s
+  LDFLAGS =-Wl,-rpath=${libmemcached:location}/lib
+
+[memstrikesource]
+recipe=plone.recipe.command
+location = ${buildout:parts-directory}/${:_buildout_section_name_} 
+stop-on-error = true
+revision = 39d7a99e8bb7eea6df8b
+command =
+  rm -rf ${:location} &&
+  ${git:location}/bin/git clone --quiet git://github.com/frsyuki/memstrike.git ${:location} &&
+  cd ${:location} &&
+  ${git:location}/bin/git checkout --quiet ${:revision}

setup.py

@@ -78,5 +78,9 @@ setup(name=name,
           'vifib = slapos.recipe.vifib:Recipe',
           'xwiki = slapos.recipe.xwiki:Recipe',
           'zabbixagent = slapos.recipe.zabbixagent:Recipe',
-      ]},
+        ],
+        'slapos.recipe.nosqltestbed.plugin': [
+          'kumo = slapos.recipe.nosqltestbed.kumo:KumoTestBed',
+        ],
+      },
     )

slapos/recipe/nosqltestbed/__init__.py

@@ -25,202 +25,31 @@
 #
 ##############################################################################
 
-import os
-import urllib
-import urllib2
+import sys
 import pkg_resources
+from logging import Formatter
 from slapos.recipe.librecipe import BaseSlapRecipe
 
 class NoSQLTestBed(BaseSlapRecipe):
 
   def _install(self):
     self.parameter_dict = self.computer_partition.getInstanceParameterDict()
+    try:
+      entry_point = pkg_resources.iter_entry_points(group='slapos.recipe.nosqltestbed.plugin',
+                                                    name=self.parameter_dict.get('plugin', 'kumo')).next()
+      plugin_class = entry_point.load()
+
+      testbed = plugin_class()
+    except:
+      print Formatter().formatException(sys.exc_info())
+      return None
+
     software_type = self.parameter_dict.get('slap_software_type', 'default')
     if software_type is None or software_type == 'RootSoftwareInstance':
-      software_type = 'kumo_cloud'
-    if "run_%s" % software_type in dir(self) and \
-       callable(getattr(self, "run_%s" % software_type)):
-      return getattr(self, "run_%s" % software_type)()
+      software_type = 'default'
+    if "run_%s" % software_type in dir(testbed) and \
+       callable(getattr(testbed, "run_%s" % software_type)):
+      return getattr(testbed, "run_%s" % software_type)(self)
     else:
       raise NotImplementedError("Do not support %s" % software_type)
 
-  def run_kumo_cloud(self):
-    """ Deploy kumofs systeom on a cloud. """
-
-    kumo_cloud_config = {}
-    kumo_cloud_config.update(self.options)
-    kumo_cloud_config.update(self.parameter_dict)
-
-    kumo_cloud_config['address'] = self.getGlobalIPv6Address()
-    kumo_cloud_config['report_path'] = self.log_directory
-    
-    if 'nb_server_max' not in kumo_cloud_config:
-      kumo_cloud_config['nb_server_max'] = 3
-    
-    if 'nb_tester_max' not in kumo_cloud_config:
-      kumo_cloud_config['nb_tester_max'] = 3
-    
-    if 'nb_thread' not in kumo_cloud_config:
-      kumo_cloud_config['nb_thread'] = 1
-    
-    if 'nb_request' not in kumo_cloud_config:
-      kumo_cloud_config['nb_request'] = 1000
-    
-    kumo_cloud_config['software_release_url'] = self.software_release_url
-    kumo_cloud_config['server_url'] = self.server_url
-    kumo_cloud_config['key_file'] = self.key_file
-    kumo_cloud_config['cert_file'] = self.cert_file
-    kumo_cloud_config['computer_id'] = self.computer_id
-    kumo_cloud_config['computer_partition_id'] = self.computer_partition_id
-    kumo_cloud_config['plugin_name'] = 'kumo'
-
-    kumo_cloud_connection = {}
-    kumo_cloud_connection['url'] = "http://["+kumo_cloud_config['address']+"]:5000/"
-    self.computer_partition.setConnectionDict(kumo_cloud_connection)
-
-    nosqltester_manager_wrapper_template_location = pkg_resources.resource_filename(
-                                             __name__, os.path.join(
-                                             'template', 'kumotester_manager_run.in'))
-    nosqltester_manager_runner_path = self.createRunningWrapper("kumotester_manager",
-          self.substituteTemplate(nosqltester_manager_wrapper_template_location, kumo_cloud_config))
-
-    return [nosqltester_manager_runner_path]
-
-  def run_all(self):
-    """ Runs all services on one machine. """
-    all_config = {}
-    all_config.update(self.options)
-
-    ipaddress = "[%s]" % self.getGlobalIPv6Address()
-
-    all_config['manager_address'] = ipaddress
-    all_config['manager_port'] = 19700
-    all_config['server_address'] = ipaddress
-    all_config['server_port'] = 19800
-    all_config['server_listen_port'] = 19900
-    all_config['server_storage'] = os.path.join(self.data_root_directory, "kumodb.tch")
-    all_config['gateway_address'] = ipaddress
-    all_config['gateway_port'] = 11411
-    all_config['manager_log'] = os.path.join(self.log_directory, "kumo-manager.log")
-    all_config['server_log'] = os.path.join(self.log_directory, "kumo-server.log")
-    all_config['gateway_log'] = os.path.join(self.log_directory, "kumo-gateway.log")
-
-    manager_wrapper_template_location = pkg_resources.resource_filename(
-                                             __name__, os.path.join(
-                                             'template', 'kumo_manager_run.in'))
-    manager_runner_path = self.createRunningWrapper("kumo-manager",
-          self.substituteTemplate(manager_wrapper_template_location, all_config))
-    server_wrapper_template_location = pkg_resources.resource_filename(
-                                             __name__, os.path.join(
-                                             'template', 'kumo_server_run.in'))
-    server_runner_path = self.createRunningWrapper("kumo-server",
-          self.substituteTemplate(server_wrapper_template_location, all_config))
-    gateway_wrapper_template_location = pkg_resources.resource_filename(
-                                             __name__, os.path.join(
-                                             'template', 'kumo_gateway_run.in'))
-    gateway_runner_path = self.createRunningWrapper("kumo-gateway",
-          self.substituteTemplate(gateway_wrapper_template_location, all_config))
-
-    return [manager_runner_path, server_runner_path, gateway_runner_path]
-
-  def run_kumo_manager(self, ipaddress=None):
-    """ Runs the kumofs manager. """
-    manager_config = {}
-    manager_config.update(self.options)
-
-    if ipaddress is None:
-      manager_config['manager_address'] = "[%s]" % self.getGlobalIPv6Address()
-    else:
-      manager_config['manager_address'] = ipaddress
-    
-    manager_config['manager_port'] = 19700
-    manager_config['manager_log'] = os.path.join(self.log_directory, "kumo-manager.log")
-
-    manager_connection = {}
-    manager_connection['address'] = manager_config['manager_address']
-    manager_connection['port'] = manager_config['manager_port']
-    self.computer_partition.setConnectionDict(manager_connection)
-
-    manager_wrapper_template_location = pkg_resources.resource_filename(
-                                             __name__, os.path.join(
-                                             'template', 'kumo_manager_run.in'))
-    manager_runner_path = self.createRunningWrapper("kumo-manager",
-          self.substituteTemplate(manager_wrapper_template_location, manager_config))
-
-    return [manager_runner_path]
-
-  def run_kumo_server(self):
-    """ Runs the kumofs server. """
-    server_config = {}
-    server_config.update(self.options)
-    server_config.update(self.parameter_dict)
-
-    server_config['server_address'] = "[%s]" % self.getGlobalIPv6Address()
-    server_config['server_port'] = 19800
-    server_config['server_listen_port'] = 19900
-    server_config['server_storage'] = os.path.join(self.var_directory,"kumodb.tch")
-    server_config['server_log'] = os.path.join(self.log_directory, "kumo-server.log")
-
-    server_connection = {}
-    server_connection['address'] = server_config['server_address']
-    self.computer_partition.setConnectionDict(server_connection)
-
-    server_wrapper_template_location = pkg_resources.resource_filename(
-                                             __name__, os.path.join(
-                                             'template', 'kumo_server_run.in'))
-    server_runner_path = self.createRunningWrapper("kumo-server",
-          self.substituteTemplate(server_wrapper_template_location, server_config))
-
-    return [server_runner_path]
-
-  def run_kumo_gateway(self):
-    """ Runs the kumofs gateway. """
-    gateway_config = {}
-    gateway_config.update(self.options)
-    gateway_config.update(self.parameter_dict)
-
-    gateway_config['gateway_address'] = "[%s]" % self.getGlobalIPv6Address()
-    gateway_config['gateway_port'] = 11411
-    gateway_config['gateway_log'] = os.path.join(self.log_directory, "kumo-gateway.log")
-
-    gateway_connection = {}
-    gateway_connection['address'] = gateway_config['gateway_address']
-    gateway_connection['port'] = gateway_config['gateway_port']
-    self.computer_partition.setConnectionDict(gateway_connection)
-
-    gateway_wrapper_template_location = pkg_resources.resource_filename(
-                                             __name__, os.path.join(
-                                             'template', 'kumo_gateway_run.in'))
-    gateway_runner_path = self.createRunningWrapper("kumo-gateway",
-          self.substituteTemplate(gateway_wrapper_template_location, gateway_config))
-
-    return [gateway_runner_path]
-
-  def run_kumo_tester(self):
-    """ Runs the kumofs tester. """
-    tester_config = {}
-    tester_config.update(self.options)
-    tester_config.update(self.parameter_dict)
-
-    tester_config['tester_address'] = self.getGlobalIPv6Address()
-    # tester_config['url'] = "http://%s:5000/" % tester_config['tester_address']
-    # tester_config['start_url'] = "http://%s:5000/start" % tester_config['tester_address']
-    tester_config['report_path'] = self.log_directory
-    tester_config['binary'] = tester_config['memstrike_binary'] + " -l " + \
-                              tester_config['gateway_address'].strip("[]") + " -p " + \
-                              tester_config['gateway_port'] + " -t " + \
-                              tester_config['nb_thread'] + " " + \
-                              tester_config['nb_request'] #" 1000" " -t 32 1024000"
-
-    tester_connection = {}
-    tester_connection['start_url'] = "http://%s:5000/start" % tester_config['tester_address']
-    self.computer_partition.setConnectionDict(tester_connection)
-
-    tester_wrapper_template_location = pkg_resources.resource_filename(
-                                             __name__, os.path.join(
-                                             'template', 'nosqltester_run.in'))
-    tester_runner_path = self.createRunningWrapper("nosqltester",
-          self.substituteTemplate(tester_wrapper_template_location, tester_config))
-
-    return [tester_runner_path]
-

slapos/recipe/nosqltestbed/template/kumo_gateway_run.in → slapos/recipe/nosqltestbed/kumo/template/kumo_gateway_run.in

 #!/bin/sh
 # BEWARE: This file is operated by slapgrid
 # BEWARE: It will be overwritten automatically
-exec %(gateway_binary)s -F -E -m %(manager_address)s:%(manager_port)s -t %(gateway_address)s:%(gateway_port)s --verbose -o %(gateway_log)s
+exec %(gateway_binary)s -F -E -m %(manager_address)s:%(manager_port)s \
+  -t %(gateway_address)s:%(gateway_port)s --verbose -o %(gateway_log)s

slapos/recipe/nosqltestbed/template/kumo_manager_run.in → slapos/recipe/nosqltestbed/kumo/template/kumo_manager_run.in

 #!/bin/sh
 # BEWARE: This file is operated by slapgrid
 # BEWARE: It will be overwritten automatically
-exec %(manager_binary)s -a -l %(manager_address)s:%(manager_port)s --verbose -o %(manager_log)s
+exec %(manager_binary)s -a -l %(manager_address)s:%(manager_port)s \
+  --verbose -o %(manager_log)s

slapos/recipe/nosqltestbed/kumo/template/kumo_server_run.in

+#!/bin/sh
+# BEWARE: This file is operated by slapgrid
+# BEWARE: It will be overwritten automatically
+exec %(server_binary)s -l %(server_address)s:%(server_port)s \
+  -L %(server_listen_port)s -m %(manager_address)s:%(manager_port)s \
+  -s %(server_storage)s  --verbose -o %(server_log)s

slapos/recipe/nosqltestbed/kumo/template/kumotester_manager_run.in

+#!/bin/sh
+# BEWARE: This file is operated by slapgrid
+# BEWARE: It will be overwritten automatically
+exec %(nosqltester_manager_binary)s %(plugin_name)s -a %(address)s \
+  -r %(report_path)s -s %(max_server)s -t %(max_tester)s \
+  --erp5-publish-url "%(erp5_publish_url)s" --erp5-publish-project "%(erp5_publish_project)s" \
+  %(software_release_url)s %(server_url)s "%(key_file)s" "%(cert_file)s" %(computer_id)s %(computer_partition_id)s \
+  %(nb_thread)s %(nb_request)s

slapos/recipe/nosqltestbed/template/kumo_server_run.in → slapos/recipe/nosqltestbed/kumo/template/memstrike_run.in

 #!/bin/sh
 # BEWARE: This file is operated by slapgrid
 # BEWARE: It will be overwritten automatically
-exec %(server_binary)s -l %(server_address)s:%(server_port)s -L %(server_listen_port)s -m %(manager_address)s:%(manager_port)s -s %(server_storage)s  --verbose -o %(server_log)s
+exec %(memstrike_binary)s -s -l %(gateway_address)s -p %(gateway_port)s -t %(nb_thread)s %(nb_request)s

slapos/recipe/nosqltestbed/template/kumotester_manager_run.in

-#!/bin/sh
-# BEWARE: This file is operated by slapgrid
-# BEWARE: It will be overwritten automatically
-exec %(nosqltester_manager_binary)s -a %(address)s -r %(report_path)s -s %(nb_server_max)s -t %(nb_tester_max)s %(software_release_url)s %(server_url)s "%(key_file)s" "%(cert_file)s" %(computer_id)s %(computer_partition_id)s %(plugin_name)s %(nb_thread)s %(nb_request)s

slapos/recipe/nosqltestbed/template/nosqltester_manager_run.in

 #!/bin/sh
 # BEWARE: This file is operated by slapgrid
 # BEWARE: It will be overwritten automatically
-exec %(nosqltester_manager_binary)s -a %(address)s -r %(report_path)s -s %(nb_server_max)s -t %(nb_tester_max)s %(software_release_url)s %(server_url)s "%(key_file)s" "%(cert_file)s" %(computer_id)s %(computer_partition_id)s %(plugin_name)s
+exec %(nosqltester_manager_binary)s %(plugin_name)s -a %(address)s \
+  -r %(report_path)s -s %(max_server)s -t %(max_tester)s \
+  --erp5-publish-url "%(erp5_publish_url)s" --erp5-publish-project "%(erp5_publish_project)s" \
+  %(software_release_url)s %(server_url)s "%(key_file)s" "%(cert_file)s" %(computer_id)s %(computer_partition_id)s

slapos/recipe/nosqltestbed/template/nosqltester_run.in

 #!/bin/sh
 # BEWARE: This file is operated by slapgrid
 # BEWARE: It will be overwritten automatically
-exec %(nosqltester_binary)s -h %(host_address)s -a %(tester_address)s -r %(report_path)s -b "%(binary)s"
+exec %(nosqltester_binary)s -m %(host_address)s -a %(tester_address)s \
+  -r %(report_path)s -b "%(binary)s" -l %(log_directory)s \
+  -c "%(compress_method)s"

software/nosqltestbed/instance.cfg

+[buildout]
+parts =
+  kumoinstance
+
+eggs-directory = ${buildout:eggs-directory}
+develop-eggs-directory = ${buildout:develop-eggs-directory}
+
+[kumoinstance]
+<=slap_connection
+recipe = slapos.cookbook:nosqltestbed
+manager_binary = ${kumo:location}/bin/kumo-manager
+server_binary = ${kumo:location}/bin/kumo-server
+gateway_binary = ${kumo:location}/bin/kumo-gateway
+nosqltester_binary = ${buildout:bin-directory}/nosqltester
+memstrike_binary = ${memstrike:location}/bin/memstrike
+nosqltester_manager_binary = ${buildout:bin-directory}/nosqltester_manager

software/nosqltestbed/software.cfg

+[buildout]
+recipe_location = ${:parts-directory}/slapos.cookbook
+develop = ${:recipe_location}
+
+extends =
+  ../../component/kumo/buildout.cfg
+  ../../component/memstrike/buildout.cfg
+  ../../component/lxml-python/buildout.cfg
+
+find-links =
+  http://www.nexedi.org/static/packages/source/slapos.buildout/
+  http://www.nexedi.org/static/packages/source/
+
+parts =
+  template
+  eggs
+  kumo
+  memstrike
+  checkrecipe
+
+versions = versions
+
+# separate from system python
+include-site-packages = false
+exec-sitecustomize = false
+allowed-eggs-from-site-packages =
+
+[kumo]
+environment += CXXFLAGS=-DKUMO_IPV6
+
+[checkrecipe]
+recipe = plone.recipe.command
+stop-on-error = true
+update-command = ${:command}
+command = grep parts ${buildout:develop-eggs-directory}/slapos.cookbook.egg-link
+
+[slapos.cookbook]
+recipe = plone.recipe.command
+stop-on-error = true
+location = ${buildout:parts-directory}/${:_buildout_section_name_}
+command = ${git:location}/bin/git clone -b nosqltestbed --quiet http://git.erp5.org/repos/slapos.git ${:location}
+update-command = cd ${:location} && ${git:location}/bin/git pull --quiet
+
+[eggs]
+dummy = ${slapos.cookbook:location}
+recipe = zc.recipe.egg
+eggs =
+  ${lxml-python:egg}
+  PyXML
+  Zope2
+  collective.recipe.template
+  netaddr
+  slapos.core
+  xml_marshaller
+  slapos.cookbook
+  slapos.tool.nosqltester
+  slapos.tool.nosqltester_manager
+  slapos.tool.nosqltester_manager.kumo
+  erp5.utils.benchmark
+
+[template]
+recipe = slapos.recipe.template
+url = ${:_profile_base_location_}/instance.cfg
+md5sum = 1c1000ea9829f23a897d256373191df6
+output = ${buildout:directory}/template.cfg
+mode = 0644
+
+[versions]
+# Use SlapOS patched zc.buildout
+zc.buildout = 1.5.3-dev-SlapOS-001