Starting refactor to run cloudoootestnode into new changes. This still early draft

slapos/recipe/cloudoootestnode/__init__.py

@@ -24,136 +24,72 @@
 # Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.
 #
 ##############################################################################
-from slapos.recipe.librecipe import BaseSlapRecipe
+import ConfigParser
+import json
 import os
-import pkg_resources
-import zc.buildout
-import zc.recipe.egg
-import sys
+import StringIO
 
-CONFIG = dict(
-  proxy_port='5000',
-  computer_id='COMPUTER',
-  partition_reference='test0',
-)
+from slapos.recipe.librecipe import GenericBaseRecipe
 
-class Recipe(BaseSlapRecipe):
-  def __init__(self, buildout, name, options):
-    self.egg = zc.recipe.egg.Egg(buildout, options['recipe'], options)
-    BaseSlapRecipe.__init__(self, buildout, name, options)
+class Recipe(GenericBaseRecipe):
+  def install(self):
+    path_list = []
+    options = self.options.copy()
+    del options['recipe']
+    CONFIG = {k.replace('-', '_'): v for k, v in options.iteritems()}
+    CONFIG['PATH'] = os.environ['PATH']
 
-  def installSlapOs(self):
-    CONFIG['slapos_directory'] = self.createDataDirectory('slapos')
-    CONFIG['working_directory'] = self.createDataDirectory('testnode')
-    CONFIG['software_root'] = os.path.join(CONFIG['slapos_directory'],
-        'software')
-    CONFIG['instance_root'] = os.path.join(CONFIG['slapos_directory'],
-        'instance')
-    CONFIG['proxy_database'] = os.path.join(CONFIG['slapos_directory'],
-        'proxy.db')
-    CONFIG['proxy_host'] = self.getLocalIPv4Address()
-    CONFIG['master_url'] = 'http://%s:%s' % (CONFIG['proxy_host'],
-        CONFIG['proxy_port'])
-    self._createDirectory(CONFIG['software_root'])
-    self._createDirectory(CONFIG['instance_root'])
-    CONFIG['slapos_config'] = self.createConfigurationFile('slapos.cfg',
-        self.substituteTemplate(pkg_resources.resource_filename(__name__,
-          'template/slapos.cfg.in'), CONFIG))
-    self.path_list.append(CONFIG['slapos_config'])
+    if self.options['instance-dict']:
+      config_instance_dict = ConfigParser.ConfigParser()
+      config_instance_dict.add_section('instance_dict')
+      instance_dict = json.loads(self.options['instance-dict'])
 
-  def setupRunningWrapper(self):
-    self.path_list.extend(zc.buildout.easy_install.scripts([(
-      'testnode',
-        __name__+'.testnode', 'run')], self.ws,
-          sys.executable, self.wrapper_directory, arguments=[
-            dict(
-              computer_id=CONFIG['computer_id'],
-              instance_dict=eval(self.parameter_dict.get('instance_dict', '{}')),
-              instance_root=CONFIG['instance_root'],
-              ipv4_address=self.getLocalIPv4Address(),
-              ipv6_address=self.getGlobalIPv6Address(),
-              master_url=CONFIG['master_url'],
-              profile_path=eval(self.parameter_dict['vcs_repository_list'])[0]['profile_path'],
-              proxy_database=CONFIG['proxy_database'],
-              proxy_port=CONFIG['proxy_port'],
-              slapgrid_partition_binary=self.options['slapgrid_partition_binary'],
-              slapgrid_software_binary=self.options['slapgrid_software_binary'],
-              slapos_config=CONFIG['slapos_config'],
-              slapproxy_binary=self.options['slapproxy_binary'],
-              git_binary=self.options['git_binary'],
-              software_root=CONFIG['software_root'],
-              working_directory=CONFIG['working_directory'],
-              vcs_repository_list=eval(self.parameter_dict.get('vcs_repository_list'),),
-              node_quantity=self.parameter_dict.get('node_quantity', '1'),
-              test_suite_master_url=self.parameter_dict.get(
-                                'test_suite_master_url', None),
-              test_suite=self.parameter_dict.get('test_suite'),
-              test_suite_title=self.parameter_dict.get('test_suite_title'),
-              test_node_title=self.parameter_dict.get('test_node_title'),
-              project_title=self.parameter_dict.get('project_title'),
-              bin_directory=self.bin_directory,
-              # botenvironemnt is splittable string of key=value to substitute
-              # environment of running bot
-              bot_environment=self.parameter_dict.get('bot_environment', ''),
-              partition_reference=CONFIG['partition_reference'],
-              environment=dict(PATH=os.environ['PATH']),
-              vcs_authentication_list=eval(self.parameter_dict.get(
-                     'vcs_authentication_list', 'None')),
-            )
-          ]))
-
-  def installLocalGit(self):
-    git_dict = dict(git_binary = self.options['git_binary'])
-    git_dict.update(self.parameter_dict)
-    double_slash_end_position = 1
-    # XXX, this should be provided by slapos
-    print "bin_directory : %r" % self.bin_directory
-    home_directory = os.path.join(*os.path.split(self.bin_directory)[0:-1])
-    print "home_directory : %r" % home_directory
-    git_dict.setdefault("git_server_name", "git.erp5.org")
-    if git_dict.get('vcs_authentication_list', None) is not None:
-      vcs_authentication_list = eval(git_dict['vcs_authentication_list'])
-      netrc_file = open(os.path.join(home_directory, '.netrc'), 'w')
-      for vcs_authentication_dict in vcs_authentication_list:
-        netrc_file.write("""
-machine %(host)s
-login %(user_name)s
-password %(password)s
-""" % vcs_authentication_dict)
-      netrc_file.close()
-
-  def installLocalRepository(self):
-    self.installLocalGit()
-
-  def installLocalZip(self):
-    zip = os.path.join(self.bin_directory, 'zip')
-    if os.path.lexists(zip):
-      os.unlink(zip)
-    os.symlink(self.options['zip_binary'], zip)
+      for k ,v in instance_dict.iteritems():
+        config_instance_dict.set('instance_dict', k, v)
+      value = StringIO.StringIO()
+      config_instance_dict.write(value)
+      CONFIG['instance_dict'] = value.getvalue()
 
-  def installLocalPython(self):
-    """Installs local python fully featured with eggs"""
-    self.path_list.extend(zc.buildout.easy_install.scripts([], self.ws,
-          sys.executable, self.bin_directory, scripts=None,
-          interpreter='python'))
+    vcs_repository_list = json.loads(self.options['repository-list'])
+    config_repository_list = ConfigParser.ConfigParser()
+    i = 0
+    for repository in vcs_repository_list:
+      section_name = 'vcs_repository_%d' % i
+      config_repository_list.add_section(section_name)
+      config_repository_list.set(section_name, 'url', repository['url'])
+      if 'branch' in repository:
+        config_repository_list.set(section_name, 'branch', repository['branch'])
+      if 'profile_path' in repository:
+        config_repository_list.set(section_name, 'profile_path',
+                                   repository['profile_path'])
+      if 'buildout_section_id' in repository:
+        config_repository_list.set(section_name, 'buildout_section_id',
+                                   repository['buildout_section_id'])
+      i += 1
+    value = StringIO.StringIO()
+    config_repository_list.write(value)
+    CONFIG['repository_list'] = value.getvalue()
 
-  def installLocalRunUnitTest(self):
-    link = os.path.join(self.bin_directory, 'runCloudoooUnitTest')
-    destination = os.path.join(CONFIG['instance_root'],
-        CONFIG['partition_reference'], 'bin', 'runCloudoooUnitTest')
-    if os.path.lexists(link):
-      if not os.readlink(link) != destination:
-        os.unlink(link)
-    if not os.path.lexists(link):
-      os.symlink(destination, link)
-
-  def _install(self):
-    self.requirements, self.ws = self.egg.working_set()
-    self.path_list = []
-    self.installSlapOs()
-    self.setupRunningWrapper()
-    self.installLocalRepository()
-    self.installLocalZip()
-    self.installLocalPython()
-    self.installLocalRunUnitTest()
-    return self.path_list
+    configuration_file = self.createFile(
+      self.options['configuration-file'],
+      self.substituteTemplate(
+        self.getTemplateFilename('cloudoootestnode.cfg.in'),
+        CONFIG
+      ),
+    )
+    path_list.append(configuration_file)
+    path_list.append(
+      self.createPythonScript(
+        self.options['wrapper'],
+        'slapos.recipe.librecipe.execute.executee',
+        [ # Executable
+          [ self.options['testnode'], '-l', self.options['log-file'],
+            configuration_file],
+          # Environment
+          {
+            'GIT_SSL_NO_VERIFY': '1',
+          }
+        ],
+      )
+    )
+    return path_list

slapos/recipe/cloudoootestnode/template/cloudoootestnode.cfg.in

+[testnode]
+slapos_directory = %(slapos_directory)s
+working_directory = %(slapos_directory)s
+log_directory = %(log_directory)s
+run_directory = %(run_directory)s
+proxy_host = %(proxy_host)s
+proxy_port = %(proxy_port)s
+test_suite_master_url = %(test_suite_master_url)s
+test_suite_directory = %(test_suite_directory)s
+test_suite_title = %(test_suite_title)s
+test_suite = %(test_suite)s
+test_node_title = %(test_node_title)s
+node_quantity = %(node_quantity)s
+project_title= %(project_title)s
+ipv4_address = %(ipv4_address)s
+ipv6_address = %(ipv6_address)s
+
+# Binaries
+git_binary = %(git_binary)s
+slapgrid_partition_binary = %(slapgrid_partition_binary)s
+slapgrid_software_binary = %(slapgrid_software_binary)s
+slapproxy_binary = %(slapproxy_binary)s
+zip_binary = %(zip_binary)s
+
+[environment]
+PATH = %(PATH)s
+
+%(instance_dict)s
+
+%(repository_list)s

slapos/recipe/cloudoootestnode/template/slapos.cfg.in

-[slapos]
-software_root = %(software_root)s
-instance_root = %(instance_root)s
-master_url = %(master_url)s
-computer_id = %(computer_id)s
-
-[slapproxy]
-host = %(proxy_host)s
-port = %(proxy_port)s
-database_uri = %(proxy_database)s

software/cloudoootestnode/instance.cfg

 [buildout]
-parts =
-  testnode
 
 eggs-directory = ${buildout:eggs-directory}
 develop-eggs-directory = ${buildout:develop-eggs-directory}
+offline = true
+
+parts =
+  testnode
+  certificate-authority
 
 [testnode]
 recipe = slapos.cookbook:cloudoootestnode
 
-buildbot_binary = ${buildout:bin-directory}/buildbot
-slapgrid_partition_binary = ${buildout:bin-directory}/slapgrid-cp
-slapgrid_software_binary = ${buildout:bin-directory}/slapgrid-sr
-slapproxy_binary = ${buildout:bin-directory}/slapproxy
-svn_binary = ${subversion:location}/bin/svn
-git_binary = ${git:location}/bin/git
-svnversion_binary = ${subversion:location}/bin/svnversion
-zip_binary = ${zip:location}/bin/zip
+slapos-directory = $${directory:slapos}
+working-directory = $${directory:testnode}
+proxy-host = $${slap-network-information:global-ipv6}
+proxy-port = 5000
+log-directory = $${directory:log}
+run-directory = $${directory:run}
+test-suite-master-url = $${slap-parameter:test-suite-master-url}
+test-suite-directory = $${directory:test-suite}
+test-suite-title = $${slap-parameter:test-suite-title}
+test-node-title = $${slap-parameter:test-node-title}
+test-suite = $${slap-parameter:test-suite}
+node-quantity = $${slap-parameter:node-quantity}
+project-title = $${slap-parameter:project-title}
+ipv4-address = $${slap-network-information:local-ipv4}
+ipv6-address = $${slap-network-information:global-ipv6}
+instance-dict = $${slap-parameter:instance-dict}
+repository-list = $${slap-parameter:vcs-repository-list}
+
+git-binary = ${git:location}/bin/git
+slapgrid-partition-binary = ${buildout:bin-directory}/slapgrid-cp
+slapgrid-software-binary = ${buildout:bin-directory}/slapgrid-sr
+slapproxy-binary = ${buildout:bin-directory}/slapproxy
+svn-binary = ${subversion:location}/bin/svn
+svnversion-binary = ${subversion:location}/bin/svnversion
+testnode = ${buildout:bin-directory}/testnode
+zip-binary = ${zip:location}/bin/zip
+
+configuration-file = $${rootdirectory:etc}/cloudoootestnode.cfg
+log-file = $${basedirectory:log}/cloudoootestnode.log
+wrapper = $${basedirectory:services}/cloudoootestnode
+
+
+[certificate-authority]
+recipe = slapos.cookbook:certificate_authority
+openssl-binary = ${openssl:location}/bin/openssl
+ca-dir = $${directory:ca-dir}
+requests-directory = $${cadirectory:requests}
+wrapper = $${basedirectory:services}/ca
+ca-private = $${cadirectory:private}
+ca-certs = $${cadirectory:certs}
+ca-newcerts = $${cadirectory:newcerts}
+ca-crl = $${cadirectory:crl}
+
+[cadirectory]
+recipe = slapos.cookbook:mkdirectory
+requests = $${directory:ca-dir}/requests/
+private = $${directory:ca-dir}/private/
+certs = $${directory:ca-dir}/certs/
+newcerts = $${directory:ca-dir}/newcerts/
+crl = $${directory:ca-dir}/crl/
+
+[rootdirectory]
+recipe = slapos.cookbook:mkdirectory
+etc = $${buildout:directory}/etc/
+var = $${buildout:directory}/var/
+srv = $${buildout:directory}/srv/
+bin = $${buildout:directory}/bin/
+tmp = $${buildout:directory}/tmp/
+
+[basedirectory]
+recipe = slapos.cookbook:mkdirectory
+log = $${rootdirectory:var}/log/
+services = $${rootdirectory:etc}/run/
+run = $${rootdirectory:var}/run/
+promises = $${rootdirectory:etc}/promise/
+
+[directory]
+recipe = slapos.cookbook:mkdirectory
+slapos = $${rootdirectory:srv}/slapos/
+testnode = $${rootdirectory:srv}/testnode/
+test-suite = $${rootdirectory:srv}/test_suite/
+log = $${basedirectory:log}/testnode/
+run = $${basedirectory:run}/testnode/
+
+[slap-parameter]
+node-quantity = 1
+test-suite-master-url =
+instance-dict =
+vcs-repository-list = []
+test-suite-title = $${:test-suite}

software/cloudoootestnode/software.cfg

 [buildout]
-extends = ../erp5testnode/software.cfg
+
+find-links = http://www.nexedi.org/static/packages/source/slapos.buildout/
+    http://dist.repoze.org
+    http://www.nexedi.org/static/packages/source/
+
+# Separate from site eggs
+allowed-eggs-from-site-packages =
+include-site-packages = false
+exec-sitecustomize = false
+
+versions = versions
+
+extends =
+  ../../component/python-2.7/buildout.cfg
+  ../../component/subversion/buildout.cfg
+  ../../component/git/buildout.cfg
+  ../../component/lxml-python/buildout.cfg
+  ../../component/zip/buildout.cfg
+
+# Local development
+develop =
+  ${:parts-directory}/slapos.cookbook-repository
+parts =
+# Local development
+  slapos.cookbook-repository
+  check-recipe
+  template
+  lxml-python
+  eggs
+  subversion
+  zip
+  git
+
+
+# Use only quite well working sites.
+allow-hosts =
+  *.nexedi.org
+  *.python.org
+  *.sourceforge.net
+  dist.repoze.org
+  effbot.org
+  github.com
+  peak.telecommunity.com
+  psutil.googlecode.com
+  www.dabeaz.com
+  launchpad.net
+
+
+# XXX: Workaround of SlapOS limitation
+# Unzippig of eggs is required, as SlapOS do not yet provide nicely working
+# development / fast switching environment for whole software
+unzip = true
+
+# Local development
+[slapos.cookbook-repository]
+recipe = plone.recipe.command
+stop-on-error = true
+location = ${buildout:parts-directory}/${:_buildout_section_name_}
+command = ${git:location}/bin/git clone --quiet http://git.erp5.org/repos/slapos.git ${:location} && ${git:location}/bin/git checkout cloudooo
+update-command = cd ${:location} && ${git:location}/bin/git pull --quiet
+
+[check-recipe]
+recipe = plone.recipe.command
+stop-on-error = true
+update-command = ${:command}
+command = grep parts ${buildout:develop-eggs-directory}/slapos.cookbook.egg-link
+
+[eggs]
+recipe = zc.recipe.egg
+eggs =
+  ${lxml-python:egg}
+  zc.buildout
+  slapos.core
+  inotifyx
+  slapos.cookbook
+scripts =
+  slapgrid-cp = slapos.grid.slapgrid:runComputerPartition
+  slapgrid-sr = slapos.grid.slapgrid:runSoftwareRelease
+  slapproxy = slapos.proxy:main
+python = python2.7
+
+[lxml-python]
+python = python2.7
 
 [template]
 recipe = slapos.recipe.template
 url = ${:_profile_base_location_}/instance.cfg
+output = ${buildout:directory}/template.cfg
+mode = 0644
+md5sum = 525477aabca7349c328b69793d16b082
+
+[versions]
+# Use SlapOS patched zc.buildout
+zc.buildout = 1.6.0-dev-SlapOS-003
+
+Jinja2 = 2.6
+Werkzeug = 0.8.2
+hexagonit.recipe.cmmi = 1.5.0
+lxml = 2.3.2
+meld3 = 0.6.7
+slapos.cookbook = 0.38
+slapos.core = 0.21
+slapos.recipe.template = 2.2
+
+# Required by:
+# slapos.core==0.21
+Flask = 0.8
+
+# Required by:
+# slapos.cookbook==0.38
+PyXML = 0.8.5
+
+# Required by:
+# hexagonit.recipe.cmmi==1.5.0
+hexagonit.recipe.download = 1.5.0
+
+# Required by:
+# slapos.cookbook==0.38
+netaddr = 0.7.6
+
+# Required by:
+# slapos.core==0.21
+netifaces = 0.6
+
+# Required by:
+# slapos.cookbook==0.38
+# slapos.core==0.21
+# zc.buildout==1.6.0-dev-SlapOS-003
+setuptools = 0.6c12dev-r88846
+
+# Required by:
+# slapos.core==0.21
+supervisor = 3.0a12
+
+# Required by:
+# slapos.cookbook==0.38
+xml-marshaller = 0.9.7
+
+# Required by:
+# slapos.cookbook==0.38
+zc.recipe.egg = 1.3.2
+
+# Required by:
+# slapos.core==0.21
+zope.interface = 3.8.0