generate_erp5_tidstorage.py 9.02 KB
Newer Older
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28
##############################################################################
#
# Copyright (c) 2011 Vifib SARL and Contributors. All Rights Reserved.
#
# WARNING: This program as such is intended to be used by professional
# programmers who take the whole responsibility of assessing all potential
# consequences resulting from its eventual inadequacies and bugs
# End users who are looking for a ready-to-use solution with commercial
# guarantees and support are strongly adviced to contract a Free Software
# Service Company
#
# This program is Free Software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License
# as published by the Free Software Foundation; either version 3
# of the License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.
#
##############################################################################
from slapos.recipe.librecipe import GenericSlapRecipe
import os
Łukasz Nowak's avatar
WIP  
Łukasz Nowak committed
29
import json
30
import traceback
Łukasz Nowak's avatar
WIP  
Łukasz Nowak committed
31

Łukasz Nowak's avatar
Łukasz Nowak committed
32 33
SECTION_BACKEND_PUBLISHER = """[publish-apache-backend-list]
recipe = slapos.cookbook:publish"""
Łukasz Nowak's avatar
WIP  
Łukasz Nowak committed
34 35 36 37
ZOPE_PORT_BASE = 12000
ZEO_PORT_BASE = 15000
HAPROXY_PORT_BASE = 11000
APACHE_PORT_BASE = 10000
38 39 40 41 42

class Recipe(GenericSlapRecipe):
  def _options(self, options):
    self.dirname = os.path.join(self.buildout['buildout']['parts-directory'],
      self.name)
Łukasz Nowak's avatar
Łukasz Nowak committed
43
    options['output'] = os.path.join(self.dirname, self.name + '.cfg')
44 45

  def _generateRealTemplate(self):
Łukasz Nowak's avatar
WIP  
Łukasz Nowak committed
46 47
    current_zeo_port = ZEO_PORT_BASE
    current_zope_port = ZOPE_PORT_BASE
Łukasz Nowak's avatar
Łukasz Nowak committed
48 49
    current_apache_port = APACHE_PORT_BASE
    current_haproxy_port = HAPROXY_PORT_BASE
Łukasz Nowak's avatar
WIP  
Łukasz Nowak committed
50 51 52 53 54
    json_data = json.loads(self.parameter_dict['json'])
    site_id = str(json_data['site-id'])
    # prepare zeo
    output = ''
    part_list = []
Łukasz Nowak's avatar
Łukasz Nowak committed
55
    zope_dict = {}
Łukasz Nowak's avatar
Łukasz Nowak committed
56
    zope_connection_dict = {}
57
    known_tid_storage_identifier_dict = {}
Łukasz Nowak's avatar
Łukasz Nowak committed
58
    snippet_zeo = open(self.options['snippet-zeo']).read()
59 60 61 62 63 64 65 66 67 68 69
    for zeo_id, zeo_configuration_list in json_data['zeo'].iteritems():
      storage_list = []
      a = storage_list.append
      for zeo_slave in zeo_configuration_list:
        zope_connection_dict[zeo_slave['storage-name']] = {
          'zope-cache-size': zeo_slave['zope-cache-size'],
          'zeo-cache-size': zeo_slave['zeo-cache-size'],
          'mount-point': zeo_slave['mount-point'] % {'site-id': site_id},
          'storage-name': zeo_slave['storage-name'],
          'server': '${zeo-instance-%(zeo-id)s:ip}:${zeo-instance-%(zeo-id)s:port}' % {'zeo-id': zeo_id}
        }
70 71 72 73
        zodb_path = os.path.join('${directory:zodb}', zeo_slave['storage-name'] + '.fs')
        a('  storage-name=%(storage-name)s zodb-path=%(zodb-path)s' % {'zodb-path': zodb_path, 'storage-name': zeo_slave['storage-name']})
        known_tid_storage_identifier_dict[
          "((('%(ip)s', %(port)s),), '%(storage_name)s')" % dict(
Łukasz Nowak's avatar
Łukasz Nowak committed
74 75
            ip='${zeo-instance-%s:ip}' % zeo_id,
            port='${zeo-instance-%s:port}' % zeo_id,
76
            storage_name=zeo_slave['storage-name'])] = (zodb_path, '${directory:zodb-backup}/%s/' % zeo_slave['storage-name'], zeo_slave['serialize-path'] % {'site-id': site_id})
Łukasz Nowak's avatar
WIP  
Łukasz Nowak committed
77
      current_zeo_port += 1
Łukasz Nowak's avatar
Łukasz Nowak committed
78
      output += snippet_zeo % dict(
Łukasz Nowak's avatar
WIP  
Łukasz Nowak committed
79 80
        zeo_id=zeo_id,
        zeo_port=current_zeo_port,
81 82
        storage_list='\n'.join(storage_list)
      )
Łukasz Nowak's avatar
WIP  
Łukasz Nowak committed
83 84 85 86
      part_list.extend([
        "zeo-instance-%s" % zeo_id,
        "logrotate-entry-zeo-%s" % zeo_id
      ])
Łukasz Nowak's avatar
Łukasz Nowak committed
87

Łukasz Nowak's avatar
Łukasz Nowak committed
88 89 90
    zeo_connection_list = []
    a = zeo_connection_list.append
    for k, v in zope_connection_dict.iteritems():
91
      a('  zeo-cache-size=%(zeo-cache-size)s zope-cache-size=%(zope-cache-size)s server=%(server)s mount-point=%(mount-point)s storage-name=%(storage-name)s' % v)
Łukasz Nowak's avatar
Łukasz Nowak committed
92
    zeo_connection_string = '\n'.join(zeo_connection_list)
Łukasz Nowak's avatar
Łukasz Nowak committed
93
    zope_dict.update(
Łukasz Nowak's avatar
Łukasz Nowak committed
94 95
      timezone=json_data['timezone'],
      zeo_connection_string=zeo_connection_string
Łukasz Nowak's avatar
Łukasz Nowak committed
96
    )
Łukasz Nowak's avatar
Łukasz Nowak committed
97
    # always one distribution node
Łukasz Nowak's avatar
Łukasz Nowak committed
98
    current_zope_port += 1
Łukasz Nowak's avatar
Łukasz Nowak committed
99
    snippet_zope = open(self.options['snippet-zope']).read()
Łukasz Nowak's avatar
Łukasz Nowak committed
100 101
    zope_id = 'zope-distribution'
    part_list.append(zope_id)
102
    part_list.append('logrotate-entry-%s' % zope_id)
103
    output += snippet_zope % dict(zope_thread_amount=1, zope_id=zope_id,
104 105 106
      zope_port=current_zope_port, zope_timeserver=True, 
      longrequest_logger_file='', longrequest_logger_timeout='', 
      longrequest_logger_interval='', **zope_dict)
Łukasz Nowak's avatar
Łukasz Nowak committed
107
    # always one admin node
Łukasz Nowak's avatar
Łukasz Nowak committed
108 109 110
    current_zope_port += 1
    zope_id = 'zope-admin'
    part_list.append(zope_id)
111
    part_list.append('logrotate-entry-%s' % zope_id)
112
    output += snippet_zope % dict(zope_thread_amount=1, zope_id=zope_id,
113 114 115
      zope_port=current_zope_port, zope_timeserver=False, 
      longrequest_logger_file='', longrequest_logger_timeout='', 
      longrequest_logger_interval='', **zope_dict)
Łukasz Nowak's avatar
Łukasz Nowak committed
116 117
    # handle activity key
    for q in range(1, json_data['activity']['zopecount'] + 1):
Łukasz Nowak's avatar
Łukasz Nowak committed
118
      current_zope_port += 1
Łukasz Nowak's avatar
Łukasz Nowak committed
119 120
      part_name = 'zope-activity-%s' % q
      part_list.append(part_name)
121
      part_list.append('logrotate-entry-%s' % part_name)
122
      output += snippet_zope % dict(zope_thread_amount=1, zope_id=part_name,
123 124 125
        zope_port=current_zope_port, zope_timeserver=True,
        longrequest_logger_file='', longrequest_logger_timeout='', 
        longrequest_logger_interval='', **zope_dict)
Łukasz Nowak's avatar
Łukasz Nowak committed
126
    # handle backend key
Łukasz Nowak's avatar
Łukasz Nowak committed
127
    snippet_backend = open(self.options['snippet-backend']).read()
Łukasz Nowak's avatar
Łukasz Nowak committed
128
    publish_url_list = []
Łukasz Nowak's avatar
Łukasz Nowak committed
129
    for backend_name, backend_configuration in json_data['backend'].iteritems():
Łukasz Nowak's avatar
Łukasz Nowak committed
130
      haproxy_backend_list = []
Łukasz Nowak's avatar
Łukasz Nowak committed
131
      for q in range(1, backend_configuration['zopecount'] + 1):
Łukasz Nowak's avatar
Łukasz Nowak committed
132
        current_zope_port += 1
Łukasz Nowak's avatar
Łukasz Nowak committed
133
        part_name = 'zope-%s-%s' % (backend_name, q)
Łukasz Nowak's avatar
Łukasz Nowak committed
134
        part_list.append(part_name)
135
        part_list.append('logrotate-entry-%s' % part_name)
136 137 138 139 140 141 142 143 144 145
        longrequest_logger = backend_configuration.get("longrequest-logger", None)
        if longrequest_logger is not None:
          longrequest_part_name = '%s-longrequest' %part_name 
          longrequest_logger_file = '${basedirectory:log}/%s.log' \
                                      %longrequest_part_name
          longrequest_logger_timeout = longrequest_logger.get('timeout', '4')
          longrequest_logger_interval = longrequest_logger.get('interval', '2')
        else:
          longrequest_logger_file = longrequest_logger_timeout = \
            longrequest_logger_interval = ''
146 147 148
        output += snippet_zope % dict(
          zope_thread_amount=backend_configuration['thread-amount'],
          zope_id=part_name, zope_port=current_zope_port, zope_timeserver=False,
149 150 151
          longrequest_logger_file=longrequest_logger_file, 
          longrequest_logger_timeout=longrequest_logger_timeout, 
          longrequest_logger_interval=longrequest_logger_interval,
152
          **zope_dict)
Łukasz Nowak's avatar
Łukasz Nowak committed
153
        haproxy_backend_list.append('${%(part_name)s:ip}:${%(part_name)s:port}' % dict(part_name=part_name))
Łukasz Nowak's avatar
Łukasz Nowak committed
154
      # now generate backend access
Łukasz Nowak's avatar
Łukasz Nowak committed
155 156 157
      current_apache_port += 1
      current_haproxy_port += 1
      part_list.append('apache-%(backend_name)s ca-apache-%(backend_name)s logrotate-entry-apache-%(backend_name)s haproxy-%(backend_name)s' % dict(backend_name=backend_name))
Łukasz Nowak's avatar
Łukasz Nowak committed
158
      backend_dict = dict(
Łukasz Nowak's avatar
Łukasz Nowak committed
159 160 161 162 163
        backend_name=backend_name,
        apache_port=current_apache_port,
        haproxy_port=current_haproxy_port,
        access_control_string=backend_configuration['access-control-string'],
        maxconn=backend_configuration['maxconn'],
Łukasz Nowak's avatar
Łukasz Nowak committed
164 165
        server_check_path='/%s/getId' % site_id,
        haproxy_backend_list=' '.join(haproxy_backend_list)
Łukasz Nowak's avatar
Łukasz Nowak committed
166
      )
Łukasz Nowak's avatar
Łukasz Nowak committed
167 168
      publish_url_list.append('url-%(backend_name)s = https://[${apache-%(backend_name)s:ip}]:${apache-%(backend_name)s:port}' % dict(
        backend_name=backend_name))
Łukasz Nowak's avatar
Łukasz Nowak committed
169
      output += snippet_backend % backend_dict
Łukasz Nowak's avatar
Łukasz Nowak committed
170 171 172
    output += SECTION_BACKEND_PUBLISHER + '\n'
    output += '\n'.join(publish_url_list)
    part_list.append('publish-apache-backend-list')
173
    prepend = open(self.options['snippet-master']).read() % dict(
Łukasz Nowak's avatar
Łukasz Nowak committed
174 175 176
        part_list='  \n'.join(['  '+q for q in part_list]),
        known_tid_storage_identifier_dict=known_tid_storage_identifier_dict,
        haproxy_section="haproxy-%s" % backend_name,
177 178 179
        zope_section=zope_id,
        site_id=site_id,
        **self.parameter_dict
Łukasz Nowak's avatar
Łukasz Nowak committed
180
        )
Łukasz Nowak's avatar
WIP  
Łukasz Nowak committed
181
    output = prepend + output
182 183
    with open(self.options['output'], 'w') as f:
      f.write(output)
184 185 186 187 188 189

  def _install(self):
    if not os.path.exists(self.dirname):
      os.mkdir(self.dirname)
    if not "json" in self.parameter_dict:
      # no json transimtted, nothing to do
Łukasz Nowak's avatar
Łukasz Nowak committed
190
      with open(self.options['output'], 'w') as f:
191 192
        f.write("[buildout]\nparts =\n")
    else:
193 194 195 196 197
      try:
        self._generateRealTemplate()
      except Exception:
        print 'Ignored issue during template generation:\n%s' % \
          traceback.format_exc()
198
    return [self.dirname]