diff --git a/slapos/recipe/README.lamp.txt b/slapos/recipe/README.lamp.txt new file mode 100644 index 0000000000000000000000000000000000000000..7a75a2033a3ed733ff5198bbce92e65b3fcb21ab --- /dev/null +++ b/slapos/recipe/README.lamp.txt @@ -0,0 +1,104 @@ +lamp +===== + +the lamp recipe help you to deploy simply a php based application on slapos. This recipe is +able to setup mariadb, apache and apache-php for your php application, is also capable to +configure your software during installation to ensure a full compatibility. + + +How to use? +----------- + +just add this part in your software.cfg to use the lamp.simple module + +[instance-recipe] +egg = slapos.cookbook +module = lamp.simple + +you also need to extend lamp.cfg + +extends = + http://git.erp5.org/gitweb/slapos.git/blob_plain/refs/tags/slapos-0.50:/stack/lamp.cfg + + +lamp.runner +===== + +When you install some software (such as prestashop) you need to remove or rename folder, with slapos you can not +access to the www-data directory. to do this, you need to tell to lamp recipe to remove or/and it when software +will be instantiated. Some software requires more than rename or delete a folder (manualy create database etc...) +in this case you need to write a python script and lamp recipe must run it when installing your software. + + + +How to use? +----------- + +CONDITION +-------- +the action (move, rename, launch script) only starts when the condition is filled. +in instance.cfg, add + +file_token = path_of_file + +and the action will begin when path_of_www-data/path_of_file will be created +you can also use database to check condition. add + +table_name = name_of_table +constraint = sql_where_condition + +name_of_table is the full or partial name(in some cases we can not know the prefix used to create tables) of table +into mariadb databse for example table_name = admin. if you use +name_of_table = **, the action will begin when database is ready. +constraint is the sql_condition to use when search entry into name_of_table for example constraint = `admin_id`=1 + +you can no use file_token and table_name at the same time, otherwise file_token will be used in priority. attention +to the conditions that will never be satisfied. + + + +ACTION +------- +the action start when condition is true +1- delete file or folder +into instance.cfg, use + +delete = file_or_folder1, file_or_folder2, file_or_folder3 ... + +for example delete = admin + +2- rename file or folder +into instance.cfg, use + +rename = old_name1 => new_name1, old_name2 => new_name2, ... you can also use + +rename = old_name1, old_name2 => new_name2, ... in this case old_name1 will be rename and the new name will be chose +by joining old_name1 and mysql_user: this should give +rename = old_name1 => old_name1-mysql_user, old_name2 => new_name2, ... + +3- launch python script + +use script = ${configure-script:location}/${configure-script:filename} into instance.cfg, add part configure-script +into software.cfg + +parts = configure-script + +[configure-script] +recipe = hexagonit.recipe.download +location = ${buildout:parts-directory}/${:_buildout_section_name_} +url = url_of_script_name.py +filename = script_name.py +download-only = True + +the script_name.py should contain a main module, sys.argv is passed to the main. you can write script_name.py like this +.... +def setup(args): + base_url, htdocs, renamed, mysql_user, mysql_password, mysql_database, mysql_host = args + ....... + +if __name__ == '__main__': + setup(sys.argv[1:]) + +base_url: is the url of php software +htdocs: is the path of www-data directory +mysql_user, mysql_password, mysql_database, mysql_host: is the mariadb parameters diff --git a/slapos/recipe/README.mkdirectory.txt b/slapos/recipe/README.mkdirectory.txt new file mode 100644 index 0000000000000000000000000000000000000000..426282a219dd9e230774b64334f8e8ecf8d6bb87 --- /dev/null +++ b/slapos/recipe/README.mkdirectory.txt @@ -0,0 +1,13 @@ +mkdirectory +=========== + +mkdirectory loops on its options and create the directory joined + +.. Note:: + + Use a slash ``/`` as directory separator. Don't use system dependent separator. + The slash will be parsed and replace by the operating system right separator. + + Only use relative directory to the buildout root directory. + +The created directory won't be added to path list. diff --git a/slapos/recipe/certificate_authority/__init__.py b/slapos/recipe/certificate_authority/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..2bf43ed728565ac7ff4415e8c04a509fd9938c21 --- /dev/null +++ b/slapos/recipe/certificate_authority/__init__.py @@ -0,0 +1,132 @@ +############################################################################## +# +# Copyright (c) 2010 Vifib SARL and Contributors. All Rights Reserved. +# +# WARNING: This program as such is intended to be used by professional +# programmers who take the whole responsibility of assessing all potential +# consequences resulting from its eventual inadequacies and bugs +# End users who are looking for a ready-to-use solution with commercial +# guarantees and support are strongly adviced to contract a Free Software +# Service Company +# +# This program is Free Software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either version 3 +# of the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. +# +############################################################################## +import os +import hashlib +import ConfigParser + +from slapos.recipe.librecipe import GenericBaseRecipe + +class Recipe(GenericBaseRecipe): + + def setPath(self): + self.ca_dir = self.options['ca-dir'] + self.request_directory = self.options['requests-directory'] + self.ca_private = self.options['ca-private'] + self.ca_certs = self.options['ca-certs'] + self.ca_newcerts = self.options['ca-newcerts'] + self.ca_crl = self.options['ca-crl'] + self.ca_key_ext = '.key' + self.ca_crt_ext = '.crt' + + def install(self): + path_list = [] + + # XXX: We gotta find better a way to get these options + ca_country_code = 'XX' + ca_email = 'xx@example.com' + ca_state = 'State', + ca_city = 'City' + ca_company = 'Company' + # XXX: end + + self.setPath() + + config = dict(ca_dir=self.ca_dir, request_dir=self.request_directory) + + for f in ['crlnumber', 'serial']: + if not os.path.exists(os.path.join(self.ca_dir, f)): + open(os.path.join(self.ca_dir, f), 'w').write('01') + if not os.path.exists(os.path.join(self.ca_dir, 'index.txt')): + open(os.path.join(self.ca_dir, 'index.txt'), 'w').write('') + openssl_configuration = os.path.join(self.ca_dir, 'openssl.cnf') + config.update( + working_directory=self.ca_dir, + country_code=ca_country_code, + state=ca_state, + city=ca_city, + company=ca_company, + email_address=ca_email, + ) + self.createFile(openssl_configuration, self.substituteTemplate( + self.getTemplateFilename('openssl.cnf.ca.in'), config)) + + ca_wrapper = self.createPythonScript( + self.options['wrapper'], + '%s.certificate_authority.runCertificateAuthority' % __name__, + dict( + openssl_configuration=openssl_configuration, + openssl_binary=self.options['openssl-binary'], + certificate=os.path.join(self.ca_dir, 'cacert.pem'), + key=os.path.join(self.ca_private, 'cakey.pem'), + crl=self.ca_crl, + request_dir=self.request_directory + ) + ) + path_list.append(ca_wrapper) + + return path_list + +class Request(Recipe): + + def _options(self, options): + if 'name' not in options: + options['name'] = self.name + + def install(self): + self.setPath() + + key_file = self.options['key-file'] + cert_file = self.options['cert-file'] + + name = self.options['name'] + hash_ = hashlib.sha512(name).hexdigest() + key = os.path.join(self.ca_private, hash_ + self.ca_key_ext) + certificate = os.path.join(self.ca_certs, hash_ + self.ca_crt_ext) + parser = ConfigParser.RawConfigParser() + parser.add_section('certificate') + parser.set('certificate', 'name', name) + parser.set('certificate', 'key_file', key) + parser.set('certificate', 'certificate_file', certificate) + parser.write(open(os.path.join(self.request_directory, hash_), 'w')) + + for link in [key_file, cert_file]: + if os.path.islink(link): + os.unlink(link) + elif os.path.exists(link): + raise OSError("%r file should be a symbolic link.") + + os.symlink(key, key_file) + os.symlink(certificate, cert_file) + + wrapper = self.createPythonScript( + self.options['wrapper'], + 'slapos.recipe.librecipe.execute.execute_wait', + [ [self.options['executable']], + [certificate, key] ], + ) + + return [key_file, cert_file, wrapper] diff --git a/slapos/recipe/certificate_authority/certificate_authority.py b/slapos/recipe/certificate_authority/certificate_authority.py new file mode 100644 index 0000000000000000000000000000000000000000..a4af4ab76df89af085dd779e12b5d1a64af37ff6 --- /dev/null +++ b/slapos/recipe/certificate_authority/certificate_authority.py @@ -0,0 +1,117 @@ +import os +import subprocess +import time +import ConfigParser + + +def popenCommunicate(command_list, input=None): + subprocess_kw = dict(stdout=subprocess.PIPE, stderr=subprocess.STDOUT) + if input is not None: + subprocess_kw.update(stdin=subprocess.PIPE) + popen = subprocess.Popen(command_list, **subprocess_kw) + result = popen.communicate(input)[0] + if popen.returncode is None: + popen.kill() + if popen.returncode != 0: + raise ValueError('Issue during calling %r, result was:\n%s' % ( + command_list, result)) + return result + + +class CertificateAuthority: + def __init__(self, key, certificate, openssl_binary, + openssl_configuration, request_dir): + self.key = key + self.certificate = certificate + self.openssl_binary = openssl_binary + self.openssl_configuration = openssl_configuration + self.request_dir = request_dir + + def checkAuthority(self): + file_list = [ self.key, self.certificate ] + ca_ready = True + for f in file_list: + if not os.path.exists(f): + ca_ready = False + break + if ca_ready: + return + for f in file_list: + if os.path.exists(f): + os.unlink(f) + try: + # no CA, let us create new one + popenCommunicate([self.openssl_binary, 'req', '-nodes', '-config', + self.openssl_configuration, '-new', '-x509', '-extensions', + 'v3_ca', '-keyout', self.key, '-out', self.certificate, + '-days', '10950'], 'Automatic Certificate Authority\n') + except: + try: + for f in file_list: + if os.path.exists(f): + os.unlink(f) + except: + # do not raise during cleanup + pass + raise + + def _checkCertificate(self, common_name, key, certificate): + file_list = [key, certificate] + ready = True + for f in file_list: + if not os.path.exists(f): + ready = False + break + if ready: + return False + for f in file_list: + if os.path.exists(f): + os.unlink(f) + csr = certificate + '.csr' + try: + popenCommunicate([self.openssl_binary, 'req', '-config', + self.openssl_configuration, '-nodes', '-new', '-keyout', + key, '-out', csr, '-days', '3650'], + common_name + '\n') + try: + popenCommunicate([self.openssl_binary, 'ca', '-batch', '-config', + self.openssl_configuration, '-out', certificate, + '-infiles', csr]) + finally: + if os.path.exists(csr): + os.unlink(csr) + except: + try: + for f in file_list: + if os.path.exists(f): + os.unlink(f) + except: + # do not raise during cleanup + pass + raise + else: + return True + + def checkRequestDir(self): + for request_file in os.listdir(self.request_dir): + parser = ConfigParser.RawConfigParser() + parser.readfp(open(os.path.join(self.request_dir, request_file), 'r')) + if self._checkCertificate(parser.get('certificate', 'name'), + parser.get('certificate', 'key_file'), parser.get('certificate', + 'certificate_file')): + print 'Created certificate %r' % parser.get('certificate', 'name') + +def runCertificateAuthority(ca_conf): + ca = CertificateAuthority(ca_conf['key'], ca_conf['certificate'], + ca_conf['openssl_binary'], ca_conf['openssl_configuration'], + ca_conf['request_dir']) + while True: + ca.checkAuthority() + ca.checkRequestDir() + # XXX + # Antoine: I really don't like that at all. It wastes useful CPU time. + # I think it would be a greater idea to use pyinotify + # <http://pyinotify.sourceforge.net/> + # Or we could use select() with socket as well. + time.sleep(60) + # end XXX diff --git a/slapos/recipe/certificate_authority/template/openssl.cnf.ca.in b/slapos/recipe/certificate_authority/template/openssl.cnf.ca.in new file mode 100644 index 0000000000000000000000000000000000000000..8a450a68762145e72923635273e06a00e80d34ca --- /dev/null +++ b/slapos/recipe/certificate_authority/template/openssl.cnf.ca.in @@ -0,0 +1,350 @@ +# +# OpenSSL example configuration file. +# This is mostly being used for generation of certificate requests. +# + +# This definition stops the following lines choking if HOME isn't +# defined. +HOME = . +RANDFILE = $ENV::HOME/.rnd + +# Extra OBJECT IDENTIFIER info: +#oid_file = $ENV::HOME/.oid +oid_section = new_oids + +# To use this configuration file with the "-extfile" option of the +# "openssl x509" utility, name here the section containing the +# X.509v3 extensions to use: +# extensions = +# (Alternatively, use a configuration file that has only +# X.509v3 extensions in its main [= default] section.) + +[ new_oids ] + +# We can add new OIDs in here for use by 'ca', 'req' and 'ts'. +# Add a simple OID like this: +# testoid1=1.2.3.4 +# Or use config file substitution like this: +# testoid2=${testoid1}.5.6 + +# Policies used by the TSA examples. +tsa_policy1 = 1.2.3.4.1 +tsa_policy2 = 1.2.3.4.5.6 +tsa_policy3 = 1.2.3.4.5.7 + +#################################################################### +[ ca ] +default_ca = CA_default # The default ca section + +#################################################################### +[ CA_default ] + +dir = %(working_directory)s # Where everything is kept +certs = $dir/certs # Where the issued certs are kept +crl_dir = $dir/crl # Where the issued crl are kept +database = $dir/index.txt # database index file. +#unique_subject = no # Set to 'no' to allow creation of + # several ctificates with same subject. +new_certs_dir = $dir/newcerts # default place for new certs. + +certificate = $dir/cacert.pem # The CA certificate +serial = $dir/serial # The current serial number +crlnumber = $dir/crlnumber # the current crl number + # must be commented out to leave a V1 CRL +crl = $dir/crl.pem # The current CRL +private_key = $dir/private/cakey.pem # The private key +RANDFILE = $dir/private/.rand # private random number file + +x509_extensions = usr_cert # The extentions to add to the cert + +# Comment out the following two lines for the "traditional" +# (and highly broken) format. +name_opt = ca_default # Subject Name options +cert_opt = ca_default # Certificate field options + +# Extension copying option: use with caution. +# copy_extensions = copy + +# Extensions to add to a CRL. Note: Netscape communicator chokes on V2 CRLs +# so this is commented out by default to leave a V1 CRL. +# crlnumber must also be commented out to leave a V1 CRL. +# crl_extensions = crl_ext + +default_days = 3650 # how long to certify for +default_crl_days= 30 # how long before next CRL +default_md = default # use public key default MD +preserve = no # keep passed DN ordering + +# A few difference way of specifying how similar the request should look +# For type CA, the listed attributes must be the same, and the optional +# and supplied fields are just that :-) +policy = policy_match + +# For the CA policy +[ policy_match ] +countryName = match +stateOrProvinceName = match +organizationName = match +organizationalUnitName = optional +commonName = supplied +emailAddress = optional + +# For the 'anything' policy +# At this point in time, you must list all acceptable 'object' +# types. +[ policy_anything ] +countryName = optional +stateOrProvinceName = optional +localityName = optional +organizationName = optional +organizationalUnitName = optional +commonName = supplied +emailAddress = optional + +#################################################################### +[ req ] +default_bits = 2048 +default_md = sha1 +default_keyfile = privkey.pem +distinguished_name = req_distinguished_name +#attributes = req_attributes +x509_extensions = v3_ca # The extentions to add to the self signed cert + +# Passwords for private keys if not present they will be prompted for +# input_password = secret +# output_password = secret + +# This sets a mask for permitted string types. There are several options. +# default: PrintableString, T61String, BMPString. +# pkix : PrintableString, BMPString (PKIX recommendation before 2004) +# utf8only: only UTF8Strings (PKIX recommendation after 2004). +# nombstr : PrintableString, T61String (no BMPStrings or UTF8Strings). +# MASK:XXXX a literal mask value. +# WARNING: ancient versions of Netscape crash on BMPStrings or UTF8Strings. +string_mask = utf8only + +# req_extensions = v3_req # The extensions to add to a certificate request + +[ req_distinguished_name ] +countryName = Country Name (2 letter code) +countryName_value = %(country_code)s +countryName_min = 2 +countryName_max = 2 + +stateOrProvinceName = State or Province Name (full name) +stateOrProvinceName_value = %(state)s + +localityName = Locality Name (eg, city) +localityName_value = %(city)s + +0.organizationName = Organization Name (eg, company) +0.organizationName_value = %(company)s + +# we can do this but it is not needed normally :-) +#1.organizationName = Second Organization Name (eg, company) +#1.organizationName_default = World Wide Web Pty Ltd + +commonName = Common Name (eg, your name or your server\'s hostname) +commonName_max = 64 + +emailAddress = Email Address +emailAddress_value = %(email_address)s +emailAddress_max = 64 + +# SET-ex3 = SET extension number 3 + +#[ req_attributes ] +#challengePassword = A challenge password +#challengePassword_min = 4 +#challengePassword_max = 20 +# +#unstructuredName = An optional company name + +[ usr_cert ] + +# These extensions are added when 'ca' signs a request. + +# This goes against PKIX guidelines but some CAs do it and some software +# requires this to avoid interpreting an end user certificate as a CA. + +basicConstraints=CA:FALSE + +# Here are some examples of the usage of nsCertType. If it is omitted +# the certificate can be used for anything *except* object signing. + +# This is OK for an SSL server. +# nsCertType = server + +# For an object signing certificate this would be used. +# nsCertType = objsign + +# For normal client use this is typical +# nsCertType = client, email + +# and for everything including object signing: +# nsCertType = client, email, objsign + +# This is typical in keyUsage for a client certificate. +# keyUsage = nonRepudiation, digitalSignature, keyEncipherment + +# This will be displayed in Netscape's comment listbox. +nsComment = "OpenSSL Generated Certificate" + +# PKIX recommendations harmless if included in all certificates. +subjectKeyIdentifier=hash +authorityKeyIdentifier=keyid,issuer + +# This stuff is for subjectAltName and issuerAltname. +# Import the email address. +# subjectAltName=email:copy +# An alternative to produce certificates that aren't +# deprecated according to PKIX. +# subjectAltName=email:move + +# Copy subject details +# issuerAltName=issuer:copy + +#nsCaRevocationUrl = http://www.domain.dom/ca-crl.pem +#nsBaseUrl +#nsRevocationUrl +#nsRenewalUrl +#nsCaPolicyUrl +#nsSslServerName + +# This is required for TSA certificates. +# extendedKeyUsage = critical,timeStamping + +[ v3_req ] + +# Extensions to add to a certificate request + +basicConstraints = CA:FALSE +keyUsage = nonRepudiation, digitalSignature, keyEncipherment + +[ v3_ca ] + + +# Extensions for a typical CA + + +# PKIX recommendation. + +subjectKeyIdentifier=hash + +authorityKeyIdentifier=keyid:always,issuer + +# This is what PKIX recommends but some broken software chokes on critical +# extensions. +#basicConstraints = critical,CA:true +# So we do this instead. +basicConstraints = CA:true + +# Key usage: this is typical for a CA certificate. However since it will +# prevent it being used as an test self-signed certificate it is best +# left out by default. +# keyUsage = cRLSign, keyCertSign + +# Some might want this also +# nsCertType = sslCA, emailCA + +# Include email address in subject alt name: another PKIX recommendation +# subjectAltName=email:copy +# Copy issuer details +# issuerAltName=issuer:copy + +# DER hex encoding of an extension: beware experts only! +# obj=DER:02:03 +# Where 'obj' is a standard or added object +# You can even override a supported extension: +# basicConstraints= critical, DER:30:03:01:01:FF + +[ crl_ext ] + +# CRL extensions. +# Only issuerAltName and authorityKeyIdentifier make any sense in a CRL. + +# issuerAltName=issuer:copy +authorityKeyIdentifier=keyid:always + +[ proxy_cert_ext ] +# These extensions should be added when creating a proxy certificate + +# This goes against PKIX guidelines but some CAs do it and some software +# requires this to avoid interpreting an end user certificate as a CA. + +basicConstraints=CA:FALSE + +# Here are some examples of the usage of nsCertType. If it is omitted +# the certificate can be used for anything *except* object signing. + +# This is OK for an SSL server. +# nsCertType = server + +# For an object signing certificate this would be used. +# nsCertType = objsign + +# For normal client use this is typical +# nsCertType = client, email + +# and for everything including object signing: +# nsCertType = client, email, objsign + +# This is typical in keyUsage for a client certificate. +# keyUsage = nonRepudiation, digitalSignature, keyEncipherment + +# This will be displayed in Netscape's comment listbox. +nsComment = "OpenSSL Generated Certificate" + +# PKIX recommendations harmless if included in all certificates. +subjectKeyIdentifier=hash +authorityKeyIdentifier=keyid,issuer + +# This stuff is for subjectAltName and issuerAltname. +# Import the email address. +# subjectAltName=email:copy +# An alternative to produce certificates that aren't +# deprecated according to PKIX. +# subjectAltName=email:move + +# Copy subject details +# issuerAltName=issuer:copy + +#nsCaRevocationUrl = http://www.domain.dom/ca-crl.pem +#nsBaseUrl +#nsRevocationUrl +#nsRenewalUrl +#nsCaPolicyUrl +#nsSslServerName + +# This really needs to be in place for it to be a proxy certificate. +proxyCertInfo=critical,language:id-ppl-anyLanguage,pathlen:3,policy:foo + +#################################################################### +[ tsa ] + +default_tsa = tsa_config1 # the default TSA section + +[ tsa_config1 ] + +# These are used by the TSA reply generation only. +dir = /etc/pki/tls # TSA root directory +serial = $dir/tsaserial # The current serial number (mandatory) +crypto_device = builtin # OpenSSL engine to use for signing +signer_cert = $dir/tsacert.pem # The TSA signing certificate + # (optional) +certs = $dir/cacert.pem # Certificate chain to include in reply + # (optional) +signer_key = $dir/private/tsakey.pem # The TSA private key (optional) + +default_policy = tsa_policy1 # Policy if request did not specify it + # (optional) +other_policies = tsa_policy2, tsa_policy3 # acceptable policies (optional) +digests = md5, sha1 # Acceptable message digests (mandatory) +accuracy = secs:1, millisecs:500, microsecs:100 # (optional) +clock_precision_digits = 0 # number of digits after dot. (optional) +ordering = yes # Is ordering defined for timestamps? + # (optional, default: no) +tsa_name = yes # Must the TSA name be included in the reply? + # (optional, default: no) +ess_cert_id_chain = no # Must the ESS cert id chain be included? + # (optional, default: no) diff --git a/slapos/recipe/davstorage/__init__.py b/slapos/recipe/davstorage/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..ae79dd33f2f063a795b6e3b0929659fc29b8ea98 --- /dev/null +++ b/slapos/recipe/davstorage/__init__.py @@ -0,0 +1,125 @@ +############################################################################## +# +# Copyright (c) 2010 Vifib SARL and Contributors. All Rights Reserved. +# +# WARNING: This program as such is intended to be used by professional +# programmers who take the whole responsibility of assessing all potential +# consequences resulting from its eventual inadequacies and bugs +# End users who are looking for a ready-to-use solution with commercial +# guarantees and support are strongly adviced to contract a Free Software +# Service Company +# +# This program is Free Software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either version 3 +# of the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. +# +############################################################################## +from slapos.recipe.librecipe import BaseSlapRecipe +import os +import subprocess +import pkg_resources +import zc.buildout +import zc.recipe.egg +import sys + +class Recipe(BaseSlapRecipe): + def getTemplateFilename(self, template_name): + return pkg_resources.resource_filename(__name__, + 'template/%s' % template_name) + + def _install(self): + self.path_list = [] + self.requirements, self.ws = self.egg.working_set() + document_root = self.createDataDirectory('www') + apache_config = self.installApache(document_root) + self.setConnectionUrl(scheme='webdavs', + host=apache_config['ip'], + port=apache_config['port'], + auth=(apache_config['user'], + apache_config['password'])) + return self.path_list + + def installApache(self, document_root, ip=None, port=None): + if ip is None: + ip=self.getGlobalIPv6Address() + if port is None: + port = '9080' + + htpasswd_config = self.createHtpasswd() + ssl_config = self.createCertificate(size=2048) + + apache_config = dict( + pid_file=os.path.join(self.run_directory, 'httpd.pid'), + lock_file=os.path.join(self.run_directory, 'httpd.lock'), + davlock_db=os.path.join(self.run_directory, 'davdb.lock'), + ip=ip, + port=port, + error_log=os.path.join(self.log_directory, 'httpd-error.log'), + access_log=os.path.join(self.log_directory, 'httpd-access.log'), + document_root=document_root, + modules_dir=self.options['apache_modules_dir'], + mime_types=self.options['apache_mime_file'], + server_root=self.work_directory, + email_address='admin@vifib.net', + htpasswd_file=htpasswd_config['htpasswd_file'], + ssl_certificate=ssl_config['certificate'], + ssl_key=ssl_config['key'], + ) + httpd_config_file = self.createConfigurationFile('httpd.conf', + self.substituteTemplate(self.getTemplateFilename('httpd.conf.in'), + apache_config)) + self.path_list.append(httpd_config_file) + apache_runner = zc.buildout.easy_install.scripts( + [('httpd', 'slapos.recipe.librecipe.execute', 'execute')], + self.ws, sys.executable, self.wrapper_directory, + arguments=[self.options['apache_binary'], + '-f', httpd_config_file, + '-DFOREGROUND', + ] + )[0] + self.path_list.append(apache_runner) + return dict(ip=apache_config['ip'], + port=apache_config['port'], + user=htpasswd_config['user'], + password=htpasswd_config['password'] + ) + + def createHtpasswd(self): + htpasswd = self.createConfigurationFile('htpasswd', '') + self.path_list.append(htpasswd) + password = self.generatePassword() + user = 'user' + subprocess.check_call([self.options['apache_htpasswd'], + '-bc', htpasswd, + user, password + ]) + return dict(htpasswd_file=htpasswd, + user=user, + password=password) + + def createCertificate(self, size=1024, subject='/C=FR/L=Marcq-en-Baroeul/O=Nexedi'): + key_file = os.path.join(self.etc_directory, 'httpd.key') + self.path_list.append(key_file) + + certificate_file = os.path.join(self.etc_directory, 'httpd.crt') + self.path_list.append(certificate_file) + + subprocess.check_call([self.options['openssl_binary'], + 'req', '-x509', '-nodes', + '-newkey', 'rsa:%s' % size, + '-subj', str(subject), + '-out', certificate_file, + '-keyout', key_file + ]) + return dict(key=key_file, + certificate=certificate_file) diff --git a/slapos/recipe/davstorage/template/httpd.conf.in b/slapos/recipe/davstorage/template/httpd.conf.in new file mode 100644 index 0000000000000000000000000000000000000000..515593e8cff5e37dbb88bc8ff15b6c78a843a490 --- /dev/null +++ b/slapos/recipe/davstorage/template/httpd.conf.in @@ -0,0 +1,82 @@ +ServerRoot "%(server_root)s" + +Listen [%(ip)s]:%(port)s + +# Needed modules +LoadModule authn_file_module "%(modules_dir)s/mod_authn_file.so" +LoadModule authz_host_module "%(modules_dir)s/mod_authz_host.so" +LoadModule authz_user_module "%(modules_dir)s/mod_authz_user.so" +LoadModule auth_basic_module "%(modules_dir)s/mod_auth_basic.so" +LoadModule auth_digest_module "%(modules_dir)s/mod_auth_digest.so" +LoadModule log_config_module "%(modules_dir)s/mod_log_config.so" +LoadModule headers_module "%(modules_dir)s/mod_headers.so" +LoadModule setenvif_module "%(modules_dir)s/mod_setenvif.so" +LoadModule ssl_module "%(modules_dir)s/mod_ssl.so" +LoadModule mime_module "%(modules_dir)s/mod_mime.so" +LoadModule dav_module "%(modules_dir)s/mod_dav.so" +LoadModule dav_fs_module "%(modules_dir)s/mod_dav_fs.so" +LoadModule dir_module "%(modules_dir)s/mod_dir.so" + +ServerAdmin %(email_address)s + +# Quiet Server header (if not, Apache give its life history) +# It's safer +ServerTokens ProductOnly + +DocumentRoot "%(document_root)s" +PidFile "%(pid_file)s" +LockFile "%(lock_file)s" +DavLockDB "%(davlock_db)s" + +<Directory /> + Options FollowSymLinks + AllowOverride None + Order deny,allow + Deny from all +</Directory> + +<Directory %(document_root)s> + Options Indexes MultiViews + AllowOverride None + Order allow,deny + Allow from all + + Dav On + # Security Rules to avoid DDoS Attacks + DavDepthInfinity Off + LimitXMLRequestBody 0 + + # Cross-Origin Resources Sharing + Header always set Access-Control-Max-Age "0" + Header always set Access-Control-Allow-Origin "*" + Header always set Access-Control-Allow-Methods "OPTIONS, GET, HEAD, POST, PUT, DELETE, PROPFIND" + Header always set Access-Control-Allow-Headers "Content-Type, X-Requested-With, X-HTTP-Method-Override, Accept, Authorization, Depth" + SetEnvIf Origin "(.+)" ORIGIN=$1 + Header always set Access-Control-Allow-Origin %%{ORIGIN}e + + AuthType Basic + AuthName "WebDAV Storage" + AuthUserFile "%(htpasswd_file)s" + <LimitExcept OPTIONS> + Require valid-user + </LimitExcept> + +</Directory> + +ErrorLog "%(error_log)s" +LogLevel warn + +LogFormat "%%h %%l %%u %%t \"%%r\" %%>s %%b \"%%{Referer}i\" \"%%{User-Agent}i\"" combined +LogFormat "%%h %%l %%u %%t \"%%r\" %%>s %%b" common +CustomLog "%(access_log)s" common + +DefaultType text/plain +TypesConfig "%(mime_types)s" +AddType application/x-compress .Z +AddType application/x-gzip .gz .tgz + +SSLRandomSeed startup builtin +SSLRandomSeed connect builtin +SSLEngine on +SSLCertificateFile "%(ssl_certificate)s" +SSLCertificateKeyFile "%(ssl_key)s" diff --git a/slapos/recipe/dcron.py b/slapos/recipe/dcron.py new file mode 100644 index 0000000000000000000000000000000000000000..3bb1336c597ca3d61b8760eec020ef66a3fd6e4e --- /dev/null +++ b/slapos/recipe/dcron.py @@ -0,0 +1,75 @@ +############################################################################## +# +# Copyright (c) 2010 Vifib SARL and Contributors. All Rights Reserved. +# +# WARNING: This program as such is intended to be used by professional +# programmers who take the whole responsibility of assessing all potential +# consequences resulting from its eventual inadequacies and bugs +# End users who are looking for a ready-to-use solution with commercial +# guarantees and support are strongly adviced to contract a Free Software +# Service Company +# +# This program is Free Software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either version 3 +# of the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. +# +############################################################################## +import os + +from slapos.recipe.librecipe import GenericBaseRecipe + +class Recipe(GenericBaseRecipe): + + def install(self): + self.logger.info("Installing dcron...") + + path_list = [] + + cronstamps = self.options['cronstamps'] + cron_d = self.options['cron-entries'] + crontabs = self.options['crontabs'] + catcher = self.options['catcher'] + + binary = self.options['binary'] + + script = self.createPythonScript(binary, + 'slapos.recipe.librecipe.execute.execute', + [self.options['dcrond-binary'].strip(), '-s', cron_d, '-c', crontabs, + '-t', cronstamps, '-f', '-l', '5', '-M', catcher] + ) + path_list.append(script) + self.logger.debug('Main cron executable created at : %r', script) + + self.logger.info("dcron successfully installed.") + + return path_list + + + +class Part(GenericBaseRecipe): + + def _options(self, options): + if 'name' not in options: + options['name'] = self.name + + def install(self): + cron_d = self.options['cron-entries'] + filename = os.path.join(cron_d, 'name') + + with open(filename, 'w') as part: + part.write('%(frequency)s %(command)s\n' % { + 'frequency': self.options['frequency'], + 'command': self.options['command'], + }) + + return [filename] diff --git a/slapos/recipe/duplicity.py b/slapos/recipe/duplicity.py new file mode 100644 index 0000000000000000000000000000000000000000..2e5d5d56887e67e30e7596068ba7040a21c7dcf4 --- /dev/null +++ b/slapos/recipe/duplicity.py @@ -0,0 +1,44 @@ +############################################################################## +# +# Copyright (c) 2010 Vifib SARL and Contributors. All Rights Reserved. +# +# WARNING: This program as such is intended to be used by professional +# programmers who take the whole responsibility of assessing all potential +# consequences resulting from its eventual inadequacies and bugs +# End users who are looking for a ready-to-use solution with commercial +# guarantees and support are strongly adviced to contract a Free Software +# Service Company +# +# This program is Free Software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either version 3 +# of the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. +# +############################################################################## +from slapos.recipe.librecipe import GenericBaseRecipe + +class Recipe(GenericBaseRecipe): + + def install(self): + + remote_url = self.options['remote_backup'] + backup_directory = self.options['directory'] + + wrapper = self.createPythonScript( + self.options['wrapper'], + 'slapos.recipe.librecipe.execute.execute', + [self.options['duplicity_binary'], '--no-encryption', + backup_directory, remote_url] + ) + return [wrapper] + + diff --git a/slapos/recipe/erp5/__init__.py b/slapos/recipe/erp5/__init__.py index f9c478a1b632311514dcccbc3531721e6ad46ffc..03c7fb227c56cc4f450e889ef8ecfcb96a2622d2 100644 --- a/slapos/recipe/erp5/__init__.py +++ b/slapos/recipe/erp5/__init__.py @@ -74,10 +74,11 @@ class Recipe(BaseSlapRecipe): if self.parameter_dict.get("slap_software_type", "").lower() == "cluster": # Site access is done by HAProxy - zope_access, site_access = self.installZopeCluster() + zope_access, site_access, key_access = self.installZopeCluster(ca_conf) else: zope_access = self.installZopeStandalone() site_access = zope_access + key_access = None key, certificate = self.requestCertificate('Login Based Access') apache_conf = dict( @@ -96,9 +97,9 @@ class Recipe(BaseSlapRecipe): self.requestCertificate(frontend_name) connection_dict["site_url"] = self.installFrontendZopeApache( - ip=self.getGlobalIPv6Address(), port=13001, name=frontend_name, - frontend_path='/%s' % self.site_id, backend_path='/%s' % self.site_id, - backend_url="http://%s" % site_access, key=frontend_key, + ip=self.getGlobalIPv6Address(), port=4443, name=frontend_name, + frontend_path='/', backend_path='', + backend_url=apache_conf['apache_login'], key=frontend_key, certificate=frontend_certificate) default_bt5_list = [] @@ -107,7 +108,7 @@ class Recipe(BaseSlapRecipe): self.installERP5Site(user, password, zope_access, mysql_conf, conversion_server_conf, memcached_conf, kumo_conf, - self.site_id, default_bt5_list) + self.site_id, default_bt5_list, ca_conf) self.installTestRunner(ca_conf, mysql_conf, conversion_server_conf, memcached_conf, kumo_conf) @@ -120,6 +121,11 @@ class Recipe(BaseSlapRecipe): memcached_url=memcached_conf['memcached_url'], kumo_url=kumo_conf['kumo_address'] )) + if key_access is not None: + connection_dict['key_access'] = key_access + if self.options.get('fulltext_search', None) == 'sphinx': + sphinx_searchd = self.installSphinxSearchd(ip=self.getLocalIPv4Address()) + connection_dict.update(**sphinx_searchd) self.setConnectionDict(connection_dict) return self.path_list @@ -128,19 +134,82 @@ class Recipe(BaseSlapRecipe): """ zodb_dir = os.path.join(self.data_root_directory, 'zodb') self._createDirectory(zodb_dir) - zodb_root_path = os.path.join(zodb_dir, 'root.fs') + zodb_root_path = os.path.join(zodb_dir, 'main.fs') thread_amount_per_zope = int(self.options.get( 'single_zope_thread_amount', 4)) + zodb_cache_size = int(self.options.get('zodb_cache_size', 5000)) + return self.installZope(ip=self.getLocalIPv4Address(), port=12000 + 1, name='zope_%s' % 1, zodb_configuration_string=self.substituteTemplate( self.getTemplateFilename('zope-zodb-snippet.conf.in'), - dict(zodb_root_path=zodb_root_path)), with_timerservice=True, + dict(zodb_root_path=zodb_root_path, + zodb_cache_size=zodb_cache_size)), + with_timerservice=True, thread_amount=thread_amount_per_zope) - def installZopeCluster(self): + def installKeyAuthorisationApache(self, ipv6, port, backend, key, certificate, + ca_conf, key_auth_path='/'): + if ipv6: + ip = self.getGlobalIPv6Address() + else: + ip = self.getLocalIPv4Address() + ssl_template = """SSLEngine on +SSLVerifyClient require +RequestHeader set REMOTE_USER %%{SSL_CLIENT_S_DN_CN}s +SSLCertificateFile %(key_auth_certificate)s +SSLCertificateKeyFile %(key_auth_key)s +SSLCACertificateFile %(ca_certificate)s +SSLCARevocationPath %(ca_crl)s""" + apache_conf = self._getApacheConfigurationDict('key_auth_apache', ip, port) + apache_conf['ssl_snippet'] = ssl_template % dict( + key_auth_certificate=certificate, + key_auth_key=key, + ca_certificate=ca_conf['ca_certificate'], + ca_crl=ca_conf['ca_crl'] + ) + prefix = 'ssl_key_auth_apache' + rewrite_rule_template = \ + "RewriteRule (.*) http://%(backend)s%(key_auth_path)s$1 [L,P]" + path_template = pkg_resources.resource_string('slapos.recipe.erp5', + 'template/apache.zope.conf.path.in') + path = path_template % dict(path='/') + d = dict( + path=path, + backend=backend, + backend_path='/', + port=apache_conf['port'], + vhname=path.replace('/', ''), + key_auth_path=key_auth_path, + ) + rewrite_rule = rewrite_rule_template % d + apache_conf.update(**dict( + path_enable=path, + rewrite_rule=rewrite_rule + )) + apache_config_file = self.createConfigurationFile(prefix + '.conf', + pkg_resources.resource_string('slapos.recipe.erp5', + 'template/apache.zope.conf.in') % apache_conf) + self.path_list.append(apache_config_file) + self.path_list.extend(zc.buildout.easy_install.scripts([( + 'key_auth_apache', + 'slapos.recipe.erp5.apache', 'runApache')], self.ws, + sys.executable, self.wrapper_directory, arguments=[ + dict( + required_path_list=[certificate, key, ca_conf['ca_certificate'], + ca_conf['ca_crl']], + binary=self.options['httpd_binary'], + config=apache_config_file + ) + ])) + if ipv6: + return 'https://[%(ip)s:%(port)s]' % apache_conf + else: + return 'https://%(ip)s:%(port)s' % apache_conf + + def installZopeCluster(self, ca_conf=None): """ Install ERP5 using ZEO Cluster """ site_check_path = '/%s/getId' % self.site_id @@ -153,6 +222,9 @@ class Recipe(BaseSlapRecipe): user_node_amount = int(self.options.get( "cluster_user_node_amount", 2)) + key_auth_node_amount = int(self.options.get( + "key_auth_node_amount", 0)) + ip = self.getLocalIPv4Address() storage_dict = self._requestZeoFileStorage('Zeo Server 1', 'main') @@ -161,6 +233,8 @@ class Recipe(BaseSlapRecipe): # XXX How to define good values for this? mount_point = '/' + zodb_cache_size = 5000 + zeo_client_cache_size = '20MB' check_path = '/erp5/account_module' known_tid_storage_identifier_dict = {} @@ -172,7 +246,8 @@ class Recipe(BaseSlapRecipe): self.getTemplateFilename('zope-zeo-snippet.conf.in'), dict( storage_name=storage_dict['storage_name'], address='%s:%s' % (storage_dict['ip'], storage_dict['port']), - mount_point=mount_point + mount_point=mount_point, zodb_cache_size=zodb_cache_size, + zeo_client_cache_size=zeo_client_cache_size )) zope_port = 12000 @@ -203,11 +278,28 @@ class Recipe(BaseSlapRecipe): login_haproxy = self.installHaproxy(ip, 15001, 'login', site_check_path, login_url_list) + key_access = None + if key_auth_node_amount > 0: + service_url_list = [] + for i in range(key_auth_node_amount): + zope_port += 1 + service_url_list.append(self.installZope(ip, zope_port, + 'zope_service_%s' % i, with_timerservice=False, + zodb_configuration_string=zodb_configuration_string, + tidstorage_config=tidstorage_config)) + service_haproxy = self.installHaproxy(ip, 15000, 'service', + site_check_path, service_url_list) + + key_auth_key, key_auth_certificate = self.requestCertificate( + 'Key Based Access') + key_access = self.installKeyAuthorisationApache(True, 15500, + service_haproxy, key_auth_key, key_auth_certificate, ca_conf) + self.installTidStorage(tidstorage_config['host'], tidstorage_config['port'], known_tid_storage_identifier_dict, 'http://' + login_haproxy) - return login_url_list[-1], login_haproxy + return login_url_list[-1], login_haproxy, key_access def _requestZeoFileStorage(self, server_name, storage_name): """Local, slap.request compatible, call to ask for filestorage on Zeo @@ -342,6 +434,27 @@ class Recipe(BaseSlapRecipe): memcached_ip=config['memcached_ip'], memcached_port=config['memcached_port']) + def installSphinxSearchd(self, ip, port=9312, sql_port=9306): + data_directory = self.createDataDirectory('sphinx') + sphinx_conf_path = self.createConfigurationFile('sphinx.conf', + self.substituteTemplate(self.getTemplateFilename('sphinx.conf.in'), dict( + ip_address=ip, + port=port, + sql_port=sql_port, + data_directory=data_directory, + log_directory=self.log_directory, + ))) + self.path_list.append(sphinx_conf_path) + wrapper = zc.buildout.easy_install.scripts([('sphinx_searchd', + 'slapos.recipe.librecipe.execute', 'execute')], self.ws, sys.executable, + self.wrapper_directory, arguments=[ + self.options['sphinx_searchd_binary'].strip(), '-c', sphinx_conf_path, '--nodetach'] + )[0] + self.path_list.append(wrapper) + return dict(sphinx_searchd_ip=ip, + sphinx_searchd_port=port, + sphinx_searchd_sql_port=sql_port) + def installTestRunner(self, ca_conf, mysql_conf, conversion_server_conf, memcached_conf, kumo_conf): """Installs bin/runUnitTest executable to run all tests using @@ -546,14 +659,24 @@ class Recipe(BaseSlapRecipe): } def installHaproxy(self, ip, port, name, server_check_path, url_list): - server_template = """ server %(name)s %(address)s cookie %(name)s check inter 20s rise 2 fall 4""" + # inter must be quite short in order to detect quickly an unresponsive node + # and to detect quickly a node which is back + # rise must be minimal possible : 1, indeed, a node which is back don't need + # to sleep more time and we can give him work immediately + # fall should be quite sort. with inter at 3, and fall at 2, a node will be + # considered as dead after 6 seconds. + # maxconn should be set as the maximum thread we have per zope, like this + # haproxy will manage the queue of request with the possibility to + # move a request to another node if the initially selected one is dead + server_template = """ server %(name)s %(address)s cookie %(name)s check inter 3s rise 1 fall 2 maxconn %(cluster_zope_thread_amount)s""" config = dict(name=name, ip=ip, port=port, server_check_path=server_check_path,) i = 1 server_list = [] + cluster_zope_thread_amount = self.options.get('cluster_zope_thread_amount', 1) for url in url_list: server_list.append(server_template % dict(name='%s_%s' % (name, i), - address=url)) + address=url, cluster_zope_thread_amount=cluster_zope_thread_amount)) i += 1 config['server_text'] = '\n'.join(server_list) haproxy_conf_path = self.createConfigurationFile('haproxy_%s.cfg' % name, @@ -640,10 +763,14 @@ class Recipe(BaseSlapRecipe): return user, password def installERP5Site(self, user, password, zope_access, mysql_conf, - conversion_server_conf=None, memcached_conf=None, kumo_conf=None, - erp5_site_id='erp5', default_bt5_list=[]): - """ Create a script controlled by supervisor, which creates a erp5 - site on current available zope and mysql environment""" + conversion_server_conf=None, memcached_conf=None, + kumo_conf=None, + erp5_site_id='erp5', default_bt5_list=[], ca_conf={}, + supervisor_controlled=True): + """ + Create a script to automatically set up an erp5 site (controlled by + supervisor by default) on available zope and mysql environments. + """ conversion_server = None if conversion_server_conf is not None: conversion_server = "%s:%s" % (conversion_server_conf['conversion_server_ip'], @@ -660,9 +787,12 @@ class Recipe(BaseSlapRecipe): bt5_repository_list = self.parameter_dict.get("bt5_repository_list", "").split() \ or getattr(self, 'bt5_repository_list', []) - self.path_list.extend(zc.buildout.easy_install.scripts([('erp5_update', + erp5_update_directory = supervisor_controlled and self.wrapper_directory or \ + self.bin_directory + + script = zc.buildout.easy_install.scripts([('erp5_update', __name__ + '.erp5', 'updateERP5')], self.ws, - sys.executable, self.wrapper_directory, + sys.executable, erp5_update_directory, arguments=[erp5_site_id, mysql_connection_string, [user, password, zope_access], @@ -670,7 +800,12 @@ class Recipe(BaseSlapRecipe): conversion_server, kumo_conf.get("kumo_address"), bt5_list, - bt5_repository_list])) + bt5_repository_list, + ca_conf.get('certificate_authority_path'), + self.options.get('openssl_binary')]) + + self.path_list.extend(script) + return [] def installZeo(self, ip): @@ -713,6 +848,26 @@ class Recipe(BaseSlapRecipe): self.path_list.append(wrapper) return zeo_configuration_dict + def installRepozo(self, zodb_root_path): + """ + Add only repozo to cron (e.g. without tidstorage) allowing full + and incremental backups. + """ + backup_path = self.createBackupDirectory('zodb') + repozo_cron_path = os.path.join(self.cron_d, 'repozo') + repozo_cron_file = open(repozo_cron_path, 'w') + try: + repozo_cron_file.write(''' +0 0 * * 0 %(repozo_binary)s --backup --full --file="%(zodb_root_path)s" --repository="%(backup_path)s" +0 * * * * %(repozo_binary)s --backup --file="%(zodb_root_path)s" --repository="%(backup_path)s" +''' % dict(repozo_binary=self.options['repozo_binary'], + zodb_root_path=zodb_root_path, + backup_path=backup_path)) + finally: + repozo_cron_file.close() + + self.path_list.append(repozo_cron_path) + def installTidStorage(self, ip, port, known_tid_storage_identifier_dict, access_url): """Install TidStorage with all required backup tools @@ -825,10 +980,6 @@ class Recipe(BaseSlapRecipe): self.erp5_directory, 'Products')) zope_config['products'] = '\n'.join(prefixed_products) zope_config['address'] = '%s:%s' % (ip, port) - zope_environment_list = [] - for envk, envv in zope_environment.iteritems(): - zope_environment_list.append('%s %s' % (envk, envv)) - zope_config['environment'] = "\n".join(zope_environment_list) zope_wrapper_template_location = self.getTemplateFilename('zope.conf.in') zope_conf_content = self.substituteTemplate( @@ -851,10 +1002,11 @@ class Recipe(BaseSlapRecipe): self.path_list.append(zope_conf_path) # Create init script wrapper = zc.buildout.easy_install.scripts([(name, - 'slapos.recipe.librecipe.execute', 'execute')], self.ws, sys.executable, + 'slapos.recipe.librecipe.execute', 'executee')], self.ws, sys.executable, self.wrapper_directory, arguments=[ - self.options['runzope_binary'].strip(), '-C', zope_conf_path] - )[0] + [self.options['runzope_binary'].strip(), '-C', zope_conf_path], + zope_environment + ])[0] self.path_list.append(wrapper) return zope_config['address'] @@ -915,9 +1067,6 @@ class Recipe(BaseSlapRecipe): 'template/apache.ssl-snippet.conf.in') % dict( login_certificate=certificate, login_key=key) - rewrite_rule_template = \ - "RewriteRule ^%(path)s($|/.*) %(backend_url)s/VirtualHostBase/https/%(server_name)s:%(port)s%(backend_path)s/VirtualHostRoot/_vh_%(vhname)s$1 [L,P]\n" - path = pkg_resources.resource_string(__name__, 'template/apache.zope.conf.path-protected.in') % \ dict(path='/', access_control_string='none') @@ -931,14 +1080,24 @@ class Recipe(BaseSlapRecipe): 'template/apache.zope.conf.path-protected.in') path += path_template % dict(path=frontend_path, access_control_string=access_control_string) - d = dict( + + rewrite_rule_template = \ + "RewriteRule ^%(path)s($|/.*) %(backend_url)s/VirtualHostBase/https/%(server_name)s:%(port)s%(backend_path)s/VirtualHostRoot/%(vhname)s$1 [L,P]\n" + + if frontend_path not in ["", None, "/"]: + vhname = "_vh_%s" % frontend_path.replace('/', '') + else: + vhname = "" + frontend_path = "" + + rewrite_rule = rewrite_rule_template % dict( path=frontend_path, backend_url=backend_url, backend_path=backend_path, port=apache_conf['port'], - vhname=frontend_path.replace('/', ''), + vhname=vhname, server_name=name) - rewrite_rule = rewrite_rule_template % d + apache_conf.update(**dict( path_enable=path, rewrite_rule=rewrite_rule @@ -987,7 +1146,7 @@ class Recipe(BaseSlapRecipe): def installMysqlServer(self, ip, port, database='erp5', user='user', test_database='test_erp5', test_user='test_user', template_filename=None, parallel_test_database_amount=100, mysql_conf=None, with_backup=True, - with_maatkit=True): + with_percona_toolkit=True): if mysql_conf is None: mysql_conf = {} backup_directory = self.createBackupDirectory('mysql') @@ -1096,29 +1255,46 @@ class Recipe(BaseSlapRecipe): open(mysql_backup_cron, 'w').write('0 0 * * * ' + backup_controller) self.path_list.append(mysql_backup_cron) - if with_maatkit: + if with_percona_toolkit: # maatkit installation - for mk_script_name in ( - 'mk-variable-advisor', - 'mk-table-usage', - 'mk-visual-explain', - 'mk-config-diff', - 'mk-deadlock-logger', - 'mk-error-log', - 'mk-index-usage', - 'mk-query-advisor', + for pt_script_name in ( + 'pt-archiver', + 'pt-config-diff', + 'pt-deadlock-logger', + 'pt-duplicate-key-checker', + 'pt-fifo-split', + 'pt-find', + 'pt-fk-error-logger', + 'pt-heartbeat', + 'pt-index-usage', + 'pt-kill', + 'pt-log-player', + 'pt-online-schema-change', + 'pt-query-advisor', + 'pt-query-digest', + 'pt-show-grants', + 'pt-slave-delay', + 'pt-slave-find', + 'pt-slave-restart', + 'pt-table-checksum', + 'pt-table-sync', + 'pt-tcp-model', + 'pt-trend', + 'pt-upgrade', + 'pt-variable-advisor', + 'pt-visual-explain', ): - mk_argument_list = [self.options['perl_binary'], - self.options['%s_binary' % mk_script_name], + pt_argument_list = [self.options['perl_binary'], + self.options['%s_binary' % pt_script_name], '--defaults-file=%s' % mysql_conf_path, '--socket=%s' %mysql_conf['socket'].strip(), '--user=root', ] environment = dict(PATH='%s' % self.bin_directory) - mk_exe = zc.buildout.easy_install.scripts([( - mk_script_name,'slapos.recipe.librecipe.execute', 'executee')], + pt_exe = zc.buildout.easy_install.scripts([( + pt_script_name,'slapos.recipe.librecipe.execute', 'executee')], self.ws, sys.executable, self.bin_directory, arguments=[ - mk_argument_list, environment])[0] - self.path_list.append(mk_exe) + pt_argument_list, environment])[0] + self.path_list.append(pt_exe) # The return could be more explicit database, user ... return mysql_conf diff --git a/slapos/recipe/erp5/erp5.py b/slapos/recipe/erp5/erp5.py index bf99ad1150549a9c3ec602d00248b0b71c84d572..c5cadbade50f191ca6927d6aac0dc06a199976f8 100644 --- a/slapos/recipe/erp5/erp5.py +++ b/slapos/recipe/erp5/erp5.py @@ -37,12 +37,14 @@ class ERP5Updater(object): erp5_catalog_storage = "erp5_mysql_innodb_catalog" header_dict = {} - sleeping_time = 120 + sleeping_time = 300 + short_sleeping_time = 60 def __init__(self, user, password, host, site_id, mysql_url, memcached_address, conversion_server_address, persistent_cache_address, - bt5_list, bt5_repository_list): + bt5_list, bt5_repository_list, certificate_authority_path, + openssl_binary): authentication_string = '%s:%s' % (user, password) base64string = base64.encodestring(authentication_string).strip() @@ -54,13 +56,17 @@ class ERP5Updater(object): self.business_template_repository_list = bt5_repository_list self.business_template_list = bt5_list self.memcached_address = memcached_address - self.persintent_cached_address = persistent_cache_address + self.persistent_cached_address = persistent_cache_address self.mysql_url = mysql_url host, port = conversion_server_address.split(":") self.conversion_server_address = host self.conversion_server_port = int(port) + # Certificate Authority Tool configuration + self.certificate_authority_path = certificate_authority_path + self.openssl_binary = openssl_binary + def log(self, level, message): date = time.strftime("%a, %d %b %Y %H:%M:%S +0000") print "%s - %s : %s" % (date, level, message) @@ -147,34 +153,25 @@ class ERP5Updater(object): return [i for i in self.business_template_repository_list if i not in found_list] - def getMissingBusinessTemplateList(self): - bt5_dict = self.getSystemSignatureDict("business_template_dict", []) - found_bt5_list = bt5_dict.keys() - return [bt for bt in self.business_template_list\ - if bt not in found_bt5_list] - - def isBusinessTemplateUpdated(self): - return len(self.getMissingBusinessTemplateList()) == 0 - - def isBusinessTemplateRepositoryUpdated(self): - return len(self.getMissingBusinessTemplateRepositoryList()) == 0 + def getMissingBusinessTemplateSet(self): + found_dict = self.getSystemSignatureDict("business_template_dict", {}) + return set(self.business_template_list).difference(found_dict) def updateBusinessTemplateList(self): """ Update Business Template Configuration, including the repositories """ - if not self.isBusinessTemplateUpdated(): - # Before update the business templates, it is required to make - # sure the repositories are updated. - if not self.isBusinessTemplateRepositoryUpdated(): - # Require to update Business template Repository - repository_list = self.getSystemSignatureDict( - "business_template_repository_list", []) - repository_list.extend(self.getMissingBusinessTemplateRepositoryList()) - self._setRepositoryList(repository_list) + missing_business_template_set = self.getMissingBusinessTemplateSet() + if missing_business_template_set: + # Before updating the business templates, it is required to make sure + # the repositories are updated, thus update them even if they are + # already present because there may be new business templates... + repository_list = self.getSystemSignatureDict( + "business_template_repository_list", []) + repository_list.extend(self.getMissingBusinessTemplateRepositoryList()) + self._setRepositoryList(repository_list) # Require to update Business template - for bt in self.getMissingBusinessTemplateList(): - self._installBusinessTemplateList([bt]) + self._installBusinessTemplateList(list(missing_business_template_set)) return True return False @@ -186,18 +183,20 @@ class ERP5Updater(object): def _installBusinessTemplateList(self, name_list, update_catalog=False): """ Install a Business Template on Remote ERP5 setup """ - set_path = "/%s/portal_templates/installBusinessTemplatesFromRepositories" % self.site_id + set_path = "/%s/portal_templates/installBusinessTemplateListFromRepository" % self.site_id self.POST(set_path, {"template_list": name_list, "only_newer": 1, - "update_catalog": int(update_catalog)}) + "update_catalog": int(update_catalog), + "activate": 1, + "install_dependency": 1}) - def _createActiveSystemPreference(self): + def _createActiveSystemPreference(self, edit_kw={}): """ Assert that at least one enabled System Preference is present on the erp5 instance. """ self.log("INFO", "Try to create New System Preference into ERP5!") path = "/%s/portal_preferences/createActiveSystemPreference" % self.site_id - status, data = self.POST(path, {}) + status, data = self.POST(path, edit_kw) if status != 200: self.log("ERROR", "Unable to create System Preference, an error ocurred %s." % data) @@ -217,18 +216,62 @@ class ERP5Updater(object): if None in [host_key, port_key]: self.log("ERROR", "Unable to find the Active System Preference to Update!") - self._createActiveSystemPreference() + self._createActiveSystemPreference( + {"preferred_ooodoc_server_address" : self.conversion_server_address, + "preferred_ooodoc_server_port_number": self.conversion_server_port }) return True is_updated = self._assertAndUpdateDocument(host_key, self.conversion_server_address, "setPreferredOoodocServerAddress") - is_updated = is_updated or self._assertAndUpdateDocument(port_key, + is_updated = self._assertAndUpdateDocument(port_key, self.conversion_server_port, - "setPreferredOoodocServerPortNumber") + "setPreferredOoodocServerPortNumber") or is_updated return is_updated + def updateCertificateAuthority(self): + """ Update the certificate authority only if is not configured yet """ + if self.isCertificateAuthorityAvailable(): + if self.isCertificateAuthorityConfigured(): + return True + + path = "/%s/portal_certificate_authority/" \ + "manage_editCertificateAuthorityTool" % self.site_id + self.POST(path, {"certificate_authority_path": self.certificate_authority_path, + "openssl_binary": self.openssl_binary}) + + + def isCertificateAuthorityAvailable(self): + """ Check if certificate Authority is available. """ + external_connection_dict = self.system_signature_dict[ + 'external_connection_dict'] + if 'portal_certificate_authority/certificate_authority_path' in \ + external_connection_dict: + return True + return False + + def isCertificateAuthorityConfigured(self): + """ Check if certificate Authority is configured correctly. """ + external_connection_dict = self.system_signature_dict[ + 'external_connection_dict'] + if self.certificate_authority_path == external_connection_dict.get( + 'portal_certificate_authority/certificate_authority_path') and \ + self.openssl_binary == external_connection_dict.get( + 'portal_certificate_authority/openssl_binary'): + return True + return False + def isCertificateAuthorityConfigured(self): + """ Check if certificate Authority is configured correctly. """ + external_connection_dict = self.system_signature_dict[ + 'external_connection_dict'] + if self.certificate_authority_path == external_connection_dict.get( + 'portal_certificate_authority/certificate_authority_path') and \ + self.openssl_binary == external_connection_dict.get( + 'portal_certificate_authority/openssl_binary'): + return True + return False + def updateMemcached(self): # Assert Memcached configuration self._assertAndUpdateDocument( @@ -239,7 +282,7 @@ class ERP5Updater(object): # Assert Persistent cache configuration (Kumofs) self._assertAndUpdateDocument( "portal_memcached/persistent_memcached_plugin/getUrlString", - self.persintent_cached_address, + self.persistent_cached_address, "setUrlString") def _assertAndUpdateDocument(self, key, expected_value, update_method): @@ -258,24 +301,9 @@ class ERP5Updater(object): return True return False - def updateMysql(self): - """ This API is not implemented yet, because it is not needed to - update Mysql Connection on ERP5 Sites. - """ - pass - - def updatePortalActivities(self): - """ This API is not implemented yet, because it is not needed for - a single instance configuration. This method should define which - instances will handle activities, which one will distribute - activities - """ - pass - def updateERP5Site(self): if not self.isERP5Present(): - url = '/manage_addProduct/ERP5/manage_addERP5Site' - self.POST(url, { + self.POST('/manage_addProduct/ERP5/manage_addERP5Site', { "id": self.site_id, "erp5_catalog_storage": self.erp5_catalog_storage, "erp5_sql_connection_string": self.mysql_url, @@ -286,54 +314,30 @@ class ERP5Updater(object): def _hasActivityPresent(self): activity_dict = self.getSystemSignatureDict("activity_dict") if activity_dict["total"] > 0: + self.log("DEBUG", "Waiting for activities on ERP5...") return True + return False def _hasFailureActivity(self): activity_dict = self.getSystemSignatureDict("activity_dict") if activity_dict["failure"] > 0: + self.log("ERROR", "Update progress found Failure activities" +\ + "and it will not be able to progress until" +\ + " activites issue be solved") return True - - def _updatePreRequiredBusinessTemplateList(self): - """ Update only the first part of bt5.""" - - # This list contains the minimal set of bt5 required to install - # portal_introspections. Move portal_introspection to erp5_core - # can remove this set. - pre_required_business_template_list = [i for i in self.business_template_list\ - if i.startswith("erp5_full_text") or i == "erp5_base"] - - if len(self.business_template_repository_list) > 0 and \ - len(pre_required_business_template_list): - pre_required_business_template_list.insert(0, "erp5_core_proxy_field_legacy") - self._setRepositoryList(self.business_template_repository_list) - time.sleep(30) - for bt in pre_required_business_template_list: - update_catalog = bt.endswith("_catalog") - self._installBusinessTemplateList([bt], update_catalog) - else: - self.log("ERROR", "Unable to install erp5_base, it is not on your " +\ - "requested business templates list. Once it is installed " +\ - "setup will continue") + return False def run(self): """ Keep running until kill""" while 1: - time.sleep(30) + time.sleep(self.short_sleeping_time) if not self.updateERP5Site(): self.loadSystemSignatureDict() - if self.getSystemSignatureDict() is None: - self.log("INFO", "The erp5_base is not installed yet, trying to " +\ - "install it before continue.") - self._updatePreRequiredBusinessTemplateList() - time.sleep(60) + if self._hasFailureActivity(): + time.sleep(self.sleeping_time) continue - + if self._hasActivityPresent(): - self.log("DEBUG", "Waiting for activities on ERP5...") - if self._hasFailureActivity(): - self.log("ERROR", "Update progress found " +\ - "Failure activities and it will not progress until " +\ - " activites issue be solved") continue if self.updateBusinessTemplateList(): @@ -341,11 +345,8 @@ class ERP5Updater(object): self.updateMemcached() if self.updateConversionServer(): - # If update Conversion Server adds a bit more delay to continue - # To wait for activiies. - time.sleep(60) continue - + self.updateCertificateAuthority() time.sleep(self.sleeping_time) def updateERP5(argument_list): @@ -356,6 +357,8 @@ def updateERP5(argument_list): conversion_server_address = argument_list[4] persistent_cache_provider = argument_list[5] bt5_list = argument_list[6] + certificate_authority_path = argument_list[8] + openssl_binary = argument_list[9] bt5_repository_list = [] if len(argument_list) > 7: @@ -374,6 +377,8 @@ def updateERP5(argument_list): conversion_server_address=conversion_server_address, persistent_cache_address=persistent_cache_provider, bt5_list=bt5_list, - bt5_repository_list=bt5_repository_list) + bt5_repository_list=bt5_repository_list, + certificate_authority_path=certificate_authority_path, + openssl_binary=openssl_binary) erp5_upgrader.run() diff --git a/slapos/recipe/erp5/template/apache.ssl-snippet.conf.in b/slapos/recipe/erp5/template/apache.ssl-snippet.conf.in index 0dd6e653d59bd2e9e85f5fd29b140d96814cd8a0..f85a164cb87fee8995a4ee4bfe8f99a18906cc80 100644 --- a/slapos/recipe/erp5/template/apache.ssl-snippet.conf.in +++ b/slapos/recipe/erp5/template/apache.ssl-snippet.conf.in @@ -4,3 +4,4 @@ SSLCertificateKeyFile %(login_key)s SSLRandomSeed startup builtin SSLRandomSeed connect builtin +SSLProxyEngine On diff --git a/slapos/recipe/erp5/template/apache.zope.conf.in b/slapos/recipe/erp5/template/apache.zope.conf.in index 131040d535278991f7d2f71b68c90c7224bf3953..f24a5327b68651531f1f7e25832dfd36d85cded3 100644 --- a/slapos/recipe/erp5/template/apache.zope.conf.in +++ b/slapos/recipe/erp5/template/apache.zope.conf.in @@ -34,9 +34,9 @@ RequestHeader unset REMOTE_USER # Log configuration ErrorLog "%(error_log)s" -LogFormat "%%h %%{REMOTE_USER}i %%l %%u %%t \"%%r\" %%>s %%b \"%%{Referer}i\" \"%%{User-Agent}i\"" combined -LogFormat "%%h %%{REMOTE_USER}i %%l %%u %%t \"%%r\" %%>s %%b" common -CustomLog "%(access_log)s" common +# Default apache log format with request time in microsecond at the end +LogFormat "%%h %%l %%u %%t \"%%r\" %%>s %%b \"%%{Referer}i\" \"%%{User-Agent}i\" %%D" combined +CustomLog "%(access_log)s" combined # Directory protection <Directory /> diff --git a/slapos/recipe/erp5/template/haproxy.cfg.in b/slapos/recipe/erp5/template/haproxy.cfg.in index 4087c1a4eacdc46a4af9c374417471ed265a7062..aa8f8a865ecb58d166cfebcc911c681780651ae8 100644 --- a/slapos/recipe/erp5/template/haproxy.cfg.in +++ b/slapos/recipe/erp5/template/haproxy.cfg.in @@ -9,10 +9,25 @@ defaults retries 1 option redispatch maxconn 2000 - timeout server 3000s - timeout queue 5s - timeout connect 10s - timeout client 3600s + # it is useless to have timeout much bigger than the one of apache. + # By default apache use 300s, so we set slightly more in order to + # make sure that apache will first stop the connection. + timeout server 305s + # Stop waiting in queue for a zope to become available. + # If no zope can be reached after one minute, consider the request will + # never succeed. + timeout queue 60s + # The connection should be immediate on LAN, + # so we should not set more than 5 seconds, and it could be already too much + timeout connect 5s + # As requested in haproxy doc, make this "at least equal to timeout server". + timeout client 305s + # Use "option httpclose" to not preserve client & server persistent connections + # while handling every incoming request individually, dispatching them one after + # another to servers, in HTTP close mode. This is really needed when haproxy + # is configured with maxconn to 1, without this options browser are unable + # to render a page + option httpclose listen %(name)s %(ip)s:%(port)s cookie SERVERID insert diff --git a/slapos/recipe/erp5/template/initmysql.sql.in b/slapos/recipe/erp5/template/initmysql.sql.in index 93256efae53bda3e52a30142bb01cf7bbed66312..981ffbc65a0d3ee40c2f7f565899f0ce25fb5e53 100644 --- a/slapos/recipe/erp5/template/initmysql.sql.in +++ b/slapos/recipe/erp5/template/initmysql.sql.in @@ -1,2 +1,3 @@ CREATE DATABASE IF NOT EXISTS %(mysql_database)s; GRANT ALL PRIVILEGES ON %(mysql_database)s.* TO %(mysql_user)s@'%%' IDENTIFIED BY '%(mysql_password)s'; +GRANT ALL PRIVILEGES ON %(mysql_database)s.* TO %(mysql_user)s@'localhost' IDENTIFIED BY '%(mysql_password)s'; diff --git a/slapos/recipe/erp5/template/logrotate_entry.in b/slapos/recipe/erp5/template/logrotate_entry.in index bfa2abf0970af28f8cab98793db9b09db85d1847..14461dccf875d9ae57ef4728bc77525855a20041 100644 --- a/slapos/recipe/erp5/template/logrotate_entry.in +++ b/slapos/recipe/erp5/template/logrotate_entry.in @@ -1,7 +1,7 @@ %(file_list)s { daily dateext - rotate 30 + rotate 3650 compress notifempty sharedscripts diff --git a/slapos/recipe/erp5/template/sphinx.conf.in b/slapos/recipe/erp5/template/sphinx.conf.in new file mode 100644 index 0000000000000000000000000000000000000000..984c4b0b605b35009678dc07dfea2ac2f052934e --- /dev/null +++ b/slapos/recipe/erp5/template/sphinx.conf.in @@ -0,0 +1,596 @@ +############################################################################# +## index definition +############################################################################# + +# realtime index +# +# you can run INSERT, REPLACE, and DELETE on this index on the fly +# using MySQL protocol (see 'listen' directive below) +index erp5 +{ + # 'rt' index type must be specified to use RT index + type = rt + + # index files path and file name, without extension + # mandatory, path must be writable, extensions will be auto-appended + path = %(data_directory)s/erp5 + + # RAM chunk size limit + # RT index will keep at most this much data in RAM, then flush to disk + # optional, default is 32M + # + # rt_mem_limit = 512M + + # full-text field declaration + # multi-value, mandatory + rt_field = SearchableText + + # unsigned integer attribute declaration + # multi-value (an arbitrary number of attributes is allowed), optional + # declares an unsigned 32-bit attribute + rt_attr_uint = uid + + # RT indexes currently support the following attribute types: + # uint, bigint, float, timestamp, string + # + # rt_attr_bigint = guid + # rt_attr_float = gpa + # rt_attr_timestamp = ts_added + # rt_attr_string = author + + # document attribute values (docinfo) storage mode + # optional, default is 'extern' + # known values are 'none', 'extern' and 'inline' + # docinfo = extern + + # memory locking for cached data (.spa and .spi), to prevent swapping + # optional, default is 0 (do not mlock) + # requires searchd to be run from root + # mlock = 0 + + # a list of morphology preprocessors to apply + # optional, default is empty + # + # builtin preprocessors are 'none', 'stem_en', 'stem_ru', 'stem_enru', + # 'soundex', and 'metaphone'; additional preprocessors available from + # libstemmer are 'libstemmer_XXX', where XXX is algorithm code + # (see libstemmer_c/libstemmer/modules.txt) + # + # morphology = stem_en, stem_ru, soundex + # morphology = libstemmer_german + # morphology = libstemmer_sv + morphology = stem_en + + # minimum word length at which to enable stemming + # optional, default is 1 (stem everything) + # + # min_stemming_len = 1 + + # stopword files list (space separated) + # optional, default is empty + # contents are plain text, charset_table and stemming are both applied + # + # stopwords = %(data_directory)s/erp5/stopwords.txt + + # wordforms file, in "mapfrom > mapto" plain text format + # optional, default is empty + # + # wordforms = %(data_directory)s/erp5/wordforms.txt + + # tokenizing exceptions file + # optional, default is empty + # + # plain text, case sensitive, space insensitive in map-from part + # one "Map Several Words => ToASingleOne" entry per line + # + # exceptions = %(data_directory)s/erp5/exceptions.txt + + # minimum indexed word length + # default is 1 (index everything) + min_word_len = 1 + + # charset encoding type + # optional, default is 'sbcs' + # known types are 'sbcs' (Single Byte CharSet) and 'utf-8' + charset_type = utf-8 + + # charset definition and case folding rules "table" + # optional, default value depends on charset_type + # + # defaults are configured to include English and Russian characters only + # you need to change the table to include additional ones + # this behavior MAY change in future versions + # + # 'sbcs' default value is + # charset_table = 0..9, A..Z->a..z, _, a..z, U+A8->U+B8, U+B8, U+C0..U+DF->U+E0..U+FF, U+E0..U+FF + # + # 'utf-8' default value is + # charset_table = 0..9, A..Z->a..z, _, a..z, U+410..U+42F->U+430..U+44F, U+430..U+44F + charset_table = \ + U+00C0->a, U+00C1->a, U+00C2->a, U+00C3->a, U+00C4->a, U+00C5->a, U+00E0->a, U+00E1->a, U+00E2->a, U+00E3->a, U+00E4->a, U+00E5->a, U+0100->a, U+0101->a, U+0102->a, U+0103->a, U+010300->a, U+0104->a, U+0105->a, U+01CD->a, U+01CE->a, U+01DE->a, U+01DF->a, \ + U+01E0->a, U+01E1->a, U+01FA->a, U+01FB->a, U+0200->a, U+0201->a, U+0202->a, U+0203->a, U+0226->a, U+0227->a, U+023A->a, U+0250->a, U+04D0->a, U+04D1->a, U+1D2C->a, U+1D43->a, U+1D44->a, U+1D8F->a, U+1E00->a, U+1E01->a, U+1E9A->a, U+1EA0->a, U+1EA1->a, \ + U+1EA2->a, U+1EA3->a, U+1EA4->a, U+1EA5->a, U+1EA6->a, U+1EA7->a, U+1EA8->a, U+1EA9->a, U+1EAA->a, U+1EAB->a, U+1EAC->a, U+1EAD->a, U+1EAE->a, U+1EAF->a, U+1EB0->a, U+1EB1->a, U+1EB2->a, U+1EB3->a, U+1EB4->a, U+1EB5->a, U+1EB6->a, U+1EB7->a, U+2090->a, \ + U+2C65->a, U+0180->b, U+0181->b, U+0182->b, U+0183->b, U+0243->b, U+0253->b, U+0299->b, U+16D2->b, U+1D03->b, U+1D2E->b, U+1D2F->b, U+1D47->b, U+1D6C->b, U+1D80->b, U+1E02->b, U+1E03->b, U+1E04->b, U+1E05->b, U+1E06->b, U+1E07->b, U+00C7->c, U+00E7->c, \ + U+0106->c, U+0107->c, U+0108->c, U+0109->c, U+010A->c, U+010B->c, U+010C->c, U+010D->c, U+0187->c, U+0188->c, U+023B->c, U+023C->c, U+0255->c, U+0297->c, U+1D9C->c, U+1D9D->c, U+1E08->c, U+1E09->c, U+212D->c, U+2184->c, U+010E->d, U+010F->d, U+0110->d, \ + U+0111->d, U+0189->d, U+018A->d, U+018B->d, U+018C->d, U+01C5->d, U+01F2->d, U+0221->d, U+0256->d, U+0257->d, U+1D05->d, U+1D30->d, U+1D48->d, U+1D6D->d, U+1D81->d, U+1D91->d, U+1E0A->d, U+1E0B->d, U+1E0C->d, U+1E0D->d, U+1E0E->d, U+1E0F->d, U+1E10->d, \ + U+1E11->d, U+1E12->d, U+1E13->d, U+00C8->e, U+00C9->e, U+00CA->e, U+00CB->e, U+00E8->e, U+00E9->e, U+00EA->e, U+00EB->e, U+0112->e, U+0113->e, U+0114->e, U+0115->e, U+0116->e, U+0117->e, U+0118->e, U+0119->e, U+011A->e, U+011B->e, U+018E->e, U+0190->e, \ + U+01DD->e, U+0204->e, U+0205->e, U+0206->e, U+0207->e, U+0228->e, U+0229->e, U+0246->e, U+0247->e, U+0258->e, U+025B->e, U+025C->e, U+025D->e, U+025E->e, U+029A->e, U+1D07->e, U+1D08->e, U+1D31->e, U+1D32->e, U+1D49->e, U+1D4B->e, U+1D4C->e, U+1D92->e, \ + U+1D93->e, U+1D94->e, U+1D9F->e, U+1E14->e, U+1E15->e, U+1E16->e, U+1E17->e, U+1E18->e, U+1E19->e, U+1E1A->e, U+1E1B->e, U+1E1C->e, U+1E1D->e, U+1EB8->e, U+1EB9->e, U+1EBA->e, U+1EBB->e, U+1EBC->e, U+1EBD->e, U+1EBE->e, U+1EBF->e, U+1EC0->e, U+1EC1->e, \ + U+1EC2->e, U+1EC3->e, U+1EC4->e, U+1EC5->e, U+1EC6->e, U+1EC7->e, U+2091->e, U+0191->f, U+0192->f, U+1D6E->f, U+1D82->f, U+1DA0->f, U+1E1E->f, U+1E1F->f, U+011C->g, U+011D->g, U+011E->g, U+011F->g, U+0120->g, U+0121->g, U+0122->g, U+0123->g, U+0193->g, \ + U+01E4->g, U+01E5->g, U+01E6->g, U+01E7->g, U+01F4->g, U+01F5->g, U+0260->g, U+0261->g, U+0262->g, U+029B->g, U+1D33->g, U+1D4D->g, U+1D77->g, U+1D79->g, U+1D83->g, U+1DA2->g, U+1E20->g, U+1E21->g, U+0124->h, U+0125->h, U+0126->h, U+0127->h, U+021E->h, \ + U+021F->h, U+0265->h, U+0266->h, U+029C->h, U+02AE->h, U+02AF->h, U+02B0->h, U+02B1->h, U+1D34->h, U+1DA3->h, U+1E22->h, U+1E23->h, U+1E24->h, U+1E25->h, U+1E26->h, U+1E27->h, U+1E28->h, U+1E29->h, U+1E2A->h, U+1E2B->h, U+1E96->h, U+210C->h, U+2C67->h, \ + U+2C68->h, U+2C75->h, U+2C76->h, U+00CC->i, U+00CD->i, U+00CE->i, U+00CF->i, U+00EC->i, U+00ED->i, U+00EE->i, U+00EF->i, U+010309->i, U+0128->i, U+0129->i, U+012A->i, U+012B->i, U+012C->i, U+012D->i, U+012E->i, U+012F->i, U+0130->i, U+0131->i, U+0197->i, \ + U+01CF->i, U+01D0->i, U+0208->i, U+0209->i, U+020A->i, U+020B->i, U+0268->i, U+026A->i, U+040D->i, U+0418->i, U+0419->i, U+0438->i, U+0439->i, U+0456->i, U+1D09->i, U+1D35->i, U+1D4E->i, U+1D62->i, U+1D7B->i, U+1D96->i, U+1DA4->i, U+1DA6->i, U+1DA7->i, \ + U+1E2C->i, U+1E2D->i, U+1E2E->i, U+1E2F->i, U+1EC8->i, U+1EC9->i, U+1ECA->i, U+1ECB->i, U+2071->i, U+2111->i, U+0134->j, U+0135->j, U+01C8->j, U+01CB->j, U+01F0->j, U+0237->j, U+0248->j, U+0249->j, U+025F->j, U+0284->j, U+029D->j, U+02B2->j, U+1D0A->j, \ + U+1D36->j, U+1DA1->j, U+1DA8->j, U+0136->k, U+0137->k, U+0198->k, U+0199->k, U+01E8->k, U+01E9->k, U+029E->k, U+1D0B->k, U+1D37->k, U+1D4F->k, U+1D84->k, U+1E30->k, U+1E31->k, U+1E32->k, U+1E33->k, U+1E34->k, U+1E35->k, U+2C69->k, U+2C6A->k, U+0139->l, \ + U+013A->l, U+013B->l, U+013C->l, U+013D->l, U+013E->l, U+013F->l, U+0140->l, U+0141->l, U+0142->l, U+019A->l, U+01C8->l, U+0234->l, U+023D->l, U+026B->l, U+026C->l, U+026D->l, U+029F->l, U+02E1->l, U+1D0C->l, U+1D38->l, U+1D85->l, U+1DA9->l, U+1DAA->l, \ + U+1DAB->l, U+1E36->l, U+1E37->l, U+1E38->l, U+1E39->l, U+1E3A->l, U+1E3B->l, U+1E3C->l, U+1E3D->l, U+2C60->l, U+2C61->l, U+2C62->l, U+019C->m, U+026F->m, U+0270->m, U+0271->m, U+1D0D->m, U+1D1F->m, U+1D39->m, U+1D50->m, U+1D5A->m, U+1D6F->m, U+1D86->m, \ + U+1DAC->m, U+1DAD->m, U+1E3E->m, U+1E3F->m, U+1E40->m, U+1E41->m, U+1E42->m, U+1E43->m, U+00D1->n, U+00F1->n, U+0143->n, U+0144->n, U+0145->n, U+0146->n, U+0147->n, U+0148->n, U+0149->n, U+019D->n, U+019E->n, U+01CB->n, U+01F8->n, U+01F9->n, U+0220->n, \ + U+0235->n, U+0272->n, U+0273->n, U+0274->n, U+1D0E->n, U+1D3A->n, U+1D3B->n, U+1D70->n, U+1D87->n, U+1DAE->n, U+1DAF->n, U+1DB0->n, U+1E44->n, U+1E45->n, U+1E46->n, U+1E47->n, U+1E48->n, U+1E49->n, U+1E4A->n, U+1E4B->n, U+207F->n, U+00D2->o, U+00D3->o, \ + U+00D4->o, U+00D5->o, U+00D6->o, U+00D8->o, U+00F2->o, U+00F3->o, U+00F4->o, U+00F5->o, U+00F6->o, U+00F8->o, U+01030F->o, U+014C->o, U+014D->o, U+014E->o, U+014F->o, U+0150->o, U+0151->o, U+0186->o, U+019F->o, U+01A0->o, U+01A1->o, U+01D1->o, U+01D2->o, \ + U+01EA->o, U+01EB->o, U+01EC->o, U+01ED->o, U+01FE->o, U+01FF->o, U+020C->o, U+020D->o, U+020E->o, U+020F->o, U+022A->o, U+022B->o, U+022C->o, U+022D->o, U+022E->o, U+022F->o, U+0230->o, U+0231->o, U+0254->o, U+0275->o, U+043E->o, U+04E6->o, U+04E7->o, \ + U+04E8->o, U+04E9->o, U+04EA->o, U+04EB->o, U+1D0F->o, U+1D10->o, U+1D11->o, U+1D12->o, U+1D13->o, U+1D16->o, U+1D17->o, U+1D3C->o, U+1D52->o, U+1D53->o, U+1D54->o, U+1D55->o, U+1D97->o, U+1DB1->o, U+1E4C->o, U+1E4D->o, U+1E4E->o, U+1E4F->o, U+1E50->o, \ + U+1E51->o, U+1E52->o, U+1E53->o, U+1ECC->o, U+1ECD->o, U+1ECE->o, U+1ECF->o, U+1ED0->o, U+1ED1->o, U+1ED2->o, U+1ED3->o, U+1ED4->o, U+1ED5->o, U+1ED6->o, U+1ED7->o, U+1ED8->o, U+1ED9->o, U+1EDA->o, U+1EDB->o, U+1EDC->o, U+1EDD->o, U+1EDE->o, U+1EDF->o, \ + U+1EE0->o, U+1EE1->o, U+1EE2->o, U+1EE3->o, U+2092->o, U+2C9E->o, U+2C9F->o, U+01A4->p, U+01A5->p, U+1D18->p, U+1D3E->p, U+1D56->p, U+1D71->p, U+1D7D->p, U+1D88->p, U+1E54->p, U+1E55->p, U+1E56->p, U+1E57->p, U+2C63->p, U+024A->q, U+024B->q, U+02A0->q, \ + U+0154->r, U+0155->r, U+0156->r, U+0157->r, U+0158->r, U+0159->r, U+0210->r, U+0211->r, U+0212->r, U+0213->r, U+024C->r, U+024D->r, U+0279->r, U+027A->r, U+027B->r, U+027C->r, U+027D->r, U+027E->r, U+027F->r, U+0280->r, U+0281->r, U+02B3->r, U+02B4->r, \ + U+02B5->r, U+02B6->r, U+1D19->r, U+1D1A->r, U+1D3F->r, U+1D63->r, U+1D72->r, U+1D73->r, U+1D89->r, U+1DCA->r, U+1E58->r, U+1E59->r, U+1E5A->r, U+1E5B->r, U+1E5C->r, U+1E5D->r, U+1E5E->r, U+1E5F->r, U+211C->r, U+2C64->r, U+00DF->s, U+015A->s, U+015B->s, \ + U+015C->s, U+015D->s, U+015E->s, U+015F->s, U+0160->s, U+0161->s, U+017F->s, U+0218->s, U+0219->s, U+023F->s, U+0282->s, U+02E2->s, U+1D74->s, U+1D8A->s, U+1DB3->s, U+1E60->s, U+1E61->s, U+1E62->s, U+1E63->s, U+1E64->s, U+1E65->s, U+1E66->s, U+1E67->s, \ + U+1E68->s, U+1E69->s, U+1E9B->s, U+0162->t, U+0163->t, U+0164->t, U+0165->t, U+0166->t, U+0167->t, U+01AB->t, U+01AC->t, U+01AD->t, U+01AE->t, U+021A->t, U+021B->t, U+0236->t, U+023E->t, U+0287->t, U+0288->t, U+1D1B->t, U+1D40->t, U+1D57->t, U+1D75->t, \ + U+1DB5->t, U+1E6A->t, U+1E6B->t, U+1E6C->t, U+1E6D->t, U+1E6E->t, U+1E6F->t, U+1E70->t, U+1E71->t, U+1E97->t, U+2C66->t, U+00D9->u, U+00DA->u, U+00DB->u, U+00DC->u, U+00F9->u, U+00FA->u, U+00FB->u, U+00FC->u, U+010316->u, U+0168->u, U+0169->u, U+016A->u, \ + U+016B->u, U+016C->u, U+016D->u, U+016E->u, U+016F->u, U+0170->u, U+0171->u, U+0172->u, U+0173->u, U+01AF->u, U+01B0->u, U+01D3->u, U+01D4->u, U+01D5->u, U+01D6->u, U+01D7->u, U+01D8->u, U+01D9->u, U+01DA->u, U+01DB->u, U+01DC->u, U+0214->u, U+0215->u, \ + U+0216->u, U+0217->u, U+0244->u, U+0289->u, U+1D1C->u, U+1D1D->u, U+1D1E->u, U+1D41->u, U+1D58->u, U+1D59->u, U+1D64->u, U+1D7E->u, U+1D99->u, U+1DB6->u, U+1DB8->u, U+1E72->u, U+1E73->u, U+1E74->u, U+1E75->u, U+1E76->u, U+1E77->u, U+1E78->u, U+1E79->u, \ + U+1E7A->u, U+1E7B->u, U+1EE4->u, U+1EE5->u, U+1EE6->u, U+1EE7->u, U+1EE8->u, U+1EE9->u, U+1EEA->u, U+1EEB->u, U+1EEC->u, U+1EED->u, U+1EEE->u, U+1EEF->u, U+1EF0->u, U+1EF1->u, U+01B2->v, U+0245->v, U+028B->v, U+028C->v, U+1D20->v, U+1D5B->v, U+1D65->v, \ + U+1D8C->v, U+1DB9->v, U+1DBA->v, U+1E7C->v, U+1E7D->v, U+1E7E->v, U+1E7F->v, U+2C74->v, U+0174->w, U+0175->w, U+028D->w, U+02B7->w, U+1D21->w, U+1D42->w, U+1E80->w, U+1E81->w, U+1E82->w, U+1E83->w, U+1E84->w, U+1E85->w, U+1E86->w, U+1E87->w, U+1E88->w, \ + U+1E89->w, U+1E98->w, U+02E3->x, U+1D8D->x, U+1E8A->x, U+1E8B->x, U+1E8C->x, U+1E8D->x, U+2093->x, U+00DD->y, U+00FD->y, U+00FF->y, U+0176->y, U+0177->y, U+0178->y, U+01B3->y, U+01B4->y, U+0232->y, U+0233->y, U+024E->y, U+024F->y, U+028E->y, U+028F->y, \ + U+02B8->y, U+1E8E->y, U+1E8F->y, U+1E99->y, U+1EF2->y, U+1EF3->y, U+1EF4->y, U+1EF5->y, U+1EF6->y, U+1EF7->y, U+1EF8->y, U+1EF9->y, U+0179->z, U+017A->z, U+017B->z, U+017C->z, U+017D->z, U+017E->z, U+01B5->z, U+01B6->z, U+0224->z, U+0225->z, U+0240->z, \ + U+0290->z, U+0291->z, U+1D22->z, U+1D76->z, U+1D8E->z, U+1DBB->z, U+1DBC->z, U+1DBD->z, U+1E90->z, U+1E91->z, U+1E92->z, U+1E93->z, U+1E94->z, U+1E95->z, U+2128->z, U+2C6B->z, U+2C6C->z, U+00C6->U+00E6, U+01E2->U+00E6, U+01E3->U+00E6, U+01FC->U+00E6, \ + U+01FD->U+00E6, U+1D01->U+00E6, U+1D02->U+00E6, U+1D2D->U+00E6, U+1D46->U+00E6, U+00E6, U+0622->U+0627, U+0623->U+0627, U+0624->U+0648, U+0625->U+0627, U+0626->U+064A, U+06C0->U+06D5, U+06C2->U+06C1, U+06D3->U+06D2, U+FB50->U+0671, U+FB51->U+0671, U+FB52->U+067B, \ + U+FB53->U+067B, U+FB54->U+067B, U+FB56->U+067E, U+FB57->U+067E, U+FB58->U+067E, U+FB5A->U+0680, U+FB5B->U+0680, U+FB5C->U+0680, U+FB5E->U+067A, U+FB5F->U+067A, U+FB60->U+067A, U+FB62->U+067F, U+FB63->U+067F, U+FB64->U+067F, U+FB66->U+0679, U+FB67->U+0679, \ + U+FB68->U+0679, U+FB6A->U+06A4, U+FB6B->U+06A4, U+FB6C->U+06A4, U+FB6E->U+06A6, U+FB6F->U+06A6, U+FB70->U+06A6, U+FB72->U+0684, U+FB73->U+0684, U+FB74->U+0684, U+FB76->U+0683, U+FB77->U+0683, U+FB78->U+0683, U+FB7A->U+0686, U+FB7B->U+0686, U+FB7C->U+0686, \ + U+FB7E->U+0687, U+FB7F->U+0687, U+FB80->U+0687, U+FB82->U+068D, U+FB83->U+068D, U+FB84->U+068C, U+FB85->U+068C, U+FB86->U+068E, U+FB87->U+068E, U+FB88->U+0688, U+FB89->U+0688, U+FB8A->U+0698, U+FB8B->U+0698, U+FB8C->U+0691, U+FB8D->U+0691, U+FB8E->U+06A9, \ + U+FB8F->U+06A9, U+FB90->U+06A9, U+FB92->U+06AF, U+FB93->U+06AF, U+FB94->U+06AF, U+FB96->U+06B3, U+FB97->U+06B3, U+FB98->U+06B3, U+FB9A->U+06B1, U+FB9B->U+06B1, U+FB9C->U+06B1, U+FB9E->U+06BA, U+FB9F->U+06BA, U+FBA0->U+06BB, U+FBA1->U+06BB, U+FBA2->U+06BB, \ + U+FBA4->U+06C0, U+FBA5->U+06C0, U+FBA6->U+06C1, U+FBA7->U+06C1, U+FBA8->U+06C1, U+FBAA->U+06BE, U+FBAB->U+06BE, U+FBAC->U+06BE, U+FBAE->U+06D2, U+FBAF->U+06D2, U+FBB0->U+06D3, U+FBB1->U+06D3, U+FBD3->U+06AD, U+FBD4->U+06AD, U+FBD5->U+06AD, U+FBD7->U+06C7, \ + U+FBD8->U+06C7, U+FBD9->U+06C6, U+FBDA->U+06C6, U+FBDB->U+06C8, U+FBDC->U+06C8, U+FBDD->U+0677, U+FBDE->U+06CB, U+FBDF->U+06CB, U+FBE0->U+06C5, U+FBE1->U+06C5, U+FBE2->U+06C9, U+FBE3->U+06C9, U+FBE4->U+06D0, U+FBE5->U+06D0, U+FBE6->U+06D0, U+FBE8->U+0649, \ + U+FBFC->U+06CC, U+FBFD->U+06CC, U+FBFE->U+06CC, U+0621, U+0627..U+063A, U+0641..U+064A, U+0660..U+0669, U+066E, U+066F, U+0671..U+06BF, U+06C1, U+06C3..U+06D2, U+06D5, U+06EE..U+06FC, U+06FF, U+0750..U+076D, U+FB55, U+FB59, U+FB5D, U+FB61, U+FB65, U+FB69, \ + U+FB6D, U+FB71, U+FB75, U+FB79, U+FB7D, U+FB81, U+FB91, U+FB95, U+FB99, U+FB9D, U+FBA3, U+FBA9, U+FBAD, U+FBD6, U+FBE7, U+FBE9, U+FBFF, U+0531..U+0556->U+0561..U+0586, U+0561..U+0586, U+0587, U+09DC->U+09A1, U+09DD->U+09A2, U+09DF->U+09AF, U+09F0->U+09AC, \ + U+09F1->U+09AC, U+0985..U+0990, U+0993..U+09B0, U+09B2, U+09B6..U+09B9, U+09CE, U+09E0, U+09E1, U+09E6..U+09EF, U+F900->U+8C48, U+F901->U+66F4, U+F902->U+8ECA, U+F903->U+8CC8, U+F904->U+6ED1, U+F905->U+4E32, U+F906->U+53E5, U+F907->U+9F9C, U+F908->U+9F9C, \ + U+F909->U+5951, U+F90A->U+91D1, U+F90B->U+5587, U+F90C->U+5948, U+F90D->U+61F6, U+F90E->U+7669, U+F90F->U+7F85, U+F910->U+863F, U+F911->U+87BA, U+F912->U+88F8, U+F913->U+908F, U+F914->U+6A02, U+F915->U+6D1B, U+F916->U+70D9, U+F917->U+73DE, U+F918->U+843D, \ + U+F919->U+916A, U+F91A->U+99F1, U+F91B->U+4E82, U+F91C->U+5375, U+F91D->U+6B04, U+F91E->U+721B, U+F91F->U+862D, U+F920->U+9E1E, U+F921->U+5D50, U+F922->U+6FEB, U+F923->U+85CD, U+F924->U+8964, U+F925->U+62C9, U+F926->U+81D8, U+F927->U+881F, U+F928->U+5ECA, \ + U+F929->U+6717, U+F92A->U+6D6A, U+F92B->U+72FC, U+F92C->U+90CE, U+F92D->U+4F86, U+F92E->U+51B7, U+F92F->U+52DE, U+F930->U+64C4, U+F931->U+6AD3, U+F932->U+7210, U+F933->U+76E7, U+F934->U+8001, U+F935->U+8606, U+F936->U+865C, U+F937->U+8DEF, U+F938->U+9732, \ + U+F939->U+9B6F, U+F93A->U+9DFA, U+F93B->U+788C, U+F93C->U+797F, U+F93D->U+7DA0, U+F93E->U+83C9, U+F93F->U+9304, U+F940->U+9E7F, U+F941->U+8AD6, U+F942->U+58DF, U+F943->U+5F04, U+F944->U+7C60, U+F945->U+807E, U+F946->U+7262, U+F947->U+78CA, U+F948->U+8CC2, \ + U+F949->U+96F7, U+F94A->U+58D8, U+F94B->U+5C62, U+F94C->U+6A13, U+F94D->U+6DDA, U+F94E->U+6F0F, U+F94F->U+7D2F, U+F950->U+7E37, U+F951->U+964B, U+F952->U+52D2, U+F953->U+808B, U+F954->U+51DC, U+F955->U+51CC, U+F956->U+7A1C, U+F957->U+7DBE, U+F958->U+83F1, \ + U+F959->U+9675, U+F95A->U+8B80, U+F95B->U+62CF, U+F95C->U+6A02, U+F95D->U+8AFE, U+F95E->U+4E39, U+F95F->U+5BE7, U+F960->U+6012, U+F961->U+7387, U+F962->U+7570, U+F963->U+5317, U+F964->U+78FB, U+F965->U+4FBF, U+F966->U+5FA9, U+F967->U+4E0D, U+F968->U+6CCC, \ + U+F969->U+6578, U+F96A->U+7D22, U+F96B->U+53C3, U+F96C->U+585E, U+F96D->U+7701, U+F96E->U+8449, U+F96F->U+8AAA, U+F970->U+6BBA, U+F971->U+8FB0, U+F972->U+6C88, U+F973->U+62FE, U+F974->U+82E5, U+F975->U+63A0, U+F976->U+7565, U+F977->U+4EAE, U+F978->U+5169, \ + U+F979->U+51C9, U+F97A->U+6881, U+F97B->U+7CE7, U+F97C->U+826F, U+F97D->U+8AD2, U+F97E->U+91CF, U+F97F->U+52F5, U+F980->U+5442, U+F981->U+5973, U+F982->U+5EEC, U+F983->U+65C5, U+F984->U+6FFE, U+F985->U+792A, U+F986->U+95AD, U+F987->U+9A6A, U+F988->U+9E97, \ + U+F989->U+9ECE, U+F98A->U+529B, U+F98B->U+66C6, U+F98C->U+6B77, U+F98D->U+8F62, U+F98E->U+5E74, U+F98F->U+6190, U+F990->U+6200, U+F991->U+649A, U+F992->U+6F23, U+F993->U+7149, U+F994->U+7489, U+F995->U+79CA, U+F996->U+7DF4, U+F997->U+806F, U+F998->U+8F26, \ + U+F999->U+84EE, U+F99A->U+9023, U+F99B->U+934A, U+F99C->U+5217, U+F99D->U+52A3, U+F99E->U+54BD, U+F99F->U+70C8, U+F9A0->U+88C2, U+F9A1->U+8AAA, U+F9A2->U+5EC9, U+F9A3->U+5FF5, U+F9A4->U+637B, U+F9A5->U+6BAE, U+F9A6->U+7C3E, U+F9A7->U+7375, U+F9A8->U+4EE4, \ + U+F9A9->U+56F9, U+F9AA->U+5BE7, U+F9AB->U+5DBA, U+F9AC->U+601C, U+F9AD->U+73B2, U+F9AE->U+7469, U+F9AF->U+7F9A, U+F9B0->U+8046, U+F9B1->U+9234, U+F9B2->U+96F6, U+F9B3->U+9748, U+F9B4->U+9818, U+F9B5->U+4F8B, U+F9B6->U+79AE, U+F9B7->U+91B4, U+F9B8->U+96B8, \ + U+F9B9->U+60E1, U+F9BA->U+4E86, U+F9BB->U+50DA, U+F9BC->U+5BEE, U+F9BD->U+5C3F, U+F9BE->U+6599, U+F9BF->U+6A02, U+F9C0->U+71CE, U+F9C1->U+7642, U+F9C2->U+84FC, U+F9C3->U+907C, U+F9C4->U+9F8D, U+F9C5->U+6688, U+F9C6->U+962E, U+F9C7->U+5289, U+F9C8->U+677B, \ + U+F9C9->U+67F3, U+F9CA->U+6D41, U+F9CB->U+6E9C, U+F9CC->U+7409, U+F9CD->U+7559, U+F9CE->U+786B, U+F9CF->U+7D10, U+F9D0->U+985E, U+F9D1->U+516D, U+F9D2->U+622E, U+F9D3->U+9678, U+F9D4->U+502B, U+F9D5->U+5D19, U+F9D6->U+6DEA, U+F9D7->U+8F2A, U+F9D8->U+5F8B, \ + U+F9D9->U+6144, U+F9DA->U+6817, U+F9DB->U+7387, U+F9DC->U+9686, U+F9DD->U+5229, U+F9DE->U+540F, U+F9DF->U+5C65, U+F9E0->U+6613, U+F9E1->U+674E, U+F9E2->U+68A8, U+F9E3->U+6CE5, U+F9E4->U+7406, U+F9E5->U+75E2, U+F9E6->U+7F79, U+F9E7->U+88CF, U+F9E8->U+88E1, \ + U+F9E9->U+91CC, U+F9EA->U+96E2, U+F9EB->U+533F, U+F9EC->U+6EBA, U+F9ED->U+541D, U+F9EE->U+71D0, U+F9EF->U+7498, U+F9F0->U+85FA, U+F9F1->U+96A3, U+F9F2->U+9C57, U+F9F3->U+9E9F, U+F9F4->U+6797, U+F9F5->U+6DCB, U+F9F6->U+81E8, U+F9F7->U+7ACB, U+F9F8->U+7B20, \ + U+F9F9->U+7C92, U+F9FA->U+72C0, U+F9FB->U+7099, U+F9FC->U+8B58, U+F9FD->U+4EC0, U+F9FE->U+8336, U+F9FF->U+523A, U+FA00->U+5207, U+FA01->U+5EA6, U+FA02->U+62D3, U+FA03->U+7CD6, U+FA04->U+5B85, U+FA05->U+6D1E, U+FA06->U+66B4, U+FA07->U+8F3B, U+FA08->U+884C, \ + U+FA09->U+964D, U+FA0A->U+898B, U+FA0B->U+5ED3, U+FA0C->U+5140, U+FA0D->U+55C0, U+FA10->U+585A, U+FA12->U+6674, U+FA15->U+51DE, U+FA16->U+732A, U+FA17->U+76CA, U+FA18->U+793C, U+FA19->U+795E, U+FA1A->U+7965, U+FA1B->U+798F, U+FA1C->U+9756, U+FA1D->U+7CBE, \ + U+FA1E->U+7FBD, U+FA20->U+8612, U+FA22->U+8AF8, U+FA25->U+9038, U+FA26->U+90FD, U+FA2A->U+98EF, U+FA2B->U+98FC, U+FA2C->U+9928, U+FA2D->U+9DB4, U+FA30->U+4FAE, U+FA31->U+50E7, U+FA32->U+514D, U+FA33->U+52C9, U+FA34->U+52E4, U+FA35->U+5351, U+FA36->U+559D, \ + U+FA37->U+5606, U+FA38->U+5668, U+FA39->U+5840, U+FA3A->U+58A8, U+FA3B->U+5C64, U+FA3C->U+5C6E, U+FA3D->U+6094, U+FA3E->U+6168, U+FA3F->U+618E, U+FA40->U+61F2, U+FA41->U+654F, U+FA42->U+65E2, U+FA43->U+6691, U+FA44->U+6885, U+FA45->U+6D77, U+FA46->U+6E1A, \ + U+FA47->U+6F22, U+FA48->U+716E, U+FA49->U+722B, U+FA4A->U+7422, U+FA4B->U+7891, U+FA4C->U+793E, U+FA4D->U+7949, U+FA4E->U+7948, U+FA4F->U+7950, U+FA50->U+7956, U+FA51->U+795D, U+FA52->U+798D, U+FA53->U+798E, U+FA54->U+7A40, U+FA55->U+7A81, U+FA56->U+7BC0, \ + U+FA57->U+7DF4, U+FA58->U+7E09, U+FA59->U+7E41, U+FA5A->U+7F72, U+FA5B->U+8005, U+FA5C->U+81ED, U+FA5D->U+8279, U+FA5E->U+8279, U+FA5F->U+8457, U+FA60->U+8910, U+FA61->U+8996, U+FA62->U+8B01, U+FA63->U+8B39, U+FA64->U+8CD3, U+FA65->U+8D08, U+FA66->U+8FB6, \ + U+FA67->U+9038, U+FA68->U+96E3, U+FA69->U+97FF, U+FA6A->U+983B, U+FA70->U+4E26, U+FA71->U+51B5, U+FA72->U+5168, U+FA73->U+4F80, U+FA74->U+5145, U+FA75->U+5180, U+FA76->U+52C7, U+FA77->U+52FA, U+FA78->U+559D, U+FA79->U+5555, U+FA7A->U+5599, U+FA7B->U+55E2, \ + U+FA7C->U+585A, U+FA7D->U+58B3, U+FA7E->U+5944, U+FA7F->U+5954, U+FA80->U+5A62, U+FA81->U+5B28, U+FA82->U+5ED2, U+FA83->U+5ED9, U+FA84->U+5F69, U+FA85->U+5FAD, U+FA86->U+60D8, U+FA87->U+614E, U+FA88->U+6108, U+FA89->U+618E, U+FA8A->U+6160, U+FA8B->U+61F2, \ + U+FA8C->U+6234, U+FA8D->U+63C4, U+FA8E->U+641C, U+FA8F->U+6452, U+FA90->U+6556, U+FA91->U+6674, U+FA92->U+6717, U+FA93->U+671B, U+FA94->U+6756, U+FA95->U+6B79, U+FA96->U+6BBA, U+FA97->U+6D41, U+FA98->U+6EDB, U+FA99->U+6ECB, U+FA9A->U+6F22, U+FA9B->U+701E, \ + U+FA9C->U+716E, U+FA9D->U+77A7, U+FA9E->U+7235, U+FA9F->U+72AF, U+FAA0->U+732A, U+FAA1->U+7471, U+FAA2->U+7506, U+FAA3->U+753B, U+FAA4->U+761D, U+FAA5->U+761F, U+FAA6->U+76CA, U+FAA7->U+76DB, U+FAA8->U+76F4, U+FAA9->U+774A, U+FAAA->U+7740, U+FAAB->U+78CC, \ + U+FAAC->U+7AB1, U+FAAD->U+7BC0, U+FAAE->U+7C7B, U+FAAF->U+7D5B, U+FAB0->U+7DF4, U+FAB1->U+7F3E, U+FAB2->U+8005, U+FAB3->U+8352, U+FAB4->U+83EF, U+FAB5->U+8779, U+FAB6->U+8941, U+FAB7->U+8986, U+FAB8->U+8996, U+FAB9->U+8ABF, U+FABA->U+8AF8, U+FABB->U+8ACB, \ + U+FABC->U+8B01, U+FABD->U+8AFE, U+FABE->U+8AED, U+FABF->U+8B39, U+FAC0->U+8B8A, U+FAC1->U+8D08, U+FAC2->U+8F38, U+FAC3->U+9072, U+FAC4->U+9199, U+FAC5->U+9276, U+FAC6->U+967C, U+FAC7->U+96E3, U+FAC8->U+9756, U+FAC9->U+97DB, U+FACA->U+97FF, U+FACB->U+980B, \ + U+FACC->U+983B, U+FACD->U+9B12, U+FACE->U+9F9C, U+FACF->U+2284A, U+FAD0->U+22844, U+FAD1->U+233D5, U+FAD2->U+3B9D, U+FAD3->U+4018, U+FAD4->U+4039, U+FAD5->U+25249, U+FAD6->U+25CD0, U+FAD7->U+27ED3, U+FAD8->U+9F43, U+FAD9->U+9F8E, U+2F800->U+4E3D, U+2F801->U+4E38, \ + U+2F802->U+4E41, U+2F803->U+20122, U+2F804->U+4F60, U+2F805->U+4FAE, U+2F806->U+4FBB, U+2F807->U+5002, U+2F808->U+507A, U+2F809->U+5099, U+2F80A->U+50E7, U+2F80B->U+50CF, U+2F80C->U+349E, U+2F80D->U+2063A, U+2F80E->U+514D, U+2F80F->U+5154, U+2F810->U+5164, \ + U+2F811->U+5177, U+2F812->U+2051C, U+2F813->U+34B9, U+2F814->U+5167, U+2F815->U+518D, U+2F816->U+2054B, U+2F817->U+5197, U+2F818->U+51A4, U+2F819->U+4ECC, U+2F81A->U+51AC, U+2F81B->U+51B5, U+2F81C->U+291DF, U+2F81D->U+51F5, U+2F81E->U+5203, U+2F81F->U+34DF, \ + U+2F820->U+523B, U+2F821->U+5246, U+2F822->U+5272, U+2F823->U+5277, U+2F824->U+3515, U+2F825->U+52C7, U+2F826->U+52C9, U+2F827->U+52E4, U+2F828->U+52FA, U+2F829->U+5305, U+2F82A->U+5306, U+2F82B->U+5317, U+2F82C->U+5349, U+2F82D->U+5351, U+2F82E->U+535A, \ + U+2F82F->U+5373, U+2F830->U+537D, U+2F831->U+537F, U+2F832->U+537F, U+2F833->U+537F, U+2F834->U+20A2C, U+2F835->U+7070, U+2F836->U+53CA, U+2F837->U+53DF, U+2F838->U+20B63, U+2F839->U+53EB, U+2F83A->U+53F1, U+2F83B->U+5406, U+2F83C->U+549E, U+2F83D->U+5438, \ + U+2F83E->U+5448, U+2F83F->U+5468, U+2F840->U+54A2, U+2F841->U+54F6, U+2F842->U+5510, U+2F843->U+5553, U+2F844->U+5563, U+2F845->U+5584, U+2F846->U+5584, U+2F847->U+5599, U+2F848->U+55AB, U+2F849->U+55B3, U+2F84A->U+55C2, U+2F84B->U+5716, U+2F84C->U+5606, \ + U+2F84D->U+5717, U+2F84E->U+5651, U+2F84F->U+5674, U+2F850->U+5207, U+2F851->U+58EE, U+2F852->U+57CE, U+2F853->U+57F4, U+2F854->U+580D, U+2F855->U+578B, U+2F856->U+5832, U+2F857->U+5831, U+2F858->U+58AC, U+2F859->U+214E4, U+2F85A->U+58F2, U+2F85B->U+58F7, \ + U+2F85C->U+5906, U+2F85D->U+591A, U+2F85E->U+5922, U+2F85F->U+5962, U+2F860->U+216A8, U+2F861->U+216EA, U+2F862->U+59EC, U+2F863->U+5A1B, U+2F864->U+5A27, U+2F865->U+59D8, U+2F866->U+5A66, U+2F867->U+36EE, U+2F868->U+36FC, U+2F869->U+5B08, U+2F86A->U+5B3E, \ + U+2F86B->U+5B3E, U+2F86C->U+219C8, U+2F86D->U+5BC3, U+2F86E->U+5BD8, U+2F86F->U+5BE7, U+2F870->U+5BF3, U+2F871->U+21B18, U+2F872->U+5BFF, U+2F873->U+5C06, U+2F874->U+5F53, U+2F875->U+5C22, U+2F876->U+3781, U+2F877->U+5C60, U+2F878->U+5C6E, U+2F879->U+5CC0, \ + U+2F87A->U+5C8D, U+2F87B->U+21DE4, U+2F87C->U+5D43, U+2F87D->U+21DE6, U+2F87E->U+5D6E, U+2F87F->U+5D6B, U+2F880->U+5D7C, U+2F881->U+5DE1, U+2F882->U+5DE2, U+2F883->U+382F, U+2F884->U+5DFD, U+2F885->U+5E28, U+2F886->U+5E3D, U+2F887->U+5E69, U+2F888->U+3862, \ + U+2F889->U+22183, U+2F88A->U+387C, U+2F88B->U+5EB0, U+2F88C->U+5EB3, U+2F88D->U+5EB6, U+2F88E->U+5ECA, U+2F88F->U+2A392, U+2F890->U+5EFE, U+2F891->U+22331, U+2F892->U+22331, U+2F893->U+8201, U+2F894->U+5F22, U+2F895->U+5F22, U+2F896->U+38C7, U+2F897->U+232B8, \ + U+2F898->U+261DA, U+2F899->U+5F62, U+2F89A->U+5F6B, U+2F89B->U+38E3, U+2F89C->U+5F9A, U+2F89D->U+5FCD, U+2F89E->U+5FD7, U+2F89F->U+5FF9, U+2F8A0->U+6081, U+2F8A1->U+393A, U+2F8A2->U+391C, U+2F8A3->U+6094, U+2F8A4->U+226D4, U+2F8A5->U+60C7, U+2F8A6->U+6148, \ + U+2F8A7->U+614C, U+2F8A8->U+614E, U+2F8A9->U+614C, U+2F8AA->U+617A, U+2F8AB->U+618E, U+2F8AC->U+61B2, U+2F8AD->U+61A4, U+2F8AE->U+61AF, U+2F8AF->U+61DE, U+2F8B0->U+61F2, U+2F8B1->U+61F6, U+2F8B2->U+6210, U+2F8B3->U+621B, U+2F8B4->U+625D, U+2F8B5->U+62B1, \ + U+2F8B6->U+62D4, U+2F8B7->U+6350, U+2F8B8->U+22B0C, U+2F8B9->U+633D, U+2F8BA->U+62FC, U+2F8BB->U+6368, U+2F8BC->U+6383, U+2F8BD->U+63E4, U+2F8BE->U+22BF1, U+2F8BF->U+6422, U+2F8C0->U+63C5, U+2F8C1->U+63A9, U+2F8C2->U+3A2E, U+2F8C3->U+6469, U+2F8C4->U+647E, \ + U+2F8C5->U+649D, U+2F8C6->U+6477, U+2F8C7->U+3A6C, U+2F8C8->U+654F, U+2F8C9->U+656C, U+2F8CA->U+2300A, U+2F8CB->U+65E3, U+2F8CC->U+66F8, U+2F8CD->U+6649, U+2F8CE->U+3B19, U+2F8CF->U+6691, U+2F8D0->U+3B08, U+2F8D1->U+3AE4, U+2F8D2->U+5192, U+2F8D3->U+5195, \ + U+2F8D4->U+6700, U+2F8D5->U+669C, U+2F8D6->U+80AD, U+2F8D7->U+43D9, U+2F8D8->U+6717, U+2F8D9->U+671B, U+2F8DA->U+6721, U+2F8DB->U+675E, U+2F8DC->U+6753, U+2F8DD->U+233C3, U+2F8DE->U+3B49, U+2F8DF->U+67FA, U+2F8E0->U+6785, U+2F8E1->U+6852, U+2F8E2->U+6885, \ + U+2F8E3->U+2346D, U+2F8E4->U+688E, U+2F8E5->U+681F, U+2F8E6->U+6914, U+2F8E7->U+3B9D, U+2F8E8->U+6942, U+2F8E9->U+69A3, U+2F8EA->U+69EA, U+2F8EB->U+6AA8, U+2F8EC->U+236A3, U+2F8ED->U+6ADB, U+2F8EE->U+3C18, U+2F8EF->U+6B21, U+2F8F0->U+238A7, U+2F8F1->U+6B54, \ + U+2F8F2->U+3C4E, U+2F8F3->U+6B72, U+2F8F4->U+6B9F, U+2F8F5->U+6BBA, U+2F8F6->U+6BBB, U+2F8F7->U+23A8D, U+2F8F8->U+21D0B, U+2F8F9->U+23AFA, U+2F8FA->U+6C4E, U+2F8FB->U+23CBC, U+2F8FC->U+6CBF, U+2F8FD->U+6CCD, U+2F8FE->U+6C67, U+2F8FF->U+6D16, U+2F900->U+6D3E, \ + U+2F901->U+6D77, U+2F902->U+6D41, U+2F903->U+6D69, U+2F904->U+6D78, U+2F905->U+6D85, U+2F906->U+23D1E, U+2F907->U+6D34, U+2F908->U+6E2F, U+2F909->U+6E6E, U+2F90A->U+3D33, U+2F90B->U+6ECB, U+2F90C->U+6EC7, U+2F90D->U+23ED1, U+2F90E->U+6DF9, U+2F90F->U+6F6E, \ + U+2F910->U+23F5E, U+2F911->U+23F8E, U+2F912->U+6FC6, U+2F913->U+7039, U+2F914->U+701E, U+2F915->U+701B, U+2F916->U+3D96, U+2F917->U+704A, U+2F918->U+707D, U+2F919->U+7077, U+2F91A->U+70AD, U+2F91B->U+20525, U+2F91C->U+7145, U+2F91D->U+24263, U+2F91E->U+719C, \ + U+2F91F->U+243AB, U+2F920->U+7228, U+2F921->U+7235, U+2F922->U+7250, U+2F923->U+24608, U+2F924->U+7280, U+2F925->U+7295, U+2F926->U+24735, U+2F927->U+24814, U+2F928->U+737A, U+2F929->U+738B, U+2F92A->U+3EAC, U+2F92B->U+73A5, U+2F92C->U+3EB8, U+2F92D->U+3EB8, \ + U+2F92E->U+7447, U+2F92F->U+745C, U+2F930->U+7471, U+2F931->U+7485, U+2F932->U+74CA, U+2F933->U+3F1B, U+2F934->U+7524, U+2F935->U+24C36, U+2F936->U+753E, U+2F937->U+24C92, U+2F938->U+7570, U+2F939->U+2219F, U+2F93A->U+7610, U+2F93B->U+24FA1, U+2F93C->U+24FB8, \ + U+2F93D->U+25044, U+2F93E->U+3FFC, U+2F93F->U+4008, U+2F940->U+76F4, U+2F941->U+250F3, U+2F942->U+250F2, U+2F943->U+25119, U+2F944->U+25133, U+2F945->U+771E, U+2F946->U+771F, U+2F947->U+771F, U+2F948->U+774A, U+2F949->U+4039, U+2F94A->U+778B, U+2F94B->U+4046, \ + U+2F94C->U+4096, U+2F94D->U+2541D, U+2F94E->U+784E, U+2F94F->U+788C, U+2F950->U+78CC, U+2F951->U+40E3, U+2F952->U+25626, U+2F953->U+7956, U+2F954->U+2569A, U+2F955->U+256C5, U+2F956->U+798F, U+2F957->U+79EB, U+2F958->U+412F, U+2F959->U+7A40, U+2F95A->U+7A4A, \ + U+2F95B->U+7A4F, U+2F95C->U+2597C, U+2F95D->U+25AA7, U+2F95E->U+25AA7, U+2F95F->U+7AEE, U+2F960->U+4202, U+2F961->U+25BAB, U+2F962->U+7BC6, U+2F963->U+7BC9, U+2F964->U+4227, U+2F965->U+25C80, U+2F966->U+7CD2, U+2F967->U+42A0, U+2F968->U+7CE8, U+2F969->U+7CE3, \ + U+2F96A->U+7D00, U+2F96B->U+25F86, U+2F96C->U+7D63, U+2F96D->U+4301, U+2F96E->U+7DC7, U+2F96F->U+7E02, U+2F970->U+7E45, U+2F971->U+4334, U+2F972->U+26228, U+2F973->U+26247, U+2F974->U+4359, U+2F975->U+262D9, U+2F976->U+7F7A, U+2F977->U+2633E, U+2F978->U+7F95, \ + U+2F979->U+7FFA, U+2F97A->U+8005, U+2F97B->U+264DA, U+2F97C->U+26523, U+2F97D->U+8060, U+2F97E->U+265A8, U+2F97F->U+8070, U+2F980->U+2335F, U+2F981->U+43D5, U+2F982->U+80B2, U+2F983->U+8103, U+2F984->U+440B, U+2F985->U+813E, U+2F986->U+5AB5, U+2F987->U+267A7, \ + U+2F988->U+267B5, U+2F989->U+23393, U+2F98A->U+2339C, U+2F98B->U+8201, U+2F98C->U+8204, U+2F98D->U+8F9E, U+2F98E->U+446B, U+2F98F->U+8291, U+2F990->U+828B, U+2F991->U+829D, U+2F992->U+52B3, U+2F993->U+82B1, U+2F994->U+82B3, U+2F995->U+82BD, U+2F996->U+82E6, \ + U+2F997->U+26B3C, U+2F998->U+82E5, U+2F999->U+831D, U+2F99A->U+8363, U+2F99B->U+83AD, U+2F99C->U+8323, U+2F99D->U+83BD, U+2F99E->U+83E7, U+2F99F->U+8457, U+2F9A0->U+8353, U+2F9A1->U+83CA, U+2F9A2->U+83CC, U+2F9A3->U+83DC, U+2F9A4->U+26C36, U+2F9A5->U+26D6B, \ + U+2F9A6->U+26CD5, U+2F9A7->U+452B, U+2F9A8->U+84F1, U+2F9A9->U+84F3, U+2F9AA->U+8516, U+2F9AB->U+273CA, U+2F9AC->U+8564, U+2F9AD->U+26F2C, U+2F9AE->U+455D, U+2F9AF->U+4561, U+2F9B0->U+26FB1, U+2F9B1->U+270D2, U+2F9B2->U+456B, U+2F9B3->U+8650, U+2F9B4->U+865C, \ + U+2F9B5->U+8667, U+2F9B6->U+8669, U+2F9B7->U+86A9, U+2F9B8->U+8688, U+2F9B9->U+870E, U+2F9BA->U+86E2, U+2F9BB->U+8779, U+2F9BC->U+8728, U+2F9BD->U+876B, U+2F9BE->U+8786, U+2F9BF->U+45D7, U+2F9C0->U+87E1, U+2F9C1->U+8801, U+2F9C2->U+45F9, U+2F9C3->U+8860, \ + U+2F9C4->U+8863, U+2F9C5->U+27667, U+2F9C6->U+88D7, U+2F9C7->U+88DE, U+2F9C8->U+4635, U+2F9C9->U+88FA, U+2F9CA->U+34BB, U+2F9CB->U+278AE, U+2F9CC->U+27966, U+2F9CD->U+46BE, U+2F9CE->U+46C7, U+2F9CF->U+8AA0, U+2F9D0->U+8AED, U+2F9D1->U+8B8A, U+2F9D2->U+8C55, \ + U+2F9D3->U+27CA8, U+2F9D4->U+8CAB, U+2F9D5->U+8CC1, U+2F9D6->U+8D1B, U+2F9D7->U+8D77, U+2F9D8->U+27F2F, U+2F9D9->U+20804, U+2F9DA->U+8DCB, U+2F9DB->U+8DBC, U+2F9DC->U+8DF0, U+2F9DD->U+208DE, U+2F9DE->U+8ED4, U+2F9DF->U+8F38, U+2F9E0->U+285D2, U+2F9E1->U+285ED, \ + U+2F9E2->U+9094, U+2F9E3->U+90F1, U+2F9E4->U+9111, U+2F9E5->U+2872E, U+2F9E6->U+911B, U+2F9E7->U+9238, U+2F9E8->U+92D7, U+2F9E9->U+92D8, U+2F9EA->U+927C, U+2F9EB->U+93F9, U+2F9EC->U+9415, U+2F9ED->U+28BFA, U+2F9EE->U+958B, U+2F9EF->U+4995, U+2F9F0->U+95B7, \ + U+2F9F1->U+28D77, U+2F9F2->U+49E6, U+2F9F3->U+96C3, U+2F9F4->U+5DB2, U+2F9F5->U+9723, U+2F9F6->U+29145, U+2F9F7->U+2921A, U+2F9F8->U+4A6E, U+2F9F9->U+4A76, U+2F9FA->U+97E0, U+2F9FB->U+2940A, U+2F9FC->U+4AB2, U+2F9FD->U+29496, U+2F9FE->U+980B, U+2F9FF->U+980B, \ + U+2FA00->U+9829, U+2FA01->U+295B6, U+2FA02->U+98E2, U+2FA03->U+4B33, U+2FA04->U+9929, U+2FA05->U+99A7, U+2FA06->U+99C2, U+2FA07->U+99FE, U+2FA08->U+4BCE, U+2FA09->U+29B30, U+2FA0A->U+9B12, U+2FA0B->U+9C40, U+2FA0C->U+9CFD, U+2FA0D->U+4CCE, U+2FA0E->U+4CED, \ + U+2FA0F->U+9D67, U+2FA10->U+2A0CE, U+2FA11->U+4CF8, U+2FA12->U+2A105, U+2FA13->U+2A20E, U+2FA14->U+2A291, U+2FA15->U+9EBB, U+2FA16->U+4D56, U+2FA17->U+9EF9, U+2FA18->U+9EFE, U+2FA19->U+9F05, U+2FA1A->U+9F0F, U+2FA1B->U+9F16, U+2FA1C->U+9F3B, U+2FA1D->U+2A600, \ + U+2F00->U+4E00, U+2F01->U+4E28, U+2F02->U+4E36, U+2F03->U+4E3F, U+2F04->U+4E59, U+2F05->U+4E85, U+2F06->U+4E8C, U+2F07->U+4EA0, U+2F08->U+4EBA, U+2F09->U+513F, U+2F0A->U+5165, U+2F0B->U+516B, U+2F0C->U+5182, U+2F0D->U+5196, U+2F0E->U+51AB, U+2F0F->U+51E0, \ + U+2F10->U+51F5, U+2F11->U+5200, U+2F12->U+529B, U+2F13->U+52F9, U+2F14->U+5315, U+2F15->U+531A, U+2F16->U+5338, U+2F17->U+5341, U+2F18->U+535C, U+2F19->U+5369, U+2F1A->U+5382, U+2F1B->U+53B6, U+2F1C->U+53C8, U+2F1D->U+53E3, U+2F1E->U+56D7, U+2F1F->U+571F, \ + U+2F20->U+58EB, U+2F21->U+5902, U+2F22->U+590A, U+2F23->U+5915, U+2F24->U+5927, U+2F25->U+5973, U+2F26->U+5B50, U+2F27->U+5B80, U+2F28->U+5BF8, U+2F29->U+5C0F, U+2F2A->U+5C22, U+2F2B->U+5C38, U+2F2C->U+5C6E, U+2F2D->U+5C71, U+2F2E->U+5DDB, U+2F2F->U+5DE5, \ + U+2F30->U+5DF1, U+2F31->U+5DFE, U+2F32->U+5E72, U+2F33->U+5E7A, U+2F34->U+5E7F, U+2F35->U+5EF4, U+2F36->U+5EFE, U+2F37->U+5F0B, U+2F38->U+5F13, U+2F39->U+5F50, U+2F3A->U+5F61, U+2F3B->U+5F73, U+2F3C->U+5FC3, U+2F3D->U+6208, U+2F3E->U+6236, U+2F3F->U+624B, \ + U+2F40->U+652F, U+2F41->U+6534, U+2F42->U+6587, U+2F43->U+6597, U+2F44->U+65A4, U+2F45->U+65B9, U+2F46->U+65E0, U+2F47->U+65E5, U+2F48->U+66F0, U+2F49->U+6708, U+2F4A->U+6728, U+2F4B->U+6B20, U+2F4C->U+6B62, U+2F4D->U+6B79, U+2F4E->U+6BB3, U+2F4F->U+6BCB, \ + U+2F50->U+6BD4, U+2F51->U+6BDB, U+2F52->U+6C0F, U+2F53->U+6C14, U+2F54->U+6C34, U+2F55->U+706B, U+2F56->U+722A, U+2F57->U+7236, U+2F58->U+723B, U+2F59->U+723F, U+2F5A->U+7247, U+2F5B->U+7259, U+2F5C->U+725B, U+2F5D->U+72AC, U+2F5E->U+7384, U+2F5F->U+7389, \ + U+2F60->U+74DC, U+2F61->U+74E6, U+2F62->U+7518, U+2F63->U+751F, U+2F64->U+7528, U+2F65->U+7530, U+2F66->U+758B, U+2F67->U+7592, U+2F68->U+7676, U+2F69->U+767D, U+2F6A->U+76AE, U+2F6B->U+76BF, U+2F6C->U+76EE, U+2F6D->U+77DB, U+2F6E->U+77E2, U+2F6F->U+77F3, \ + U+2F70->U+793A, U+2F71->U+79B8, U+2F72->U+79BE, U+2F73->U+7A74, U+2F74->U+7ACB, U+2F75->U+7AF9, U+2F76->U+7C73, U+2F77->U+7CF8, U+2F78->U+7F36, U+2F79->U+7F51, U+2F7A->U+7F8A, U+2F7B->U+7FBD, U+2F7C->U+8001, U+2F7D->U+800C, U+2F7E->U+8012, U+2F7F->U+8033, \ + U+2F80->U+807F, U+2F81->U+8089, U+2F82->U+81E3, U+2F83->U+81EA, U+2F84->U+81F3, U+2F85->U+81FC, U+2F86->U+820C, U+2F87->U+821B, U+2F88->U+821F, U+2F89->U+826E, U+2F8A->U+8272, U+2F8B->U+8278, U+2F8C->U+864D, U+2F8D->U+866B, U+2F8E->U+8840, U+2F8F->U+884C, \ + U+2F90->U+8863, U+2F91->U+897E, U+2F92->U+898B, U+2F93->U+89D2, U+2F94->U+8A00, U+2F95->U+8C37, U+2F96->U+8C46, U+2F97->U+8C55, U+2F98->U+8C78, U+2F99->U+8C9D, U+2F9A->U+8D64, U+2F9B->U+8D70, U+2F9C->U+8DB3, U+2F9D->U+8EAB, U+2F9E->U+8ECA, U+2F9F->U+8F9B, \ + U+2FA0->U+8FB0, U+2FA1->U+8FB5, U+2FA2->U+9091, U+2FA3->U+9149, U+2FA4->U+91C6, U+2FA5->U+91CC, U+2FA6->U+91D1, U+2FA7->U+9577, U+2FA8->U+9580, U+2FA9->U+961C, U+2FAA->U+96B6, U+2FAB->U+96B9, U+2FAC->U+96E8, U+2FAD->U+9751, U+2FAE->U+975E, U+2FAF->U+9762, \ + U+2FB0->U+9769, U+2FB1->U+97CB, U+2FB2->U+97ED, U+2FB3->U+97F3, U+2FB4->U+9801, U+2FB5->U+98A8, U+2FB6->U+98DB, U+2FB7->U+98DF, U+2FB8->U+9996, U+2FB9->U+9999, U+2FBA->U+99AC, U+2FBB->U+9AA8, U+2FBC->U+9AD8, U+2FBD->U+9ADF, U+2FBE->U+9B25, U+2FBF->U+9B2F, \ + U+2FC0->U+9B32, U+2FC1->U+9B3C, U+2FC2->U+9B5A, U+2FC3->U+9CE5, U+2FC4->U+9E75, U+2FC5->U+9E7F, U+2FC6->U+9EA5, U+2FC7->U+9EBB, U+2FC8->U+9EC3, U+2FC9->U+9ECD, U+2FCA->U+9ED1, U+2FCB->U+9EF9, U+2FCC->U+9EFD, U+2FCD->U+9F0E, U+2FCE->U+9F13, U+2FCF->U+9F20, \ + U+2FD0->U+9F3B, U+2FD1->U+9F4A, U+2FD2->U+9F52, U+2FD3->U+9F8D, U+2FD4->U+9F9C, U+2FD5->U+9FA0, U+3042->U+3041, U+3044->U+3043, U+3046->U+3045, U+3048->U+3047, U+304A->U+3049, U+304C->U+304B, U+304E->U+304D, U+3050->U+304F, U+3052->U+3051, U+3054->U+3053, \ + U+3056->U+3055, U+3058->U+3057, U+305A->U+3059, U+305C->U+305B, U+305E->U+305D, U+3060->U+305F, U+3062->U+3061, U+3064->U+3063, U+3065->U+3063, U+3067->U+3066, U+3069->U+3068, U+3070->U+306F, U+3071->U+306F, U+3073->U+3072, U+3074->U+3072, U+3076->U+3075, \ + U+3077->U+3075, U+3079->U+3078, U+307A->U+3078, U+307C->U+307B, U+307D->U+307B, U+3084->U+3083, U+3086->U+3085, U+3088->U+3087, U+308F->U+308E, U+3094->U+3046, U+3095->U+304B, U+3096->U+3051, U+30A2->U+30A1, U+30A4->U+30A3, U+30A6->U+30A5, U+30A8->U+30A7, \ + U+30AA->U+30A9, U+30AC->U+30AB, U+30AE->U+30AD, U+30B0->U+30AF, U+30B2->U+30B1, U+30B4->U+30B3, U+30B6->U+30B5, U+30B8->U+30B7, U+30BA->U+30B9, U+30BC->U+30BB, U+30BE->U+30BD, U+30C0->U+30BF, U+30C2->U+30C1, U+30C5->U+30C4, U+30C7->U+30C6, U+30C9->U+30C8, \ + U+30D0->U+30CF, U+30D1->U+30CF, U+30D3->U+30D2, U+30D4->U+30D2, U+30D6->U+30D5, U+30D7->U+30D5, U+30D9->U+30D8, U+30DA->U+30D8, U+30DC->U+30DB, U+30DD->U+30DB, U+30E4->U+30E3, U+30E6->U+30E5, U+30E8->U+30E7, U+30EF->U+30EE, U+30F4->U+30A6, U+30AB->U+30F5, \ + U+30B1->U+30F6, U+30F7->U+30EF, U+30F8->U+30F0, U+30F9->U+30F1, U+30FA->U+30F2, U+30AF->U+31F0, U+30B7->U+31F1, U+30B9->U+31F2, U+30C8->U+31F3, U+30CC->U+31F4, U+30CF->U+31F5, U+30D2->U+31F6, U+30D5->U+31F7, U+30D8->U+31F8, U+30DB->U+31F9, U+30E0->U+31FA, \ + U+30E9->U+31FB, U+30EA->U+31FC, U+30EB->U+31FD, U+30EC->U+31FE, U+30ED->U+31FF, U+FF66->U+30F2, U+FF67->U+30A1, U+FF68->U+30A3, U+FF69->U+30A5, U+FF6A->U+30A7, U+FF6B->U+30A9, U+FF6C->U+30E3, U+FF6D->U+30E5, U+FF6E->U+30E7, U+FF6F->U+30C3, U+FF71->U+30A1, \ + U+FF72->U+30A3, U+FF73->U+30A5, U+FF74->U+30A7, U+FF75->U+30A9, U+FF76->U+30AB, U+FF77->U+30AD, U+FF78->U+30AF, U+FF79->U+30B1, U+FF7A->U+30B3, U+FF7B->U+30B5, U+FF7C->U+30B7, U+FF7D->U+30B9, U+FF7E->U+30BB, U+FF7F->U+30BD, U+FF80->U+30BF, U+FF81->U+30C1, \ + U+FF82->U+30C3, U+FF83->U+30C6, U+FF84->U+30C8, U+FF85->U+30CA, U+FF86->U+30CB, U+FF87->U+30CC, U+FF88->U+30CD, U+FF89->U+30CE, U+FF8A->U+30CF, U+FF8B->U+30D2, U+FF8C->U+30D5, U+FF8D->U+30D8, U+FF8E->U+30DB, U+FF8F->U+30DE, U+FF90->U+30DF, U+FF91->U+30E0, \ + U+FF92->U+30E1, U+FF93->U+30E2, U+FF94->U+30E3, U+FF95->U+30E5, U+FF96->U+30E7, U+FF97->U+30E9, U+FF98->U+30EA, U+FF99->U+30EB, U+FF9A->U+30EC, U+FF9B->U+30ED, U+FF9C->U+30EF, U+FF9D->U+30F3, U+FFA0->U+3164, U+FFA1->U+3131, U+FFA2->U+3132, U+FFA3->U+3133, \ + U+FFA4->U+3134, U+FFA5->U+3135, U+FFA6->U+3136, U+FFA7->U+3137, U+FFA8->U+3138, U+FFA9->U+3139, U+FFAA->U+313A, U+FFAB->U+313B, U+FFAC->U+313C, U+FFAD->U+313D, U+FFAE->U+313E, U+FFAF->U+313F, U+FFB0->U+3140, U+FFB1->U+3141, U+FFB2->U+3142, U+FFB3->U+3143, \ + U+FFB4->U+3144, U+FFB5->U+3145, U+FFB6->U+3146, U+FFB7->U+3147, U+FFB8->U+3148, U+FFB9->U+3149, U+FFBA->U+314A, U+FFBB->U+314B, U+FFBC->U+314C, U+FFBD->U+314D, U+FFBE->U+314E, U+FFC2->U+314F, U+FFC3->U+3150, U+FFC4->U+3151, U+FFC5->U+3152, U+FFC6->U+3153, \ + U+FFC7->U+3154, U+FFCA->U+3155, U+FFCB->U+3156, U+FFCC->U+3157, U+FFCD->U+3158, U+FFCE->U+3159, U+FFCF->U+315A, U+FFD2->U+315B, U+FFD3->U+315C, U+FFD4->U+315D, U+FFD5->U+315E, U+FFD6->U+315F, U+FFD7->U+3160, U+FFDA->U+3161, U+FFDB->U+3162, U+FFDC->U+3163, \ + U+3131->U+1100, U+3132->U+1101, U+3133->U+11AA, U+3134->U+1102, U+3135->U+11AC, U+3136->U+11AD, U+3137->U+1103, U+3138->U+1104, U+3139->U+1105, U+313A->U+11B0, U+313B->U+11B1, U+313C->U+11B2, U+313D->U+11B3, U+313E->U+11B4, U+313F->U+11B5, U+3140->U+111A, \ + U+3141->U+1106, U+3142->U+1107, U+3143->U+1108, U+3144->U+1121, U+3145->U+1109, U+3146->U+110A, U+3147->U+110B, U+3148->U+110C, U+3149->U+110D, U+314A->U+110E, U+314B->U+110F, U+314C->U+1110, U+314D->U+1111, U+314E->U+1112, U+314F->U+1161, U+3150->U+1162, \ + U+3151->U+1163, U+3152->U+1164, U+3153->U+1165, U+3154->U+1166, U+3155->U+1167, U+3156->U+1168, U+3157->U+1169, U+3158->U+116A, U+3159->U+116B, U+315A->U+116C, U+315B->U+116D, U+315C->U+116E, U+315D->U+116F, U+315E->U+1170, U+315F->U+1171, U+3160->U+1172, \ + U+3161->U+1173, U+3162->U+1174, U+3163->U+1175, U+3165->U+1114, U+3166->U+1115, U+3167->U+11C7, U+3168->U+11C8, U+3169->U+11CC, U+316A->U+11CE, U+316B->U+11D3, U+316C->U+11D7, U+316D->U+11D9, U+316E->U+111C, U+316F->U+11DD, U+3170->U+11DF, U+3171->U+111D, \ + U+3172->U+111E, U+3173->U+1120, U+3174->U+1122, U+3175->U+1123, U+3176->U+1127, U+3177->U+1129, U+3178->U+112B, U+3179->U+112C, U+317A->U+112D, U+317B->U+112E, U+317C->U+112F, U+317D->U+1132, U+317E->U+1136, U+317F->U+1140, U+3180->U+1147, U+3181->U+114C, \ + U+3182->U+11F1, U+3183->U+11F2, U+3184->U+1157, U+3185->U+1158, U+3186->U+1159, U+3187->U+1184, U+3188->U+1185, U+3189->U+1188, U+318A->U+1191, U+318B->U+1192, U+318C->U+1194, U+318D->U+119E, U+318E->U+11A1, U+A490->U+A408, U+A491->U+A1B9, U+4E00..U+9FBB, \ + U+3400..U+4DB5, U+20000..U+2A6D6, U+FA0E, U+FA0F, U+FA11, U+FA13, U+FA14, U+FA1F, U+FA21, U+FA23, U+FA24, U+FA27, U+FA28, U+FA29, U+3105..U+312C, U+31A0..U+31B7, U+3041, U+3043, U+3045, U+3047, U+3049, U+304B, U+304D, U+304F, U+3051, U+3053, U+3055, U+3057, \ + U+3059, U+305B, U+305D, U+305F, U+3061, U+3063, U+3066, U+3068, U+306A..U+306F, U+3072, U+3075, U+3078, U+307B, U+307E..U+3083, U+3085, U+3087, U+3089..U+308E, U+3090..U+3093, U+30A1, U+30A3, U+30A5, U+30A7, U+30A9, U+30AD, U+30AF, U+30B3, U+30B5, U+30BB, \ + U+30BD, U+30BF, U+30C1, U+30C3, U+30C4, U+30C6, U+30CA, U+30CB, U+30CD, U+30CE, U+30DE, U+30DF, U+30E1, U+30E2, U+30E3, U+30E5, U+30E7, U+30EE, U+30F0..U+30F3, U+30F5, U+30F6, U+31F0, U+31F1, U+31F2, U+31F3, U+31F4, U+31F5, U+31F6, U+31F7, U+31F8, U+31F9, \ + U+31FA, U+31FB, U+31FC, U+31FD, U+31FE, U+31FF, U+AC00..U+D7A3, U+1100..U+1159, U+1161..U+11A2, U+11A8..U+11F9, U+A000..U+A48C, U+A492..U+A4C6, U+2C80->U+2C81, U+2C81, U+2C82->U+2C83, U+2C83, U+2C84->U+2C85, U+2C85, U+2C86->U+2C87, U+2C87, U+2C88->U+2C89, \ + U+2C89, U+2C8A->U+2C8B, U+2C8B, U+2C8C->U+2C8D, U+2C8D, U+2C8E->U+2C8F, U+2C8F, U+2C90->U+2C91, U+2C91, U+2C92->U+2C93, U+2C93, U+2C94->U+2C95, U+2C95, U+2C96->U+2C97, U+2C97, U+2C98->U+2C99, U+2C99, U+2C9A->U+2C9B, U+2C9B, U+2C9C->U+2C9D, U+2C9D, U+2C9E->U+2C9F, \ + U+2C9F, U+2CA0->U+2CA1, U+2CA1, U+2CA2->U+2CA3, U+2CA3, U+2CA4->U+2CA5, U+2CA5, U+2CA6->U+2CA7, U+2CA7, U+2CA8->U+2CA9, U+2CA9, U+2CAA->U+2CAB, U+2CAB, U+2CAC->U+2CAD, U+2CAD, U+2CAE->U+2CAF, U+2CAF, U+2CB0->U+2CB1, U+2CB1, U+2CB2->U+2CB3, U+2CB3, U+2CB4->U+2CB5, \ + U+2CB5, U+2CB6->U+2CB7, U+2CB7, U+2CB8->U+2CB9, U+2CB9, U+2CBA->U+2CBB, U+2CBB, U+2CBC->U+2CBD, U+2CBD, U+2CBE->U+2CBF, U+2CBF, U+2CC0->U+2CC1, U+2CC1, U+2CC2->U+2CC3, U+2CC3, U+2CC4->U+2CC5, U+2CC5, U+2CC6->U+2CC7, U+2CC7, U+2CC8->U+2CC9, U+2CC9, U+2CCA->U+2CCB, \ + U+2CCB, U+2CCC->U+2CCD, U+2CCD, U+2CCE->U+2CCF, U+2CCF, U+2CD0->U+2CD1, U+2CD1, U+2CD2->U+2CD3, U+2CD3, U+2CD4->U+2CD5, U+2CD5, U+2CD6->U+2CD7, U+2CD7, U+2CD8->U+2CD9, U+2CD9, U+2CDA->U+2CDB, U+2CDB, U+2CDC->U+2CDD, U+2CDD, U+2CDE->U+2CDF, U+2CDF, U+2CE0->U+2CE1, \ + U+2CE1, U+2CE2->U+2CE3, U+2CE3, U+0400->U+0435, U+0401->U+0435, U+0402->U+0452, U+0452, U+0403->U+0433, U+0404->U+0454, U+0454, U+0405->U+0455, U+0455, U+0406->U+0456, U+0407->U+0456, U+0457->U+0456, U+0456, U+0408..U+040B->U+0458..U+045B, U+0458..U+045B, \ + U+040C->U+043A, U+040D->U+0438, U+040E->U+0443, U+040F->U+045F, U+045F, U+0450->U+0435, U+0451->U+0435, U+0453->U+0433, U+045C->U+043A, U+045D->U+0438, U+045E->U+0443, U+0460->U+0461, U+0461, U+0462->U+0463, U+0463, U+0464->U+0465, U+0465, U+0466->U+0467, \ + U+0467, U+0468->U+0469, U+0469, U+046A->U+046B, U+046B, U+046C->U+046D, U+046D, U+046E->U+046F, U+046F, U+0470->U+0471, U+0471, U+0472->U+0473, U+0473, U+0474->U+0475, U+0476->U+0475, U+0477->U+0475, U+0475, U+0478->U+0479, U+0479, U+047A->U+047B, U+047B, \ + U+047C->U+047D, U+047D, U+047E->U+047F, U+047F, U+0480->U+0481, U+0481, U+048A->U+0438, U+048B->U+0438, U+048C->U+044C, U+048D->U+044C, U+048E->U+0440, U+048F->U+0440, U+0490->U+0433, U+0491->U+0433, U+0490->U+0433, U+0491->U+0433, U+0492->U+0433, U+0493->U+0433, \ + U+0494->U+0433, U+0495->U+0433, U+0496->U+0436, U+0497->U+0436, U+0498->U+0437, U+0499->U+0437, U+049A->U+043A, U+049B->U+043A, U+049C->U+043A, U+049D->U+043A, U+049E->U+043A, U+049F->U+043A, U+04A0->U+043A, U+04A1->U+043A, U+04A2->U+043D, U+04A3->U+043D, \ + U+04A4->U+043D, U+04A5->U+043D, U+04A6->U+043F, U+04A7->U+043F, U+04A8->U+04A9, U+04A9, U+04AA->U+0441, U+04AB->U+0441, U+04AC->U+0442, U+04AD->U+0442, U+04AE->U+0443, U+04AF->U+0443, U+04B0->U+0443, U+04B1->U+0443, U+04B2->U+0445, U+04B3->U+0445, U+04B4->U+04B5, \ + U+04B5, U+04B6->U+0447, U+04B7->U+0447, U+04B8->U+0447, U+04B9->U+0447, U+04BA->U+04BB, U+04BB, U+04BC->U+04BD, U+04BE->U+04BD, U+04BF->U+04BD, U+04BD, U+04C0->U+04CF, U+04CF, U+04C1->U+0436, U+04C2->U+0436, U+04C3->U+043A, U+04C4->U+043A, U+04C5->U+043B, \ + U+04C6->U+043B, U+04C7->U+043D, U+04C8->U+043D, U+04C9->U+043D, U+04CA->U+043D, U+04CB->U+0447, U+04CC->U+0447, U+04CD->U+043C, U+04CE->U+043C, U+04D0->U+0430, U+04D1->U+0430, U+04D2->U+0430, U+04D3->U+0430, U+04D4->U+00E6, U+04D5->U+00E6, U+04D6->U+0435, \ + U+04D7->U+0435, U+04D8->U+04D9, U+04DA->U+04D9, U+04DB->U+04D9, U+04D9, U+04DC->U+0436, U+04DD->U+0436, U+04DE->U+0437, U+04DF->U+0437, U+04E0->U+04E1, U+04E1, U+04E2->U+0438, U+04E3->U+0438, U+04E4->U+0438, U+04E5->U+0438, U+04E6->U+043E, U+04E7->U+043E, \ + U+04E8->U+043E, U+04E9->U+043E, U+04EA->U+043E, U+04EB->U+043E, U+04EC->U+044D, U+04ED->U+044D, U+04EE->U+0443, U+04EF->U+0443, U+04F0->U+0443, U+04F1->U+0443, U+04F2->U+0443, U+04F3->U+0443, U+04F4->U+0447, U+04F5->U+0447, U+04F6->U+0433, U+04F7->U+0433, \ + U+04F8->U+044B, U+04F9->U+044B, U+04FA->U+0433, U+04FB->U+0433, U+04FC->U+0445, U+04FD->U+0445, U+04FE->U+0445, U+04FF->U+0445, U+0410..U+0418->U+0430..U+0438, U+0419->U+0438, U+0430..U+0438, U+041A..U+042F->U+043A..U+044F, U+043A..U+044F, U+0929->U+0928, \ + U+0931->U+0930, U+0934->U+0933, U+0958->U+0915, U+0959->U+0916, U+095A->U+0917, U+095B->U+091C, U+095C->U+0921, U+095D->U+0922, U+095E->U+092B, U+095F->U+092F, U+0904..U+0928, U+092A..U+0930, U+0932, U+0933, U+0935..U+0939, U+0960, U+0961, U+0966..U+096F, \ + U+097B..U+097F, U+10FC->U+10DC, U+10D0..U+10FA, U+10A0..U+10C5->U+2D00..U+2D25, U+2D00..U+2D25, U+0386->U+03B1, U+0388->U+03B5, U+0389->U+03B7, U+038A->U+03B9, U+038C->U+03BF, U+038E->U+03C5, U+038F->U+03C9, U+0390->U+03B9, U+03AA->U+03B9, U+03AB->U+03C5, \ + U+03AC->U+03B1, U+03AD->U+03B5, U+03AE->U+03B7, U+03AF->U+03B9, U+03B0->U+03C5, U+03CA->U+03B9, U+03CB->U+03C5, U+03CC->U+03BF, U+03CD->U+03C5, U+03CE->U+03C9, U+03D0->U+03B2, U+03D1->U+03B8, U+03D2->U+03C5, U+03D3->U+03C5, U+03D4->U+03C5, U+03D5->U+03C6, \ + U+03D6->U+03C0, U+03D8->U+03D9, U+03DA->U+03DB, U+03DC->U+03DD, U+03DE->U+03DF, U+03E0->U+03E1, U+03E2->U+03E3, U+03E4->U+03E5, U+03E6->U+03E7, U+03E8->U+03E9, U+03EA->U+03EB, U+03EC->U+03ED, U+03EE->U+03EF, U+03F0->U+03BA, U+03F1->U+03C1, U+03F2->U+03C3, \ + U+03F4->U+03B8, U+03F5->U+03B5, U+03F6->U+03B5, U+03F7->U+03F8, U+03F9->U+03C3, U+03FA->U+03FB, U+1F00->U+03B1, U+1F01->U+03B1, U+1F02->U+03B1, U+1F03->U+03B1, U+1F04->U+03B1, U+1F05->U+03B1, U+1F06->U+03B1, U+1F07->U+03B1, U+1F08->U+03B1, U+1F09->U+03B1, \ + U+1F0A->U+03B1, U+1F0B->U+03B1, U+1F0C->U+03B1, U+1F0D->U+03B1, U+1F0E->U+03B1, U+1F0F->U+03B1, U+1F10->U+03B5, U+1F11->U+03B5, U+1F12->U+03B5, U+1F13->U+03B5, U+1F14->U+03B5, U+1F15->U+03B5, U+1F18->U+03B5, U+1F19->U+03B5, U+1F1A->U+03B5, U+1F1B->U+03B5, \ + U+1F1C->U+03B5, U+1F1D->U+03B5, U+1F20->U+03B7, U+1F21->U+03B7, U+1F22->U+03B7, U+1F23->U+03B7, U+1F24->U+03B7, U+1F25->U+03B7, U+1F26->U+03B7, U+1F27->U+03B7, U+1F28->U+03B7, U+1F29->U+03B7, U+1F2A->U+03B7, U+1F2B->U+03B7, U+1F2C->U+03B7, U+1F2D->U+03B7, \ + U+1F2E->U+03B7, U+1F2F->U+03B7, U+1F30->U+03B9, U+1F31->U+03B9, U+1F32->U+03B9, U+1F33->U+03B9, U+1F34->U+03B9, U+1F35->U+03B9, U+1F36->U+03B9, U+1F37->U+03B9, U+1F38->U+03B9, U+1F39->U+03B9, U+1F3A->U+03B9, U+1F3B->U+03B9, U+1F3C->U+03B9, U+1F3D->U+03B9, \ + U+1F3E->U+03B9, U+1F3F->U+03B9, U+1F40->U+03BF, U+1F41->U+03BF, U+1F42->U+03BF, U+1F43->U+03BF, U+1F44->U+03BF, U+1F45->U+03BF, U+1F48->U+03BF, U+1F49->U+03BF, U+1F4A->U+03BF, U+1F4B->U+03BF, U+1F4C->U+03BF, U+1F4D->U+03BF, U+1F50->U+03C5, U+1F51->U+03C5, \ + U+1F52->U+03C5, U+1F53->U+03C5, U+1F54->U+03C5, U+1F55->U+03C5, U+1F56->U+03C5, U+1F57->U+03C5, U+1F59->U+03C5, U+1F5B->U+03C5, U+1F5D->U+03C5, U+1F5F->U+03C5, U+1F60->U+03C9, U+1F61->U+03C9, U+1F62->U+03C9, U+1F63->U+03C9, U+1F64->U+03C9, U+1F65->U+03C9, \ + U+1F66->U+03C9, U+1F67->U+03C9, U+1F68->U+03C9, U+1F69->U+03C9, U+1F6A->U+03C9, U+1F6B->U+03C9, U+1F6C->U+03C9, U+1F6D->U+03C9, U+1F6E->U+03C9, U+1F6F->U+03C9, U+1F70->U+03B1, U+1F71->U+03B1, U+1F72->U+03B5, U+1F73->U+03B5, U+1F74->U+03B7, U+1F75->U+03B7, \ + U+1F76->U+03B9, U+1F77->U+03B9, U+1F78->U+03BF, U+1F79->U+03BF, U+1F7A->U+03C5, U+1F7B->U+03C5, U+1F7C->U+03C9, U+1F7D->U+03C9, U+1F80->U+03B1, U+1F81->U+03B1, U+1F82->U+03B1, U+1F83->U+03B1, U+1F84->U+03B1, U+1F85->U+03B1, U+1F86->U+03B1, U+1F87->U+03B1, \ + U+1F88->U+03B1, U+1F89->U+03B1, U+1F8A->U+03B1, U+1F8B->U+03B1, U+1F8C->U+03B1, U+1F8D->U+03B1, U+1F8E->U+03B1, U+1F8F->U+03B1, U+1F90->U+03B7, U+1F91->U+03B7, U+1F92->U+03B7, U+1F93->U+03B7, U+1F94->U+03B7, U+1F95->U+03B7, U+1F96->U+03B7, U+1F97->U+03B7, \ + U+1F98->U+03B7, U+1F99->U+03B7, U+1F9A->U+03B7, U+1F9B->U+03B7, U+1F9C->U+03B7, U+1F9D->U+03B7, U+1F9E->U+03B7, U+1F9F->U+03B7, U+1FA0->U+03C9, U+1FA1->U+03C9, U+1FA2->U+03C9, U+1FA3->U+03C9, U+1FA4->U+03C9, U+1FA5->U+03C9, U+1FA6->U+03C9, U+1FA7->U+03C9, \ + U+1FA8->U+03C9, U+1FA9->U+03C9, U+1FAA->U+03C9, U+1FAB->U+03C9, U+1FAC->U+03C9, U+1FAD->U+03C9, U+1FAE->U+03C9, U+1FAF->U+03C9, U+1FB0->U+03B1, U+1FB1->U+03B1, U+1FB2->U+03B1, U+1FB3->U+03B1, U+1FB4->U+03B1, U+1FB6->U+03B1, U+1FB7->U+03B1, U+1FB8->U+03B1, \ + U+1FB9->U+03B1, U+1FBA->U+03B1, U+1FBB->U+03B1, U+1FBC->U+03B1, U+1FC2->U+03B7, U+1FC3->U+03B7, U+1FC4->U+03B7, U+1FC6->U+03B7, U+1FC7->U+03B7, U+1FC8->U+03B5, U+1FC9->U+03B5, U+1FCA->U+03B7, U+1FCB->U+03B7, U+1FCC->U+03B7, U+1FD0->U+03B9, U+1FD1->U+03B9, \ + U+1FD2->U+03B9, U+1FD3->U+03B9, U+1FD6->U+03B9, U+1FD7->U+03B9, U+1FD8->U+03B9, U+1FD9->U+03B9, U+1FDA->U+03B9, U+1FDB->U+03B9, U+1FE0->U+03C5, U+1FE1->U+03C5, U+1FE2->U+03C5, U+1FE3->U+03C5, U+1FE4->U+03C1, U+1FE5->U+03C1, U+1FE6->U+03C5, U+1FE7->U+03C5, \ + U+1FE8->U+03C5, U+1FE9->U+03C5, U+1FEA->U+03C5, U+1FEB->U+03C5, U+1FEC->U+03C1, U+1FF2->U+03C9, U+1FF3->U+03C9, U+1FF4->U+03C9, U+1FF6->U+03C9, U+1FF7->U+03C9, U+1FF8->U+03BF, U+1FF9->U+03BF, U+1FFA->U+03C9, U+1FFB->U+03C9, U+1FFC->U+03C9, U+0391..U+03A1->U+03B1..U+03C1, \ + U+03B1..U+03C1, U+03A3..U+03A9->U+03C3..U+03C9, U+03C3..U+03C9, U+03C2, U+03D9, U+03DB, U+03DD, U+03DF, U+03E1, U+03E3, U+03E5, U+03E7, U+03E9, U+03EB, U+03ED, U+03EF, U+03F3, U+03F8, U+03FB, U+0A85..U+0A8C, U+0A8F, U+0A90, U+0A93..U+0AB0, U+0AB2, U+0AB3, \ + U+0AB5..U+0AB9, U+0AE0, U+0AE1, U+0AE6..U+0AEF, U+0A33->U+0A32, U+0A36->U+0A38, U+0A59->U+0A16, U+0A5A->U+0A17, U+0A5B->U+0A1C, U+0A5E->U+0A2B, U+0A05..U+0A0A, U+0A0F, U+0A10, U+0A13..U+0A28, U+0A2A..U+0A30, U+0A32, U+0A35, U+0A38, U+0A39, U+0A5C, U+0A66..U+0A6F, \ + U+FB1D->U+05D9, U+FB1F->U+05F2, U+FB20->U+05E2, U+FB21->U+05D0, U+FB22->U+05D3, U+FB23->U+05D4, U+FB24->U+05DB, U+FB25->U+05DC, U+FB26->U+05DD, U+FB27->U+05E8, U+FB28->U+05EA, U+FB2A->U+05E9, U+FB2B->U+05E9, U+FB2C->U+05E9, U+FB2D->U+05E9, U+FB2E->U+05D0, \ + U+FB2F->U+05D0, U+FB30->U+05D0, U+FB31->U+05D1, U+FB32->U+05D2, U+FB33->U+05D3, U+FB34->U+05D4, U+FB35->U+05D5, U+FB36->U+05D6, U+FB38->U+05D8, U+FB39->U+05D9, U+FB3A->U+05DA, U+FB3B->U+05DB, U+FB3C->U+05DC, U+FB3E->U+05DE, U+FB40->U+05E0, U+FB41->U+05E1, \ + U+FB43->U+05E3, U+FB44->U+05E4, U+FB46->U+05E6, U+FB47->U+05E7, U+FB48->U+05E8, U+FB49->U+05E9, U+FB4A->U+05EA, U+FB4B->U+05D5, U+FB4C->U+05D1, U+FB4D->U+05DB, U+FB4E->U+05E4, U+FB4F->U+05D0, U+05D0..U+05F2, U+0C85..U+0C8C, U+0C8E..U+0C90, U+0C92..U+0CA8, \ + U+0CAA..U+0CB3, U+0CB5..U+0CB9, U+0CE0, U+0CE1, U+0CE6..U+0CEF, U+1900..U+191C, U+1930..U+1938, U+1946..U+194F, U+0D05..U+0D0C, U+0D0E..U+0D10, U+0D12..U+0D28, U+0D2A..U+0D39, U+0D60, U+0D61, U+0D66..U+0D6F, U+0B94->U+0B92, U+0B85..U+0B8A, U+0B8E..U+0B90, \ + U+0B92, U+0B93, U+0B95, U+0B99, U+0B9A, U+0B9C, U+0B9E, U+0B9F, U+0BA3, U+0BA4, U+0BA8..U+0BAA, U+0BAE..U+0BB9, U+0BE6..U+0BEF, U+0E01..U+0E30, U+0E32, U+0E33, U+0E40..U+0E46, U+0E50..U+0E5B, U+FF10..U+FF19->0..9, U+FF21..U+FF3A->a..z, U+FF41..U+FF5A->a..z, \ + 0..9, A..Z->a..z, a..z + + # ignored characters list + # optional, default value is empty + # + # ignore_chars = U+00AD + + # minimum word prefix length to index + # optional, default is 0 (do not index prefixes) + # + # min_prefix_len = 0 + + # minimum word infix length to index + # optional, default is 0 (do not index infixes) + # + # min_infix_len = 0 + + # list of fields to limit prefix/infix indexing to + # optional, default value is empty (index all fields in prefix/infix mode) + # + # prefix_fields = filename + # infix_fields = url, domain + + # enable star-syntax (wildcards) when searching prefix/infix indexes + # search-time only, does not affect indexing, can be 0 or 1 + # optional, default is 0 (do not use wildcard syntax) + # + # enable_star = 1 + + # expand keywords with exact forms and/or stars when searching fit indexes + # search-time only, does not affect indexing, can be 0 or 1 + # optional, default is 0 (do not expand keywords) + # + # expand_keywords = 1 + + # n-gram length to index, for CJK indexing + # only supports 0 and 1 for now, other lengths to be implemented + # optional, default is 0 (disable n-grams) + # + ngram_len = 1 + + # n-gram characters list, for CJK indexing + # optional, default is empty + # + ngram_chars = U+4E00..U+9FBB, U+3400..U+4DB5, U+20000..U+2A6D6, U+FA0E, U+FA0F, U+FA11, U+FA13, U+FA14, U+FA1F, U+FA21, U+FA23, U+FA24, U+FA27, U+FA28, U+FA29, U+3105..U+312C, U+31A0..U+31B7, U+3041, U+3043, U+3045, U+3047, U+3049, U+304B, U+304D, U+304F, U+3051, U+3053, U+3055, U+3057, U+3059, U+305B, U+305D, U+305F, U+3061, U+3063, U+3066, U+3068, U+306A..U+306F, U+3072, U+3075, U+3078, U+307B, U+307E..U+3083, U+3085, U+3087, U+3089..U+308E, U+3090..U+3093, U+30A1, U+30A3, U+30A5, U+30A7, U+30A9, U+30AD, U+30AF, U+30B3, U+30B5, U+30BB, U+30BD, U+30BF, U+30C1, U+30C3, U+30C4, U+30C6, U+30CA, U+30CB, U+30CD, U+30CE, U+30DE, U+30DF, U+30E1, U+30E2, U+30E3, U+30E5, U+30E7, U+30EE, U+30F0..U+30F3, U+30F5, U+30F6, U+31F0, U+31F1, U+31F2, U+31F3, U+31F4, U+31F5, U+31F6, U+31F7, U+31F8, U+31F9, U+31FA, U+31FB, U+31FC, U+31FD, U+31FE, U+31FF, U+AC00..U+D7A3, U+1100..U+1159, U+1161..U+11A2, U+11A8..U+11F9, U+A000..U+A48C, U+A492..U+A4C6 + + # phrase boundary characters list + # optional, default is empty + # + # phrase_boundary = ., ?, !, U+2026 # horizontal ellipsis + + # phrase boundary word position increment + # optional, default is 0 + # + # phrase_boundary_step = 100 + + # blended characters list + # blended chars are indexed both as separators and valid characters + # for instance, AT&T will results in 3 tokens ("at", "t", and "at&t") + # optional, default is empty + # + # blend_chars = +, &, U+23 + + # blended token indexing mode + # a comma separated list of blended token indexing variants + # known variants are trim_none, trim_head, trim_tail, trim_both, skip_pure + # optional, default is trim_none + # + # blend_mode = trim_tail, skip_pure + + # whether to strip HTML tags from incoming documents + # known values are 0 (do not strip) and 1 (do strip) + # optional, default is 0 + html_strip = 0 + + # what HTML attributes to index if stripping HTML + # optional, default is empty (do not index anything) + # + # html_index_attrs = img=alt,title; a=title; + + # what HTML elements contents to strip + # optional, default is empty (do not strip element contents) + # + # html_remove_elements = style, script + + # whether to preopen index data files on startup + # optional, default is 0 (do not preopen), searchd-only + # + # preopen = 1 + + # whether to keep dictionary (.spi) on disk, or cache it in RAM + # optional, default is 0 (cache in RAM), searchd-only + # + # ondisk_dict = 1 + + # whether to enable in-place inversion (2x less disk, 90-95% speed) + # optional, default is 0 (use separate temporary files), indexer-only + # + # inplace_enable = 1 + + # in-place fine-tuning options + # optional, defaults are listed below + # + # inplace_hit_gap = 0 # preallocated hitlist gap size + # inplace_docinfo_gap = 0 # preallocated docinfo gap size + # inplace_reloc_factor = 0.1 # relocation buffer size within arena + # inplace_write_factor = 0.1 # write buffer size within arena + + # whether to index original keywords along with stemmed versions + # enables "=exactform" operator to work + # optional, default is 0 + # + # index_exact_words = 1 + + # position increment on overshort (less that min_word_len) words + # optional, allowed values are 0 and 1, default is 1 + # + # overshort_step = 1 + + # position increment on stopword + # optional, allowed values are 0 and 1, default is 1 + # + # stopword_step = 1 + + # hitless words list + # positions for these keywords will not be stored in the index + # optional, allowed values are 'all', or a list file name + # + # hitless_words = all + # hitless_words = hitless.txt + + # detect and index sentence and paragraph boundaries + # required for the SENTENCE and PARAGRAPH operators to work + # optional, allowed values are 0 and 1, default is 0 + # + # index_sp = 1 + + # index zones, delimited by HTML/XML tags + # a comma separated list of tags and wildcards + # required for the ZONE operator to work + # optional, default is empty string (do not index zones) + # + # index_zones = title, h*, th +} + +############################################################################# +## searchd settings +############################################################################# + +searchd +{ + # [hostname:]port[:protocol], or /unix/socket/path to listen on + # known protocols are 'sphinx' (SphinxAPI) and 'mysql41' (SphinxQL) + # + # multi-value, multiple listen points are allowed + # optional, defaults are 9312:sphinx and 9306:mysql41, as below + # + # listen = 127.0.0.1 + # listen = 192.168.0.1:9312 + # listen = 9312 + # listen = /var/run/searchd.sock + listen = %(ip_address)s:%(port)s:sphinx + listen = %(ip_address)s:%(sql_port)s:mysql41 + + # log file, searchd run info is logged here + # optional, default is 'searchd.log' + log = %(log_directory)s/sphinx-searchd.log + + # query log file, all search queries are logged here + # optional, default is empty (do not log queries) + query_log = %(log_directory)s/sphinx-query.log + + # client read timeout, seconds + # optional, default is 5 + read_timeout = 5 + + # request timeout, seconds + # optional, default is 5 minutes + client_timeout = 300 + + # maximum amount of children to fork (concurrent searches to run) + # optional, default is 0 (unlimited) + max_children = 30 + + # PID file, searchd process ID file name + # mandatory + pid_file = %(data_directory)s/sphinx-searchd.pid + + # max amount of matches the daemon ever keeps in RAM, per-index + # WARNING, THERE'S ALSO PER-QUERY LIMIT, SEE SetLimits() API CALL + # default is 1000 (just like Google) + max_matches = 1000 + + # seamless rotate, prevents rotate stalls if precaching huge datasets + # optional, default is 1 + seamless_rotate = 1 + + # whether to forcibly preopen all indexes on startup + # optional, default is 0 (do not preopen) + preopen_indexes = 0 + + # whether to unlink .old index copies on succesful rotation. + # optional, default is 1 (do unlink) + unlink_old = 1 + + # attribute updates periodic flush timeout, seconds + # updates will be automatically dumped to disk this frequently + # optional, default is 0 (disable periodic flush) + # + # attr_flush_period = 900 + + # instance-wide ondisk_dict defaults (per-index value take precedence) + # optional, default is 0 (precache all dictionaries in RAM) + # + # ondisk_dict_default = 1 + + # MVA updates pool size + # shared between all instances of searchd, disables attr flushes! + # optional, default size is 1M + mva_updates_pool = 1M + + # max allowed network packet size + # limits both query packets from clients, and responses from agents + # optional, default size is 8M + max_packet_size = 8M + + # crash log path + # searchd will (try to) log crashed query to 'crash_log_path.PID' file + # optional, default is empty (do not create crash logs) + # + # crash_log_path = %(log_directory)s + + # max allowed per-query filter count + # optional, default is 256 + max_filters = 256 + + # max allowed per-filter values count + # optional, default is 4096 + max_filter_values = 4096 + + # socket listen queue length + # optional, default is 5 + # + # listen_backlog = 5 + + # per-keyword read buffer size + # optional, default is 256K + # + # read_buffer = 256K + + # unhinted read size (currently used when reading hits) + # optional, default is 32K + # + # read_unhinted = 32K + + # max allowed per-batch query count (aka multi-query count) + # optional, default is 32 + max_batch_queries = 32 + + # max common subtree document cache size, per-query + # optional, default is 0 (disable subtree optimization) + # + # subtree_docs_cache = 4M + + # max common subtree hit cache size, per-query + # optional, default is 0 (disable subtree optimization) + # + # subtree_hits_cache = 8M + + # multi-processing mode (MPM) + # known values are none, fork, prefork, and threads + # optional, default is fork + # + workers = threads # for RT to work + + # max threads to create for searching local parts of a distributed index + # optional, default is 0, which means disable multi-threaded searching + # should work with all MPMs (ie. does NOT require workers=threads) + # + # dist_threads = 4 + + # binlog files path; use empty string to disable binlog + # optional, default is build-time configured data directory + # + binlog_path = # disable logging + # binlog_path = %(data_directory)s # binlog.001 etc will be created there + + # binlog flush/sync mode + # 0 means flush and sync every second + # 1 means flush and sync every transaction + # 2 means flush every transaction, sync every second + # optional, default is 2 + # + # binlog_flush = 2 + + # binlog per-file size limit + # optional, default is 128M, 0 means no limit + # + # binlog_max_log_size = 256M +} diff --git a/slapos/recipe/erp5/template/zeo.conf.in b/slapos/recipe/erp5/template/zeo.conf.in index a37dd6cd78993affe4e027ab60a6d9816567fb9b..94ffbb7859acf2d1f92306b0557cb985497d115e 100644 --- a/slapos/recipe/erp5/template/zeo.conf.in +++ b/slapos/recipe/erp5/template/zeo.conf.in @@ -1,8 +1,6 @@ # ZEO configuration file generated by SlapOS <zeo> address %(zeo_ip)s:%(zeo_port)s - read-only false - invalidation-queue-size 100 pid-filename %(zeo_pid)s </zeo> @@ -10,6 +8,7 @@ <eventlog> <logfile> + dateformat path %(zeo_event_log)s </logfile> </eventlog> diff --git a/slapos/recipe/erp5/template/zope-zeo-snippet.conf.in b/slapos/recipe/erp5/template/zope-zeo-snippet.conf.in index 71ff1567956b72075cd497c164bf7464dbab9397..dbd9bcac222f08ad6697e74c5e8cd62017ce9648 100644 --- a/slapos/recipe/erp5/template/zope-zeo-snippet.conf.in +++ b/slapos/recipe/erp5/template/zope-zeo-snippet.conf.in @@ -1,6 +1,8 @@ <zodb_db %(storage_name)s> + cache-size %(zodb_cache_size)d mount-point %(mount_point)s <zeoclient> + cache-size %(zeo_client_cache_size)s server %(address)s storage %(storage_name)s name %(storage_name)s diff --git a/slapos/recipe/erp5/template/zope-zodb-snippet.conf.in b/slapos/recipe/erp5/template/zope-zodb-snippet.conf.in index d0b3ed1357779e9f4db7c93b2df16848874eb8ce..4eb6a051bdcbc2a575d34bf9f299dfdfb43b1596 100644 --- a/slapos/recipe/erp5/template/zope-zodb-snippet.conf.in +++ b/slapos/recipe/erp5/template/zope-zodb-snippet.conf.in @@ -1,4 +1,5 @@ <zodb_db root> + cache-size %(zodb_cache_size)d <filestorage> path %(zodb_root_path)s </filestorage> diff --git a/slapos/recipe/erp5/template/zope.conf.in b/slapos/recipe/erp5/template/zope.conf.in index 90a1ac96b1f9d5e3f50c93ee149939b91b845d57..e8576401314fb57a5154abd98bba69cb744f43d6 100644 --- a/slapos/recipe/erp5/template/zope.conf.in +++ b/slapos/recipe/erp5/template/zope.conf.in @@ -7,10 +7,8 @@ instancehome $INSTANCE # Used products %(products)s -# Environment override -<environment> -%(environment)s -</environment> +# Environment is setup in running wrapper script +# Reason: zope.conf is read too late for some componets # No need to debug debug-mode off @@ -34,11 +32,13 @@ lock-filename %(lock-filename)s # Logging configuration <eventlog> <logfile> + dateformat path %(event_log)s </logfile> </eventlog> <logger access> <logfile> + dateformat path %(z2_log)s </logfile> </logger> diff --git a/slapos/recipe/kvm/__init__.py b/slapos/recipe/kvm/__init__.py index 24ea90a059fc4643496d03bc14c9d7b348da26c5..40c123135c85fddd8b483295c3664e09872f2899 100644 --- a/slapos/recipe/kvm/__init__.py +++ b/slapos/recipe/kvm/__init__.py @@ -37,6 +37,9 @@ import hashlib class Recipe(BaseSlapRecipe): + # To avoid magic numbers + VNC_BASE_PORT = 5900 + def _install(self): """ Set the connection dictionnary for the computer partition and create a list @@ -49,43 +52,58 @@ class Recipe(BaseSlapRecipe): self.path_list = [] self.requirements, self.ws = self.egg.working_set() - self.cron_d = self.installCrond() + self.cron_d = self.installCrond() self.ca_conf = self.installCertificateAuthority() self.key_path, self.certificate_path = self.requestCertificate('noVNC') - + + # Install the socket_connection_attempt script + catcher = zc.buildout.easy_install.scripts( + [('check_port_listening', __name__ + 'socket_connection_attempt', 'connection_attempt')], + self.ws, + sys.executable, + self.bin_directory, + ) + # Save the check_port_listening script path + check_port_listening_script = catcher[0] + # Get the port_listening_promise template path, and save it + self.port_listening_promise_path = pkg_resources.resource_filename( + __name__, 'template/port_listening_promise.in') + self.port_listening_promise_conf = dict( + check_port_listening_script=check_port_listening_script, + ) + kvm_conf = self.installKvm(vnc_ip = self.getLocalIPv4Address()) - - vnc_port = 5900 + kvm_conf['vnc_display'] - + + vnc_port = Recipe.VNC_BASE_PORT + kvm_conf['vnc_display'] + noVNC_conf = self.installNoVnc(source_ip = self.getGlobalIPv6Address(), source_port = 6080, target_ip = kvm_conf['vnc_ip'], - target_port = vnc_port, - python_path = kvm_conf['python_path']) - + target_port = vnc_port) + self.linkBinary() self.computer_partition.setConnectionDict(dict( url = "https://[%s]:%s/vnc.html?host=[%s]&port=%s&encrypt=1" % (noVNC_conf['source_ip'], noVNC_conf['source_port'], noVNC_conf['source_ip'], noVNC_conf['source_port'] - ), + ), password = kvm_conf['vnc_passwd'])) - + return self.path_list def installKvm(self, vnc_ip): """ - Create kvm configuration dictionnary and instanciate a wrapper for kvm and - kvm controller + Create kvm configuration dictionnary and instanciate a wrapper for kvm and + kvm controller Parameters : IP the vnc server is listening on Returns : Dictionnary kvm_conf """ kvm_conf = dict(vnc_ip = vnc_ip) - + connection_found = False for tap_interface, dummy in self.parameter_dict['ip_list']: # Get an ip associated to a tap interface @@ -95,13 +113,13 @@ class Recipe(BaseSlapRecipe): raise NotImplementedError("Do not support ip without tap interface") kvm_conf['tap_interface'] = tap_interface - + # Disk path kvm_conf['disk_path'] = os.path.join(self.data_root_directory, 'virtual.qcow2') kvm_conf['socket_path'] = os.path.join(self.var_directory, 'qmp_socket') # XXX Weak password - ##XXX -Vivien: add an option to generate one password for all instances + ##XXX -Vivien: add an option to generate one password for all instances # and/or to input it yourself kvm_conf['vnc_passwd'] = binascii.hexlify(os.urandom(4)) @@ -120,7 +138,7 @@ class Recipe(BaseSlapRecipe): int(self.options['disk_size']))], shell=True) if retcode != 0: raise OSError, "Disk creation failed!" - + # Options nbd_ip and nbd_port are provided by slapos master kvm_conf['nbd_ip'] = self.parameter_dict['nbd_ip'] kvm_conf['nbd_port'] = self.parameter_dict['nbd_port'] @@ -134,41 +152,48 @@ class Recipe(BaseSlapRecipe): kvm_conf['ram_size'] = self.options['ram_size'] kvm_conf['vnc_display'] = 1 - + # Instanciate KVM - kvm_template_location = pkg_resources.resource_filename( - __name__, os.path.join( - 'template', 'kvm_run.in')) - - kvm_runner_path = self.createRunningWrapper("kvm", + kvm_template_location = pkg_resources.resource_filename( + __name__, 'template/kvm_run.in') + + kvm_runner_path = self.createRunningWrapper("kvm", self.substituteTemplate(kvm_template_location, kvm_conf)) - + self.path_list.append(kvm_runner_path) # Instanciate KVM controller - kvm_controller_template_location = pkg_resources.resource_filename( - __name__, os.path.join( - 'template', - 'kvm_controller_run.in' )) - - kvm_controller_runner_path = self.createRunningWrapper("kvm_controller", + kvm_controller_template_location = pkg_resources.resource_filename( + __name__, 'template/kvm_controller_run.in') + + kvm_controller_runner_path = self.createRunningWrapper("kvm_controller", self.substituteTemplate(kvm_controller_template_location, kvm_conf)) - + self.path_list.append(kvm_controller_runner_path) - + # Instanciate Slapmonitor ##slapmonitor_runner_path = self.instanciate_wrapper("slapmonitor", # [database_path, pid_file_path, python_path]) # Instanciate Slapreport ##slapreport_runner_path = self.instanciate_wrapper("slapreport", # [database_path, python_path]) - + + # Add VNC promise + self.port_listening_promise_conf.update( + hostname=kvm_conf['vnc_ip'], + port=Recipe.VNC_BASE_PORT + kvm_conf['vnc_display'], + ) + self.createPromiseWrapper("vnc_promise", + self.substituteTemplate(self.port_listening_promise_path, + self.port_listening_promise_conf, + ) + ) + return kvm_conf - def installNoVnc(self, source_ip, source_port, target_ip, target_port, - python_path): + def installNoVnc(self, source_ip, source_port, target_ip, target_port): """ Create noVNC configuration dictionnary and instanciate Websockify proxy @@ -181,14 +206,20 @@ class Recipe(BaseSlapRecipe): noVNC_conf = {} - noVNC_conf['source_ip'] = source_ip + noVNC_conf['source_ip'] = source_ip noVNC_conf['source_port'] = source_port + # Install numpy. + # XXX-Cedric : this looks like a hack. Do we have better solution, knowing + # That websockify is not an egg? + numpy = zc.buildout.easy_install.install(['numpy'], self.options['eggs-directory']) + environment = dict(PYTHONPATH='%s' % numpy.entries[0]) + # Instanciate Websockify websockify_runner_path = zc.buildout.easy_install.scripts([('websockify', - 'slapos.recipe.librecipe.execute', 'execute_wait')], self.ws, + 'slapos.recipe.librecipe.execute', 'executee_wait')], self.ws, sys.executable, self.wrapper_directory, arguments=[ - [python_path.strip(), + [sys.executable.strip(), self.options['websockify_path'], '--web', self.options['noVNC_location'], @@ -197,11 +228,22 @@ class Recipe(BaseSlapRecipe): '--ssl-only', '%s:%s' % (source_ip, source_port), '%s:%s' % (target_ip, target_port)], - [self.certificate_path, self.key_path]] + [self.certificate_path, self.key_path], + environment] )[0] - + self.path_list.append(websockify_runner_path) - + + # Add noVNC promise + self.port_listening_promise_conf.update(hostname=noVNC_conf['source_ip'], + port=noVNC_conf['source_port'], + ) + self.createPromiseWrapper("novnc_promise", + self.substituteTemplate(self.port_listening_promise_path, + self.port_listening_promise_conf, + ) + ) + return noVNC_conf def linkBinary(self): @@ -225,7 +267,7 @@ class Recipe(BaseSlapRecipe): os.symlink(target, link) self.logger.debug('Created link %r -> %r' % (link, target)) self.path_list.append(link) - + def installCertificateAuthority(self, ca_country_code='XX', ca_email='xx@example.com', ca_state='State', ca_city='City', ca_company='Company'): @@ -284,7 +326,7 @@ class Recipe(BaseSlapRecipe): ca_crl=os.path.join(config['ca_dir'], 'crl'), certificate_authority_path=config['ca_dir'] ) - + def requestCertificate(self, name): hash = hashlib.sha512(name).hexdigest() key = os.path.join(self.ca_private, hash + self.ca_key_ext) @@ -296,7 +338,7 @@ class Recipe(BaseSlapRecipe): parser.set('certificate', 'certificate_file', certificate) parser.write(open(os.path.join(self.ca_request_dir, hash), 'w')) return key, certificate - + def installCrond(self): timestamps = self.createDataDirectory('cronstamps') cron_output = os.path.join(self.log_directory, 'cron-output') diff --git a/slapos/recipe/kvm/socket_connection_attempt.py b/slapos/recipe/kvm/socket_connection_attempt.py new file mode 100644 index 0000000000000000000000000000000000000000..dfd9fad4b930518c6ef94a88c69ed40ce6b22539 --- /dev/null +++ b/slapos/recipe/kvm/socket_connection_attempt.py @@ -0,0 +1,26 @@ +import socket +import sys + +def connection_attempt(): + + try: + hostname, port = sys.argv[1:3] + except ValueError: + print >> sys.stderr, """Bad command line. + Usage: %s hostname|ip port""" % sys.argv[0] + sys.exit(1) + + connection_okay = False + + try: + s = socket.create_connection((hostname, port)) + connection_okay = True + s.close() + except (socket.error, socket.timeout): + connection_okay = False + + if not connection_okay: + print >> sys.stderr, "%(port)s on %(ip)s isn't listening" % { + 'port': port, 'ip': hostname + } + sys.exit(127) diff --git a/slapos/recipe/kvm/template/port_listening_promise.in b/slapos/recipe/kvm/template/port_listening_promise.in new file mode 100644 index 0000000000000000000000000000000000000000..15fa390d01e38096cfb8a9d01356bf1687e3f740 --- /dev/null +++ b/slapos/recipe/kvm/template/port_listening_promise.in @@ -0,0 +1,4 @@ +#!/usr/bin/env sh + +"%(check_port_listening_script)s" "%(hostname)s" "%(port)s" +exit $? diff --git a/slapos/recipe/lamp/__init__.py b/slapos/recipe/lamp/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..64af4334163cc01532300f91a6e5b810861ea6e8 --- /dev/null +++ b/slapos/recipe/lamp/__init__.py @@ -0,0 +1,300 @@ +############################################################################## +# +# Copyright (c) 2010 Vifib SARL and Contributors. All Rights Reserved. +# +# WARNING: This program as such is intended to be used by professional +# programmers who take the whole responsibility of assessing all potential +# consequences resulting from its eventual inadequacies and bugs +# End users who are looking for a ready-to-use solution with commercial +# guarantees and support are strongly adviced to contract a Free Software +# Service Company +# +# This program is Free Software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either version 3 +# of the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. +# +############################################################################## +from slapos.recipe.librecipe import BaseSlapRecipe +import os +import shutil +import pkg_resources +import zc.buildout +import sys +import zc.recipe.egg +import urlparse + +class BaseRecipe(BaseSlapRecipe): + def getTemplateFilename(self, template_name): + return pkg_resources.resource_filename(__name__, + 'template/%s' % template_name) + + def installMysqlServer(self, ip=None, port=None): + if ip is None: + ip = self.getLocalIPv4Address() + if port is None: + port = '3306' + mysql_conf = dict( + ip=ip, + data_directory=os.path.join(self.data_root_directory, + 'mysql'), + tcp_port=port, + pid_file=os.path.join(self.run_directory, 'mysqld.pid'), + socket=os.path.join(self.run_directory, 'mysqld.sock'), + error_log=os.path.join(self.log_directory, 'mysqld.log'), + slow_query_log=os.path.join(self.log_directory, + 'mysql-slow.log'), + database='appdb', + user='appuser', + password=self.generatePassword(), + ) + self._createDirectory(mysql_conf['data_directory']) + + mysql_conf_path = self.createConfigurationFile("my.cnf", + self.substituteTemplate(pkg_resources.resource_filename(__name__, + 'template/my.cnf.in'), mysql_conf)) + + mysql_script = pkg_resources.resource_string(__name__, + 'template/mysqlinit.sql.in') % mysql_conf + self.path_list.extend(zc.buildout.easy_install.scripts([('mysql_update', + __name__ + '.mysql', 'updateMysql')], self.ws, + sys.executable, self.wrapper_directory, arguments=[dict( + mysql_script=mysql_script, + mysql_binary=self.options['mysql_binary'].strip(), + mysql_upgrade_binary=self.options['mysql_upgrade_binary'].strip(), + socket=mysql_conf['socket'], + )])) + self.path_list.extend(zc.buildout.easy_install.scripts([('mysqld', + __name__ + '.mysql', 'runMysql')], self.ws, + sys.executable, self.wrapper_directory, arguments=[dict( + mysql_install_binary=self.options['mysql_install_binary'].strip(), + mysqld_binary=self.options['mysqld_binary'].strip(), + data_directory=mysql_conf['data_directory'].strip(), + mysql_binary=self.options['mysql_binary'].strip(), + socket=mysql_conf['socket'].strip(), + configuration_file=mysql_conf_path, + )])) + self.path_list.extend([mysql_conf_path]) + return dict( + mysql_host=mysql_conf['ip'], + mysql_port=mysql_conf['tcp_port'], + mysql_user=mysql_conf['user'], + mysql_password=mysql_conf['password'], + mysql_database=mysql_conf['database'], + ) + + def createHtdocs(self, source, document_root): + source = self.options['source'].strip() + document_root = self.createDataDirectory('htdocs') + for p in os.listdir(document_root): + path = os.path.join(document_root, p) + if os.path.isdir(path): + shutil.rmtree(path) + else: + os.unlink(path) + for p in os.listdir(source): + path = os.path.join(source, p) + if os.path.isdir(path): + shutil.copytree(path, os.path.join(document_root, p)) + else: + shutil.copy2(path, os.path.join(document_root, p)) + + def installApache(self, document_root, ip=None, port=None): + if ip is None: + ip=self.getGlobalIPv6Address() + if port is None: + port = '9080' + apache_config = dict( + pid_file=os.path.join(self.run_directory, 'httpd.pid'), + lock_file=os.path.join(self.run_directory, 'httpd.lock'), + ip=ip, + port=port, + error_log=os.path.join(self.log_directory, 'httpd-error.log'), + access_log=os.path.join(self.log_directory, 'httpd-access.log'), + document_root=document_root, + php_ini_dir=self.etc_directory + ) + config_file = self.createConfigurationFile('httpd.conf', + self.substituteTemplate(pkg_resources.resource_filename(__name__, + 'template/apache.in'), apache_config)) + self.path_list.append(config_file) + self.path_list.append(self.createConfigurationFile('php.ini', + self.substituteTemplate(pkg_resources.resource_filename(__name__, + 'template/php.ini.in'), dict(tmp_directory=self.tmp_directory)))) + self.path_list.extend(zc.buildout.easy_install.scripts([( + 'httpd', + __name__ + '.apache', 'runApache')], self.ws, + sys.executable, self.wrapper_directory, arguments=[ + dict( + required_path_list=[], + binary=self.options['httpd_binary'], + config=config_file + ) + ])) + return 'http://[%s]:%s' % (ip, port) + + def createConfiguration(self, template, document_root, destination, d): + directory = os.path.dirname(destination) + file = os.path.basename(destination) + path = document_root + if directory: + path = os.path.join(document_root, directory) + if not os.path.exists(path): + os.makedirs(path) + destination = os.path.join(path, file) + open(destination, 'w').write(open(template, 'r').read() % d) + + def configureInstallation(self, document_root, mysql_conf, url): + """Start process which can launch python scripts, move or remove files or + directories when installing software. + """ + if not self.options.has_key('delete') and not self.options.has_key('rename') and not\ + self.options.has_key('chmod') and not self.options.has_key('script'): + return + delete = [] + chmod = [] + data = [] + rename = [] + rename_list = "" + argument = [self.options['lampconfigure_directory'].strip()] + if not self.options.has_key('file_token'): + argument = argument + ["-d", mysql_conf['mysql_database'], + "-H", mysql_conf['mysql_host'], "-P", mysql_conf['mysql_port'], + "-p", mysql_conf['mysql_password'], "-u", mysql_conf['mysql_user'], + "--table", self.options['table_name'].strip(), "--cond", + self.options['constraint'].strip()] + else: + argument = argument + ["-f", self.options['file_token'].strip()] + argument += ["-t", document_root] + + if self.options.has_key('delete'): + delete = ["delete"] + for fname in self.options['delete'].split(','): + delete.append(fname.strip()) + if self.options.has_key('rename'): + for fname in self.options['rename'].split(','): + if fname.find("=>") < 0: + old_name = fname + fname = [] + fname.append(old_name) + fname.append(old_name + '-' + mysql_conf['mysql_user']) + else: + fname = fname.split("=>") + cmd = ["rename"] + if self.options.has_key('rename_chmod'): + cmd += ["--chmod", self.options['rename_chmod'].strip()] + rename.append(cmd + [fname[0].strip(), fname[1].strip()]) + rename_list += fname[0] + "=>" + fname[1] + " " + if self.options.has_key('chmod'): + chmod = ["chmod ", self.options['mode'].strip()] + for fname in self.options['chmod'].split(','): + chmod.append(fname.strip()) + if self.options.has_key('script') and \ + self.options['script'].strip().endswith(".py"): + data = ["run", self.options['script'].strip(), "-v", mysql_conf['mysql_database'], url, document_root] + self.path_list.extend(zc.buildout.easy_install.scripts( + [('configureInstall', __name__ + '.runner', 'executeRunner')], self.ws, + sys.executable, self.wrapper_directory, arguments=[argument, delete, rename, + chmod, data])) + return rename_list + +class Static(BaseRecipe): + def _install(self): + self.path_list = [] + self.requirements, self.ws = self.egg.working_set() + document_root = self.createDataDirectory('htdocs') + self.createHtdocs(self.options['source'].strip(), document_root) + url = self.installApache(document_root) + self.setConnectionDict(dict(url = url)) + return self.path_list + +class Simple(BaseRecipe): + def _install(self): + self.path_list = [] + self.requirements, self.ws = self.egg.working_set() + document_root = self.createDataDirectory('htdocs') + self.createHtdocs(self.options['source'].strip(), document_root) + mysql_conf = self.installMysqlServer() + url = self.installApache(document_root) + renamed = self.configureInstallation(document_root, mysql_conf, url) + self.setConnectionDict(dict( + url=url, + rename=renamed, + **mysql_conf + )) + if self.options.has_key('template') and self.options.has_key('configuration'): + self.createConfiguration(self.options['template'], document_root, + self.options['configuration'], mysql_conf) + return self.path_list + +class Request(BaseRecipe): + def _install(self): + self.path_list = [] + self.requirements, self.ws = self.egg.working_set() + software_type = self.parameter_dict['slap_software_type'] + + document_root = self.createDataDirectory('htdocs') + self.createHtdocs(self.options['source'].strip(), document_root) + + if software_type == 'Backuped': + davstorage = self.request(self.options['davstorage-software-url'], + software_type, 'Backup Server').getConnectionParameter('url') + + parameters = {'remote_backup': davstorage} + elif software_type == 'PersonnalBackup': + parameters = {'remote_backup': self.parameter_dict['remote_backup']} + else: + parameters = {} + + mysql = self.request(self.options['mariadb-software-url'], + software_type, 'MariaDB Server', partition_parameter_kw=parameters + ).getConnectionParameter('url') + mysql_parsed = urlparse.urlparse(mysql) + + mysql_host, mysql_port = mysql_parsed.hostname, mysql_parsed.port + if mysql_parsed.scheme == 'mysqls': # Listen over stunnel + mysql_host, mysql_port = self.installStunnelClient(mysql_host, + mysql_port) + + mysql_conf = dict(mysql_database=mysql_parsed.path.strip('/'), + mysql_user=mysql_parsed.username, + mysql_password=mysql_parsed.password, + mysql_host='%s:%s' % (mysql_host,mysql_port)) + + url = self.installApache(document_root) + + self.setConnectionDict(dict( + url=url, + )) + + self.createConfiguration(self.options['template'], document_root, + self.options['configuration'], mysql_conf) + return self.path_list + + def installStunnelClient(self, remote_host, remote_port): + local_host = self.getLocalIPv4Address() + local_port = 8888 + stunnel_conf_path = self.createConfigurationFile('stunnel.conf', + self.substituteTemplate( + self.getTemplateFilename('stunnel.conf.in'), { + 'log': os.path.join(self.log_directory, 'stunnel.log'), + 'pid_file': os.path.join(self.run_directory, 'stunnel.pid'), + 'remote_host': remote_host, 'remote_port': remote_port, + 'local_host': local_host, 'local_port': local_port, + })) + wrapper = zc.buildout.easy_install.scripts([('stunnel', + 'slapos.recipe.librecipe.execute', 'execute')], self.ws, + sys.executable, self.wrapper_directory, arguments=[ + self.options['stunnel_binary'].strip(), stunnel_conf_path] + )[0] + self.path_list.append(wrapper) + return (local_host, local_port,) diff --git a/slapos/recipe/lamp/apache.py b/slapos/recipe/lamp/apache.py new file mode 100644 index 0000000000000000000000000000000000000000..861f787d09fed9c80b38f561528fbd86b5c7ad44 --- /dev/null +++ b/slapos/recipe/lamp/apache.py @@ -0,0 +1,22 @@ +import os +import sys +import time + + +def runApache(args): + sleep = 60 + conf = args[0] + while True: + ready = True + for f in conf.get('required_path_list', []): + if not os.path.exists(f): + print 'File %r does not exists, sleeping for %s' % (f, sleep) + ready = False + if ready: + break + time.sleep(sleep) + apache_wrapper_list = [conf['binary'], '-f', conf['config'], '-DFOREGROUND'] + apache_wrapper_list.extend(sys.argv[1:]) + sys.stdout.flush() + sys.stderr.flush() + os.execl(apache_wrapper_list[0], *apache_wrapper_list) diff --git a/slapos/recipe/lamp/mysql.py b/slapos/recipe/lamp/mysql.py new file mode 100644 index 0000000000000000000000000000000000000000..c0f399084738ea3aee5020e33d1cf2b5f3db730d --- /dev/null +++ b/slapos/recipe/lamp/mysql.py @@ -0,0 +1,72 @@ +import os +import subprocess +import time +import sys + + +def runMysql(args): + sleep = 60 + conf = args[0] + mysqld_wrapper_list = [conf['mysqld_binary'], '--defaults-file=%s' % + conf['configuration_file']] + # we trust mysql_install that if mysql directory is available mysql was + # correctly initalised + if not os.path.isdir(os.path.join(conf['data_directory'], 'mysql')): + while True: + # XXX: Protect with proper root password + popen = subprocess.Popen([conf['mysql_install_binary'], + '--skip-name-resolve', '--no-defaults', '--datadir=%s' % + conf['data_directory']], + stdout=subprocess.PIPE, stderr=subprocess.STDOUT) + result = popen.communicate()[0] + if popen.returncode is None or popen.returncode != 0: + print "Failed to initialise server.\nThe error was: %s" % result + print "Waiting for %ss and retrying" % sleep + time.sleep(sleep) + else: + print "Mysql properly initialised" + break + else: + print "MySQL already initialised" + print "Starting %r" % mysqld_wrapper_list[0] + sys.stdout.flush() + sys.stderr.flush() + os.execl(mysqld_wrapper_list[0], *mysqld_wrapper_list) + + +def updateMysql(args): + conf = args[0] + sleep = 30 + is_succeed = False + while True: + if not is_succeed: + mysql_upgrade_list = [conf['mysql_upgrade_binary'], '--no-defaults', '--user=root', '--socket=%s' % conf['socket']] + mysql_upgrade = subprocess.Popen(mysql_upgrade_list, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) + result = mysql_upgrade.communicate()[0] + if mysql_upgrade.returncode is None: + mysql_upgrade.kill() + if mysql_upgrade.returncode != 0 and not 'is already upgraded' in result: + print "Command %r failed with result:\n%s" % (mysql_upgrade_list, result) + print 'Sleeping for %ss and retrying' % sleep + else: + if mysql_upgrade.returncode == 0: + print "MySQL database upgraded with result:\n%s" % result + else: + print "No need to upgrade MySQL database" + mysql_script = conf.get('mysql_script') + if mysql_script: + mysql_list = [conf['mysql_binary'].strip(), '--no-defaults', '-B', '--user=root', '--socket=%s' % conf['socket']] + mysql = subprocess.Popen(mysql_list, stdin=subprocess.PIPE, + stdout=subprocess.PIPE, stderr=subprocess.STDOUT) + result = mysql.communicate(conf['mysql_script'])[0] + if mysql.returncode is None: + mysql.kill() + if mysql.returncode != 0: + print 'Command %r failed with:\n%s' % (mysql_list, result) + print 'Sleeping for %ss and retrying' % sleep + else: + is_succeed = True + print 'SlapOS initialisation script succesfully applied on database.' + sys.stdout.flush() + sys.stderr.flush() + time.sleep(sleep) diff --git a/slapos/recipe/lamp/runner.py b/slapos/recipe/lamp/runner.py new file mode 100644 index 0000000000000000000000000000000000000000..8ec60ce88a9ec26dda687520d288b148509019c3 --- /dev/null +++ b/slapos/recipe/lamp/runner.py @@ -0,0 +1,27 @@ +import sys +import subprocess + +def executeRunner(args): + """Start the instance configure. this may run a python script, move or/and rename + file or directory when dondition is filled. the condition may be when file exist or when an entry + exist into database. + """ + arguments, delete, rename, chmod, data = args + if delete != []: + print "Calling lampconfigure with 'delete' arguments" + result = subprocess.Popen(arguments + delete) + result.wait() + if rename != []: + for parameters in rename: + print "Calling lampconfigure with 'rename' arguments" + result = subprocess.Popen(arguments + parameters) + result.wait() + if chmod != []: + print "Calling lampconfigure with 'chmod' arguments" + result = subprocess.Popen(arguments + chmod) + result.wait() + if data != []: + print "Calling lampconfigure with 'run' arguments" + result = subprocess.Popen(arguments + data) + result.wait() + return diff --git a/slapos/recipe/lamp/template/apache.in b/slapos/recipe/lamp/template/apache.in new file mode 100644 index 0000000000000000000000000000000000000000..1d7d6a94d878c91a13533a3af66c426a2dde3490 --- /dev/null +++ b/slapos/recipe/lamp/template/apache.in @@ -0,0 +1,58 @@ +# Apache static configuration +# Automatically generated + +# Basic server configuration +PidFile "%(pid_file)s" +LockFile "%(lock_file)s" +Listen %(ip)s:%(port)s +PHPINIDir %(php_ini_dir)s +ServerAdmin someone@email +DefaultType text/plain +TypesConfig conf/mime.types +AddType application/x-compress .Z +AddType application/x-gzip .gz .tgz +AddType application/x-httpd-php .php .phtml .php5 .php4 +AddType application/x-httpd-php-source .phps + +# Log configuration +ErrorLog "%(error_log)s" +LogLevel warn +LogFormat "%%h %%{REMOTE_USER}i %%l %%u %%t \"%%r\" %%>s %%b \"%%{Referer}i\" \"%%{User-Agent}i\"" combined +LogFormat "%%h %%{REMOTE_USER}i %%l %%u %%t \"%%r\" %%>s %%b" common +CustomLog "%(access_log)s" common + +# Directory protection +<Directory /> + Options FollowSymLinks + AllowOverride None + Order deny,allow + Deny from all +</Directory> + +<Directory %(document_root)s> + Options FollowSymLinks + AllowOverride All + Order allow,deny + Allow from all +</Directory> +DocumentRoot %(document_root)s +DirectoryIndex index.html index.php + +# List of modules +LoadModule authz_host_module modules/mod_authz_host.so +LoadModule log_config_module modules/mod_log_config.so +LoadModule setenvif_module modules/mod_setenvif.so +LoadModule version_module modules/mod_version.so +LoadModule proxy_module modules/mod_proxy.so +LoadModule proxy_http_module modules/mod_proxy_http.so +LoadModule mime_module modules/mod_mime.so +LoadModule dav_module modules/mod_dav.so +LoadModule dav_fs_module modules/mod_dav_fs.so +LoadModule negotiation_module modules/mod_negotiation.so +LoadModule rewrite_module modules/mod_rewrite.so +LoadModule headers_module modules/mod_headers.so +LoadModule dir_module modules/mod_dir.so +LoadModule php5_module modules/libphp5.so +LoadModule alias_module modules/mod_alias.so +LoadModule env_module modules/mod_env.so +LoadModule autoindex_module modules/mod_autoindex.so diff --git a/slapos/recipe/lamp/template/my.cnf.in b/slapos/recipe/lamp/template/my.cnf.in new file mode 100644 index 0000000000000000000000000000000000000000..043fb3ad56bbea3b3e35766a9b019cd8dbf7cc77 --- /dev/null +++ b/slapos/recipe/lamp/template/my.cnf.in @@ -0,0 +1,52 @@ +# ERP5 buildout my.cnf template based on my-huge.cnf shipped with mysql +# The MySQL server +[mysqld] +# ERP5 by default requires InnoDB storage. MySQL by default fallbacks to using +# different engine, like MyISAM. Such behaviour generates problems only, when +# tables requested as InnoDB are silently created with MyISAM engine. +# +# Loud fail is really required in such case. +sql-mode="NO_ENGINE_SUBSTITUTION" + +skip-show-database +port = %(tcp_port)s +bind-address = %(ip)s +socket = %(socket)s +datadir = %(data_directory)s +pid-file = %(pid_file)s +log-error = %(error_log)s +log-slow-file = %(slow_query_log)s +long_query_time = 5 +max_allowed_packet = 128M +query_cache_size = 32M + +plugin-load = ha_innodb_plugin.so + +# The following are important to configure and depend a lot on to the size of +# your database and the available resources. +#innodb_buffer_pool_size = 4G +#innodb_log_file_size = 256M +#innodb_log_buffer_size = 8M + +# Some dangerous settings you may want to uncomment if you only want +# performance or less disk access. Useful for unit tests. +#innodb_flush_log_at_trx_commit = 0 +#innodb_flush_method = nosync +#innodb_doublewrite = 0 +#sync_frm = 0 + +# Uncomment the following if you need binary logging, which is recommended +# on production instances (either for replication or incremental backups). +#log-bin=mysql-bin + +# Force utf8 usage +collation_server = utf8_unicode_ci +character_set_server = utf8 +skip-character-set-client-handshake + +[mysql] +no-auto-rehash +socket = %(socket)s + +[mysqlhotcopy] +interactive-timeout diff --git a/slapos/recipe/lamp/template/mysqlinit.sql.in b/slapos/recipe/lamp/template/mysqlinit.sql.in new file mode 100644 index 0000000000000000000000000000000000000000..9189d8d13005b57f36817409f9f873eaec5ef189 --- /dev/null +++ b/slapos/recipe/lamp/template/mysqlinit.sql.in @@ -0,0 +1,7 @@ +CREATE DATABASE IF NOT EXISTS %(database)s; +GRANT ALL PRIVILEGES ON %(database)s.* TO %(user)s@localhost IDENTIFIED BY %(password)r; +GRANT ALL PRIVILEGES ON %(database)s.* TO %(user)s@'%%' IDENTIFIED BY %(password)r; +GRANT SHOW DATABASES ON *.* TO %(user)s@localhost IDENTIFIED BY %(password)r; +GRANT SHOW DATABASES ON *.* TO %(user)s@'%%' IDENTIFIED BY %(password)r; +FLUSH PRIVILEGES; +EXIT diff --git a/slapos/recipe/lamp/template/php.ini.in b/slapos/recipe/lamp/template/php.ini.in new file mode 100644 index 0000000000000000000000000000000000000000..76dfd3a88e64902fd08f2bcc5f2318a3c7ed6a8b --- /dev/null +++ b/slapos/recipe/lamp/template/php.ini.in @@ -0,0 +1,18 @@ +[PHP] +engine = On +safe_mode = Off +expose_php = Off +error_reporting = E_ALL & ~(E_DEPRECATED|E_NOTICE|E_WARNING) +display_errors = On +display_startup_errors = Off +log_errors = On +log_errors_max_len = 1024 +ignore_repeated_errors = Off +ignore_repeated_source = Off +session.save_path = "%(tmp_directory)s" +session.auto_start = 0 +date.timezone = Europe/Paris +file_uploads = On +upload_max_filesize = 16M +post_max_size = 16M +magic_quotes_gpc=Off diff --git a/slapos/recipe/lamp/template/stunnel.conf.in b/slapos/recipe/lamp/template/stunnel.conf.in new file mode 100644 index 0000000000000000000000000000000000000000..f72634ec3d8f258c5b7caf4ea1e8adf47e9f8847 --- /dev/null +++ b/slapos/recipe/lamp/template/stunnel.conf.in @@ -0,0 +1,9 @@ +foreground = yes +output = %(log)s +pid = %(pid_file)s +syslog = no + +[service] +client = yes +accept = %(local_host)s:%(local_port)s +connect = %(remote_host)s:%(remote_port)s diff --git a/slapos/recipe/librecipe/__init__.py b/slapos/recipe/librecipe/__init__.py index e0eb9d547d421d05f9067d909e8ae1a008c54228..9a892bf0d85c78dfb6aa432c78e2643c1e43ea25 100644 --- a/slapos/recipe/librecipe/__init__.py +++ b/slapos/recipe/librecipe/__init__.py @@ -33,9 +33,16 @@ from hashlib import md5 import stat import netaddr import time +import re +import urlparse + +# Use to do from slapos.recipe.librecipe import GenericBaseRecipe +from generic import GenericBaseRecipe +from genericslap import GenericSlapRecipe class BaseSlapRecipe: """Base class for all slap.recipe.*""" + def __init__(self, buildout, name, options): """Default initialisation""" self.name = name @@ -60,6 +67,7 @@ class BaseSlapRecipe: 'xml_report') self.destroy_script_location = os.path.join(self, self.work_directory, 'sbin', 'destroy') + self.promise_directory = os.path.join(self.etc_directory, 'promise') # default directory structure information self.default_directory_list = [ @@ -71,6 +79,7 @@ class BaseSlapRecipe: self.etc_directory, # CP/etc - configuration container self.wrapper_directory, # CP/etc/run - for wrappers self.wrapper_report_directory, # CP/etc/report - for report wrappers + self.promise_directory, # CP/etc/promise - for promise checking scripts self.var_directory, # CP/var - partition "internal" container for logs, # and another metadata self.wrapper_xml_report_directory, # CP/var/xml_report - for xml_report wrappers @@ -81,16 +90,19 @@ class BaseSlapRecipe: # SLAP related information slap_connection = buildout['slap_connection'] - self.computer_id=slap_connection['computer_id'] - self.computer_partition_id=slap_connection['partition_id'] - self.server_url=slap_connection['server_url'] - self.software_release_url=slap_connection['software_release_url'] - self.key_file=slap_connection.get('key_file') - self.cert_file=slap_connection.get('cert_file') + self.computer_id = slap_connection['computer_id'] + self.computer_partition_id = slap_connection['partition_id'] + self.server_url = slap_connection['server_url'] + self.software_release_url = slap_connection['software_release_url'] + self.key_file = slap_connection.get('key_file') + self.cert_file = slap_connection.get('cert_file') # setup egg to give possibility to generate scripts self.egg = zc.recipe.egg.Egg(buildout, options['recipe'], options) + # Hook options + self._options(options) + # setup auto uninstall/install self._setupAutoInstallUninstall() @@ -243,3 +255,55 @@ class BaseSlapRecipe: def _install(self): """Hook which shall be implemented in children class""" raise NotImplementedError('Shall be implemented by subclass') + + def _options(self, options): + """Hook which can be implemented in children class""" + pass + + def createPromiseWrapper(self, promise_name, file_content): + """Create a promise wrapper. + + This wrapper aim to check if the software release is doing its job. + + Return the promise file path. + """ + promise_path = os.path.join(self.promise_directory, promise_name) + self._writeExecutable(promise_path, file_content) + return promise_path + + def setConnectionUrl(self, *args, **kwargs): + url = self._unparseUrl(*args, **kwargs) + self.setConnectionDict(dict(url=url)) + + def _unparseUrl(self, scheme, host, path='', params='', query='', + fragment='', port=None, auth=None): + """Join a url with auth, host, and port. + + * auth can be either a login string or a tuple (login, password). + * if the host is an ipv6 address, brackets will be added to surround it. + + """ + # XXX-Antoine: I didn't find any standard module to join an url with + # login, password, ipv6 host and port. + # So instead of copy and past in every recipe I factorized it right here. + netloc = '' + if auth is not None: + auth = tuple(auth) + netloc = str(auth[0]) # Login + if len(auth) > 1: + netloc += ':%s' % auth[1] # Password + netloc += '@' + + # host is an ipv6 address whithout brackets + if ':' in host and not re.match(r'^\[.*\]$', host): + netloc += '[%s]' % host + else: + netloc += str(host) + + if port is not None: + netloc += ':%s' % port + + url = urlparse.urlunparse((scheme, netloc, path, params, query, fragment)) + + return url + diff --git a/slapos/recipe/librecipe/execute.py b/slapos/recipe/librecipe/execute.py index ec2862116b0787b16c75287cbc89bdf860d73d94..d9e04caafc0ac84b76458ae72c98aef9bd4d1010 100644 --- a/slapos/recipe/librecipe/execute.py +++ b/slapos/recipe/librecipe/execute.py @@ -23,6 +23,8 @@ def execute_wait(args): ready = False if ready: break + # XXX: It's the same as ../ca/certificate_authoritiy.py + # We should use pyinotify as well. Or select() on socket. time.sleep(sleep) os.execv(exec_list[0], exec_list + sys.argv[1:]) @@ -39,6 +41,25 @@ def executee(args): env[k] = v os.execve(exec_list[0], exec_list + sys.argv[1:], env) +def executee_wait(args): + """Portable execution with process replacement and environment manipulation""" + exec_list = list(args[0]) + file_list = list(args[1]) + environment = args[2] + env = os.environ.copy() + for k,v in environment.iteritems(): + env[k] = v + sleep = 60 + while True: + ready = True + for f in file_list: + if not os.path.exists(f): + print 'File %r does not exists, sleeping for %s' % (f, sleep) + ready = False + if ready: + break + time.sleep(sleep) + os.execve(exec_list[0], exec_list + sys.argv[1:], env) def sig_handler(signal, frame): print 'Received signal %r, killing children and exiting' % signal diff --git a/slapos/recipe/librecipe/generic.py b/slapos/recipe/librecipe/generic.py new file mode 100644 index 0000000000000000000000000000000000000000..a203ce9b4714b772996843c8ae6001827e1dc61a --- /dev/null +++ b/slapos/recipe/librecipe/generic.py @@ -0,0 +1,123 @@ +############################################################################## +# +# Copyright (c) 2010 Vifib SARL and Contributors. All Rights Reserved. +# +# WARNING: This program as such is intended to be used by professional +# programmers who take the whole responsibility of assessing all potential +# consequences resulting from its eventual inadequacies and bugs +# End users who are looking for a ready-to-use solution with commercial +# guarantees and support are strongly adviced to contract a Free Software +# Service Company +# +# This program is Free Software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either version 3 +# of the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. +# +############################################################################## +import logging +import os +import sys +import inspect + +import pkg_resources +import zc.buildout + +class GenericBaseRecipe(object): + + TRUE_VALUES = ['y', 'yes', '1', 'true'] + + def __init__(self, buildout, name, options): + """Recipe initialisation""" + self.name = name + self.options = options + self.buildout = buildout + self.logger = logging.getLogger(name) + + self._options(options) # Options Hook + + self._ws = self.getWorkingSet() + + def update(self): + """By default update method does the same thing than install""" + return self.install() + + def install(self): + """Install method of the recipe. This must be overriden in child + classes """ + raise NotImplementedError("install method is not implemented.") + + def getWorkingSet(self): + """If you want do override the default working set""" + egg = zc.recipe.egg.Egg(self.buildout, 'slapos.cookbook', + self.options.copy()) + requirements, ws = egg.working_set() + return ws + + def _options(self, options): + """Options Hook method. This method can be overriden in child classes""" + return + + def createFile(self, name, content, mode=0600): + """Create a file with content + + The parent directory should exists, else it would raise IOError""" + with open(name, 'w') as fileobject: + fileobject.write(content) + os.chmod(fileobject.name, mode) + return os.path.abspath(name) + + def createExecutable(self, name, content, mode=0700): + return self.createFile(name, content, mode) + + def createPythonScript(self, name, absolute_function, arguments=''): + """Create a python script using zc.buildout.easy_install.scripts + + * function should look like 'module.function', or only 'function' + if it is a builtin function.""" + absolute_function = tuple(absolute_function.rsplit('.', 1)) + if len(absolute_function) == 1: + absolute_function = ('__builtin__',) + absolute_function + if len(absolute_function) != 2: + raise ValueError("A non valid function was given") + + module, function = absolute_function + path, filename = os.path.split(os.path.abspath(name)) + + script = zc.buildout.easy_install.scripts( + [(filename, module, function)], self._ws, sys.executable, + path, arguments=arguments)[0] + return script + + def substituteTemplate(self, template_location, mapping_dict): + """Read from file template_location an substitute content with + mapping_dict douing a dummy python format.""" + with open(template_location, 'r') as template: + return template.read() % mapping_dict + + def getTemplateFilename(self, template_name): + caller = inspect.stack()[1] + caller_frame = caller[0] + name = caller_frame.f_globals['__name__'] + return pkg_resources.resource_filename(name, + 'template/%s' % template_name) + + def generatePassword(self, len_=32): + # TODO: implement a real password generator which remember the last + # call. + return "insecure" + + def isTrueValue(self, value): + return str(value).lower() in GenericBaseRecipe.TRUE_VALUES + + def optionIsTrue(self, optionname, default=None): + return self.isTrueValue(self.options[optionname]) diff --git a/slapos/recipe/librecipe/genericslap.py b/slapos/recipe/librecipe/genericslap.py new file mode 100644 index 0000000000000000000000000000000000000000..0181a0da6c78852e5051ca30bef9e97e20fb34e3 --- /dev/null +++ b/slapos/recipe/librecipe/genericslap.py @@ -0,0 +1,129 @@ +############################################################################## +# +# Copyright (c) 2010 Vifib SARL and Contributors. All Rights Reserved. +# +# WARNING: This program as such is intended to be used by professional +# programmers who take the whole responsibility of assessing all potential +# consequences resulting from its eventual inadequacies and bugs +# End users who are looking for a ready-to-use solution with commercial +# guarantees and support are strongly adviced to contract a Free Software +# Service Company +# +# This program is Free Software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either version 3 +# of the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. +# +############################################################################## +import logging +from slapos import slap +import zc.buildout +import zc.recipe.egg +import time +import re +import urlparse + +class GenericSlapRecipe(object): + """Base class for all slap.recipe.*""" + + def __init__(self, buildout, name, options): + """Default initialisation""" + self.name = name + options['eggs'] = 'slapos.cookbook' + self.options = options + self.logger = logging.getLogger(self.name) + self.slap = slap.slap() + + # SLAP related information + slap_connection = buildout['slap-connection'] + self.computer_id = slap_connection['computer-id'] + self.computer_partition_id = slap_connection['partition-id'] + self.server_url = slap_connection['server-url'] + self.software_release_url = slap_connection['software-release-url'] + self.key_file = slap_connection.get('key-file') + self.cert_file = slap_connection.get('cert-file') + + # setup egg to give possibility to generate scripts + self.egg = zc.recipe.egg.Egg(buildout, options['recipe'], options) + + # Hook options + self._options(options) + + # setup auto uninstall/install + self._setupAutoInstallUninstall() + + def _setupAutoInstallUninstall(self): + """By default SlapOS recipes are reinstalled each time""" + # Note: It is possible to create in future subclass which will do no-op in + # this method + self.options['slapos-timestamp'] = str(time.time()) + + def install(self): + self.slap.initializeConnection(self.server_url, self.key_file, + self.cert_file) + self.computer_partition = self.slap.registerComputerPartition( + self.computer_id, + self.computer_partition_id) + self.request = self.computer_partition.request + self.setConnectionDict = self.computer_partition.setConnectionDict + self.parameter_dict = self.computer_partition.getInstanceParameterDict() + + # call children part of install + path_list = self._install() + + return path_list + + update = install + + def _install(self): + """Hook which shall be implemented in children class""" + raise NotImplementedError('Shall be implemented by subclass') + + def _options(self, options): + """Hook which can be implemented in children class""" + pass + + def setConnectionUrl(self, *args, **kwargs): + url = self._unparseUrl(*args, **kwargs) + self.setConnectionDict(dict(url=url)) + + def _unparseUrl(self, scheme, host, path='', params='', query='', + fragment='', port=None, auth=None): + """Join a url with auth, host, and port. + + * auth can be either a login string or a tuple (login, password). + * if the host is an ipv6 address, brackets will be added to surround it. + + """ + # XXX-Antoine: I didn't find any standard module to join an url with + # login, password, ipv6 host and port. + # So instead of copy and past in every recipe I factorized it right here. + netloc = '' + if auth is not None: + auth = tuple(auth) + netloc = str(auth[0]) # Login + if len(auth) > 1: + netloc += ':%s' % auth[1] # Password + netloc += '@' + + # host is an ipv6 address whithout brackets + if ':' in host and not re.match(r'^\[.*\]$', host): + netloc += '[%s]' % host + else: + netloc += str(host) + + if port is not None: + netloc += ':%s' % port + + url = urlparse.urlunparse((scheme, netloc, path, params, query, fragment)) + + return url diff --git a/slapos/recipe/logrotate.py b/slapos/recipe/logrotate.py new file mode 100644 index 0000000000000000000000000000000000000000..9551952d80696883462224718d686139ccfda66e --- /dev/null +++ b/slapos/recipe/logrotate.py @@ -0,0 +1,120 @@ +############################################################################## +# +# Copyright (c) 2010 Vifib SARL and Contributors. All Rights Reserved. +# +# WARNING: This program as such is intended to be used by professional +# programmers who take the whole responsibility of assessing all potential +# consequences resulting from its eventual inadequacies and bugs +# End users who are looking for a ready-to-use solution with commercial +# guarantees and support are strongly adviced to contract a Free Software +# Service Company +# +# This program is Free Software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either version 3 +# of the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. +# +############################################################################## +import os + +from slapos.recipe.librecipe import GenericBaseRecipe + +class Recipe(GenericBaseRecipe): + + def _options(self, options): + if 'name' not in options: + options['name'] = self.name + + def install(self): + path_list = [] + + logrotate_backup = self.options['backup'] + logrotate_d = self.options['logrotate-entries'] + logrotate_conf_file = self.options['conf'] + + logrotate_conf = [] + logrotate_conf.append("include %s" % logrotate_d) + logrotate_conf.append("olddir %s" % logrotate_backup) + logrotate_conf.append("dateext") + + frequency = 'daily' + if 'frequency' in self.options: + frequency = self.options['frequency'] + logrotate_conf.append(frequency) + + num_rotate = 30 + if 'num-rotate' in self.options: + num_rotate = self.options['num-rotate'] + logrotate_conf.append("rotate %s" % num_rotate) + + logrotate_conf.append("compress") + logrotate_conf.append("compresscmd %s" % self.options['gzip-binary']) + logrotate_conf.append("compressoptions -9") + logrotate_conf.append("uncompresscmd %s" % self.options['gunzip-binary']) + + logrotate_conf_file = self.createFile(logrotate_conf_file, '\n'.join(logrotate_conf)) + logrotate_conf.append(logrotate_conf_file) + + state_file = self.options['state-file'] + + logrotate = self.createPythonScript( + self.options['wrapper'], + 'slapos.recipe.librecipe.exceute.execute', + [self.options['logrotate-binary'], '-s', state_file, logrotate_conf_file, ] + ) + path_list.append(logrotate) + + return path_list + +class Part(GenericBaseRecipe): + + def _options(self, options): + if 'name' not in options: + options['name'] = self.name + + def install(self): + + logrotate_d = self.options['logrotate-entries'] + + part_path = os.path.join(logrotate_d, self.options['name']) + + conf = [] + + if 'frequency' in self.options: + conf.append(self.options['frequency']) + if 'num-rotate' in self.options: + conf.append('rotate %s' % self.options['num-rotate']) + + if 'post' in self.options: + conf.append("postrotate\n%s\nendscript" % self.options['post']) + if 'pre' in self.options: + conf.append("prerotate\n%s\nendscript" % self.options['pre']) + + if self.optionIsTrue('sharedscripts', False): + conf.append("sharedscripts") + + if self.optionIsTrue('notifempty', False): + conf.append('notifempty') + + if self.optionIsTrue('create', True): + conf.append('create') + + log = self.options['log'] + + self.createFile(os.path.join(logrotate_d, self.options['name']), + "%(logfiles)s {\n%(conf)s\n}" % { + 'logfiles': log, + 'conf': '\n'.join(conf), + } + ) + + return [part_path] diff --git a/slapos/recipe/mkdirectory.py b/slapos/recipe/mkdirectory.py new file mode 100644 index 0000000000000000000000000000000000000000..d715da3d5932d5d0ecac8b69813cb47bdc057335 --- /dev/null +++ b/slapos/recipe/mkdirectory.py @@ -0,0 +1,53 @@ +############################################################################## +# +# Copyright (c) 2010 Vifib SARL and Contributors. All Rights Reserved. +# +# WARNING: This program as such is intended to be used by professional +# programmers who take the whole responsibility of assessing all potential +# consequences resulting from its eventual inadequacies and bugs +# End users who are looking for a ready-to-use solution with commercial +# guarantees and support are strongly adviced to contract a Free Software +# Service Company +# +# This program is Free Software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either version 3 +# of the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. +# +############################################################################## +import os + +from slapos.recipe.librecipe import GenericBaseRecipe + +class Recipe(GenericBaseRecipe): + + def _options(self, options): + self.directory = options.copy() + del self.directory['recipe'] + + str_mode = '0700' + if 'mode' in self.directory: + str_mode = self.directory['mode'] + del self.directory['mode'] + self.mode = int(str_mode, 8) + + def install(self): + + for directory in self.directory.values(): + path = directory + + if not os.path.exists(path): + os.mkdir(path, self.mode) + elif not os.path.isdir(path): + raise OSError("%s path exits, but it's not a directory.") + + return [] diff --git a/slapos/recipe/mysql/__init__.py b/slapos/recipe/mysql/__init__.py index 0976d0411d1a17a09ca65920d0fecffa308ed5ac..14c63140ff21d62b50dcf3218bd8083eaec151bc 100644 --- a/slapos/recipe/mysql/__init__.py +++ b/slapos/recipe/mysql/__init__.py @@ -24,312 +24,135 @@ # Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. # ############################################################################## -from slapos.recipe.librecipe import BaseSlapRecipe -import hashlib +from slapos.recipe.librecipe import GenericBaseRecipe import os -import pkg_resources -import sys -import zc.buildout -import ConfigParser -class Recipe(BaseSlapRecipe): - def getTemplateFilename(self, template_name): - return pkg_resources.resource_filename(__name__, - 'template/%s' % template_name) +class Recipe(GenericBaseRecipe): - def _install(self): - self.path_list = [] + def _options(self, options): + options['password'] = self.generatePassword() - self.requirements, self.ws = self.egg.working_set() - # self.cron_d is a directory, where cron jobs can be registered - self.cron_d = self.installCrond() - self.logrotate_d, self.logrotate_backup = self.installLogrotate() - - mysql_conf = self.installMysqlServer(self.getLocalIPv4Address(), 45678) - - ca_conf = self.installCertificateAuthority() - key, certificate = self.requestCertificate('MySQL') - - stunnel_conf = self.installStunnel(self.getGlobalIPv6Address(), - self.getLocalIPv4Address(), 12345, mysql_conf['tcp_port'], - certificate, key, ca_conf['ca_crl'], - ca_conf['certificate_authority_path']) - - self.linkBinary() - self.setConnectionDict(dict( - stunnel_ip = stunnel_conf['public_ip'], - stunnel_port = stunnel_conf['public_port'], - mysql_database = mysql_conf['mysql_database'], - mysql_user = mysql_conf['mysql_user'], - mysql_password = mysql_conf['mysql_password'], - )) - return self.path_list + def install(self): + path_list = [] - def linkBinary(self): - """Links binaries to instance's bin directory for easier exposal""" - for linkline in self.options.get('link_binary_list', '').splitlines(): - if not linkline: - continue - target = linkline.split() - if len(target) == 1: - target = target[0] - path, linkname = os.path.split(target) - else: - linkname = target[1] - target = target[0] - link = os.path.join(self.bin_directory, linkname) - if os.path.lexists(link): - if not os.path.islink(link): - raise zc.buildout.UserError( - 'Target link already %r exists but it is not link' % link) - os.unlink(link) - os.symlink(target, link) - self.logger.debug('Created link %r -> %r' % (link, target)) - self.path_list.append(link) + template_filename = self.getTemplateFilename('my.cnf.in') - def installCrond(self): - timestamps = self.createDataDirectory('cronstamps') - cron_output = os.path.join(self.log_directory, 'cron-output') - self._createDirectory(cron_output) - catcher = zc.buildout.easy_install.scripts([('catchcron', - __name__ + '.catdatefile', 'catdatefile')], self.ws, sys.executable, - self.bin_directory, arguments=[cron_output])[0] - self.path_list.append(catcher) - cron_d = os.path.join(self.etc_directory, 'cron.d') - crontabs = os.path.join(self.etc_directory, 'crontabs') - self._createDirectory(cron_d) - self._createDirectory(crontabs) - wrapper = zc.buildout.easy_install.scripts([('crond', - 'slapos.recipe.librecipe.execute', 'execute')], self.ws, sys.executable, - self.wrapper_directory, arguments=[ - self.options['dcrond_binary'].strip(), '-s', cron_d, '-c', crontabs, - '-t', timestamps, '-f', '-l', '5', '-M', catcher] - )[0] - self.path_list.append(wrapper) - return cron_d - - def installLogrotate(self): - """Installs logortate main configuration file and registers its to cron""" - logrotate_d = os.path.abspath(os.path.join(self.etc_directory, - 'logrotate.d')) - self._createDirectory(logrotate_d) - logrotate_backup = self.createBackupDirectory('logrotate') - logrotate_conf = self.createConfigurationFile("logrotate.conf", - "include %s" % logrotate_d) - logrotate_cron = os.path.join(self.cron_d, 'logrotate') - state_file = os.path.join(self.data_root_directory, 'logrotate.status') - open(logrotate_cron, 'w').write('0 0 * * * %s -s %s %s' % - (self.options['logrotate_binary'], state_file, logrotate_conf)) - self.path_list.extend([logrotate_d, logrotate_conf, logrotate_cron]) - return logrotate_d, logrotate_backup + mysql_conf = dict( + ip=self.options['ip'], + data_directory=self.options['data-directory'], + tcp_port=self.options['port'], + pid_file=self.options['pid-file'], + socket=self.options['socket'], + error_log=self.options['error-log'], + slow_query_log=self.options['slow-query-log'], + mysql_database=self.options['database'], + mysql_user=self.options['user'], + mysql_password=self.options['password'], + ) - def registerLogRotation(self, name, log_file_list, postrotate_script): - """Register new log rotation requirement""" - open(os.path.join(self.logrotate_d, name), 'w').write( - self.substituteTemplate(self.getTemplateFilename( - 'logrotate_entry.in'), - dict(file_list=' '.join(['"'+q+'"' for q in log_file_list]), - postrotate=postrotate_script, olddir=self.logrotate_backup))) + mysql_binary = self.options['mysql-binary'] + socket = self.options['socket'], + post_rotate = self.createPythonScript( + self.options['logrotate-post'], + 'slapos.recipe.librecipe.execute.execute', + [mysql_binary, '--no-defaults', '-B', '--socket=%s' % socket, '-e', + 'FLUSH LOGS'] + ) + path_list.append(post_rotate) - def installCertificateAuthority(self, ca_country_code='XX', - ca_email='xx@example.com', ca_state='State', ca_city='City', - ca_company='Company'): - backup_path = self.createBackupDirectory('ca') - self.ca_dir = os.path.join(self.data_root_directory, 'ca') - self._createDirectory(self.ca_dir) - self.ca_request_dir = os.path.join(self.ca_dir, 'requests') - self._createDirectory(self.ca_request_dir) - config = dict(ca_dir=self.ca_dir, request_dir=self.ca_request_dir) - self.ca_private = os.path.join(self.ca_dir, 'private') - self.ca_certs = os.path.join(self.ca_dir, 'certs') - self.ca_crl = os.path.join(self.ca_dir, 'crl') - self.ca_newcerts = os.path.join(self.ca_dir, 'newcerts') - self.ca_key_ext = '.key' - self.ca_crt_ext = '.crt' - for d in [self.ca_private, self.ca_crl, self.ca_newcerts, self.ca_certs]: - self._createDirectory(d) - for f in ['crlnumber', 'serial']: - if not os.path.exists(os.path.join(self.ca_dir, f)): - open(os.path.join(self.ca_dir, f), 'w').write('01') - if not os.path.exists(os.path.join(self.ca_dir, 'index.txt')): - open(os.path.join(self.ca_dir, 'index.txt'), 'w').write('') - openssl_configuration = os.path.join(self.ca_dir, 'openssl.cnf') - config.update( - working_directory=self.ca_dir, - country_code=ca_country_code, - state=ca_state, - city=ca_city, - company=ca_company, - email_address=ca_email, + mysql_conf_file = self.createFile( + self.options['conf-file'], + self.substituteTemplate(template_filename, mysql_conf) ) - self._writeFile(openssl_configuration, pkg_resources.resource_string( - __name__, 'template/openssl.cnf.ca.in') % config) - self.path_list.extend(zc.buildout.easy_install.scripts([ - ('certificate_authority', - __name__ + '.certificate_authority', 'runCertificateAuthority')], - self.ws, sys.executable, self.wrapper_directory, arguments=[dict( - openssl_configuration=openssl_configuration, - openssl_binary=self.options['openssl_binary'], - certificate=os.path.join(self.ca_dir, 'cacert.pem'), - key=os.path.join(self.ca_private, 'cakey.pem'), - crl=os.path.join(self.ca_crl), - request_dir=self.ca_request_dir - )])) - # configure backup - backup_cron = os.path.join(self.cron_d, 'ca_rdiff_backup') - open(backup_cron, 'w').write( - '''0 0 * * * %(rdiff_backup)s %(source)s %(destination)s'''%dict( - rdiff_backup=self.options['rdiff_backup_binary'], - source=self.ca_dir, - destination=backup_path)) - self.path_list.append(backup_cron) + path_list.append(mysql_conf_file) + + mysql_script_list = [] - return dict( - ca_certificate=os.path.join(config['ca_dir'], 'cacert.pem'), - ca_crl=os.path.join(config['ca_dir'], 'crl'), - certificate_authority_path=config['ca_dir'] + init_script = self.substituteTemplate( + self.getTemplateFilename('initmysql.sql.in'), + { + 'mysql_database': mysql_conf['mysql_database'], + 'mysql_user': mysql_conf['mysql_user'], + 'mysql_password': mysql_conf['mysql_password'] + } ) + mysql_script_list.append(init_script) + mysql_script_list.append('EXIT') + mysql_script = '\n'.join(mysql_script_list) - def requestCertificate(self, name): - hash = hashlib.sha512(name).hexdigest() - key = os.path.join(self.ca_private, hash + self.ca_key_ext) - certificate = os.path.join(self.ca_certs, hash + self.ca_crt_ext) - parser = ConfigParser.RawConfigParser() - parser.add_section('certificate') - parser.set('certificate', 'name', name) - parser.set('certificate', 'key_file', key) - parser.set('certificate', 'certificate_file', certificate) - parser.write(open(os.path.join(self.ca_request_dir, hash), 'w')) - return key, certificate + mysql_upgrade_binary = self.options['mysql-upgrade-binary'] + mysql_update = self.createPythonScript( + self.options['update-wrapper'], + '%s.mysql.updateMysql' % __name__, + dict( + mysql_script=mysql_script, + mysql_binary=mysql_binary, + mysql_upgrade_binary=mysql_upgrade_binary, + socket=socket, + ) + ) + path_list.append(mysql_update) - def installStunnel(self, public_ip, private_ip, public_port, private_port, - ca_certificate, key, ca_crl, ca_path): - """Installs stunnel""" - template_filename = self.getTemplateFilename('stunnel.conf.in') - log = os.path.join(self.log_directory, 'stunnel.log') - pid_file = os.path.join(self.run_directory, 'stunnel.pid') - stunnel_conf = dict( - public_ip=public_ip, - private_ip=private_ip, - public_port=public_port, - pid_file=pid_file, - log=log, - cert = ca_certificate, - key = key, - ca_crl = ca_crl, - ca_path = ca_path, - private_port = private_port, + mysqld_binary = self.options['mysqld-binary'] + mysqld = self.createPythonScript( + self.options['wrapper'], + '%s.mysql.runMysql' % __name__, + dict( + mysql_install_binary=self.options['mysql-install-binary'], + mysqld_binary=mysqld_binary, + data_directory=mysql_conf['data_directory'], + mysql_binary=mysql_binary, + socket=socket, + configuration_file=mysql_conf_file, + ) ) - stunnel_conf_path = self.createConfigurationFile("stunnel.conf", - self.substituteTemplate(template_filename, - stunnel_conf)) - wrapper = zc.buildout.easy_install.scripts([('stunnel', - 'slapos.recipe.librecipe.execute', 'execute_wait')], self.ws, - sys.executable, self.wrapper_directory, arguments=[ - [self.options['stunnel_binary'].strip(), stunnel_conf_path], - [ca_certificate, key]] - )[0] - self.path_list.append(wrapper) - return stunnel_conf + path_list.append(mysqld) - - def installMysqlServer(self, ip, port, database='db', user='user', - template_filename=None, mysql_conf=None): - if mysql_conf is None: - mysql_conf = {} - backup_directory = self.createBackupDirectory('mysql') - if template_filename is None: - template_filename = self.getTemplateFilename('my.cnf.in') - error_log = os.path.join(self.log_directory, 'mysqld.log') - slow_query_log = os.path.join(self.log_directory, 'mysql-slow.log') - mysql_conf.update( - ip=ip, - data_directory=os.path.join(self.data_root_directory, - 'mysql'), - tcp_port=port, - pid_file=os.path.join(self.run_directory, 'mysqld.pid'), - socket=os.path.join(self.run_directory, 'mysqld.sock'), - error_log=error_log, - slow_query_log=slow_query_log, - mysql_database=database, - mysql_user=user, - mysql_password=self.generatePassword(), + # backup configuration + mysqldump_binary = self.options['mysqldump-binary'] + backup_directory = self.options['backup-directory'] + pending_backup_dir = self.options['backup-pending-directory'] + dump_filename = self.options['dumpname'] + + mysqldump_cmd = [mysqldump_binary, + mysql_conf['mysql_database'], + '-u', 'root', + '-S', mysql_conf['socket'].strip(), + '--single-transaction', '--opt', + ] + dump_file = os.path.join(backup_directory, dump_filename) + tmpdump_file = os.path.join(pending_backup_dir, dump_filename) + backup_script = self.createPythonScript( + self.options['backup-script'], + '%s.backup.do_backup' % __name__, + { + 'mysqldump': mysqldump_cmd, + 'gzip': self.options['gzip-binary'], + 'tmpdump': tmpdump_file, + 'dumpfile': dump_file, + }, ) - self.registerLogRotation('mysql', [error_log, slow_query_log], - '%(mysql_binary)s --no-defaults -B --user=root ' - '--socket=%(mysql_socket)s -e "FLUSH LOGS"' % dict( - mysql_binary=self.options['mysql_binary'], - mysql_socket=mysql_conf['socket'])) - self._createDirectory(mysql_conf['data_directory']) + path_list.append(backup_script) - mysql_conf_path = self.createConfigurationFile("my.cnf", - self.substituteTemplate(template_filename, - mysql_conf)) + # Recovering backup + if self.optionIsTrue('recovering', default=False): + recovering_script = self.createPythonScript( + self.options['recovering-wrapper'], + '%s.recover.import_remote_dump' % __name__, + { + 'lock_file': os.path.join(self.work_directory, + 'import_done'), + 'database': mysql_conf['mysql_database'], + 'mysql_binary': self.options['mysql-binary'], + 'mysql_socket': mysql_conf['socket'], + 'duplicity_binary': self.options['duplicity-binary'], + 'remote_backup': self.parameter_dict['remote-backup'], + 'local_directory': self.mysql_backup_directory, + 'dump_name': dump_filename, + 'zcat_binary': self.options['zcat-binary'], + } + ) + path_list.append(recovering_script) - mysql_script_list = [] - for x_database, x_user, x_password in \ - [(mysql_conf['mysql_database'], - mysql_conf['mysql_user'], - mysql_conf['mysql_password']), - ]: - mysql_script_list.append(pkg_resources.resource_string(__name__, - 'template/initmysql.sql.in') % { - 'mysql_database': x_database, - 'mysql_user': x_user, - 'mysql_password': x_password}) - mysql_script_list.append('EXIT') - mysql_script = '\n'.join(mysql_script_list) - self.path_list.extend(zc.buildout.easy_install.scripts([('mysql_update', - __name__ + '.mysql', 'updateMysql')], self.ws, - sys.executable, self.wrapper_directory, arguments=[dict( - mysql_script=mysql_script, - mysql_binary=self.options['mysql_binary'].strip(), - mysql_upgrade_binary=self.options['mysql_upgrade_binary'].strip(), - socket=mysql_conf['socket'], - )])) - self.path_list.extend(zc.buildout.easy_install.scripts([('mysqld', - __name__ + '.mysql', 'runMysql')], self.ws, - sys.executable, self.wrapper_directory, arguments=[dict( - mysql_install_binary=self.options['mysql_install_binary'].strip(), - mysqld_binary=self.options['mysqld_binary'].strip(), - data_directory=mysql_conf['data_directory'].strip(), - mysql_binary=self.options['mysql_binary'].strip(), - socket=mysql_conf['socket'].strip(), - configuration_file=mysql_conf_path, - )])) - self.path_list.extend([mysql_conf_path]) - # backup configuration - backup_directory = self.createBackupDirectory('mysql') - full_backup = os.path.join(backup_directory, 'full') - incremental_backup = os.path.join(backup_directory, 'incremental') - self._createDirectory(full_backup) - self._createDirectory(incremental_backup) - innobackupex_argument_list = [self.options['perl_binary'], - self.options['innobackupex_binary'], - '--defaults-file=%s' % mysql_conf_path, - '--socket=%s' %mysql_conf['socket'].strip(), '--user=root'] - environment = dict(PATH='%s' % self.bin_directory) - innobackupex_incremental = zc.buildout.easy_install.scripts([( - 'innobackupex_incremental', 'slapos.recipe.librecipe.execute', 'executee')], - self.ws, sys.executable, self.bin_directory, arguments=[ - innobackupex_argument_list + ['--incremental'], - environment])[0] - self.path_list.append(innobackupex_incremental) - innobackupex_full = zc.buildout.easy_install.scripts([('innobackupex_full', - 'slapos.recipe.librecipe.execute', 'executee')], self.ws, - sys.executable, self.bin_directory, arguments=[ - innobackupex_argument_list, - environment])[0] - self.path_list.append(innobackupex_full) - backup_controller = zc.buildout.easy_install.scripts([ - ('innobackupex_controller', __name__ + '.innobackupex', 'controller')], - self.ws, sys.executable, self.bin_directory, - arguments=[innobackupex_incremental, innobackupex_full, full_backup, - incremental_backup])[0] - self.path_list.append(backup_controller) - mysql_backup_cron = os.path.join(self.cron_d, 'mysql_backup') - open(mysql_backup_cron, 'w').write('0 0 * * * ' + backup_controller) - self.path_list.append(mysql_backup_cron) - # The return could be more explicit database, user ... - return mysql_conf + return path_list diff --git a/slapos/recipe/mysql/backup.py b/slapos/recipe/mysql/backup.py new file mode 100644 index 0000000000000000000000000000000000000000..590771d6b73a33bf8014c8b15b6a60fb2ba905aa --- /dev/null +++ b/slapos/recipe/mysql/backup.py @@ -0,0 +1,25 @@ +import subprocess +import os + +# Replace mysqldump | gzip > tmpdump && mv -f tmpdump dumpfile +def do_backup(kwargs): + mysqldump_cmd = kwargs['mysqldump'] + gzip_bin = kwargs['gzip'] + tmpdump = kwargs['tmpdump'] + dumpfile = kwargs['dumpfile'] + + # mysqldump | gzip > tmpdump + with open(tmpdump, 'w') as output: + mysqldump = subprocess.Popen(mysqldump_cmd, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT) + gzip = subprocess.Popen([gzip_bin], + stdin=mysqldump.stdout, + stdout=output, + stderr=subprocess.STDOUT) + mysqldump.stdout.close() + + if gzip.wait() != 0: + raise ValueError("Gzip return a non zero value.") + + os.rename(tmpdump, dumpfile) diff --git a/slapos/recipe/mysql/catdatefile.py b/slapos/recipe/mysql/catdatefile.py new file mode 100644 index 0000000000000000000000000000000000000000..d3de298b272cce0eba88570aa6734ee74115e58e --- /dev/null +++ b/slapos/recipe/mysql/catdatefile.py @@ -0,0 +1,14 @@ +import os +import sys +import time +def catdatefile(args): + directory = args[0] + try: + suffix = args[1] + except IndexError: + suffix = '.log' + f = open(os.path.join(directory, + time.strftime('%Y-%m-%d.%H:%M.%s') + suffix), 'aw') + for line in sys.stdin.read(): + f.write(line) + f.close() diff --git a/slapos/recipe/mysql/mysql.py b/slapos/recipe/mysql/mysql.py index e2036076a611cd308755a93f77c763ba91e3fb73..24812bc8503756845f016acd07089819bfdf66d9 100644 --- a/slapos/recipe/mysql/mysql.py +++ b/slapos/recipe/mysql/mysql.py @@ -4,9 +4,8 @@ import time import sys -def runMysql(args): +def runMysql(conf): sleep = 60 - conf = args[0] mysqld_wrapper_list = [conf['mysqld_binary'], '--defaults-file=%s' % conf['configuration_file']] # we trust mysql_install that if mysql directory is available mysql was @@ -16,8 +15,8 @@ def runMysql(args): # XXX: Protect with proper root password # XXX: Follow http://dev.mysql.com/doc/refman/5.0/en/default-privileges.html popen = subprocess.Popen([conf['mysql_install_binary'], - '--skip-name-resolve', '--no-defaults', '--datadir=%s' % - conf['data_directory']], + '--skip-name-resolve', '--skip-host-cache', '--no-defaults', + '--datadir=%s' % conf['data_directory']], stdout=subprocess.PIPE, stderr=subprocess.STDOUT) result = popen.communicate()[0] if popen.returncode is None or popen.returncode != 0: @@ -35,8 +34,7 @@ def runMysql(args): os.execl(mysqld_wrapper_list[0], *mysqld_wrapper_list) -def updateMysql(args): - conf = args[0] +def updateMysql(conf): sleep = 30 is_succeed = False while True: diff --git a/slapos/recipe/mysql/recover.py b/slapos/recipe/mysql/recover.py new file mode 100644 index 0000000000000000000000000000000000000000..7641321a9ddab55a290af8bb9ad0faff97fd0af6 --- /dev/null +++ b/slapos/recipe/mysql/recover.py @@ -0,0 +1,42 @@ +import sys +import os +import time +import subprocess + +def import_remote_dump(kwargs): + # Get data from kwargs + lock_file = kwargs['lock_file'] + database = kwargs['database'] + mysql_binary = kwargs['mysql_binary'] + mysql_socket = kwargs['mysql_socket'] + duplicity_binary = kwargs['duplicity_binary'] + remote_backup = kwargs['remote_backup'] + local_directory = kwargs['local_directory'] + dump_name = kwargs['dump_name'] + zcat_binary = kwargs['zcat_binary'] + + # The script start really here + if os.path.exists(lock_file): + sys.exit(127) + + while subprocess.call([mysql_binary, '--socket=%s' % mysql_socket, + '-u', 'root', '-e', 'use %s;' % database]) != 0: + time.sleep(10) + + subprocess.check_call([duplicity_binary, 'restore', '--no-encryption', + remote_backup, local_directory]) + + zcat = subprocess.Popen([zcat_binary, os.path.join(local_directory, + dump_name)], + stdout=subprocess.PIPE) + mysql = subprocess.Popen([mysql_binary, '--socket=%s' % mysql_socket, + '-D', database, '-u', 'root'], + stdin=zcat.stdout) + zcat.stdout.close() + + returncode = mysql.poll() + + if returncode == 0: + open(lock_file, 'w').close() # Just a touch + + sys.exit(returncode) diff --git a/slapos/recipe/publishurl.py b/slapos/recipe/publishurl.py new file mode 100644 index 0000000000000000000000000000000000000000..98655433a986cc414050498cf0933e5fae7fd5e0 --- /dev/null +++ b/slapos/recipe/publishurl.py @@ -0,0 +1,69 @@ +############################################################################## +# +# Copyright (c) 2010 Vifib SARL and Contributors. All Rights Reserved. +# +# WARNING: This program as such is intended to be used by professional +# programmers who take the whole responsibility of assessing all potential +# consequences resulting from its eventual inadequacies and bugs +# End users who are looking for a ready-to-use solution with commercial +# guarantees and support are strongly adviced to contract a Free Software +# Service Company +# +# This program is Free Software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either version 3 +# of the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. +# +############################################################################## +import zc.buildout + +from slapos.recipe.librecipe import GenericSlapRecipe + +class Recipe(GenericSlapRecipe): + + def _options(self, options): + + self.useparts = True + + if 'url' in options: + self.useparts = False + self.url = options['url'] + else: + self.urlparts = {} + + if 'scheme' not in options: + raise zc.buildout.UserError("No scheme specified.") + else: + self.urlparts.update(scheme=options['scheme']) + if 'host' not in options: + raise zc.buildout.UserError("No host specified.") + else: + self.urlparts.update(host=options['host']) + + def _install(self): + + if self.useparts: + for option in ['path', 'params', 'query', 'fragment', 'port']: + if option in self.options: + self.urlparts[option] = self.options[option] + + if 'user' in self.options: + self.urlparts.update(auth=(self.options['user'],)) + if 'password' in self.options: + self.urlparts.update(auth=(self.options['user'], + self.options['password'])) + + self.setConnectionUrl(**self.urlparts) + else: + self.setConnectionDict(dict(url=self.url)) + + return [] diff --git a/slapos/recipe/request.py b/slapos/recipe/request.py new file mode 100644 index 0000000000000000000000000000000000000000..66de7394bdac9e8564661ede2866d2b0009b209e --- /dev/null +++ b/slapos/recipe/request.py @@ -0,0 +1,96 @@ +############################################################################## +# +# Copyright (c) 2010 Vifib SARL and Contributors. All Rights Reserved. +# +# WARNING: This program as such is intended to be used by professional +# programmers who take the whole responsibility of assessing all potential +# consequences resulting from its eventual inadequacies and bugs +# End users who are looking for a ready-to-use solution with commercial +# guarantees and support are strongly adviced to contract a Free Software +# Service Company +# +# This program is Free Software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either version 3 +# of the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. +# +############################################################################## +import logging +import os + +from slapos import slap as slapmodule + +class Recipe(object): + + def parseMultiValues(self, string): + return dict([ [str(column).strip() for column in line.split('=', 1)] + for line in str(string).splitlines() if '=' in line]) + + def __init__(self, buildout, name, options): + self.logger = logging.getLogger(name) + + slap = slapmodule.slap() + + slap_connection = buildout['slap_connection'] + self.software_release_url = slap_connection['software_release_url'] + + # XXX: Dirty network interation stuff + slap.initializeConnection(slap_connection['server_url'], + slap_connection.get('key_file'), + slap_connection.get('cert_file'), + ) + computer_partition = slap.registerComputerPartition( + slap_connection['computer_id'], slap_connection['partition_id']) + self.request = computer_partition.request + + if 'software-url' not in options: + options['software-url'] = self.software_release_url + + if 'name' not in options: + options['name'] = name + + self.return_parameters = [] + if 'return' in options: + self.return_parameters = [str(parameter).strip() + for parameter in options['return'].splitlines()] + else: + self.logger.warning("No parameter to return to main instance." + "Be careful about that...") + + software_type = 'RootInstanceSoftware' + if 'software-type' in options: + software_type = options['software-type'] + + filter_kw = {} + if 'sla' in options: + filter_kw = self.parseMultiValues(options['sla']) + + partition_parameter_kw = {} + if 'config' in options: + partition_parameter_kw = self.parseMultiValues(options['config']) + + instance = self.request(options['software-url'], software_type, + options['name'], partition_parameter_kw=partition_parameter_kw, + filter_kw=filter_kw) + + result = {} + for param in self.return_parameters: + result[param] = instance.getConnectionParameter(param) + + # Return the connections parameters in options dict + for key, value in result.items(): + options['connection-%s' % key] = value + + def install(self): + return [] + + update = install diff --git a/slapos/recipe/simplelogger.py b/slapos/recipe/simplelogger.py new file mode 100644 index 0000000000000000000000000000000000000000..494ef81f1b8278783dc8c7bc0a4384c6d01f1e59 --- /dev/null +++ b/slapos/recipe/simplelogger.py @@ -0,0 +1,54 @@ +############################################################################## +# +# Copyright (c) 2010 Vifib SARL and Contributors. All Rights Reserved. +# +# WARNING: This program as such is intended to be used by professional +# programmers who take the whole responsibility of assessing all potential +# consequences resulting from its eventual inadequacies and bugs +# End users who are looking for a ready-to-use solution with commercial +# guarantees and support are strongly adviced to contract a Free Software +# Service Company +# +# This program is Free Software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either version 3 +# of the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. +# +############################################################################## +import shutil +import os +import sys +import time + +from slapos.recipe.librecipe import GenericBaseRecipe + +def log(args): + directory, suffix = args + filename = time.strftime('%Y-%m-%d.%H:%M.%s') + suffix + with open(os.path.join(directory, filename), 'aw') as logfile: + shutil.copyfileobj(sys.stdin, logfile) + +class Recipe(GenericBaseRecipe): + + def install(self): + self.logger.info("Simple logger installation") + binary = self.options['binary'] + output = self.options['output'] + suffix = self.options.get('suffix', '.log') + + script = self.createPythonScript(binary, + 'slapos.recipe.simplelogger.log', + arguments=[output, suffix]) + self.logger.debug("Logger script created at : %r", script) + self.logger.info("Simple logger installed.") + + return [script] diff --git a/slapos/recipe/softwaretype.py b/slapos/recipe/softwaretype.py new file mode 100644 index 0000000000000000000000000000000000000000..661adfbe1f3d9c851472744b5e690ffd848c26d3 --- /dev/null +++ b/slapos/recipe/softwaretype.py @@ -0,0 +1,134 @@ +############################################################################## +# +# Copyright (c) 2010 Vifib SARL and Contributors. All Rights Reserved. +# +# WARNING: This program as such is intended to be used by professional +# programmers who take the whole responsibility of assessing all potential +# consequences resulting from its eventual inadequacies and bugs +# End users who are looking for a ready-to-use solution with commercial +# guarantees and support are strongly adviced to contract a Free Software +# Service Company +# +# This program is Free Software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either version 3 +# of the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. +# +############################################################################## + +import os +import sys +import copy +from ConfigParser import ConfigParser +import subprocess +import slapos.slap +import netaddr +import logging + +import zc.buildout + +class Recipe: + + def __init__(self, buildout, name, options): + self.buildout = buildout + self.options = options + self.name = name + self.logger = logging.getLogger(self.name) + + def _getIpAddress(self, test_method): + """Internal helper method to fetch ip address""" + if not 'ip_list' in self.parameter_dict: + raise AttributeError + for name, ip in self.parameter_dict['ip_list']: + if test_method(ip): + return ip + raise AttributeError + + def getLocalIPv4Address(self): + """Returns local IPv4 address available on partition""" + # XXX: Lack checking for locality of address + return self._getIpAddress(netaddr.valid_ipv4) + + def getGlobalIPv6Address(self): + """Returns global IPv6 address available on partition""" + # XXX: Lack checking for globality of address + return self._getIpAddress(netaddr.valid_ipv6) + + def install(self): + slap = slapos.slap.slap() + slap_connection = self.buildout['slap_connection'] + computer_id = slap_connection['computer_id'] + computer_partition_id = slap_connection['partition_id'] + server_url = slap_connection['server_url'] + key_file = slap_connection.get('key_file') + cert_file = slap_connection.get('cert_file') + slap.initializeConnection(server_url, key_file, cert_file) + self.computer_partition = slap.registerComputerPartition( + computer_id, + computer_partition_id) + self.parameter_dict = self.computer_partition.getInstanceParameterDict() + software_type = self.parameter_dict['slap_software_type'] + + if software_type not in self.options: + if 'default' in self.options: + software_type = 'default' + else: + raise zc.buildout.UserError("This software type isn't mapped. And" + "there's no default software type.") + + instance_file_path = self.options[software_type] + + if not os.path.exists(instance_file_path): + raise zc.buildout.UserError("The specified buildout config file does not" + "exist.") + + buildout = ConfigParser() + with open(instance_file_path) as instance_path: + buildout.readfp(instance_path) + + buildout.set('buildout', 'installed', + '.installed-%s.cfg' % software_type) + + buildout.add_section('slap-parameter') + for parameter, value in self.parameter_dict.items(): + buildout.set('slap-parameter', parameter, value) + + buildout.add_section('slap-network-information') + buildout.set('slap-network-information', 'local-ipv4', + self.getLocalIPv4Address()) + buildout.set('slap-network-information', 'global-ipv6', + self.getGlobalIPv6Address()) + + # Copy/paste slap_connection + buildout.add_section('slap-connection') + for key, value in self.buildout['slap_connection'].iteritems(): + # XXX: Waiting for SlapBaseRecipe to use dash instead of underscores + buildout.set('slap-connection', key.replace('_', '-'), value) + + work_directory = os.path.abspath(self.buildout['buildout'][ + 'directory']) + buildout_filename = os.path.join(work_directory, + 'buildout-%s.cfg' % software_type) + with open(buildout_filename, 'w') as buildout_file: + buildout.write(buildout_file) + + # XXX-Antoine: We gotta find a better way to do this. I tried to check + # out how slapgrid-cp was running buildout. But it is worse than that. + command_line_args = copy.copy(sys.argv) + ['-c', buildout_filename] + + self.logger.info("Invoking commandline : '%s'", + ' '.join(command_line_args)) + + subprocess.check_call(command_line_args, cwd=work_directory, + env=os.environ.copy()) + return [] + update = install diff --git a/slapos/recipe/stunnel/__init__.py b/slapos/recipe/stunnel/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..06bad67a21d0942f8d2077777083ea709c86a0d4 --- /dev/null +++ b/slapos/recipe/stunnel/__init__.py @@ -0,0 +1,94 @@ +############################################################################## +# +# Copyright (c) 2010 Vifib SARL and Contributors. All Rights Reserved. +# +# WARNING: This program as such is intended to be used by professional +# programmers who take the whole responsibility of assessing all potential +# consequences resulting from its eventual inadequacies and bugs +# End users who are looking for a ready-to-use solution with commercial +# guarantees and support are strongly adviced to contract a Free Software +# Service Company +# +# This program is Free Software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either version 3 +# of the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. +# +############################################################################## +import itertools + +import zc.buildout + +from slapos.recipe.librecipe import GenericBaseRecipe + +class Recipe(GenericBaseRecipe): + + def _options(self, options): + self.types = ['local', 'remote'] + self.datas = ['address', 'port'] + for type_ in self.types: + for data in self.datas: + opt = '%s-%s' % (type_, data) + if opt not in options: + raise zc.buildout.UserError("No %s for %s connections." % (data, type_)) + + self.isClient = self.optionIsTrue('client', default=False) + if self.isClient: + self.logger.info("Client mode") + else: + self.logger.info("Server mode") + + if 'name' not in options: + options['name'] = self.name + + + def install(self): + path_list = [] + conf = {} + + gathered_options = ['%s-%s' % option + for option in itertools.product(self.types, + self.datas)] + for option in gathered_options: + # XXX: Because the options are using dash and the template uses + # underscore + conf[option.replace('-', '_')] = self.options[option] + + pid_file = self.options['pid-file'] + conf.update(pid_file=pid_file) + path_list.append(pid_file) + + log_file = self.options['log-file'] + conf.update(log=log_file) + + if self.isClient: + template = self.getTemplateFilename('client.conf.in') + + else: + template = self.getTemplateFilename('server.conf.in') + key = self.options['key-file'] + cert = self.options['cert-file'] + conf.update(key=key, cert=cert) + + conf_file = self.createFile( + self.options['config-file'], + self.substituteTemplate(template, conf)) + path_list.append(conf_file) + + wrapper = self.createPythonScript( + self.options['wrapper'], + 'slapos.recipe.librecipe.execute.execute', + [self.options['stunnel-binary'], conf_file] + ) + path_list.append(wrapper) + + return path_list diff --git a/slapos/recipe/stunnel/template/client.conf.in b/slapos/recipe/stunnel/template/client.conf.in new file mode 100644 index 0000000000000000000000000000000000000000..f72634ec3d8f258c5b7caf4ea1e8adf47e9f8847 --- /dev/null +++ b/slapos/recipe/stunnel/template/client.conf.in @@ -0,0 +1,9 @@ +foreground = yes +output = %(log)s +pid = %(pid_file)s +syslog = no + +[service] +client = yes +accept = %(local_host)s:%(local_port)s +connect = %(remote_host)s:%(remote_port)s diff --git a/slapos/recipe/stunnel/template/server.conf.in b/slapos/recipe/stunnel/template/server.conf.in new file mode 100644 index 0000000000000000000000000000000000000000..e53bbc5c11aadee509cae300a54086d1b957cc4b --- /dev/null +++ b/slapos/recipe/stunnel/template/server.conf.in @@ -0,0 +1,10 @@ +foreground = yes +output = %(log)s +pid = %(pid_file)s +syslog = no +key = %(key)s +cert = %(cert)s + +[service] +accept = %(remote_address)s:%(remote_port)s +connect = %(local_address)s:%(local_port)s diff --git a/slapos/recipe/vifib.py b/slapos/recipe/vifib.py index 833def71752d05f688b3272201612a750de89203..55ec2243b508aeb3e05114585825ca9e0e59f59d 100644 --- a/slapos/recipe/vifib.py +++ b/slapos/recipe/vifib.py @@ -31,60 +31,8 @@ import zc.buildout import sys class Recipe(slapos.recipe.erp5.Recipe): - - default_bt5_list = [] - def installKeyAuthorisationApache(self, ip, port, backend, key, certificate, - ca_conf, key_auth_path='/erp5/portal_slap'): - ssl_template = """SSLEngine on -SSLVerifyClient require -RequestHeader set REMOTE_USER %%{SSL_CLIENT_S_DN_CN}s -SSLCertificateFile %(key_auth_certificate)s -SSLCertificateKeyFile %(key_auth_key)s -SSLCACertificateFile %(ca_certificate)s -SSLCARevocationPath %(ca_crl)s""" - apache_conf = self._getApacheConfigurationDict('key_auth_apache', ip, port) - apache_conf['ssl_snippet'] = ssl_template % dict( - key_auth_certificate=certificate, - key_auth_key=key, - ca_certificate=ca_conf['ca_certificate'], - ca_crl=ca_conf['ca_crl'] - ) - prefix = 'ssl_key_auth_apache' - rewrite_rule_template = \ - "RewriteRule (.*) http://%(backend)s%(key_auth_path)s$1 [L,P]" - path_template = pkg_resources.resource_string('slapos.recipe.erp5', - 'template/apache.zope.conf.path.in') - path = path_template % dict(path='/') - d = dict( - path=path, - backend=backend, - backend_path='/', - port=apache_conf['port'], - vhname=path.replace('/', ''), - key_auth_path=key_auth_path, - ) - rewrite_rule = rewrite_rule_template % d - apache_conf.update(**dict( - path_enable=path, - rewrite_rule=rewrite_rule - )) - apache_config_file = self.createConfigurationFile(prefix + '.conf', - pkg_resources.resource_string('slapos.recipe.erp5', - 'template/apache.zope.conf.in') % apache_conf) - self.path_list.append(apache_config_file) - self.path_list.extend(zc.buildout.easy_install.scripts([( - 'key_auth_apache', - 'slapos.recipe.erp5.apache', 'runApache')], self.ws, - sys.executable, self.wrapper_directory, arguments=[ - dict( - required_path_list=[certificate, key, ca_conf['ca_certificate'], - ca_conf['ca_crl']], - binary=self.options['httpd_binary'], - config=apache_config_file - ) - ])) - return 'https://%(ip)s:%(port)s' % apache_conf + default_bt5_list = [] def _getZeoClusterDict(self): site_path = '/erp5/' @@ -119,8 +67,8 @@ SSLCARevocationPath %(ca_crl)s""" self.getTemplateFilename('zope-zeo-snippet.conf.in'), dict( storage_name=storage_dict['storage_name'], address='%s:%s' % (storage_dict['ip'], storage_dict['port']), - mount_point=mount_point - ))) + mount_point=mount_point, zodb_cache_size=self.zodb_cache_size, + zeo_client_cache_size=self.zeo_client_cache_size))) tidstorage_config = dict(host=self.getLocalIPv4Address(), port='6001') zodb_configuration_string = '\n'.join(zodb_configuration_list) zope_port = 12000 @@ -150,9 +98,15 @@ SSLCARevocationPath %(ca_crl)s""" login_url_list) apache_login = self.installBackendApache(self.getGlobalIPv6Address(), 15000, login_haproxy, backend_key, backend_certificate) + + # Install Frontend + frontend_domain_name = self.parameter_dict.get("domain_name", 'vifib') + frontend_key, frontend_certificate = \ + self.requestCertificate(frontend_domain_name) apache_frontend_login = self.installFrontendZopeApache( - self.getGlobalIPv6Address(), 4443, 'vifib', '/', - apache_login, '/', backend_key, backend_certificate) + self.getGlobalIPv6Address(), 4443, frontend_domain_name, '/', + apache_login, '', frontend_key, frontend_certificate) + # Four Web Service Nodes (Machine access) service_url_list = [] for i in (1, 2, 3, 4): @@ -166,9 +120,9 @@ SSLCARevocationPath %(ca_crl)s""" key_auth_key, key_auth_certificate = self.requestCertificate( 'Key Based Access') - apache_keyauth = self.installKeyAuthorisationApache( - self.getLocalIPv4Address(), 15500, service_haproxy, key_auth_key, - key_auth_certificate, ca_conf, key_auth_path=self.key_auth_path) + apache_keyauth = self.installKeyAuthorisationApache(False, 15500, + service_haproxy, key_auth_key, key_auth_certificate, ca_conf, + key_auth_path=self.key_auth_path) memcached_conf = self.installMemcached(ip=self.getLocalIPv4Address(), port=11000) kumo_conf = self.installKumo(self.getLocalIPv4Address()) @@ -179,7 +133,7 @@ SSLCARevocationPath %(ca_crl)s""" # Connect direct to Zope to create the instance. self.installERP5Site(user, password, service_url_list[-1], mysql_conf, conversion_server_conf, memcached_conf, kumo_conf, - self.site_id, self.default_bt5_list) + self.site_id, self.default_bt5_list, ca_conf) self.setConnectionDict(dict( front_end_url=apache_frontend_login, @@ -191,12 +145,6 @@ SSLCARevocationPath %(ca_crl)s""" kumo_url=kumo_conf['kumo_address'], conversion_server_url='%(conversion_server_ip)s:%(conversion_server_port)s' % conversion_server_conf, - # openssl binary might be removed, as soon as CP environment will be - # fully controlled - openssl_binary=self.options['openssl_binary'], - # As soon as there would be Vifib ERP5 configuration and possibility to - # call it over the network this can be removed - certificate_authority_path=ca_conf['certificate_authority_path'], # as installERP5Site is not trusted (yet) and this recipe is production # ready expose more information mysql_url='%(mysql_database)s@%(ip)s:%(tcp_port)s %(mysql_user)s %(mysql_password)s' % mysql_conf, @@ -213,13 +161,14 @@ SSLCARevocationPath %(ca_crl)s""" user, password = self.installERP5() zodb_dir = os.path.join(self.data_root_directory, 'zodb') self._createDirectory(zodb_dir) - zodb_root_path = os.path.join(zodb_dir, 'root.fs') + zodb_root_path = os.path.join(zodb_dir, 'main.fs') ip = self.getLocalIPv4Address() zope_port = '18080' zope_access = self.installZope(ip, zope_port, 'zope_development', zodb_configuration_string=self.substituteTemplate( self.getTemplateFilename('zope-zodb-snippet.conf.in'), - dict(zodb_root_path=zodb_root_path)), + dict(zodb_root_path=zodb_root_path, + zodb_cache_size=self.zodb_cache_size)), thread_amount=8, with_timerservice=True) service_haproxy = self.installHaproxy(ip, 15000, 'service', self.site_check_path, [zope_access]) @@ -238,7 +187,7 @@ SSLCARevocationPath %(ca_crl)s""" self.linkBinary() self.installERP5Site(user, password, zope_access, mysql_conf, conversion_server_conf, memcached_conf, kumo_conf, - self.site_id, self.default_bt5_list) + self.site_id, self.default_bt5_list, ca_conf) self.setConnectionDict(dict( development_zope='http://%s:%s/' % (ip, zope_port), @@ -249,12 +198,6 @@ SSLCARevocationPath %(ca_crl)s""" kumo_url=kumo_conf['kumo_address'], conversion_server_url='%(conversion_server_ip)s:%(conversion_server_port)s' % conversion_server_conf, - # openssl binary might be removed, as soon as CP environment will be - # fully controlled - openssl_binary=self.options['openssl_binary'], - # As soon as there would be Vifib ERP5 configuration and possibility to - # call it over the network this can be removed - certificate_authority_path=ca_conf['certificate_authority_path'], # as installERP5Site is not trusted (yet) and this recipe is production # ready expose more information mysql_url='%(mysql_database)s@%(ip)s:%(tcp_port)s %(mysql_user)s %(mysql_password)s' % mysql_conf, @@ -267,6 +210,9 @@ SSLCARevocationPath %(ca_crl)s""" self.path_list = [] self.requirements, self.ws = self.egg.working_set() # self.cron_d is a directory, where cron jobs can be registered + self.zodb_cache_size = int(self.options.get('zodb_cache_size', 5000)) + self.zeo_client_cache_size = self.options.get('zeo_client_cache_size', + '20MB') self.cron_d = self.installCrond() self.logrotate_d, self.logrotate_backup = self.installLogrotate() self.killpidfromfile = zc.buildout.easy_install.scripts( @@ -276,8 +222,6 @@ SSLCARevocationPath %(ca_crl)s""" if self.parameter_dict.get("flavour", "default") == 'configurator': self.default_bt5_list = self.options.get("configurator_bt5_list", '').split() - if self.parameter_dict.get('development', 'false').lower() == 'true': - return self.installDevelopment() if self.parameter_dict.get('production', 'false').lower() == 'true': return self.installProduction() - raise NotImplementedError('Flavour of instance have to be given.') + return self.installDevelopment() diff --git a/slapos/recipe/xwiki/__init__.py b/slapos/recipe/xwiki/__init__.py index 8dbfcd7c72ebf544072c2a6828d4c9194b62b070..5f55423d78a69e38c42c7af950decb57887aabb6 100644 --- a/slapos/recipe/xwiki/__init__.py +++ b/slapos/recipe/xwiki/__init__.py @@ -35,11 +35,12 @@ import zc.buildout class Recipe(BaseSlapRecipe): def _install(self): + self.requirements, self.ws = self.egg.working_set() parameter_dict = self.computer_partition.getInstanceParameterDict() - ipv4 = self.getLocalIPv4Address(parameter_dict) - ipv6 = self.getGlobalIPv6Address(parameter_dict) + ipv4 = self.getLocalIPv4Address() + ipv6 = self.getGlobalIPv6Address() - self.install_mysql_server_configuration(self.getLocalIPv4Address(parameter_dict)) + self.install_mysql_server_configuration(ipv4) port = '8900' tomcat_home = os.path.join(self.data_root_directory, 'tomcat') @@ -56,8 +57,8 @@ class Recipe(BaseSlapRecipe): shtuil.rmtree(dst) raise - shutil.copy(self.options['hsql_location'].strip(), os.path.join(tomcat_lib, - 'hsqldb.jar')) + shutil.copy(self.options['jdbc_location'].strip(), os.path.join(tomcat_lib, + 'jdbc.jar')) # headless mode self._writeFile(os.path.join(tomcat_home, 'bin', 'setenv.sh'), '''#!/bin/sh export JAVA_OPTS="${JAVA_OPTS} -Djava.awt.headless=true" diff --git a/slapos/recipe/xwiki/template/hibernate.cfg.xml.in b/slapos/recipe/xwiki/template/hibernate.cfg.xml.in index 91196e74a928ca7c008ed82396aa8c074eb4586e..78d25f78bfa77b23b263ef21875e237470d5995b 100644 --- a/slapos/recipe/xwiki/template/hibernate.cfg.xml.in +++ b/slapos/recipe/xwiki/template/hibernate.cfg.xml.in @@ -49,7 +49,7 @@ Uncomment if you want to use MySQL and comment out other database configurations. We need to set the sql_mode to a less strict value, see XWIKI-1945 --> - <property name="connection.url">jdbc:mysql://%(mysql_ip)s:%(mysql_port)s/xwiki?useServerPrepStmts=false&useUnicode=true&characterEncoding=UTF-8&sessionVariables=sql_mode=''</property> + <property name="connection.url">jdbc:mysql://%(mysql_ip)s:%(mysql_port)s/xwiki?useServerPrepStmts=false&useUnicode=true&characterEncoding=UTF-8&sessionVariables=&sql_mode=''</property> <property name="connection.username">xwiki</property> <property name="connection.password">xwiki</property> <property name="connection.driver_class">com.mysql.jdbc.Driver</property> diff --git a/slapos/recipe/xwiki/template/my.cnf.in b/slapos/recipe/xwiki/template/my.cnf.in index 5e2742175bab0e7406b7e9c01d586f9b2859fb8a..09171fb6d9837199f4030f574791df31c251dc54 100644 --- a/slapos/recipe/xwiki/template/my.cnf.in +++ b/slapos/recipe/xwiki/template/my.cnf.in @@ -30,9 +30,6 @@ query_cache_size = 32M # Try number of CPU's*2 for thread_concurrency thread_concurrency = 8 -# Disable Federated by default -skip-federated - # Replication Master Server (default) # binary logging is required for replication log-bin=mysql-bin diff --git a/slapos/recipe/zabbixagent/__init__.py b/slapos/recipe/zabbixagent/__init__.py index ef587e32ae39bb410ddb7107283990a9c9c22540..41d9a469b6f38417bf5ba754ea51d8d354846e7a 100644 --- a/slapos/recipe/zabbixagent/__init__.py +++ b/slapos/recipe/zabbixagent/__init__.py @@ -79,35 +79,50 @@ class Recipe(BaseSlapRecipe): self.path_list.append(wrapper) return cron_d - def _install(self): - self.path_list = [] - self.requirements, self.ws = self.egg.working_set() - # self.cron_d is a directory, where cron jobs can be registered - self.cron_d = self.installCrond() - self.logrotate_d, self.logrotate_backup = self.installLogrotate() - zabbix_log_file = os.path.join(self.log_directory, 'zabbix_agentd.log') - self.registerLogRotation('zabbix_agentd', [zabbix_log_file]) - zabbix_agentd = dict( + def installZabbixAgentd(self, ip, port, hostname, server_ip, + user_parameter_string=''): + log_file = os.path.join(self.log_directory, 'zabbix_agentd.log') + self.registerLogRotation('zabbix_agentd', [log_file]) + + zabbix_agentd_conf = dict( pid_file=os.path.join(self.run_directory, "zabbix_agentd.pid"), - log_file=zabbix_log_file, - ip=self.getGlobalIPv6Address(), - server=self.parameter_dict['server'], - hostname=self.parameter_dict['hostname'], - port='10050' - ) - zabbix_agentd_conf = self.createConfigurationFile("zabbix_agentd.conf", - pkg_resources.resource_string(__name__, - 'template/zabbix_agentd.conf.in') % zabbix_agentd) - self.path_list.append(zabbix_agentd_conf) + log_file=log_file, + ip=ip, + server=server_ip, + hostname=hostname, + port=port, + user_parameter_string=user_parameter_string) + + zabbix_agentd_path = self.createConfigurationFile( + "zabbix_agentd.conf", + pkg_resources.resource_string( + __name__, 'template/zabbix_agentd.conf.in') % zabbix_agentd_conf) + + self.path_list.append(zabbix_agentd_path) + wrapper = zc.buildout.easy_install.scripts([('zabbixagentd', 'slapos.recipe.librecipe.execute', 'execute')], self.ws, sys.executable, self.bin_directory, arguments=[ self.options['zabbix_agentd_binary'].strip(), '-c', - zabbix_agentd_conf])[0] + zabbix_agentd_path])[0] + self.path_list.extend(zc.buildout.easy_install.scripts([ ('zabbixagentd', __name__ + '.svcdaemon', 'svcdaemon')], self.ws, sys.executable, self.wrapper_directory, arguments=[dict( - real_binary=wrapper, pid_file=zabbix_agentd['pid_file'])])) - self.setConnectionDict(dict(ip=zabbix_agentd['ip'], - name=zabbix_agentd['hostname'], port=zabbix_agentd['port'])) + real_binary=wrapper, pid_file=zabbix_agentd_conf['pid_file'])])) + + return zabbix_agentd_conf + + def _install(self): + self.path_list = [] + self.requirements, self.ws = self.egg.working_set() + # self.cron_d is a directory, where cron jobs can be registered + self.cron_d = self.installCrond() + self.logrotate_d, self.logrotate_backup = self.installLogrotate() + zabbix_agentd_conf = self.installZabbixAgentd(self.getGlobalIPv6Address(), + 10050, + self.parameter_dict['hostname'], + self.parameter_dict['server']) + self.setConnectionDict(dict(ip=zabbix_agentd_conf['ip'], + name=zabbix_agentd_conf['hostname'], port=zabbix_agentd_conf['port'])) return self.path_list diff --git a/slapos/recipe/zabbixagent/template/zabbix_agentd.conf.in b/slapos/recipe/zabbixagent/template/zabbix_agentd.conf.in index 9aef9930ca937ea7e1ef55a667676b2303b0d0c5..d5ba48be5664ebecc7c7d980d583e74086cbb9fd 100644 --- a/slapos/recipe/zabbixagent/template/zabbix_agentd.conf.in +++ b/slapos/recipe/zabbixagent/template/zabbix_agentd.conf.in @@ -229,3 +229,5 @@ ListenIP=%(ip)s # Mandatory: no # Default: # UserParameter= + +%(user_parameter_string)s