diff --git a/slapos/recipe/erp5/__init__.py b/slapos/recipe/erp5/__init__.py index f1a96ecc4d9ac2a1f6c4de3f5e6a572ea15f9c09..53ac57a023914b605dfb8659e358b13fe09dd21a 100644 --- a/slapos/recipe/erp5/__init__.py +++ b/slapos/recipe/erp5/__init__.py @@ -56,7 +56,7 @@ class Recipe(BaseSlapRecipe): # kumo_conf = self.installKumo(self.getLocalIPv4Address()) # conversion_server_conf = self.installConversionServer( # self.getLocalIPv4Address(), 23000, 23060) - mysql_conf = self.installMysqlServer(self.getLocalIPv4Address(), 45678) +# mysql_conf = self.installMysqlServer(self.getLocalIPv4Address(), 45678) user, password = self.installERP5() if self.parameter_dict.get("slap_software_type", "").lower() == "cluster": @@ -848,142 +848,3 @@ SSLCARevocationPath %(ca_crl)s""" ])) # Note: IPv6 is assumed always return 'https://[%(ip)s]:%(port)s' % apache_conf - - def installMysqlServer(self, ip, port, database='erp5', user='user', - test_database='test_erp5', test_user='test_user', template_filename=None, - parallel_test_database_amount=100, mysql_conf=None, with_backup=True, - with_maatkit=True): - if mysql_conf is None: - mysql_conf = {} - backup_directory = self.createBackupDirectory('mysql') - if template_filename is None: - template_filename = self.getTemplateFilename('my.cnf.in') - error_log = os.path.join(self.log_directory, 'mysqld.log') - slow_query_log = os.path.join(self.log_directory, 'mysql-slow.log') - mysql_conf.update( - ip=ip, - data_directory=os.path.join(self.data_root_directory, - 'mysql'), - tcp_port=port, - pid_file=os.path.join(self.run_directory, 'mysqld.pid'), - socket=os.path.join(self.run_directory, 'mysqld.sock'), - error_log=error_log, - slow_query_log=slow_query_log, - mysql_database=database, - mysql_user=user, - mysql_password=self.generatePassword(), - mysql_test_password=self.generatePassword(), - mysql_test_database=test_database, - mysql_test_user=test_user, - mysql_parallel_test_dict=[ - ('test_%i' % x,)*2 + (self.generatePassword(),) \ - for x in xrange(0,parallel_test_database_amount)], - ) - self.registerLogRotation('mysql', [error_log, slow_query_log], - '%(mysql_binary)s --no-defaults -B --user=root ' - '--socket=%(mysql_socket)s -e "FLUSH LOGS"' % dict( - mysql_binary=self.options['mysql_binary'], - mysql_socket=mysql_conf['socket'])) - self._createDirectory(mysql_conf['data_directory']) - - mysql_conf_path = self.createConfigurationFile("my.cnf", - self.substituteTemplate(template_filename, - mysql_conf)) - - mysql_script_list = [] - for x_database, x_user, x_password in \ - [(mysql_conf['mysql_database'], - mysql_conf['mysql_user'], - mysql_conf['mysql_password']), - (mysql_conf['mysql_test_database'], - mysql_conf['mysql_test_user'], - mysql_conf['mysql_test_password']), - ] + mysql_conf['mysql_parallel_test_dict']: - mysql_script_list.append(pkg_resources.resource_string(__name__, - 'template/initmysql.sql.in') % { - 'mysql_database': x_database, - 'mysql_user': x_user, - 'mysql_password': x_password}) - mysql_script_list.append('EXIT') - mysql_script = '\n'.join(mysql_script_list) - self.path_list.extend(zc.buildout.easy_install.scripts([('mysql_update', - __name__ + '.mysql', 'updateMysql')], self.ws, - sys.executable, self.wrapper_directory, arguments=[dict( - mysql_script=mysql_script, - mysql_binary=self.options['mysql_binary'].strip(), - mysql_upgrade_binary=self.options['mysql_upgrade_binary'].strip(), - socket=mysql_conf['socket'], - )])) - self.path_list.extend(zc.buildout.easy_install.scripts([('mysqld', - __name__ + '.mysql', 'runMysql')], self.ws, - sys.executable, self.wrapper_directory, arguments=[dict( - mysql_install_binary=self.options['mysql_install_binary'].strip(), - mysqld_binary=self.options['mysqld_binary'].strip(), - data_directory=mysql_conf['data_directory'].strip(), - mysql_binary=self.options['mysql_binary'].strip(), - socket=mysql_conf['socket'].strip(), - configuration_file=mysql_conf_path, - )])) - self.path_list.extend([mysql_conf_path]) - - if with_backup: - # backup configuration - backup_directory = self.createBackupDirectory('mysql') - full_backup = os.path.join(backup_directory, 'full') - incremental_backup = os.path.join(backup_directory, 'incremental') - self._createDirectory(full_backup) - self._createDirectory(incremental_backup) - innobackupex_argument_list = [self.options['perl_binary'], - self.options['innobackupex_binary'], - '--defaults-file=%s' % mysql_conf_path, - '--socket=%s' %mysql_conf['socket'].strip(), '--user=root', - '--ibbackup=%s'% self.options['xtrabackup_binary']] - environment = dict(PATH='%s' % self.bin_directory) - innobackupex_incremental = zc.buildout.easy_install.scripts([( - 'innobackupex_incremental','slapos.recipe.librecipe.execute', 'executee')], - self.ws, sys.executable, self.bin_directory, arguments=[ - innobackupex_argument_list + ['--incremental'], - environment])[0] - self.path_list.append(innobackupex_incremental) - innobackupex_full = zc.buildout.easy_install.scripts([('innobackupex_full', - 'slapos.recipe.librecipe.execute', 'executee')], self.ws, - sys.executable, self.bin_directory, arguments=[ - innobackupex_argument_list, - environment])[0] - self.path_list.append(innobackupex_full) - backup_controller = zc.buildout.easy_install.scripts([ - ('innobackupex_controller', __name__ + '.innobackupex', 'controller')], - self.ws, sys.executable, self.bin_directory, - arguments=[innobackupex_incremental, innobackupex_full, full_backup, - incremental_backup])[0] - self.path_list.append(backup_controller) - mysql_backup_cron = os.path.join(self.cron_d, 'mysql_backup') - open(mysql_backup_cron, 'w').write('0 0 * * * ' + backup_controller) - self.path_list.append(mysql_backup_cron) - - if with_maatkit: - # maatkit installation - for mk_script_name in ( - 'mk-variable-advisor', - 'mk-table-usage', - 'mk-visual-explain', - 'mk-config-diff', - 'mk-deadlock-logger', - 'mk-error-log', - 'mk-index-usage', - 'mk-query-advisor', - ): - mk_argument_list = [self.options['perl_binary'], - self.options['%s_binary' % mk_script_name], - '--defaults-file=%s' % mysql_conf_path, - '--socket=%s' %mysql_conf['socket'].strip(), '--user=root', - ] - environment = dict(PATH='%s' % self.bin_directory) - mk_exe = zc.buildout.easy_install.scripts([( - mk_script_name,'slapos.recipe.librecipe.execute', 'executee')], - self.ws, sys.executable, self.bin_directory, arguments=[ - mk_argument_list, environment])[0] - self.path_list.append(mk_exe) - - # The return could be more explicit database, user ... - return mysql_conf diff --git a/slapos/recipe/generic_mysql/__init__.py b/slapos/recipe/generic_mysql/__init__.py index 9f3f2a429837643c55685e6c5d8cdc9db370e91a..0845c05de7c1698bf010932fa5583ca8ce303797 100644 --- a/slapos/recipe/generic_mysql/__init__.py +++ b/slapos/recipe/generic_mysql/__init__.py @@ -31,6 +31,9 @@ class Recipe(GenericBaseRecipe): def _options(self, options): options['password'] = self.generatePassword() + options['test_password'] = self.generatePassword() + for x in xrange(0, int(options['parallel-test-database-amount'])): + options['test_password_%s' % x] = self.generatePassword() def install(self): path_list = [] @@ -48,6 +51,9 @@ class Recipe(GenericBaseRecipe): mysql_database=self.options['database'], mysql_user=self.options['user'], mysql_password=self.options['password'], + mysql_test_database=self.options['test-database'], + mysql_test_user=self.options['test-user'], + mysql_test_password=self.options['test-password'], ) mysql_binary = self.options['mysql-binary'] @@ -68,15 +74,34 @@ class Recipe(GenericBaseRecipe): mysql_script_list = [] - init_script = self.substituteTemplate( + # real database + mysql_script_list.append(self.substituteTemplate( self.getTemplateFilename('initmysql.sql.in'), { 'mysql_database': mysql_conf['mysql_database'], 'mysql_user': mysql_conf['mysql_user'], 'mysql_password': mysql_conf['mysql_password'] } - ) - mysql_script_list.append(init_script) + )) + # default test database + mysql_script_list.append(self.substituteTemplate( + self.getTemplateFilename('initmysql.sql.in'), + { + 'mysql_database': mysql_conf['mysql_test_database'], + 'mysql_user': mysql_conf['mysql_test_user'], + 'mysql_password': mysql_conf['mysql_test_password'] + } + )) + # parallel test databases + for x in xrange(0, int(self.options['parallel-test-database-amount'])): + mysql_script_list.append(self.substituteTemplate( + self.getTemplateFilename('initmysql.sql.in'), + { + 'mysql_database': self.options['mysql-test-database-base'] + '_%s' % x, + 'mysql_user': self.options['mysql-test-user-base'] + '_%s' % x, + 'mysql_password': self.options['test-password-%s' % x] + } + )) mysql_script_list.append('EXIT') mysql_script = '\n'.join(mysql_script_list) @@ -107,5 +132,39 @@ class Recipe(GenericBaseRecipe): ) ) path_list.append(mysqld) + # backup configuration + full_backup = self.options['full-backup-path'] + incremental_backup = self.options['incremental-backup-path'] + innobackupex_argument_list = [self.options['perl-binary'], + self.options['innobackupex-binary'], + '--defaults-file=%s' % mysql_conf_file, + '--socket=%s' %mysql_conf['socket'].strip(), '--user=root', + '--ibbackup=%s'% self.options['xtrabackup-binary']] + environment = dict(PATH='%s' % self.bin_directory) + innobackupex_incremental = self.createPythonScript(self.options['innobackupex-incremental'], 'slapos.recipe.librecipe.execute.executee', [innobackupex_argument_list + ['--incremental'], environment]) + path_list.append(innobackupex_incremental) + innobackupex_full = self.createPythonScript(self.options['innobackupex-full'], 'slapos.recipe.librecipe.execute.executee', [innobackupex_argument_list, environment]) + path_list.append(innobackupex_full) + backup_controller = self.createPythonScript(self.options['innobackupex-controller'], __name__ + '.innobackupex.controller', [innobackupex_incremental, innobackupex_full, full_backup, incremental_backup]) + path_list.append(backup_controller) + # maatkit installation + for mk_script_name in ( + 'mk-variable-advisor', + 'mk-table-usage', + 'mk-visual-explain', + 'mk-config-diff', + 'mk-deadlock-logger', + 'mk-error-log', + 'mk-index-usage', + 'mk-query-advisor', + ): + mk_argument_list = [self.options['perl_binary'], + self.options['%s_binary' % mk_script_name], + '--defaults-file=%s' % mysql_conf_file, + '--socket=%s' %mysql_conf['socket'].strip(), '--user=root', + ] + environment = dict(PATH='%s' % self.bin_directory) + mk_exe = self.createPythonScript(os.path.join(self.bin_directory, 'mk_script_name,'), 'slapos.recipe.librecipe.execute.executee', [mk_argument_list, environment]) + path_list.append(mk_exe) return path_list diff --git a/slapos/recipe/erp5/innobackupex.py b/slapos/recipe/generic_mysql/innobackupex.py similarity index 100% rename from slapos/recipe/erp5/innobackupex.py rename to slapos/recipe/generic_mysql/innobackupex.py diff --git a/software/erp5/instance-mariadb.cfg b/software/erp5/instance-mariadb.cfg index 669c78d27e00a9e705ecdd74d6dd8faf528a044f..2ac83f17126fa1c65e8bb4f0a1e2b80a5ea01258 100644 --- a/software/erp5/instance-mariadb.cfg +++ b/software/erp5/instance-mariadb.cfg @@ -6,20 +6,27 @@ parts = logrotate-entry-mariadb cron cron-entry-logrotate + cron-entry-mariadb-backup eggs-directory = ${buildout:eggs-directory} develop-eggs-directory = ${buildout:develop-eggs-directory} offline = true -[url] +[publish-mariadb-url] recipe = slapos.cookbook:publishurl url = mysqls://$${mariadb:user}:$${mariadb:password}@$${mariadb:ip}:$${mariadb:port}/$${mariadb:database} +[cron-entry-mariadb-backup] +<= cron +recipe = slapos.cookbook:cron.d +name = mariadb-backup +frequency = 0 0 * * * +command = $${mariadb:innobackupex-controller} + [mariadb] recipe = slapos.cookbook:mysql # Options -recovering = false user = user port = 45678 ip = $${slap-network-information:local-ipv4} @@ -39,12 +46,20 @@ backup-pending-directory = $${directory:mariadb-backup-pending} dumpname = dump.sql.gz # Binary information +innobackupex-binary = ${xtrabackup:location}/bin/innobackupex mysql-binary = ${mariadb:location}/bin/mysql mysql-install-binary = ${mariadb:location}/bin/mysql_install_db mysql-upgrade-binary = ${mariadb:location}/bin/mysql_upgrade mysqld-binary = ${mariadb:location}/libexec/mysqld -mysqldump-binary = ${mariadb:location}/bin/mysqldump - +mk-variable-advisor_binary = ${perl:siteprefix}/bin/mk-variable-advisor +mk-table-usage_binary = ${perl:siteprefix}/bin/mk-table-usage +mk-visual-explain_binary = ${perl:siteprefix}/bin/mk-visual-explain +mk-config-diff_binary = ${perl:siteprefix}/bin/mk-config-diff +mk-deadlock-logger_binary = ${perl:siteprefix}/bin/mk-deadlock-logger +mk-error-log_binary = ${perl:siteprefix}/bin/mk-error-log +mk-index-usage_binary = ${perl:siteprefix}/bin/mk-index-usage +mk-query-advisor_binary = ${perl:siteprefix}/bin/mk-query-advisor +xtrabackup-binary = ${xtrabackup:location}/bin/xtrabackup_51 [logrotate] recipe = slapos.cookbook:logrotate diff --git a/software/erp5/instance.cfg b/software/erp5/instance.cfg index c2e427c92b287f811071b850744b72d02b13f501..c60f029e2e07f513f694cb374061a43dd1bc38c3 100644 --- a/software/erp5/instance.cfg +++ b/software/erp5/instance.cfg @@ -10,12 +10,7 @@ recipe = ${instance-recipe:egg}:${instance-recipe:module} dcrond_binary = ${dcron:location}/sbin/crond gzip_binary = ${gzip:location}/bin/gzip httpd_binary = ${apache:location}/bin/httpd -innobackupex_binary = ${xtrabackup:location}/bin/innobackupex logrotate_binary = ${logrotate:location}/usr/sbin/logrotate -mysql_binary = ${mariadb:location}/bin/mysql -mysql_install_binary = ${mariadb:location}/bin/mysql_install_db -mysql_upgrade_binary = ${mariadb:location}/bin/mysql_upgrade -mysqld_binary = ${mariadb:location}/libexec/mysqld openssl_binary = ${openssl:location}/bin/openssl perl_binary = ${perl:location}/bin/perl rdiff_backup_binary = ${buildout:bin-directory}/rdiff-backup @@ -26,16 +21,7 @@ runzope_binary = ${buildout:bin-directory}/runzope sphinx_searchd_binary = ${sphinx:location}/bin/searchd tidstorage_repozo_binary = ${buildout:bin-directory}/tidstorage_repozo tidstoraged_binary = ${buildout:bin-directory}/tidstoraged -xtrabackup_binary = ${xtrabackup:location}/bin/xtrabackup_51 zabbix_agentd_binary = ${zabbix-agent:location}/sbin/zabbix_agentd -mk-variable-advisor_binary = ${perl:siteprefix}/bin/mk-variable-advisor -mk-table-usage_binary = ${perl:siteprefix}/bin/mk-table-usage -mk-visual-explain_binary = ${perl:siteprefix}/bin/mk-visual-explain -mk-config-diff_binary = ${perl:siteprefix}/bin/mk-config-diff -mk-deadlock-logger_binary = ${perl:siteprefix}/bin/mk-deadlock-logger -mk-error-log_binary = ${perl:siteprefix}/bin/mk-error-log -mk-index-usage_binary = ${perl:siteprefix}/bin/mk-index-usage -mk-query-advisor_binary = ${perl:siteprefix}/bin/mk-query-advisor link_binary_list = ${coreutils:location}/bin/basename diff --git a/software/erp5/software.cfg b/software/erp5/software.cfg index eb621589cf67c6493648e26ec90ac68d02eabbd3..926d994e7184de8e7b5ec45fbe25162d50e7522f 100644 --- a/software/erp5/software.cfg +++ b/software/erp5/software.cfg @@ -25,7 +25,7 @@ unzip = true [template-mariadb] recipe = slapos.recipe.template url = ${:_profile_base_location_}/instance-mariadb.cfg -md5sum = 06da9043777ba8e1fda5798cc8000535 +md5sum = bf1b49d90b268ba4e69d8423c3435b65 output = ${buildout:directory}/template-mariadb.cfg mode = 0644