Commit 723e9dbb authored by Kazuhiko Shiozaki's avatar Kazuhiko Shiozaki

Merge branch 'master' into erp5-component

parents e4981ef8 1e343058
Changes
=======
0.77.1 (2013-04-18)
-------------------
* Re-release of 0.77.0.
0.77.0 (2013-04-18)
-------------------
* Allow to pass extra parameters when creating simple wrapper. [Sebastien Robin]
* Apache frontend: Append all rewrite module options to http as well. [Cedric de Saint Martin]
* Apache frontend: Add https-only support. [Cedric de Saint Martin]
* Apache frontend: make logrotate work by using "generic" component. [Cedric de Saint Martin]
0.76.0 (2013-04-03)
-------------------
......
......@@ -28,7 +28,7 @@ from setuptools import setup, find_packages
import glob
import os
version = '0.76.1.dev'
version = '0.77.2.dev'
name = 'slapos.cookbook'
long_description = open("README.txt").read() + "\n" + \
open("CHANGES.txt").read() + "\n"
......
......@@ -36,6 +36,7 @@ import ConfigParser
import re
import traceback
TRUE_VALUES = ['y', 'yes', '1', 'true']
class Recipe(BaseSlapRecipe):
......@@ -65,13 +66,13 @@ class Recipe(BaseSlapRecipe):
# self.cron_d is a directory, where cron jobs can be registered
self.cron_d = self.installCrond()
self.logrotate_d, self.logrotate_backup = self.installLogrotate()
self.killpidfromfile = zc.buildout.easy_install.scripts(
[('killpidfromfile', 'slapos.recipe.erp5.killpidfromfile',
[('killpidfromfile', 'slapos.toolbox.killpidfromfile',
'killpidfromfile')], self.ws, sys.executable, self.bin_directory)[0]
self.path_list.append(self.killpidfromfile)
rewrite_rule_list = []
rewrite_rule_https_only_list = []
rewrite_rule_zope_list = []
rewrite_rule_zope_path_list = []
slave_dict = {}
......@@ -88,19 +89,15 @@ class Recipe(BaseSlapRecipe):
# Sanitize inputs
backend_url = slave_instance.get("url", None)
reference = slave_instance.get("slave_reference")
enable_cache = slave_instance.get('enable_cache', '').lower() in TRUE_VALUES
slave_type = slave_instance.get('type', '').lower() or None
if slave_instance.haskey("enable_cache"):
enable_cache = slave_instance.get("enable_cache", "").upper() in ('1', 'TRUE')
else:
enable_cache = False
if slave_instance.haskey("type"):
slave_type = slave_instance.get("type", "").lower()
else:
slave_type = None
https_only = slave_instance.get('https-only', '').lower() in TRUE_VALUES
# Set scheme (http? https?)
# Future work may allow to choose between http and https (or both?)
if https_only:
scheme = 'https://'
else:
scheme = 'http://'
self.logger.info('Processing slave instance: %s' % reference)
......@@ -143,6 +140,10 @@ class Recipe(BaseSlapRecipe):
rewrite_rule = "%s %s" % (domain, backend_url)
# Finally, if successful, we add the rewrite rule to our list of rules
# We have 4 RewriteMaps:
# - One for generic (non-zope) websites, accepting both HTTP and HTTPS
# - One for generic websites that only accept HTTPS
# - Two for Zope-based websites
if rewrite_rule:
# We check if we have a zope slave. It requires different rewrite
# rule structure.
......@@ -153,6 +154,9 @@ class Recipe(BaseSlapRecipe):
# For Zope, we have another dict containing the path e.g '/erp5/...
rewrite_rule_path = "%s %s" % (domain, slave_instance.get('path', ''))
rewrite_rule_zope_path_list.append(rewrite_rule_path)
else:
if https_only:
rewrite_rule_https_only_list.append(rewrite_rule)
else:
rewrite_rule_list.append(rewrite_rule)
......@@ -186,6 +190,7 @@ class Recipe(BaseSlapRecipe):
plain_http_port=frontend_plain_http_port_number,
name=frontend_domain_name,
rewrite_rule_list=rewrite_rule_list,
rewrite_rule_https_only_list=rewrite_rule_https_only_list,
rewrite_rule_zope_list=rewrite_rule_zope_list,
rewrite_rule_zope_path_list=rewrite_rule_zope_path_list,
key=key, certificate=certificate)
......@@ -279,29 +284,6 @@ class Recipe(BaseSlapRecipe):
return "%s http://%s:%s" % \
(domain, varnish_ip, base_varnish_port)
def installLogrotate(self):
"""Installs logortate main configuration file and registers its to cron"""
logrotate_d = os.path.abspath(os.path.join(self.etc_directory,
'logrotate.d'))
self._createDirectory(logrotate_d)
logrotate_backup = self.createBackupDirectory('logrotate')
logrotate_conf = self.createConfigurationFile("logrotate.conf",
"include %s" % logrotate_d)
logrotate_cron = os.path.join(self.cron_d, 'logrotate')
state_file = os.path.join(self.data_root_directory, 'logrotate.status')
open(logrotate_cron, 'w').write('0 0 * * * %s -s %s %s' %
(self.options['logrotate_binary'], state_file, logrotate_conf))
self.path_list.extend([logrotate_d, logrotate_conf, logrotate_cron])
return logrotate_d, logrotate_backup
def registerLogRotation(self, name, log_file_list, postrotate_script):
"""Register new log rotation requirement"""
open(os.path.join(self.logrotate_d, name), 'w').write(
self.substituteTemplate(self.getTemplateFilename(
'logrotate_entry.in'),
dict(file_list=' '.join(['"'+q+'"' for q in log_file_list]),
postrotate=postrotate_script, olddir=self.logrotate_backup)))
def requestCertificate(self, name):
hash = hashlib.sha512(name).hexdigest()
key = os.path.join(self.ca_private, hash + self.ca_key_ext)
......@@ -418,8 +400,7 @@ class Recipe(BaseSlapRecipe):
def _getApacheConfigurationDict(self, name, ip_list, port):
apache_conf = dict()
apache_conf['server_name'] = name
apache_conf['pid_file'] = os.path.join(self.run_directory,
name + '.pid')
apache_conf['pid_file'] = self.options['pid-file']
apache_conf['lock_file'] = os.path.join(self.run_directory,
name + '.lock')
apache_conf['document_root'] = os.path.join(self.data_root_directory,
......@@ -429,13 +410,8 @@ class Recipe(BaseSlapRecipe):
apache_conf['ip_list'] = ip_list
apache_conf['port'] = port
apache_conf['server_admin'] = 'admin@'
apache_conf['error_log'] = os.path.join(self.log_directory,
'frontend-apache-error.log')
apache_conf['access_log'] = os.path.join(self.log_directory,
'frontend-apache-access.log')
self.registerLogRotation(name, [apache_conf['error_log'],
apache_conf['access_log']], self.killpidfromfile + ' ' +
apache_conf['pid_file'] + ' SIGUSR1')
apache_conf['error_log'] = self.options['error-log']
apache_conf['access_log'] = self.options['access-log']
return apache_conf
def installVarnishCache(self, name, ip, port, control_port, backend_host,
......@@ -517,10 +493,13 @@ class Recipe(BaseSlapRecipe):
port=4443, plain_http_port=8080,
rewrite_rule_list=None,
rewrite_rule_zope_list=None,
rewrite_rule_https_only_list=None,
rewrite_rule_zope_path_list=None,
access_control_string=None):
if rewrite_rule_list is None:
rewrite_rule_list = []
if rewrite_rule_https_only_list is None:
rewrite_rule_zope_path_list = []
if rewrite_rule_zope_list is None:
rewrite_rule_zope_list = []
if rewrite_rule_zope_path_list is None:
......@@ -571,15 +550,22 @@ class Recipe(BaseSlapRecipe):
self.path_list.append(backup_cron)
# Create configuration file and rewritemaps
apachemap_name = "apachemap.txt"
apachemapzope_name = "apachemapzope.txt"
apachemapzopepath_name = "apachemapzopepath.txt"
self.createConfigurationFile(apachemap_name, "\n".join(rewrite_rule_list))
self.createConfigurationFile(apachemapzope_name,
"\n".join(rewrite_rule_zope_list))
self.createConfigurationFile(apachemapzopepath_name,
"\n".join(rewrite_rule_zope_path_list))
apachemap_path = self.createConfigurationFile(
"apache_rewritemap_generic.txt",
"\n".join(rewrite_rule_list)
)
apachemap_httpsonly_path = self.createConfigurationFile(
"apache_rewritemap_httpsonly.txt",
"\n".join(rewrite_rule_https_only_list)
)
apachemap_zope_path = self.createConfigurationFile(
"apache_rewritemap_zope.txt",
"\n".join(rewrite_rule_zope_list)
)
apachemap_zopepath_path = self.createConfigurationFile(
"apache_rewritemap_zopepath.txt",
"\n".join(rewrite_rule_zope_path_list)
)
apache_conf = self._getApacheConfigurationDict(name, ip_list, port)
apache_conf['ssl_snippet'] = self.substituteTemplate(
......@@ -606,9 +592,10 @@ class Recipe(BaseSlapRecipe):
apache_conf.update(**dict(
path_enable=path,
apachemap_path=os.path.join(self.etc_directory, apachemap_name),
apachemapzope_path=os.path.join(self.etc_directory, apachemapzope_name),
apachemapzopepath_path=os.path.join(self.etc_directory, apachemapzopepath_name),
apachemap_path=apachemap_path,
apachemap_httpsonly_path=apachemap_httpsonly_path,
apachemapzope_path=apachemap_zope_path,
apachemapzopepath_path=apachemap_zopepath_path,
apache_domain=name,
https_port=port,
plain_http_port=plain_http_port,
......
......@@ -104,10 +104,12 @@ Header append Vary User-Agent
# or changed when slapgrid is ran. It can be freely customized by node admin.
Include %(custom_apache_virtualhost_conf)s
# Define the two RewriteMaps (key -> value store): one for Zope, one generic
# Define the 3 RewriteMaps (key -> value store): one for Zope, one generic,
# one generic https only,
# containing: rewritten URL -> original URL (a.k.a VirtualHostBase in Zope)
RewriteMap apachemapzope txt:%(apachemapzope_path)s
RewriteMap apachemapgeneric txt:%(apachemap_path)s
RewriteMap apachemapgenerichttpsonly txt:%(apachemap_httpsonly_path)s
# Define another RewriteMap for Zope, containing:
# rewritten URL -> VirtualHostRoot
......@@ -123,21 +125,32 @@ Header append Vary User-Agent
RewriteCond ${apachemapgeneric:%%{SERVER_NAME}} >""
# We suppose that Apache listens to 443 (even indirectly thanks to things like iptables)
RewriteRule ^/(.*)$ ${apachemapgeneric:%%{SERVER_NAME}}/$1 [L,P]
# Same for https only server
RewriteCond ${apachemapgenerichttpsonly:%%{SERVER_NAME}} >""
# We suppose that Apache listens to 443 (even indirectly thanks to things like iptables)
RewriteRule ^/(.*)$ ${apachemapgenerichttpsonly:%%{SERVER_NAME}}/$1 [L,P]
# If nothing exist : put a nice error
ErrorDocument 404 /notfound.html
</VirtualHost>
# Only accept generic (i.e not Zope) backends on http
<VirtualHost *:%(plain_http_port)s>
RewriteEngine On
SSLProxyEngine on
# Rewrite part
ProxyVia On
ProxyPreserveHost On
ProxyTimeout 600
RewriteEngine On
# Remove "Secure" from cookies, as backend may be https
Header edit Set-Cookie "(?i)^(.+);secure$" "$1"
# Include configuration file not operated by slapos. This file won't be erased
# or changed when slapgrid is ran. It can be freely customized by node admin.
Include %(custom_apache_virtualhost_conf)s
# We accept generic (i.e not lamp) backends on http
RewriteMap apachemapgeneric txt:%(apachemap_path)s
RewriteCond ${apachemapgeneric:%%{SERVER_NAME}} >""
RewriteRule ^/(.*)$ ${apachemapgeneric:%%{SERVER_NAME}}/$1 [L,P]
......@@ -148,6 +161,7 @@ Header append Vary User-Agent
RewriteRule ^/(.*)$ https://%%{SERVER_NAME}%%{REQUEST_URI}
</VirtualHost>
# Include configuration file not operated by slapos. This file won't be erased
# or changed when slapgrid is ran. It can be freely customized by node admin.
Include %(custom_apache_conf)s
......@@ -118,6 +118,13 @@ Domain name to use as frontend. The frontend will be accessible from this domain
[instancereference].[masterdomain].
Example: www.mycustomdomain.com
https-only
~~~~~~~~~~
Specify if website should be accessed using https only. If so, the frontend
will redirect the user to https if accessed from http.
Possible values: "true", "false".
This is an optional parameter. Defaults to false.
path
~~~~
Only used if type is "zope".
......
......@@ -3,6 +3,7 @@ extends =
../../component/binutils/buildout.cfg
../../component/lxml-python/buildout.cfg
../../component/apache/buildout.cfg
../../component/gzip/buildout.cfg
../../component/stunnel/buildout.cfg
../../component/varnish/buildout.cfg
../../component/dcron/buildout.cfg
......@@ -38,14 +39,15 @@ recipe = zc.recipe.egg
eggs = ${instance-recipe:egg}
[eggs]
recipe = zc.recipe.egg
recipe = z3c.recipe.scripts
eggs =
${lxml-python:egg}
slapos.toolbox
[template]
# Default template for apache instance.
recipe = slapos.recipe.template
url = ${:_profile_base_location_}/instance.cfg
md5sum = fea902a2b9dbf8c80ff201bcf73f9396
md5sum = e7b9f57da7eb1450fc15789e239388d4
output = ${buildout:directory}/template.cfg
mode = 0644
[buildout]
parts =
directory
instance
apache
configtest
logrotate
logrotate-entry-apache
eggs-directory = ${buildout:eggs-directory}
develop-eggs-directory = ${buildout:develop-eggs-directory}
......@@ -10,15 +12,23 @@ develop-eggs-directory = ${buildout:develop-eggs-directory}
# Create all needed directories
[directory]
recipe = slapos.cookbook:mkdirectory
bin = $${buildout:directory}/bin/
etc = $${buildout:directory}/etc/
var = $${buildout:directory}/var/
srv = $${buildout:directory}/srv/
bin = $${buildout:directory}/bin/
var = $${buildout:directory}/var/
backup = $${:srv}/backup
log = $${:var}/log
run = $${:var}/run
service = $${:etc}/service
logrotate-backup = $${:backup}/logrotate
logrotate-entries = $${:etc}/logrotate.d
# Deploy Apache (old way, with monolithic recipe)
[instance]
[apache]
recipe = ${instance-recipe:egg}:${instance-recipe:module}
httpd_home = ${apache-2.2:location}
httpd_binary = ${apache-2.2:location}/bin/httpd
......@@ -31,9 +41,40 @@ rdiff_backup_binary = ${buildout:bin-directory}/rdiff-backup
gcc_binary = gcc
binutils_directory = ${binutils:location}/bin/
access-log = $${directory:log}/frontend-apache-access.log
error-log = $${directory:log}/frontend-apache-error.log
pid-file = $${directory:run}/httpd.pid
# Create wrapper for "apachectl conftest" in bin
[configtest]
recipe = slapos.cookbook:wrapper
command-line = $${instance:httpd_binary} -f $${directory:etc}/apache_frontend.conf -t
output = $${directory:bin}/apache-configtest
command-line = $${apache:httpd_binary} -f $${directory:etc}/apache_frontend.conf -t
wrapper-path = $${directory:bin}/apache-configtest
# Deploy Logrotate
[logrotate]
recipe = slapos.cookbook:logrotate
# Binaries
logrotate-binary = ${logrotate:location}/usr/sbin/logrotate
gzip-binary = ${gzip:location}/bin/gzip
gunzip-binary = ${gzip:location}/bin/gunzip
# Directories
wrapper = $${directory:bin}/logrotate
conf = $${directory:etc}/logrotate.conf
logrotate-entries = $${directory:logrotate-entries}
backup = $${directory:logrotate-backup}
state-file = $${directory:srv}/logrotate.status
[logrotate-entry-apache]
<= logrotate
recipe = slapos.cookbook:logrotate.d
name = apache
log = $${apache:error-log} $${apache:access-log}
frequency = daily
rotate-num = 30
post = ${buildout:bin-directory}/killpidfromfile $${apache:pid-file} SIGUSR1
sharedscripts = true
notifempty = true
create = true
......@@ -4,67 +4,101 @@ extends = common.cfg
[versions]
Jinja2 = 2.6
Werkzeug = 0.8.3
apache-libcloud = 0.12.3
async = 0.6.1
buildout-versions = 1.7
hexagonit.recipe.cmmi = 1.6
gitdb = 0.5.4
hexagonit.recipe.cmmi = 2.0
meld3 = 0.6.10
pycrypto = 2.6
rdiff-backup = 1.0.5
slapos.cookbook = 0.71.1
slapos.recipe.build = 0.11.5
slapos.recipe.build = 0.11.6
slapos.recipe.cmmi = 0.1
slapos.recipe.template = 2.4.2
slapos.toolbox = 0.34.0
smmap = 0.8.2
z3c.recipe.scripts = 1.0.1
# Required by:
# slapos.core==0.33.1
# slapos.core==0.35.1
# slapos.toolbox==0.34.0
Flask = 0.9
# Required by:
# hexagonit.recipe.cmmi==1.6
hexagonit.recipe.download = 1.6nxd002
# slapos.toolbox==0.34.0
GitPython = 0.3.2.RC1
# Required by:
# slapos.cookbook==0.71.1
# slapos.toolbox==0.34.0
atomize = 0.1.1
# Required by:
# slapos.toolbox==0.34.0
feedparser = 5.1.3
# Required by:
# slapos.cookbook==0.77.1
inotifyx = 0.2.0
# Required by:
# slapos.cookbook==0.71.1
# slapos.core==0.33.1
# slapos.cookbook==0.77.1
# slapos.core==0.35.1
# xml-marshaller==0.9.7
lxml = 3.0.2
lxml = 3.1.2
# Required by:
# slapos.cookbook==0.71.1
# slapos.cookbook==0.77.1
netaddr = 0.7.10
# Required by:
# slapos.core==0.33.1
# slapos.core==0.35.1
netifaces = 0.8
# Required by:
# slapos.cookbook==0.71.1
pytz = 2012j
# slapos.toolbox==0.34.0
paramiko = 1.10.1
# Required by:
# slapos.cookbook==0.71.1
# slapos.core==0.33.1
# slapos.toolbox==0.34.0
psutil = 0.7.0
# Required by:
# slapos.core==0.35.1
pyflakes = 0.7
# Required by:
# slapos.cookbook==0.77.1
pytz = 2013b
# Required by:
# slapos.cookbook==0.77.1
# slapos.core==0.35.1
# slapos.toolbox==0.34.0
# zc.buildout==1.6.0-dev-SlapOS-010
# zc.recipe.egg==1.3.2
setuptools = 0.6c12dev-r88846
# Required by:
# slapos.cookbook==0.71.1
slapos.core = 0.33.1
# slapos.cookbook==0.77.1
# slapos.toolbox==0.34.0
slapos.core = 0.35.1
# Required by:
# slapos.core==0.33.1
# slapos.core==0.35.1
supervisor = 3.0b1
# Required by:
# slapos.cookbook==0.71.1
xml-marshaller = 0.9.7
# slapos.core==0.35.1
unittest2 = 0.5.1
# Required by:
# slapos.core==0.33.1
zope.interface = 4.0.3
# slapos.cookbook==0.77.1
# slapos.toolbox==0.34.0
xml-marshaller = 0.9.7
# Required by:
# slapos.core==0.35.1
zope.interface = 4.0.5
[networkcache]
# signature certificates of the following uploaders.
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment