Merge branch 'master' into erp5-component

CHANGES.txt

 Changes
 =======
 
+0.77.1 (2013-04-18)
+-------------------
+
+ * Re-release of 0.77.0.
+
+0.77.0 (2013-04-18)
+-------------------
+
+ * Allow to pass extra parameters when creating simple wrapper. [Sebastien Robin]
+ * Apache frontend: Append all rewrite module options to http as well. [Cedric de Saint Martin]
+ * Apache frontend: Add https-only support. [Cedric de Saint Martin]
+ * Apache frontend: make logrotate work by using "generic" component. [Cedric de Saint Martin]
+
 0.76.0 (2013-04-03)
 -------------------
 

setup.py

@@ -28,7 +28,7 @@ from setuptools import setup, find_packages
 import glob
 import os
 
-version = '0.76.1.dev'
+version = '0.77.2.dev'
 name = 'slapos.cookbook'
 long_description = open("README.txt").read() + "\n" + \
     open("CHANGES.txt").read() + "\n"

slapos/recipe/apache_frontend/__init__.py

@@ -36,6 +36,7 @@ import ConfigParser
 import re
 import traceback
 
+TRUE_VALUES = ['y', 'yes', '1', 'true']
 
 class Recipe(BaseSlapRecipe):
 
@@ -65,13 +66,13 @@ class Recipe(BaseSlapRecipe):
 
     # self.cron_d is a directory, where cron jobs can be registered
     self.cron_d = self.installCrond()
-    self.logrotate_d, self.logrotate_backup = self.installLogrotate()
     self.killpidfromfile = zc.buildout.easy_install.scripts(
-        [('killpidfromfile', 'slapos.recipe.erp5.killpidfromfile',
+        [('killpidfromfile', 'slapos.toolbox.killpidfromfile',
           'killpidfromfile')], self.ws, sys.executable, self.bin_directory)[0]
     self.path_list.append(self.killpidfromfile)
 
     rewrite_rule_list = []
+    rewrite_rule_https_only_list = []
     rewrite_rule_zope_list = []
     rewrite_rule_zope_path_list = []
     slave_dict = {}
@@ -88,19 +89,15 @@ class Recipe(BaseSlapRecipe):
       # Sanitize inputs
       backend_url = slave_instance.get("url", None)
       reference = slave_instance.get("slave_reference")
+      enable_cache = slave_instance.get('enable_cache', '').lower() in TRUE_VALUES
+      slave_type = slave_instance.get('type', '').lower() or None
 
-      if slave_instance.haskey("enable_cache"):
-        enable_cache = slave_instance.get("enable_cache", "").upper() in ('1', 'TRUE')
-      else:
-        enable_cache = False
-
-      if slave_instance.haskey("type"):
-        slave_type = slave_instance.get("type", "").lower()
-      else:
-        slave_type = None
+      https_only = slave_instance.get('https-only', '').lower() in TRUE_VALUES
 
       # Set scheme (http? https?)
-      # Future work may allow to choose between http and https (or both?)
+      if https_only:
+        scheme = 'https://'
+      else:
         scheme = 'http://'
 
       self.logger.info('Processing slave instance: %s' % reference)
@@ -143,6 +140,10 @@ class Recipe(BaseSlapRecipe):
       rewrite_rule = "%s %s" % (domain, backend_url)
 
       # Finally, if successful, we add the rewrite rule to our list of rules
+      # We have 4 RewriteMaps:
+      #  - One for generic (non-zope) websites, accepting both HTTP and HTTPS
+      #  - One for generic websites that only accept HTTPS
+      #  - Two for Zope-based websites
       if rewrite_rule:
         # We check if we have a zope slave. It requires different rewrite
         # rule structure.
@@ -153,6 +154,9 @@ class Recipe(BaseSlapRecipe):
           # For Zope, we have another dict containing the path e.g '/erp5/...
           rewrite_rule_path = "%s %s" % (domain, slave_instance.get('path', ''))
           rewrite_rule_zope_path_list.append(rewrite_rule_path)
+        else:
+          if https_only:
+            rewrite_rule_https_only_list.append(rewrite_rule)
           else:
             rewrite_rule_list.append(rewrite_rule)
 
@@ -186,6 +190,7 @@ class Recipe(BaseSlapRecipe):
         plain_http_port=frontend_plain_http_port_number,
         name=frontend_domain_name,
         rewrite_rule_list=rewrite_rule_list,
+        rewrite_rule_https_only_list=rewrite_rule_https_only_list,
         rewrite_rule_zope_list=rewrite_rule_zope_list,
         rewrite_rule_zope_path_list=rewrite_rule_zope_path_list,
         key=key, certificate=certificate)
@@ -279,29 +284,6 @@ class Recipe(BaseSlapRecipe):
     return "%s http://%s:%s" % \
         (domain, varnish_ip, base_varnish_port)
 
-  def installLogrotate(self):
-    """Installs logortate main configuration file and registers its to cron"""
-    logrotate_d = os.path.abspath(os.path.join(self.etc_directory,
-      'logrotate.d'))
-    self._createDirectory(logrotate_d)
-    logrotate_backup = self.createBackupDirectory('logrotate')
-    logrotate_conf = self.createConfigurationFile("logrotate.conf",
-        "include %s" % logrotate_d)
-    logrotate_cron = os.path.join(self.cron_d, 'logrotate')
-    state_file = os.path.join(self.data_root_directory, 'logrotate.status')
-    open(logrotate_cron, 'w').write('0 0 * * * %s -s %s %s' %
-        (self.options['logrotate_binary'], state_file, logrotate_conf))
-    self.path_list.extend([logrotate_d, logrotate_conf, logrotate_cron])
-    return logrotate_d, logrotate_backup
-
-  def registerLogRotation(self, name, log_file_list, postrotate_script):
-    """Register new log rotation requirement"""
-    open(os.path.join(self.logrotate_d, name), 'w').write(
-        self.substituteTemplate(self.getTemplateFilename(
-          'logrotate_entry.in'),
-          dict(file_list=' '.join(['"'+q+'"' for q in log_file_list]),
-            postrotate=postrotate_script, olddir=self.logrotate_backup)))
-
   def requestCertificate(self, name):
     hash = hashlib.sha512(name).hexdigest()
     key = os.path.join(self.ca_private, hash + self.ca_key_ext)
@@ -418,8 +400,7 @@ class Recipe(BaseSlapRecipe):
   def _getApacheConfigurationDict(self, name, ip_list, port):
     apache_conf = dict()
     apache_conf['server_name'] = name
-    apache_conf['pid_file'] = os.path.join(self.run_directory,
-        name + '.pid')
+    apache_conf['pid_file'] = self.options['pid-file']
     apache_conf['lock_file'] = os.path.join(self.run_directory,
         name + '.lock')
     apache_conf['document_root'] = os.path.join(self.data_root_directory,
@@ -429,13 +410,8 @@ class Recipe(BaseSlapRecipe):
     apache_conf['ip_list'] = ip_list
     apache_conf['port'] = port
     apache_conf['server_admin'] = 'admin@'
-    apache_conf['error_log'] = os.path.join(self.log_directory,
-        'frontend-apache-error.log')
-    apache_conf['access_log'] = os.path.join(self.log_directory,
-        'frontend-apache-access.log')
-    self.registerLogRotation(name, [apache_conf['error_log'],
-      apache_conf['access_log']], self.killpidfromfile + ' ' +
-      apache_conf['pid_file'] + ' SIGUSR1')
+    apache_conf['error_log'] = self.options['error-log']
+    apache_conf['access_log'] = self.options['access-log']
     return apache_conf
 
   def installVarnishCache(self, name, ip, port, control_port, backend_host,
@@ -517,10 +493,13 @@ class Recipe(BaseSlapRecipe):
                             port=4443, plain_http_port=8080,
                             rewrite_rule_list=None,
                             rewrite_rule_zope_list=None,
+                            rewrite_rule_https_only_list=None,
                             rewrite_rule_zope_path_list=None,
                             access_control_string=None):
     if rewrite_rule_list is None:
       rewrite_rule_list = []
+    if rewrite_rule_https_only_list is None:
+      rewrite_rule_zope_path_list = []
     if rewrite_rule_zope_list is None:
       rewrite_rule_zope_list = []
     if rewrite_rule_zope_path_list is None:
@@ -571,15 +550,22 @@ class Recipe(BaseSlapRecipe):
     self.path_list.append(backup_cron)
 
     # Create configuration file and rewritemaps
-    apachemap_name = "apachemap.txt"
-    apachemapzope_name = "apachemapzope.txt"
-    apachemapzopepath_name = "apachemapzopepath.txt"
-
-    self.createConfigurationFile(apachemap_name, "\n".join(rewrite_rule_list))
-    self.createConfigurationFile(apachemapzope_name,
-        "\n".join(rewrite_rule_zope_list))
-    self.createConfigurationFile(apachemapzopepath_name,
-        "\n".join(rewrite_rule_zope_path_list))
+    apachemap_path = self.createConfigurationFile(
+        "apache_rewritemap_generic.txt",
+        "\n".join(rewrite_rule_list)
+    )
+    apachemap_httpsonly_path = self.createConfigurationFile(
+        "apache_rewritemap_httpsonly.txt",
+        "\n".join(rewrite_rule_https_only_list)
+    )
+    apachemap_zope_path = self.createConfigurationFile(
+        "apache_rewritemap_zope.txt",
+        "\n".join(rewrite_rule_zope_list)
+    )
+    apachemap_zopepath_path = self.createConfigurationFile(
+        "apache_rewritemap_zopepath.txt",
+        "\n".join(rewrite_rule_zope_path_list)
+    )
 
     apache_conf = self._getApacheConfigurationDict(name, ip_list, port)
     apache_conf['ssl_snippet'] = self.substituteTemplate(
@@ -606,9 +592,10 @@ class Recipe(BaseSlapRecipe):
 
     apache_conf.update(**dict(
       path_enable=path,
-      apachemap_path=os.path.join(self.etc_directory, apachemap_name),
-      apachemapzope_path=os.path.join(self.etc_directory, apachemapzope_name),
-      apachemapzopepath_path=os.path.join(self.etc_directory, apachemapzopepath_name),
+      apachemap_path=apachemap_path,
+      apachemap_httpsonly_path=apachemap_httpsonly_path,
+      apachemapzope_path=apachemap_zope_path,
+      apachemapzopepath_path=apachemap_zopepath_path,
       apache_domain=name,
       https_port=port,
       plain_http_port=plain_http_port,

slapos/recipe/apache_frontend/template/apache.conf.in

@@ -104,10 +104,12 @@ Header append Vary User-Agent
   # or changed when slapgrid is ran. It can be freely customized by node admin.
   Include %(custom_apache_virtualhost_conf)s
 
-  # Define the two RewriteMaps (key -> value store): one for Zope, one generic
+  # Define the 3 RewriteMaps (key -> value store): one for Zope, one generic,
+  # one generic https only,
   # containing: rewritten URL -> original URL (a.k.a VirtualHostBase in Zope)
   RewriteMap apachemapzope txt:%(apachemapzope_path)s
   RewriteMap apachemapgeneric txt:%(apachemap_path)s
+  RewriteMap apachemapgenerichttpsonly txt:%(apachemap_httpsonly_path)s
 
   # Define another RewriteMap for Zope, containing:
   # rewritten URL -> VirtualHostRoot
@@ -123,21 +125,32 @@ Header append Vary User-Agent
   RewriteCond ${apachemapgeneric:%%{SERVER_NAME}} >""
   # We suppose that Apache listens to 443 (even indirectly thanks to things like iptables)
   RewriteRule ^/(.*)$ ${apachemapgeneric:%%{SERVER_NAME}}/$1 [L,P]
+  # Same for https only server
+  RewriteCond ${apachemapgenerichttpsonly:%%{SERVER_NAME}} >""
+  # We suppose that Apache listens to 443 (even indirectly thanks to things like iptables)
+  RewriteRule ^/(.*)$ ${apachemapgenerichttpsonly:%%{SERVER_NAME}}/$1 [L,P]
 
   # If nothing exist : put a nice error
   ErrorDocument 404 /notfound.html
 </VirtualHost>
 
+
+# Only accept generic (i.e not Zope) backends on http
 <VirtualHost *:%(plain_http_port)s>
-  RewriteEngine On
   SSLProxyEngine on
+  # Rewrite part
+  ProxyVia On
   ProxyPreserveHost On
+  ProxyTimeout 600
+  RewriteEngine On
+
+  # Remove "Secure" from cookies, as backend may be https
+  Header edit Set-Cookie "(?i)^(.+);secure$" "$1"
 
   # Include configuration file not operated by slapos. This file won't be erased
   # or changed when slapgrid is ran. It can be freely customized by node admin.
   Include %(custom_apache_virtualhost_conf)s
 
-  # We accept generic (i.e not lamp) backends on http
   RewriteMap apachemapgeneric txt:%(apachemap_path)s
   RewriteCond ${apachemapgeneric:%%{SERVER_NAME}} >""
   RewriteRule ^/(.*)$ ${apachemapgeneric:%%{SERVER_NAME}}/$1 [L,P]
@@ -148,6 +161,7 @@ Header append Vary User-Agent
   RewriteRule ^/(.*)$ https://%%{SERVER_NAME}%%{REQUEST_URI}
 </VirtualHost>
 
+
 # Include configuration file not operated by slapos. This file won't be erased
 # or changed when slapgrid is ran. It can be freely customized by node admin.
 Include %(custom_apache_conf)s

software/apache-frontend/README.apache_frontend.txt

@@ -118,6 +118,13 @@ Domain name to use as frontend. The frontend will be accessible from this domain
 [instancereference].[masterdomain].
 Example: www.mycustomdomain.com
 
+https-only
+~~~~~~~~~~
+Specify if website should be accessed using https only. If so, the frontend
+will redirect the user to https if accessed from http.
+Possible values: "true", "false".
+This is an optional parameter. Defaults to false.
+
 path
 ~~~~
 Only used if type is "zope".

software/apache-frontend/common.cfg

@@ -3,6 +3,7 @@ extends =
   ../../component/binutils/buildout.cfg
   ../../component/lxml-python/buildout.cfg
   ../../component/apache/buildout.cfg
+  ../../component/gzip/buildout.cfg
   ../../component/stunnel/buildout.cfg
   ../../component/varnish/buildout.cfg
   ../../component/dcron/buildout.cfg
@@ -38,14 +39,15 @@ recipe = zc.recipe.egg
 eggs = ${instance-recipe:egg}
 
 [eggs]
-recipe = zc.recipe.egg
+recipe = z3c.recipe.scripts
 eggs =
   ${lxml-python:egg}
+  slapos.toolbox
 
 [template]
 # Default template for apache instance.
 recipe = slapos.recipe.template
 url = ${:_profile_base_location_}/instance.cfg
-md5sum = fea902a2b9dbf8c80ff201bcf73f9396
+md5sum = e7b9f57da7eb1450fc15789e239388d4
 output = ${buildout:directory}/template.cfg
 mode = 0644

software/apache-frontend/instance.cfg

 [buildout]
 parts =
   directory
-  instance
+  apache
   configtest
+  logrotate
+  logrotate-entry-apache
 
 eggs-directory = ${buildout:eggs-directory}
 develop-eggs-directory = ${buildout:develop-eggs-directory}
@@ -10,15 +12,23 @@ develop-eggs-directory = ${buildout:develop-eggs-directory}
 # Create all needed directories
 [directory]
 recipe = slapos.cookbook:mkdirectory
+
+bin = $${buildout:directory}/bin/
 etc = $${buildout:directory}/etc/
-var = $${buildout:directory}/var/
 srv = $${buildout:directory}/srv/
-bin = $${buildout:directory}/bin/
+var = $${buildout:directory}/var/
+
+backup = $${:srv}/backup
+log = $${:var}/log
+run = $${:var}/run
 service = $${:etc}/service
 
+logrotate-backup = $${:backup}/logrotate
+logrotate-entries = $${:etc}/logrotate.d
+
 
 # Deploy Apache (old way, with monolithic recipe)
-[instance]
+[apache]
 recipe = ${instance-recipe:egg}:${instance-recipe:module}
 httpd_home = ${apache-2.2:location}
 httpd_binary = ${apache-2.2:location}/bin/httpd
@@ -31,9 +41,40 @@ rdiff_backup_binary = ${buildout:bin-directory}/rdiff-backup
 gcc_binary = gcc
 binutils_directory = ${binutils:location}/bin/
 
+access-log = $${directory:log}/frontend-apache-access.log
+error-log = $${directory:log}/frontend-apache-error.log
+pid-file = $${directory:run}/httpd.pid
+
 
 # Create wrapper for "apachectl conftest" in bin
 [configtest]
 recipe = slapos.cookbook:wrapper
-command-line = $${instance:httpd_binary} -f $${directory:etc}/apache_frontend.conf -t
-output = $${directory:bin}/apache-configtest
+command-line = $${apache:httpd_binary} -f $${directory:etc}/apache_frontend.conf -t
+wrapper-path = $${directory:bin}/apache-configtest
+
+
+# Deploy Logrotate
+[logrotate]
+recipe = slapos.cookbook:logrotate
+# Binaries
+logrotate-binary = ${logrotate:location}/usr/sbin/logrotate
+gzip-binary = ${gzip:location}/bin/gzip
+gunzip-binary = ${gzip:location}/bin/gunzip
+# Directories
+wrapper = $${directory:bin}/logrotate
+conf = $${directory:etc}/logrotate.conf
+logrotate-entries = $${directory:logrotate-entries}
+backup = $${directory:logrotate-backup}
+state-file = $${directory:srv}/logrotate.status
+
+[logrotate-entry-apache]
+<= logrotate
+recipe = slapos.cookbook:logrotate.d
+name = apache
+log = $${apache:error-log} $${apache:access-log}
+frequency = daily
+rotate-num = 30
+post = ${buildout:bin-directory}/killpidfromfile $${apache:pid-file} SIGUSR1
+sharedscripts = true
+notifempty = true
+create = true

software/apache-frontend/software.cfg

@@ -4,67 +4,101 @@ extends = common.cfg
 [versions]
 Jinja2 = 2.6
 Werkzeug = 0.8.3
+apache-libcloud = 0.12.3
+async = 0.6.1
 buildout-versions = 1.7
-hexagonit.recipe.cmmi = 1.6
+gitdb = 0.5.4
+hexagonit.recipe.cmmi = 2.0
 meld3 = 0.6.10
+pycrypto = 2.6
 rdiff-backup = 1.0.5
-slapos.cookbook = 0.71.1
-slapos.recipe.build = 0.11.5
+slapos.recipe.build = 0.11.6
+slapos.recipe.cmmi = 0.1
 slapos.recipe.template = 2.4.2
+slapos.toolbox = 0.34.0
+smmap = 0.8.2
+z3c.recipe.scripts = 1.0.1
 
 # Required by:
-# slapos.core==0.33.1
+# slapos.core==0.35.1
+# slapos.toolbox==0.34.0
 Flask = 0.9
 
 # Required by:
-# hexagonit.recipe.cmmi==1.6
-hexagonit.recipe.download = 1.6nxd002
+# slapos.toolbox==0.34.0
+GitPython = 0.3.2.RC1
 
 # Required by:
-# slapos.cookbook==0.71.1
+# slapos.toolbox==0.34.0
+atomize = 0.1.1
+
+# Required by:
+# slapos.toolbox==0.34.0
+feedparser = 5.1.3
+
+# Required by:
+# slapos.cookbook==0.77.1
 inotifyx = 0.2.0
 
 # Required by:
-# slapos.cookbook==0.71.1
-# slapos.core==0.33.1
+# slapos.cookbook==0.77.1
+# slapos.core==0.35.1
 # xml-marshaller==0.9.7
-lxml = 3.0.2
+lxml = 3.1.2
 
 # Required by:
-# slapos.cookbook==0.71.1
+# slapos.cookbook==0.77.1
 netaddr = 0.7.10
 
 # Required by:
-# slapos.core==0.33.1
+# slapos.core==0.35.1
 netifaces = 0.8
 
 # Required by:
-# slapos.cookbook==0.71.1
-pytz = 2012j
+# slapos.toolbox==0.34.0
+paramiko = 1.10.1
 
 # Required by:
-# slapos.cookbook==0.71.1
-# slapos.core==0.33.1
+# slapos.toolbox==0.34.0
+psutil = 0.7.0
+
+# Required by:
+# slapos.core==0.35.1
+pyflakes = 0.7
+
+# Required by:
+# slapos.cookbook==0.77.1
+pytz = 2013b
+
+# Required by:
+# slapos.cookbook==0.77.1
+# slapos.core==0.35.1
+# slapos.toolbox==0.34.0
 # zc.buildout==1.6.0-dev-SlapOS-010
 # zc.recipe.egg==1.3.2
 setuptools = 0.6c12dev-r88846
 
 # Required by:
-# slapos.cookbook==0.71.1
-slapos.core = 0.33.1
+# slapos.cookbook==0.77.1
+# slapos.toolbox==0.34.0
+slapos.core = 0.35.1
 
 # Required by:
-# slapos.core==0.33.1
+# slapos.core==0.35.1
 supervisor = 3.0b1
 
 # Required by:
-# slapos.cookbook==0.71.1
-xml-marshaller = 0.9.7
+# slapos.core==0.35.1
+unittest2 = 0.5.1
 
 # Required by:
-# slapos.core==0.33.1
-zope.interface = 4.0.3
+# slapos.cookbook==0.77.1
+# slapos.toolbox==0.34.0
+xml-marshaller = 0.9.7
 
+# Required by:
+# slapos.core==0.35.1
+zope.interface = 4.0.5
 
 [networkcache]
 # signature certificates of the following uploaders.