Merge master in lamp-mohamadou branch

CHANGES.txt

 Changes
 =======
 
-0.49 (Unreleased)
+0.52 (Unreleased)
 -----------------
 
-  * Slap Test Agent [Yingjie Xu]
+  * No change yet.
+
+0.51 (2012-05-14)
+-----------------
+
+  * LAMP stack bugfix: Users were losing data when slapgrid is ran (Don't
+    erase htdocs if it already exist). [Cedric de Saint Martin]
+
+0.50 (2012-05-12)
+-----------------
+
+  * LAMP stack bugfix: fix a crash where recipe was trying to restart
+    non-existent httpd process. [Cedric de Saint Martin]
+  * LAMP stack bugfix: don't erase htdocs at update [Cedric de Saint Martin]
+  * Apache Frontend: Improve Apache configuration, inspired by Nexedi
+    production frontend. [Cedric de Saint Martin]
+  * Allow sysadmin of node to customize frontend instance.
+    [Cedric de Saint Martin]
+  * Apache Frontend: Change 'zope=true' option to 'type=zope'.
+    [Cedric de Saint Martin]
+  * Apache Frontend: listens to plain http port as well to redirect to https.
+    [Cedric de Saint Martin]
+  
+0.49 (2012-05-10)
+-----------------
+
+  * Apache Frontend supports Zope and Varnish. [Cedric de Saint Martin]
 
 0.48 (2012-04-26)
 -----------------

component/apache-php/buildout.cfg

@@ -20,8 +20,8 @@ extends =
 [apache-php]
 # Note: Shall react on each build of apache and reinstall itself
 recipe = hexagonit.recipe.cmmi
-url = http://fr2.php.net/distributions/php-5.3.10.tar.gz
-md5sum = 2b3d2d0ff22175685978fb6a5cbcdc13
+url = http://fr2.php.net/distributions/php-5.3.13.tar.gz
+md5sum = 179c67ce347680f468edbfc3c425476a
 configure-options =
   --with-apxs2=${apache:location}/bin/apxs
   --with-libxml-dir=${libxml2:location}

component/apache/buildout.cfg

@@ -45,6 +45,7 @@ configure-options = --prefix=${buildout:parts-directory}/${:_buildout_section_na
                     --enable-cgid
                     --enable-charset-lite
                     --enable-disk-cache
+                    --enable-mem-cache
                     --enable-echo
                     --enable-exception-hook
                     --enable-mods-shared=all
@@ -115,3 +116,73 @@ configure-options = -c mod_antiloris.c
 make-binary = ${:configure-command}
 make-options = -i -a -n antiloris mod_antiloris.la
 make-targets =
+
+[apache-2.2]
+# inspired on http://old.aclark.net/team/aclark/blog/a-lamp-buildout-for-wordpress-and-other-php-apps/
+recipe = hexagonit.recipe.cmmi
+url = http://mir2.ovh.net/ftp.apache.org/dist//httpd/httpd-2.2.22.tar.gz
+md5sum = d77fa5af23df96a8af68ea8114fa6ce1
+patch-options = -p1
+configure-options = --disable-static
+                    --enable-authn-alias
+                    --enable-bucketeer
+                    --enable-cache
+                    --enable-case-filter
+                    --enable-case-filter-in
+                    --enable-cgid
+                    --enable-charset-lite
+                    --enable-disk-cache
+                    --enable-mem-cache
+                    --enable-echo
+                    --enable-exception-hook
+                    --enable-mods-shared=all
+                    --enable-optional-fn-export
+                    --enable-optional-fn-import
+                    --enable-optional-hook-export
+                    --enable-optional-hook-import
+                    --enable-proxy
+                    --enable-proxy-ajp
+                    --enable-proxy-balancer
+                    --enable-proxy-connect
+                    --enable-proxy-ftp
+                    --enable-proxy-http
+                    --enable-proxy-scgi
+                    --enable-dav
+                    --enable-dav-fs
+                    --enable-so
+                    --enable-ssl
+                    --with-included-apr
+                    --with-ssl=${openssl:location}
+                    --with-z=${zlib:location}
+                    --with-expat=${libexpat:location}
+                    --with-pcre=${pcre:location}
+                    --with-sqlite3=${sqlite3:location}
+                    --with-gdbm=${gdbm:location}
+                    --without-lber
+                    --without-ldap
+                    --without-ndbm
+                    --without-berkeley-db
+                    --without-pgsql
+                    --without-mysql
+                    --without-sqlite2
+                    --without-oracle
+                    --without-freedts
+                    --without-odbc
+                    --without-iconv
+
+environment =
+  PATH=${pkgconfig:location}/bin:%(PATH)s
+  PKG_CONFIG_PATH=${openssl:location}/lib/pkgconfig
+  CPPFLAGS =-I${libuuid:location}/include
+  LDFLAGS =-Wl,-rpath=${zlib:location}/lib -Wl,-rpath=${openssl:location}/lib -L${libuuid:location}/lib -Wl,-rpath=${libuuid:location}/lib -Wl,-rpath=${libexpat:location}/lib -Wl,-rpath=${pcre:location}/lib -Wl,-rpath=${sqlite3:location}/lib -Wl,-rpath=${gdbm:location}/lib
+
+[apache-antiloris-apache-2.2]
+# Note: Shall react on each build of apache and reinstall itself
+recipe = hexagonit.recipe.cmmi
+url = http://sourceforge.net/projects/mod-antiloris/files/mod_antiloris-0.4.tar.bz2/download
+md5sum = 66862bf10e9be3a023e475604a28a0b4
+configure-command = ${apache-2.2:location}/bin/apxs
+configure-options = -c mod_antiloris.c
+make-binary = ${:configure-command}
+make-options = -i -a -n antiloris mod_antiloris.la
+make-targets =

setup.py

@@ -2,7 +2,7 @@ from setuptools import setup, find_packages
 import glob
 import os
 
-version = '0.49-dev'
+version = '0.52-dev'
 name = 'slapos.cookbook'
 long_description = open("README.txt").read() + "\n" + \
     open("CHANGES.txt").read() + "\n"

slapos/recipe/README.apache_frontend.txt

-apache_frontend
-==========
-
-Frontend using Apache, allowing to rewrite and proxy URLs like
-myinstance.myfrontenddomainname.com to real IP/URL of myinstance.
-
-apache_frontend works using the master instance / slave instance design.
-It means that a single main instance of Apache will be used to act as frontend
-for many slaves.
-
-
-How to use
-========
-First, you will need to request a "master" instance of Apache Frontend with
-"domain" parameter, like : 
-<?xml version='1.0' encoding='utf-8'?>
-<instance>
- <parameter id="domain">moulefrite.com</parameter>
- <parameter id="port">443</parameter>
-</instance>
-
-Then, it is possible to request many slave instances
-(currently only from slapconsole, UI doesn't work yet)
-of Apache Frontend, like : 
-instance = request(
-       software_release=apache_frontend,
-       partition_reference='frontend2',
-       shared=True,
-       partition_parameter_kw={"url":"https://[1:2:3:4]:1234/someresource"}
-     )
-Those slave instances will be redirected to the "master" instance,
-and you will see on the "master" instance the associated RewriteRules of
-all slave instances.
-
-Finally, the slave instance will be accessible from :
-https://someidentifier.moulefrite.com.

slapos/recipe/apache_frontend/__init__.py

@@ -42,6 +42,19 @@ class Recipe(BaseSlapRecipe):
         'template/%s' % template_name)
 
   def _install(self):
+    # Check for mandatory arguments
+    frontend_domain_name = self.parameter_dict.get("domain")
+    if frontend_domain_name is None:
+      raise zc.buildout.UserError('No domain name specified. Please define '
+          'the "domain" instance parameter.')
+
+    # Define optional arguments
+    frontend_port_number = self.parameter_dict.get("port", 4443)
+    frontend_plain_http_port_number = self.parameter_dict.get(
+        "plain_http_port", 8080)
+    base_varnish_port = 26009
+    slave_instance_list = self.parameter_dict.get("slave_instance_list", [])
+
     self.path_list = []
     self.requirements, self.ws = self.egg.working_set()
 
@@ -51,72 +64,63 @@ class Recipe(BaseSlapRecipe):
     self.killpidfromfile = zc.buildout.easy_install.scripts(
         [('killpidfromfile', 'slapos.recipe.erp5.killpidfromfile',
           'killpidfromfile')], self.ws, sys.executable, self.bin_directory)[0]
-
     self.path_list.append(self.killpidfromfile)
 
-    frontend_port_number = self.parameter_dict.get("port", 4443)
-    frontend_domain_name = self.parameter_dict.get("domain",
-        "host.vifib.net")
-
-    base_varnish_port = 26009
-    slave_instance_list = self.parameter_dict.get("slave_instance_list", [])
     rewrite_rule_list = []
+    rewrite_rule_zope_list = []
     slave_dict = {}
     service_dict = {}
-    if frontend_port_number is 443:
-      base_url = "%s/" % frontend_domain_name
+
+    # Check if default port
+    if frontend_port_number is 443 or frontend_port_number is 80:
+      port_snippet = ""
     else:
-      base_url = "%s:%s/" % (frontend_domain_name, frontend_port_number)
+      port_snippet = ":%s" % frontend_port_number
+
     for slave_instance in slave_instance_list:
-      url = slave_instance.get("url")
-      if url is None:
-        continue
+      backend_url = slave_instance.get("url", None)
       reference = slave_instance.get("slave_reference")
-      subdomain = reference.replace("-", "").lower()
-      slave_dict[reference] = "https://%s.%s" % (subdomain, base_url)
-
-      enable_cache = slave_instance.get("enable_cache", "")
-      if enable_cache.upper() in ('1', 'TRUE'):
-        # Varnish should use stunnel to connect to the backend
-        base_varnish_control_port = base_varnish_port
-        base_varnish_port += 1
-        # Use regex
-        host_regex = "((\[\w*|[0-9]+\.)(\:|)).*(\]|\.[0-9]+)"
-        slave_host = re.search(host_regex, url).group(0)
-        port_regex = "\w+(\/|)$"
-        matcher = re.search(port_regex, url)
-        if matcher is not None:
-          slave_port = matcher.group(0)
-          slave_port = slave_port.replace("/", "")
-        elif url.startswith("https://"):
-          slave_port = 443
-        else:
-          slave_port = 80
-        service_name = "varnish_%s" % reference
-        varnish_ip = self.getLocalIPv4Address()
-        stunnel_port = base_varnish_port + 1
-        self.installVarnishCache(service_name,
-          ip=varnish_ip,
-          port=base_varnish_port,
-          control_port=base_varnish_control_port,
-          backend_host=varnish_ip,
-          backend_port=stunnel_port,
-          size="1G")
-        service_dict[service_name] = dict(public_ip=varnish_ip,
-            public_port=stunnel_port,
-            private_ip=slave_host.replace("[", "").replace("]", ""),
-            private_port=slave_port)
-        rewrite_rule_list.append("%s.%s http://%s:%s" % \
-            (reference.replace("-", ""), frontend_domain_name,
-            varnish_ip, base_varnish_port))
+      # Set scheme (http? https?)
+      # Future work may allow to choose between http and https (or both?)
+      scheme = 'https://'
+
+      self.logger.info('processing slave instance: %s' % reference)
+
+      # Check for mandatory slave fields
+      if backend_url is None:
+        self.logger.warn('No "url" parameter is defined for %s slave'\
+            'instance. Ignoring it.' % reference)
+        continue
+
+      # Check for custom domain (like mypersonaldomain.com)
+      # If no custom domain, use generated one
+      domain = slave_instance.get('custom_domain', 
+          "%s.%s" % (reference.replace("-", "").lower(), frontend_domain_name))
+      slave_dict[reference] = "%s%s%s/" % (scheme, domain, port_snippet)
+
+      # Check if we want varnish+stunnel cache.
+      if slave_instance.get("enable_cache", "").upper() in ('1', 'TRUE'):
+        # XXX-Cedric : need to refactor to clean code? (to many variables)
+        rewrite_rule = self.configureVarnishSlave(
+            base_varnish_port, backend_url, reference, service_dict, domain)
         base_varnish_port += 2
       else:
-        rewrite_rule_list.append("%s.%s %s" % (subdomain, frontend_domain_name,
-            url))
+        rewrite_rule = "%s %s" % (domain, backend_url)
+
+      # Finally, if successful, we add the rewrite rule to our list of rules
+      if rewrite_rule:
+        # We check if we have a zope slave. It requires different rewrite
+        # rule structure.
+        # So we will have one RewriteMap for normal websites, and one
+        # RewriteMap for Zope Virtual Host Monster websites.
+        if slave_instance.get("type", "").lower() in ('zope'):
+          rewrite_rule_zope_list.append(rewrite_rule)
+        else:
+          rewrite_rule_list.append(rewrite_rule)
 
+    # Certificate stuff
     valid_certificate_str = self.parameter_dict.get("domain_ssl_ca_cert")
     valid_key_str = self.parameter_dict.get("domain_ssl_ca_key")
-
     if valid_certificate_str is None and valid_key_str is None:
       ca_conf = self.installCertificateAuthority()
       key, certificate = self.requestCertificate(frontend_domain_name)
@@ -125,14 +129,13 @@ class Recipe(BaseSlapRecipe):
           frontend_domain_name, valid_certificate_str, valid_key_str)
       key = ca_conf.pop("key")
       certificate = ca_conf.pop("certificate")
-
     if service_dict != {}:
       if valid_certificate_str is not None and valid_key_str is not None:
         self.installCertificateAuthority()
         stunnel_key, stunnel_certificate = \
             self.requestCertificate(frontend_domain_name)
       else:
-        stunnel_key, stunnet_certificate = key, certificate
+        stunnel_key, stunnel_certificate = key, certificate
       self.installStunnel(service_dict,
         stunnel_certificate, stunnel_key,
         ca_conf["ca_crl"],
@@ -142,19 +145,87 @@ class Recipe(BaseSlapRecipe):
         ip_list=["[%s]" % self.getGlobalIPv6Address(),
                  self.getLocalIPv4Address()],
         port=frontend_port_number,
+        plain_http_port=frontend_plain_http_port_number,
         name=frontend_domain_name,
         rewrite_rule_list=rewrite_rule_list,
+        rewrite_rule_zope_list=rewrite_rule_zope_list,
         key=key, certificate=certificate)
 
+    # Send connection informations about each slave
     for reference, url in slave_dict.iteritems():
       self.setConnectionDict(dict(site_url=url), reference)
 
+    # Then set it for master instance
     self.setConnectionDict(
       dict(site_url=apache_parameter_dict["site_url"],
            domain_ipv6_address=self.getGlobalIPv6Address(),
            domain_ipv4_address=self.getLocalIPv4Address()))
+
+    # Promises
+    promise_config = dict(
+      hostname=self.getGlobalIPv6Address(),
+      port=frontend_port_number,
+      python_path=sys.executable,
+    )
+    promise_v6 = self.createPromiseWrapper(
+      'apache_ipv6',
+      self.substituteTemplate(
+          pkg_resources.resource_filename(
+              'slapos.recipe.check_port_listening',
+          'template/socket_connection_attempt.py.in'),
+        promise_config))
+    self.path_list.append(promise_v6)
+
+    promise_config = dict(
+      hostname=self.getLocalIPv4Address(),
+      port=frontend_port_number,
+      python_path=sys.executable,
+    )
+    promise_v4 = self.createPromiseWrapper(
+      'apache_ipv4',
+      self.substituteTemplate(
+          pkg_resources.resource_filename(
+              'slapos.recipe.check_port_listening',
+          'template/socket_connection_attempt.py.in'),
+        promise_config))
+    self.path_list.append(promise_v4)
+
     return self.path_list
 
+  def configureVarnishSlave(self, base_varnish_port, url, reference,
+      service_dict, domain):
+    # Varnish should use stunnel to connect to the backend
+    base_varnish_control_port = base_varnish_port
+    base_varnish_port += 1
+    # Use regex
+    host_regex = "((\[\w*|[0-9]+\.)(\:|)).*(\]|\.[0-9]+)"
+    slave_host = re.search(host_regex, url).group(0)
+    port_regex = "\w+(\/|)$"
+    matcher = re.search(port_regex, url)
+    if matcher is not None:
+      slave_port = matcher.group(0)
+      slave_port = slave_port.replace("/", "")
+    elif url.startswith("https://"):
+      slave_port = 443
+    else:
+      slave_port = 80
+    service_name = "varnish_%s" % reference
+    varnish_ip = self.getLocalIPv4Address()
+    stunnel_port = base_varnish_port + 1
+    self.installVarnishCache(service_name,
+      ip=varnish_ip,
+      port=base_varnish_port,
+      control_port=base_varnish_control_port,
+      backend_host=varnish_ip,
+      backend_port=stunnel_port,
+      size="1G")
+    service_dict[service_name] = dict(public_ip=varnish_ip,
+        public_port=stunnel_port,
+        private_ip=slave_host.replace("[", "").replace("]", ""),
+        private_port=slave_port)
+    return "%s http://%s:%s" % \
+        (domain, varnish_ip, base_varnish_port)
+
   def installLogrotate(self):
     """Installs logortate main configuration file and registers its to cron"""
     logrotate_d = os.path.abspath(os.path.join(self.etc_directory,
@@ -301,9 +372,9 @@ class Recipe(BaseSlapRecipe):
     apache_conf['port'] = port
     apache_conf['server_admin'] = 'admin@'
     apache_conf['error_log'] = os.path.join(self.log_directory,
-        name + '-error.log')
+        'frontend-apache-error.log')
     apache_conf['access_log'] = os.path.join(self.log_directory,
-        name + '-access.log')
+        'frontend-apache-access.log')
     self.registerLogRotation(name, [apache_conf['error_log'],
       apache_conf['access_log']], self.killpidfromfile + ' ' +
       apache_conf['pid_file'] + ' SIGUSR1')
@@ -383,7 +454,8 @@ class Recipe(BaseSlapRecipe):
     self.path_list.append(wrapper)
     return stunnel_conf
 
-  def installFrontendApache(self, ip_list, port, key, certificate, name,
+  def installFrontendApache(self, ip_list, key, certificate, name,
+                            port, plain_http_port=8080, 
                             rewrite_rule_list=[], rewrite_rule_zope_list=[],
                             access_control_string=None):
     # Create htdocs, populate it with default 404 document
@@ -395,10 +467,36 @@ class Recipe(BaseSlapRecipe):
     notfound_file_content = open(notfound_template_file_location, 'r').read()
     self._writeFile(notfound_file_location, notfound_file_content)
 
+    # Create mod_ssl cache directory
+    cache_directory_location = os.path.join(self.var_directory, 'cache')
+    mod_ssl_cache_location = os.path.join(cache_directory_location,
+        'httpd_mod_ssl')
+    self._createDirectory(cache_directory_location)
+    self._createDirectory(mod_ssl_cache_location)
+
+    # Create "custom" apache configuration file if it does not exist.
+    # Note : This file won't be erased or changed when slapgrid is ran.
+    # It can be freely customized by node admin.
+    custom_apache_configuration_directory = os.path.join(
+        self.data_root_directory, 'apache-conf.d')
+    self._createDirectory(custom_apache_configuration_directory)
+    custom_apache_configuration_file_location = os.path.join(
+        custom_apache_configuration_directory, 'apache_frontend.custom.conf')
+    f = open(custom_apache_configuration_file_location, 'a')
+    f.close()
+
+    # Create backup of custom apache configuration
+    backup_path = self.createBackupDirectory('custom_apache_conf_backup')
+    backup_cron = os.path.join(self.cron_d, 'custom_apache_conf_backup')
+    open(backup_cron, 'w').write(
+        '''0 0 * * * %(rdiff_backup)s %(source)s %(destination)s'''%dict(
+          rdiff_backup=self.options['rdiff_backup_binary'],
+          source=custom_apache_configuration_directory,
+          destination=backup_path))
+    self.path_list.append(backup_cron)
+
     # Create configuration file and rewritemaps
     apachemap_name = "apachemap.txt"
-    # XXX-Cedric : implement zope specific rewrites list. Current apachemap is
-    #              generic and does not use VirtualHost Monster.
     apachemapzope_name = "apachemapzope.txt"
     self.createConfigurationFile(apachemap_name, "\n".join(rewrite_rule_list))
     self.createConfigurationFile(apachemapzope_name,
@@ -406,9 +504,17 @@ class Recipe(BaseSlapRecipe):
     apache_conf = self._getApacheConfigurationDict(name, ip_list, port)
     apache_conf['ssl_snippet'] = self.substituteTemplate(
         self.getTemplateFilename('apache.ssl-snippet.conf.in'),
-        dict(login_certificate=certificate, login_key=key))
+        dict(login_certificate=certificate,
+            login_key=key,
+            httpd_mod_ssl_cache_directory=mod_ssl_cache_location,
+        )
+    )
 
-    apache_conf["listen"] = "\n".join(["Listen %s:%s" % (ip, port) for ip in ip_list])
+    apache_conf["listen"] = "\n".join([
+        "Listen %s:%s" % (ip, port)
+        for port in (plain_http_port, port)
+        for ip in ip_list
+    ])
 
     path = self.substituteTemplate(
         self.getTemplateFilename('apache.conf.path-protected.in'),
@@ -419,7 +525,9 @@ class Recipe(BaseSlapRecipe):
       apachemap_path=os.path.join(self.etc_directory, apachemap_name),
       apachemapzope_path=os.path.join(self.etc_directory, apachemapzope_name),
       apache_domain=name,
-      port=port,
+      https_port=port,
+      plain_http_port=plain_http_port,
+      custom_apache_conf=custom_apache_configuration_file_location,
     ))
 
     apache_conf_string = self.substituteTemplate(
@@ -427,11 +535,10 @@ class Recipe(BaseSlapRecipe):
 
     apache_config_file = self.createConfigurationFile('apache_frontend.conf',
         apache_conf_string)
-
-
     self.path_list.append(apache_config_file)
+
     self.path_list.extend(zc.buildout.easy_install.scripts([(
-      name, 'slapos.recipe.erp5.apache', 'runApache')], self.ws,
+      'frontend_apache', 'slapos.recipe.erp5.apache', 'runApache')], self.ws,
           sys.executable, self.wrapper_directory, arguments=[
             dict(
               required_path_list=[key, certificate],

slapos/recipe/apache_frontend/template/apache.conf.in

@@ -3,7 +3,6 @@
 
 # Basic server configuration
 PidFile "%(pid_file)s"
-LockFile "%(lock_file)s"
 ServerName %(server_name)s
 DocumentRoot %(document_root)s
 
@@ -18,8 +17,7 @@ AddType application/x-gzip .gz .tgz
 # As backend is trusting REMOTE_USER header unset it always
 RequestHeader unset REMOTE_USER
 
-# SSL Configuration
-%(ssl_snippet)s
+ServerTokens Prod
 
 # Log configuration
 ErrorLog "%(error_log)s"
@@ -28,36 +26,12 @@ LogFormat "%%h %%{REMOTE_USER}i %%l %%u %%t \"%%r\" %%>s %%b \"%%{Referer}i\" \"
 LogFormat "%%h %%{REMOTE_USER}i %%l %%u %%t \"%%r\" %%>s %%b" common
 CustomLog "%(access_log)s" common
 
-# Directory protection
-<Directory />
-    Options FollowSymLinks
-    AllowOverride None
-    Order deny,allow
-    Deny from all
-</Directory>
-
 %(path_enable)s
 
-# Rewrite part
-RewriteEngine On
-
-# Define the two rewritemaps : one for zope, one generic
-RewriteMap apachemapzope txt:%(apachemapzope_path)s
-RewriteMap apachemapgeneric txt:%(apachemap_path)s
-
-# First, we check if we have a zope backend server
-# If so, let's use Virtual Host Daemon rewrite
-#RewriteCond ${apachemapzope:%%{SERVER_NAME}} >""
-#RewriteRule ^/(\w+)($|/.*) ${apachemapzope:$1}/VirtualHostBase/https/%(apache_domain)s:%(port)s/VirtualHostRoot/_vh_$1$2 [L,P]
-
-# If we have generic backend server, let's rewrite without virtual host daemon
-RewriteCond ${apachemapgeneric:%%{SERVER_NAME}} >""
-RewriteRule ^/(.*)$ ${apachemapgeneric:%%{SERVER_NAME}}/$1 [L,P]
-
-# If nothing exist : put a nice error
-ErrorDocument 404 /notfound.html
-
 # List of modules
+#LoadModule unixd_module modules/mod_unixd.so
+#LoadModule access_compat_module modules/mod_access_compat.so
+#LoadModule authz_core_module modules/mod_authz_core.so
 LoadModule authz_host_module modules/mod_authz_host.so
 LoadModule log_config_module modules/mod_log_config.so
 LoadModule setenvif_module modules/mod_setenvif.so
@@ -71,4 +45,83 @@ LoadModule dav_fs_module modules/mod_dav_fs.so
 LoadModule negotiation_module modules/mod_negotiation.so
 LoadModule rewrite_module modules/mod_rewrite.so
 LoadModule headers_module modules/mod_headers.so
+LoadModule cache_module modules/mod_cache.so
+LoadModule mem_cache_module modules/mod_mem_cache.so
 LoadModule antiloris_module modules/mod_antiloris.so
+
+# The following directives modify normal HTTP response behavior to
+# handle known problems with browser implementations.
+BrowserMatch "Mozilla/2" nokeepalive
+BrowserMatch ".*MSIE.*" nokeepalive ssl-unclean-shutdown \
+                        downgrade-1.0 force-response-1.0
+BrowserMatch "RealPlayer 4\.0" force-response-1.0
+BrowserMatch "Java/1\.0" force-response-1.0
+BrowserMatch "JDK/1\.0" force-response-1.0
+# The following directive disables redirects on non-GET requests for
+# a directory that does not include the trailing slash.  This fixes a 
+# problem with Microsoft WebFolders which does not appropriately handle 
+# redirects for folders with DAV methods.
+# Same deal with Apple's DAV filesystem and Gnome VFS support for DAV.
+BrowserMatch "Microsoft Data Access Internet Publishing Provider" redirect-carefully
+BrowserMatch "MS FrontPage" redirect-carefully
+BrowserMatch "^WebDrive" redirect-carefully
+BrowserMatch "^WebDAVFS/1.[0123]" redirect-carefully
+BrowserMatch "^gnome-vfs" redirect-carefully
+BrowserMatch "^XML Spy" redirect-carefully
+BrowserMatch "^Dreamweaver-WebDAV-SCM1" redirect-carefully
+
+# Cache directives
+CacheEnable mem /
+CacheDefaultExpire 3600
+MCacheSize 8192
+MCacheMaxObjectCount 1000
+MCacheMaxObjectSize 8192
+MCacheRemovalAlgorithm LRU
+
+# Deflate
+AddOutputFilterByType DEFLATE text/html text/plain text/xml text/css text/javascript application/x-javascript application/javascript
+BrowserMatch ^Mozilla/4 gzip-only-text/html
+BrowserMatch ^Mozilla/4\.0[678] no-gzip
+BrowserMatch \bMSIE !no-gzip !gzip-only-text/html 
+# Make sure proxies don't deliver the wrong content
+Header append Vary User-Agent
+
+# SSL Configuration
+%(ssl_snippet)s
+
+# Dummy virtualhost redirecting to https. Note: will work only if https listens
+# on standard port (443)
+<VirtualHost *:%(plain_http_port)s>
+  RewriteEngine On
+  # Not using HTTPS? Ask that guy over there.
+  RewriteRule ^/(.*)$ https://%%{SERVER_NAME}%%{REQUEST_URI}
+</VirtualHost>
+
+<VirtualHost *:%(https_port)s>
+  SSLEngine on
+  SSLProxyEngine on
+  # Rewrite part
+  ProxyVia On
+  ProxyTimeout 600
+  RewriteEngine On
+
+  # Define the two rewritemaps : one for zope, one generic
+  RewriteMap apachemapzope txt:%(apachemapzope_path)s
+  RewriteMap apachemapgeneric txt:%(apachemap_path)s
+
+  # First, we check if we have a zope backend server
+  # If so, let's use Virtual Host Daemon rewrite
+  RewriteCond ${apachemapzope:%%{SERVER_NAME}} >""
+  RewriteRule ^/(.*)$ ${apachemapzope:%%{SERVER_NAME}}/VirtualHostBase/https/%%{SERVER_NAME}:%%{SERVER_PORT}/VirtualHostRoot/$1 [L,P]
+
+  # If we have generic backend server, let's rewrite without virtual host daemon
+  RewriteCond ${apachemapgeneric:%%{SERVER_NAME}} >""
+  RewriteRule ^/(.*)$ ${apachemapgeneric:%%{SERVER_NAME}}/$1 [L,P]
+
+  # If nothing exist : put a nice error
+  ErrorDocument 404 /notfound.html
+</VirtualHost>
+
+# Include configuration file not operated by slapos. This file won't be erased
+# or changed when slapgrid is ran. It can be freely customized by node admin.
+Include %(custom_apache_conf)s

slapos/recipe/apache_frontend/template/apache.ssl-snippet.conf.in

-SSLEngine on
-SSLProxyEngine on
 SSLCertificateFile %(login_certificate)s
 SSLCertificateKeyFile %(login_key)s
 SSLRandomSeed startup builtin
 SSLRandomSeed connect builtin
+SSLSessionCache shmcb:/%(httpd_mod_ssl_cache_directory)s/ssl_scache(512000)
+SSLSessionCacheTimeout  300
+SSLRandomSeed startup /dev/urandom 256
+SSLRandomSeed connect builtin
+SSLProtocol -ALL +SSLv3 +TLSv1
+SSLHonorCipherOrder On
+SSLCipherSuite RC4-SHA:HIGH:!ADH
+<FilesMatch "\.(cgi|shtml|phtml|php)$">
+      SSLOptions +StdEnvVars
+</FilesMatch>
+# Accept proxy to sites using self-signed SSL certificates
+SSLProxyCheckPeerCN off
+SSLProxyCheckPeerExpire off

slapos/recipe/apachephp/__init__.py

@@ -37,9 +37,9 @@ class Recipe(GenericBaseRecipe):
     path_list = []
 
     # Copy application
-    shutil.rmtree(self.options['htdocs'])
-    shutil.copytree(self.options['source'],
-                    self.options['htdocs'])
+    if not os.path.exists(self.options['htdocs']):
+      shutil.copytree(self.options['source'],
+                      self.options['htdocs'])
 
     # Install php.ini
     php_ini = self.createFile(os.path.join(self.options['php-ini-dir'],
@@ -112,6 +112,8 @@ class Recipe(GenericBaseRecipe):
       # Reload apache configuration
       with open(self.options['pid-file']) as pid_file:
         pid = int(pid_file.read().strip(), 10)
-      os.kill(pid, signal.SIGUSR1) # Graceful restart
-
+      try:
+        os.kill(pid, signal.SIGUSR1) # Graceful restart
+      except OSError:
+        pass
     return path_list

slapos/recipe/librecipe/__init__.py

@@ -236,6 +236,8 @@ class BaseSlapRecipe:
     return 'insecure'
 
   def install(self):
+    self.logger.warning("BaseSlapRecipe has been deprecated. Use " \
+        "GenericBaseRecipe or GenericSlapRecipe instead.")
     self.slap.initializeConnection(self.server_url, self.key_file,
         self.cert_file)
     self.computer_partition = self.slap.registerComputerPartition(

slapos/recipe/librecipe/generic.py

@@ -36,6 +36,8 @@ import pkg_resources
 import zc.buildout
 
 class GenericBaseRecipe(object):
+  """Boilerplate class providing helpful methods for all SlapOS recipes.
+     Can be used to extend SlapOS recipes to ease development"""
 
   TRUE_VALUES = ['y', 'yes', '1', 'true']
   FALSE_VALUES = ['n', 'no', '0', 'false']
@@ -145,9 +147,6 @@ class GenericBaseRecipe(object):
     * if the host is an ipv6 address, brackets will be added to surround it.
 
     """
-    # XXX-Antoine: I didn't find any standard module to join an url with
-    # login, password, ipv6 host and port.
-    # So instead of copy and past in every recipe I factorized it right here.
     netloc = ''
     if auth is not None:
       auth = tuple(auth)

slapos/recipe/librecipe/genericslap.py

@@ -30,7 +30,10 @@ import time
 from generic import GenericBaseRecipe
 
 class GenericSlapRecipe(GenericBaseRecipe):
-  """Base class for all slap.recipe.*"""
+  """Base class for all slap.recipe.* needing SLAP informations like instance
+     parameters.
+     recipes that don't explicitely need to retrieve from server informations
+     should use GenericBaseRecipe."""
 
   def __init__(self, buildout, name, options):
     """Default initialisation"""

software/apache-frontend/README.apache_frontend.txt

+apache_frontend
+===============
+
+Frontend system using Apache, allowing to rewrite and proxy URLs like
+myinstance.myfrontenddomainname.com to real IP/URL of myinstance.
+
+apache_frontend works using the master instance / slave instance design.
+It means that a single main instance of Apache will be used to act as frontend
+for many slaves.
+
+
+How to use
+==========
+
+First, you will need to request a "master" instance of Apache Frontend with
+"domain" parameter, like::
+  <?xml version='1.0' encoding='utf-8'?>
+  <instance>
+   <parameter id="domain">moulefrite.org</parameter>
+   <parameter id="port">443</parameter>
+  </instance>
+
+Then, it is possible to request many slave instances
+(currently only from slapconsole, UI doesn't work yet)
+of Apache Frontend, like::
+  instance = request(
+    software_release=apache_frontend,
+    partition_reference='frontend2',
+    shared=True,
+    partition_parameter_kw={"url":"https://[1:2:3:4]:1234/someresource"}
+  )
+Those slave instances will be redirected to the "master" instance,
+and you will see on the "master" instance the associated RewriteRules of
+all slave instances.
+
+Finally, the slave instance will be accessible from:
+https://someidentifier.moulefrite.org.
+
+Instance Parameters
+===================
+
+Master Instance Parameters
+--------------------------
+
+domain
+~~~~~~
+name of the domain to be used (example: mydomain.com). Subdomains of this
+domain will be used for the slave instances (example:
+instance12345.mydomain.com). It is then recommended to add a wildcard in DNS
+for the subdomains of the chosen domain like::
+  *.mydomain.com. IN A 123.123.123.123
+Using the IP given by the Master Instance.
+"domain" is a mandatory Parameter.
+
+port
+~~~~
+Port used by Apache. Optional parameter, defaults to 4443.
+
+plain_http_port
+Port used by apache to serve plain http (only used to redirect to https).
+Optional parameter, defaults to 8080.
+
+Slave Instance Parameters
+-------------------------
+
+url
+~~~
+url of backend to use.
+"url" is a mandatory parameter.
+Example: http://mybackend.com/myresource
+
+cache
+~~~~~
+Specify if slave instance should use a varnish / stunnel to connect to backend.
+Possible values: "true", "false".
+"cache" is an optional parameter. Defaults to "false". 
+Example: true
+
+type
+~~~~
+Specify if slave instance will redirect to a zope backend. If specified, Apache
+RewriteRule will use Zope's Virtual Host Daemon.
+Possible values: "zope", "default".
+"type" is an optional parameter. Defaults to "default". 
+Example: zope
+
+custom_domain
+~~~~~~~~~~~~~
+Domain name to use as frontend. The frontend will be accessible from this domain.
+"custom_domain" is an optional parameter. Defaults to
+[instancereference].[masterdomain]. 
+Example: www.mycustomdomain.com
+
+
+Advanced example
+================
+
+Request slave frontend instance using a Zope backend, with Varnish activated,
+listening to a custom domain::
+  instance = request(
+    software_release=apache_frontend,
+    partition_reference='frontend2',
+    shared=True,
+    partition_parameter_kw={
+        "url":"https://[1:2:3:4]:1234/someresource",
+        "cache":"true",
+        "type":"zope",
+        "custom_domain":"mycustomdomain.com",
+    }
+  )
+
+Notes
+=====
+
+It is not possible with slapos to listen to port <= 1024, because process are
+not run as root. It is a good idea then to go on the node where the instance is
+and set some iptables rules like (if using default ports)::
+  iptables -t nat -A PREROUTING -p tcp -d {public ip} --dport 443 -j DNAT --to-destination {listening ip}:4443
+  iptables -t nat -A PREROUTING -p tcp -d {public_ip} --dport 80 -j DNAT --to-destination {listening ip}:8080

software/apache-frontend/instance.cfg

@@ -7,7 +7,7 @@ develop-eggs-directory = ${buildout:develop-eggs-directory}
 
 [instance]
 recipe = ${instance-recipe:egg}:${instance-recipe:module}
-httpd_binary = ${apache:location}/bin/httpd
+httpd_binary = ${apache-2.2:location}/bin/httpd
 logrotate_binary = ${logrotate:location}/usr/sbin/logrotate
 openssl_binary = ${openssl:location}/bin/openssl
 dcrond_binary = ${dcron:location}/sbin/crond

software/apache-frontend/software.cfg

@@ -17,8 +17,8 @@ parts =
   template
   binutils
   gcc-java-minimal
-  apache
-  apache-antiloris
+  apache-2.2
+  apache-antiloris-apache-2.2
 
   stunnel
   varnish-2.1
@@ -50,77 +50,71 @@ eggs =
 # Default template for apache instance.
 recipe = slapos.recipe.template
 url = ${:_profile_base_location_}/instance.cfg
-md5sum = 17180caef7d1c477fbb037d28b705e8b
+md5sum = 74c0f41246d167c020854a212e919ce4
 output = ${buildout:directory}/template.cfg
 mode = 0644
 
 [versions]
-# Use SlapOS patched zc.buildout
-zc.buildout = 1.6.0-dev-SlapOS-004
 Jinja2 = 2.6
 Werkzeug = 0.8.3
 buildout-versions = 1.7
 hexagonit.recipe.cmmi = 1.5.0
 meld3 = 0.6.8
 rdiff-backup = 1.0.5
-slapos.recipe.template = 2.2
-slapos.cookbook = 0.40.1
+slapos.cookbook = 0.50
+slapos.recipe.template = 2.3
 
 # Required by:
-# slapos.core==0.23
+# slapos.core==0.24
 Flask = 0.8
 
 # Required by:
-# slapos.cookbook==0.40.1
+# slapos.cookbook==0.50
 PyXML = 0.8.4
 
 # Required by:
-# hexagonit.recipe.cmmi==1.5.0
-hexagonit.recipe.download = 1.5.0
-
-# Required by:
-# slapos.cookbook==0.40.1
+# slapos.cookbook==0.50
 inotifyx = 0.2.0
 
 # Required by:
-# slapos.cookbook==0.40.1
-# slapos.core==0.23
+# slapos.cookbook==0.50
+# slapos.core==0.24
 # xml-marshaller==0.9.7
-lxml = 2.3.3
+lxml = 2.3.4
 
 # Required by:
-# slapos.cookbook==0.40.1
+# slapos.cookbook==0.50
 netaddr = 0.7.6
 
 # Required by:
-# slapos.core==0.23
+# slapos.core==0.24
 netifaces = 0.8
 
 # Required by:
-# slapos.cookbook==0.40.1
-# slapos.core==0.23
+# slapos.cookbook==0.50
+# slapos.core==0.24
 # zc.buildout==1.6.0-dev-SlapOS-004
 # zc.recipe.egg==1.3.2
 setuptools = 0.6c12dev-r88846
 
 # Required by:
-# slapos.cookbook==0.40.1
-slapos.core = 0.23
+# slapos.cookbook==0.50
+slapos.core = 0.24
 
 # Required by:
-# slapos.core==0.23
+# slapos.core==0.24
 supervisor = 3.0a12
 
 # Required by:
-# slapos.cookbook==0.40.1
+# slapos.cookbook==0.50
 xml-marshaller = 0.9.7
 
 # Required by:
-# slapos.cookbook==0.40.1
+# slapos.cookbook==0.50
 zc.recipe.egg = 1.3.2
 
 # Required by:
-# slapos.core==0.23
+# slapos.core==0.24
 zope.interface = 3.8.0
 
 [networkcache]