PALO: polished ETL and support for ssl

software/palo/instance.cfg.in

 [buildout]
 parts =
   instance-parameter
-  directory
   palo_olap
-  publish-connection-parameter
   palo_etl
+  publish-connection-parameter
 
 eggs-directory = ${buildout:eggs-directory}
 develop-eggs-directory = ${buildout:develop-eggs-directory}
 offline = true
 
+# TODO: json schema with parameters
+# TODO: review certificate generation, file permissions and passwords
 
+# common parts
 [instance-parameter]
 recipe = slapos.cookbook:slapconfiguration
 computer = $${slap_connection:computer_id}
@@ -19,51 +21,53 @@ url = $${slap_connection:server_url}
 key = $${slap_connection:key_file}
 cert = $${slap_connection:cert_file}
 
+# erp5_url is the URL of an ERP5 instance, with erp5_palo business template installed
 configuration.erp5_url = 
+# TODO: configuration of a simple password based authentication ? 
 
-[palo_olap_parameter]
-ipv4 = $${instance-parameter:ipv4-random}
-ipv6 = $${instance-parameter:ipv6-random}
-palo_olap_port = 7777
-palo_olap_admin_port = 7778
+
+[publish-connection-parameter]
+recipe = slapos.cookbook:publish
+palo_olap = [$${palo_olap_parameter:ipv6}]:$${palo_olap_parameter:palo_olap_port}
+palo_olap_ssl = [$${palo_olap_parameter:ipv6}]:$${palo_olap_parameter:palo_olap_ssl_port}
+palo_etl_url = https://[$${palo_etl_parameter:tomcat_host}]:$${palo_etl_parameter:tomcat_port}/etlserver/services//ETL-Server?wsdl
+# palo_olap_admin_url = [$${palo_olap_parameter:ipv6}]:$${palo_olap_parameter:palo_olap_admin_port}
 
 [directory]
 recipe = slapos.cookbook:mkdirectory
 home = $${buildout:directory}
+bin = $${:home}/bin
 etc = $${:home}/etc
-var = $${:home}/var
-script = $${:etc}/run/
+script = $${:etc}/run
 service = $${:etc}/service
-promise = $${:etc}/promise/
+promise = $${:etc}/promise
+var = $${:home}/var
 log = $${:var}/log
+srv = $${:home}/srv
 
-[directory_palo_etl]
-<= directory
-data_directory = $${:var}/palo_olap/
-
-[directory_tomcat]
-<= directory
-catalina_base = $${:var}/palo_etl
-catalina_logs = $${:catalina_base}/logs
-catalina_temp = $${:catalina_base}/temp
-catalina_webapps = $${:catalina_base}/webapps
-catalina_work = $${:catalina_base}/work
-catalina_conf = $${:catalina_base}/conf
 
+# Palo olap instance
+[palo_olap_parameter]
+ipv4 = $${instance-parameter:ipv4-random}
+ipv6 = $${instance-parameter:ipv6-random}
+palo_olap_port = 7777
+palo_olap_ssl_port = 7778
+palo_olap_admin_port = 7779
+key-file = $${certificate_palo_olap:key-file}
+dh1024-file = $${certificate_palo_olap:dh1024-file}
 
-[TODO]
-todo =
-  tunnel
-  etl
-  log_rotation
+# XXX this depends on architecture. Maybe we need to patch palo_olap for that
+extensions_dir = ${palo_olap:location}/usr/lib64
 
+[directory_palo_olap]
+<= directory
+data_directory = $${:var}/palo_olap
 
 [palo_ini]
 recipe = slapos.recipe.template:jinja2
-template = ${:_profile_base_location_}/palo.ini.in
+template = ${template_palo_ini:target}
 rendered = $${directory:etc}/palo.ini
-extensions = jinja2.ext.do
-context = import json_module json
+context =
   key directory_log directory:log
   raw palo_olap_repository_location ${palo_olap-repository.git:location}
   section parameter instance-parameter
@@ -71,45 +75,69 @@ context = import json_module json
   key erp5_url instance-parameter:configuration.erp5_url
   raw erp5_login_worker_path ${erp5_login_worker:output}
 
-[palo_olap]
-recipe = slapos.cookbook:wrapper
-command-line = ${palo_olap:location}/usr/bin/palo --data-directory $${directory_palo_etl:data_directory}  --init-file $${palo_ini:rendered}
-wrapper-path = $${directory:service}/palo_olap
-
-[tomcat_palo_etl]
+[certificate_palo_olap]
 recipe = plone.recipe.command
-command = echo "needed ?"
+# Manually generate certificates as documented in palo.ini.sample
+command =
+  ${openssl:location}/bin/openssl req -x509 -nodes -days 3650 \
+  -subj "/C=AA/ST=X/L=X/O=Dis/CN=$${palo_olap_parameter:ipv6}" \
+  -newkey rsa:1024 -keyout $${:key-file} -out $${:key-file} && \
+  ${openssl:location}/bin/openssl dhparam -2 -outform PEM -out $${:dh1024-file} 1024
+key-file = $${directory:etc}/palo_olap.pem
+dh1024-file = $${directory:etc}/dh1024.pem
 
-[palo_etl]
+[palo_olap]
 recipe = slapos.cookbook:wrapper
-command-line = ${tomcat:location}/bin/catalina.sh run
-wrapper-path = $${directory:service}/palo_etl
-environment = JAVA_HOME = ${java:location}
-  CATALINA_BASE = $${directory_tomcat:catalina_base}
-dependencies = $${palo_etl_server_xml:rendered} $${tomcat_palo_etl:recipe}
+command-line = ${palo_olap:location}/usr/bin/palo --data-directory $${directory_palo_olap:data_directory}  --init-file $${palo_ini:rendered}
+wrapper-path = $${directory:service}/palo_olap
 
-[palo_etl_server_xml]
-recipe = slapos.recipe.template:jinja2
-# XXX  template = ${template_server_xml:location}/${template_server_xml:filename}
-template = ${template_server_xml:url}
-rendered = $${directory_tomcat:catalina_conf}/server.xml
-extensions = jinja2.ext.do
-context = import json_module json
-  section palo_etl_parameter palo_etl_parameter
-  raw palo_etl_server_war ${palo_etl_download:location}
 
+# Palo etl instance
 [palo_etl_parameter]
-tomcat_port = 8888
+tomcat_port = 8443
 tomcat_host = $${:ipv6}
-tomcat_server_port = 8006
 ipv4 = $${instance-parameter:ipv4-random}
 ipv6 = $${instance-parameter:ipv6-random}
 palo_etl_war = ${palo_etl_download:location}
+keystore_file = $${keystore_import:keystore_file}
+keystore_pass = $${keystore_import:keystore_pass}
 
-[publish-connection-parameter]
-recipe = slapos.cookbook:publish
-palo_olap_url = $${palo_olap_parameter:ipv4}:$${palo_olap_parameter:palo_olap_port}
-palo_olap_ipv6_url = [$${palo_olap_parameter:ipv6}]:$${palo_olap_parameter:palo_olap_port}
-palo_olap_admin_url = $${palo_olap_parameter:ipv4}:$${palo_olap_parameter:palo_olap_admin_port}
-palo_etl_url = http://[$${palo_etl_parameter:tomcat_host}]:$${palo_etl_parameter:tomcat_port}/etlserver/services/ETL-Server?wsdl
+[keystore_import]
+recipe = plone.recipe.command
+command =
+  ${java:location}/bin/keytool \
+    -genkeypair \
+    -alias "tomcat" \
+    -keyalg RSA \
+    -keypass "$${:keystore_pass}" \
+    -dname "CN=Web Server,OU=Unit,O=Organization,L=City,S=State,C=Country" \
+    -keystore "$${:keystore_file}" \
+    -storepass "$${:keystore_pass}"
+keystore_file = $${directory_palo_etl:catalina_base}/.keystore
+keystore_pass = insecure
+
+[directory_palo_etl]
+<= directory
+catalina_base = $${:var}/palo_etl
+catalina_logs = $${:catalina_base}/logs
+catalina_temp = $${:catalina_base}/temp
+catalina_webapps = $${:catalina_base}/webapps
+catalina_work = $${:catalina_base}/work
+catalina_conf = $${:catalina_base}/conf
 
+[palo_etl_server_xml]
+recipe = slapos.recipe.template:jinja2
+template = ${template_server_xml:target}
+rendered = $${directory_palo_etl:catalina_conf}/server.xml
+context =
+  section palo_etl_parameter palo_etl_parameter
+  raw palo_etl_server_war ${palo_etl_download:location}
+
+[palo_etl]
+recipe = slapos.cookbook:wrapper
+command-line = ${tomcat:location}/bin/catalina.sh run
+wrapper-path = $${directory:service}/palo_etl
+environment =
+  JAVA_HOME = ${java:location}
+  CATALINA_BASE = $${directory_palo_etl:catalina_base}
+dependencies = $${palo_etl_server_xml:rendered}

software/palo/palo.ini.in

 template-directory {{ palo_olap_repository_location }}/Api
-http {{ palo_olap_parameter.ipv4 }} {{ palo_olap_parameter.palo_olap_port }}
 http {{ palo_olap_parameter.ipv6 }} {{ palo_olap_parameter.palo_olap_port }}
+https {{ palo_olap_parameter.palo_olap_ssl_port }}
+
+encryption optional
+key-files {{ palo_olap_parameter['key-file'] }} {{ palo_olap_parameter['key-file'] }} {{ palo_olap_parameter['dh1024-file'] }}
+extensions {{ palo_olap_parameter.extensions_dir }}
+
 log {{ directory_log }}/palo_olap.log
 verbose debug
 
 {% if erp5_url %}
 workerlogin authorization
 worker {{ erp5_login_worker_path }} {{ directory_log }}/erp5_login_worker.log {{ erp5_url }}
-admin {{ palo_olap_parameter.ipv4 }} {{ palo_olap_parameter.palo_olap_admin_port }}
+#admin {{ palo_olap_parameter.ipv4 }} {{ palo_olap_parameter.palo_olap_admin_port }}
 {% else %}
 # no authorization enabled, as instance parameter erp5_url was not defined
 {% endif %}
-

software/palo/server.xml.in

 <?xml version='1.0' encoding='utf-8'?>
-<!--
-  Licensed to the Apache Software Foundation (ASF) under one or more
-  contributor license agreements.  See the NOTICE file distributed with
-  this work for additional information regarding copyright ownership.
-  The ASF licenses this file to You under the Apache License, Version 2.0
-  (the "License"); you may not use this file except in compliance with
-  the License.  You may obtain a copy of the License at
-
-      http://www.apache.org/licenses/LICENSE-2.0
-
-  Unless required by applicable law or agreed to in writing, software
-  distributed under the License is distributed on an "AS IS" BASIS,
-  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-  See the License for the specific language governing permissions and
-  limitations under the License.
--->
-<!-- Note:  A "Server" is not itself a "Container", so you may not
-     define subcomponents such as "Valves" at this level.
-     Documentation at /docs/config/server.html
-     
-     Debug: {{ repr(palo_etl_parameter) }}
- -->
 <Server port="-1" shutdown="SHUTDOWN">
-
-  <!--APR library loader. Documentation at /docs/apr.html -->
-  <Listener className="org.apache.catalina.core.AprLifecycleListener" SSLEngine="on" />
-  <!--Initialize Jasper prior to webapps are loaded. Documentation at /docs/jasper-howto.html -->
-  <Listener className="org.apache.catalina.core.JasperListener" />
-  <!-- Prevent memory leaks due to use of particular java/javax APIs-->
-  <Listener className="org.apache.catalina.core.JreMemoryLeakPreventionListener" />
-  <!-- JMX Support for the Tomcat server. Documentation at /docs/non-existent.html -->
-<!--
-  <Listener className="org.apache.catalina.mbeans.ServerLifecycleListener" />
-  <Listener className="org.apache.catalina.mbeans.GlobalResourcesLifecycleListener" />
--->
-
-<!--
-  <GlobalNamingResources>
-    <Resource name="UserDatabase" auth="Container"
-              type="org.apache.catalina.UserDatabase"
-              description="User database that can be updated and saved"
-              factory="org.apache.catalina.users.MemoryUserDatabaseFactory"
-              pathname="conf/tomcat-users.xml" />
-  </GlobalNamingResources>
--->  
-
   <Service name="Catalina">
-    <Connector address="{{ palo_etl_parameter['tomcat_host'] }}" port="{{ palo_etl_parameter['tomcat_port'] }}"
+    <Connector
+      protocol="org.apache.coyote.http11.Http11Protocol"
+      address="{{ palo_etl_parameter['tomcat_host'] }}" 
+      port="{{ palo_etl_parameter['tomcat_port'] }}"
       maxThreads="10"
-      minSpareThreads="3"
-      maxSpareThreads="7"
-      connectionTimeout="20000"
-      enableLookups="false"
-      maxHttpHeaderSize="8192"
-      protocol="HTTP/1.1"
-      useBodyEncodingForURI="true"
-      redirectPort="8443"
-      acceptCount="100"
-      disableUploadTimeout="true" />
+      scheme="https"
+      secure="true"
+      clientAuth="false"
+      sslProtocol="TLS"
+      SSLEnabled="true"
+      keypass="{{ palo_etl_parameter['keystore_pass'] }}"
+      keystore="{{ palo_etl_parameter['keystore_file'] }}"
+    />
 
     <Context path="/" docBase="{{ palo_etl_parameter['palo_etl_war'] }}/etlserver.war" 
-         debug="1" privileged="true">
-      <Realm className="org.apache.catalina.realm.UserDatabaseRealm"                
-         resourceName="UserDatabase"/> 
+       privileged="true">
     </Context>
     
     <Engine name="Catalina" defaultHost="localhost">
-      <Realm className="org.apache.catalina.realm.UserDatabaseRealm"
-             resourceName="UserDatabase"/>
-
-      <Host name="localhost"  appBase="webapps" debug="1"
+      <Host name="localhost"  appBase="webapps"
             unpackWARs="true" autoDeploy="true"
             xmlValidation="false" xmlNamespaceAware="false">
-
       </Host>
     </Engine>
   </Service>

software/palo/software.cfg

@@ -6,17 +6,17 @@ extends =
 
 parts =
   palo_olap
-  palo_etl
+  palo_etl_download
   template_server_xml
+  template_palo_ini
   slapos-cookbook
   instance-profile
 
-
 [instance-profile]
 recipe = slapos.recipe.template
 url = ${:_profile_base_location_}/instance.cfg.in
 output = ${buildout:directory}/instance.cfg
-#md5sum = 
+#md5sum = b30db17333d11ce11f486a96e68ad9b1
 mode = 0644
 
 [palo_etl_download]
@@ -32,11 +32,9 @@ mode = 644
 [template_server_xml]
 < = download-base
 filename = server.xml.in
-# md5sum = 
-
-[palo_etl]
-recipe = plone.recipe.command
-command = echo "etl downlaoded in ${palo_etl_download:location}"
-needs = ${palo_etl_download:url}
+#md5sum = 19daa17468de0940e0d10d4bd4adf86e
 
-[versions]
+[template_palo_ini]
+< = download-base
+filename = palo.ini.in
+#md5sum = d85f449ce322eba4d75b6ad28c8bece8