PALO: Slapos profile for PALO OLAP 5.1 and ETL

an old release of PALO web is also included, but not working for now

component/palo_olap/buildout.cfg

@@ -25,6 +25,7 @@ recipe = slapos.recipe.build:gitclone
 repository = https://lab.nexedi.cn/jerome/palo_olap.git
 branch = 5.1
 git-executable = ${git:location}/bin/git
+develop = true
 
 [palo_olap]
 recipe = slapos.recipe.cmmi

component/palo_olap/erp5_login_worker.py

@@ -6,6 +6,18 @@ import csv
 import hashlib
 import socket
 
+# XXX This should at least be an option.
+import ssl
+
+try:
+  _create_unverified_https_context = ssl._create_unverified_context
+except AttributeError:
+  # Legacy Python that doesn't verify HTTPS certificates by default
+  pass
+else:
+  # Handle target environment that doesn't support HTTPS verification
+  ssl._create_default_https_context = _create_unverified_https_context
+
 
 def main():
   logfile = sys.argv[1]
@@ -42,6 +54,7 @@ def main():
         login, password = args
 
         if login == 'admin':
+          # XXX in Palo 5.1, admin is anyway authenticated internally.
           # XXX better way ?
   #        admin_pass = "admin"
   #        authentication_success = hashlib.md5(admin_pass).hexdigest() == password

software/palo/instance-input-schema.json

+{
+  "$schema": "http://json-schema.org/draft-04/schema#",
+  "extends": "./schema-definitions.json#",
+  "description": "Parameters to instantiate PALO",
+  "additionalProperties": false,
+  "properties": {
+    "ssl": {
+      "description": "Use ssl for PALO OLAP",
+      "default": "false",
+      "type": "boolean"
+    },
+    "erp5_url": {
+      "description": "URL of an ERP5 instance to authenticate users",
+      "default": "",
+      "type": "string"
+    }
+  }
+}

software/palo/instance.cfg.in

 [buildout]
 parts =
   instance-parameter
-  directory
   palo_olap
-  publish-connection-parameter
-todo=
   palo_etl
+  palo_web
+  publish-connection-parameter
 
 eggs-directory = ${buildout:eggs-directory}
 develop-eggs-directory = ${buildout:develop-eggs-directory}
 offline = true
 
+# TODO: review certificate generation, file permissions and passwords
+# TODO: log rotation
+# TODO: ipv4 TCP frontend for palo (http frontend does not seem to work)
+# TODO: palo_web does not work
 
+
+# common parts
 [instance-parameter]
 recipe = slapos.cookbook:slapconfiguration
 computer = $${slap_connection:computer_id}
@@ -20,88 +25,197 @@ url = $${slap_connection:server_url}
 key = $${slap_connection:key_file}
 cert = $${slap_connection:cert_file}
 
-configuration.name = John Doe
+# TODO: name these parameter palo_olap_ ?
+# erp5_url is the URL of an ERP5 instance, with erp5_palo business template installed
 configuration.erp5_url = 
+configuration.ssl = 
+# TODO: configuration of a simple password based authentication ? 
+# TODO: certificates as input parameters ?
 
-[palo_olap_parameter]
-ipv4 = $${instance-parameter:ipv4-random}
-ipv6 = $${instance-parameter:ipv6-random}
-palo_olap_port = 7777
-palo_olap_admin_port = 7778
+
+[publish-connection-parameter]
+recipe = slapos.cookbook:publish
+palo_olap = [$${palo_olap_parameter:ipv6}]:$${palo_olap_parameter:palo_olap_port}
+palo_olap_ssl = [$${palo_olap_parameter:ipv6}]:$${palo_olap_parameter:palo_olap_ssl_port}
+palo_etl_url = https://[$${palo_etl_parameter:tomcat_host}]:$${palo_etl_parameter:tomcat_port}/etlserver/services//ETL-Server?wsdl
+palo_web_url = https://[$${palo_web_parameter:tomcat_host}]:$${palo_web_parameter:tomcat_port}/web-palo/
+# palo_olap_admin_url = [$${palo_olap_parameter:ipv6}]:$${palo_olap_parameter:palo_olap_admin_port}
 
 [directory]
 recipe = slapos.cookbook:mkdirectory
 home = $${buildout:directory}
+bin = $${:home}/bin
 etc = $${:home}/etc
-var = $${:home}/var
-script = $${:etc}/run/
+script = $${:etc}/run
 service = $${:etc}/service
-promise = $${:etc}/promise/
+promise = $${:etc}/promise
+var = $${:home}/var
 log = $${:var}/log
+srv = $${:home}/srv
 
-[directory_tomcat]
-<= directory
-catalina_base = $${:home}/var/palo_etl
-catalina_logs = $${:catalina_base}/logs
-catalina_temp = $${:catalina_base}/temp
-catalina_webapps = $${:catalina_base}/webapps
-catalina_work = $${:catalina_base}/work
-catalina_conf = $${:catalina_base}/conf
 
+# Palo olap instance
+[palo_olap_parameter]
+ipv4 = $${instance-parameter:ipv4-random}
+ipv6 = $${instance-parameter:ipv6-random}
+palo_olap_host = $${:ipv6}
+palo_olap_port = 7777
+palo_olap_ssl_port = 7778
+palo_olap_admin_port = 7779
+key-file = $${certificate_palo_olap:key-file}
+dh1024-file = $${certificate_palo_olap:dh1024-file}
 
-[TODO]
-todo =
-  tunnel
-  patch ipv6
-  etl
-  log_rotation
+# XXX this depends on architecture. Maybe we need to patch palo_olap for that
+extensions_dir = ${palo_olap:location}/usr/lib64
 
+[directory_palo_olap]
+<= directory
+data_directory = $${:var}/palo_olap
 
 [palo_ini]
 recipe = slapos.recipe.template:jinja2
-template = ${:_profile_base_location_}/palo.ini.in
+template = ${template_palo_olap_palo_ini:target}
 rendered = $${directory:etc}/palo.ini
-extensions = jinja2.ext.do
-context = import json_module json
+context =
   key directory_log directory:log
   raw palo_olap_repository_location ${palo_olap-repository.git:location}
   section parameter instance-parameter
   section palo_olap_parameter palo_olap_parameter
   key erp5_url instance-parameter:configuration.erp5_url
   raw erp5_login_worker_path ${erp5_login_worker:output}
+  key ssl instance-parameter:configuration.ssl
+
+[certificate_palo_olap]
+recipe = plone.recipe.command
+# Manually generate certificates as documented in palo.ini.sample
+command =
+  ${openssl:location}/bin/openssl req -x509 -nodes -days 3650 \
+  -subj "/C=AA/ST=X/L=X/O=Dis/CN=$${palo_olap_parameter:ipv6}" \
+  -newkey rsa:1024 -keyout $${:key-file} -out $${:key-file} && \
+  ${openssl:location}/bin/openssl dhparam -2 -outform PEM -out $${:dh1024-file} 1024
+key-file = $${directory:etc}/palo_olap.pem
+dh1024-file = $${directory:etc}/dh1024.pem
 
 [palo_olap]
 recipe = slapos.cookbook:wrapper
-command-line = ${palo_olap:location}/usr/bin/palo --data-direct $${directory:var}  --init-file $${palo_ini:rendered}
+command-line = ${palo_olap:location}/usr/bin/palo --data-directory $${directory_palo_olap:data_directory}  --init-file $${palo_ini:rendered}
 wrapper-path = $${directory:service}/palo_olap
 
+
+# Palo etl instance
+[palo_etl_parameter]
+tomcat_port = 8443
+tomcat_host = $${:ipv6}
+ipv4 = $${instance-parameter:ipv4-random}
+ipv6 = $${instance-parameter:ipv6-random}
+palo_etl_war = ${palo_etl_download:location}
+keystore_file = $${keystore_import:keystore_file}
+keystore_pass = $${keystore_import:keystore_pass}
+
+[keystore_import]
+recipe = plone.recipe.command
+command =
+  ${java:location}/bin/keytool \
+    -genkeypair \
+    -alias "tomcat" \
+    -keyalg RSA \
+    -keypass "$${:keystore_pass}" \
+    -dname "CN=Web Server,OU=Unit,O=Organization,L=City,S=State,C=Country" \
+    -keystore "$${:keystore_file}" \
+    -storepass "$${:keystore_pass}"
+keystore_file = $${directory_palo_etl:catalina_base}/.keystore
+keystore_pass = insecure
+
+[directory_palo_etl]
+<= directory
+catalina_base = $${:var}/palo_etl
+catalina_logs = $${:catalina_base}/logs
+catalina_temp = $${:catalina_base}/temp
+catalina_webapps = $${:catalina_base}/webapps
+catalina_work = $${:catalina_base}/work
+catalina_conf = $${:catalina_base}/conf
+
+[palo_etl_server_xml]
+recipe = slapos.recipe.template:jinja2
+template = ${template_palo_etl_server_xml:target}
+rendered = $${directory_palo_etl:catalina_conf}/server.xml
+context =
+  section palo_etl_parameter palo_etl_parameter
+  raw palo_etl_server_war ${palo_etl_download:location}
+
 [palo_etl]
 recipe = slapos.cookbook:wrapper
 command-line = ${tomcat:location}/bin/catalina.sh run
 wrapper-path = $${directory:service}/palo_etl
-environment = JAVA_HOME = ${java:location}
-  CATALINA_BASE = $${directory_tomcat:catalina_base}
+environment =
+  JAVA_HOME = ${java:location}
+  CATALINA_BASE = $${directory_palo_etl:catalina_base}
 dependencies = $${palo_etl_server_xml:rendered}
 
-[palo_etl_server_xml]
-recipe = slapos.recipe.template:jinja2
-template = ${:_profile_base_location_}/server.xml.in
-rendered = $${directory_tomcat:catalina_conf}/server.xml
-extensions = jinja2.ext.do
-context = import json_module json
-  section palo_etl_parameter palo_etl_parameter
-  raw palo_etl_server_war ${palo_etl_download:location}
 
-[palo_etl_parameter]
-tomcat_port = 8888
-tomcat_host = $${:ipv4}
-tomcat_server_port = 8005
-ipv4 = $${instance-parameter:ipv4-random}
+# Palo web instance
+[palo_web_parameter]
+tomcat_port = 18443
+tomcat_host = $${:ipv6}
 ipv6 = $${instance-parameter:ipv6-random}
+palo_web_war = ${palo_web_download:location}
+# XXX this is same keystore as etl
+keystore_file = $${keystore_import:keystore_file}
+keystore_pass = $${keystore_import:keystore_pass}
 
-[publish-connection-parameter]
-recipe = slapos.cookbook:publish
-palo_olap_url = $${palo_olap_parameter:ipv4}:$${palo_olap_parameter:palo_olap_port}
-palo_olap_ipv6_url = [$${palo_olap_parameter:ipv6}]:$${palo_olap_parameter:palo_olap_port}
-palo_olap_admin_url = $${palo_olap_parameter:ipv4}:$${palo_olap_parameter:palo_olap_admin_port}
+[directory_palo_web]
+<= directory
+catalina_base = $${:var}/palo_web
+catalina_logs = $${:catalina_base}/logs
+catalina_temp = $${:catalina_base}/temp
+catalina_webapps = $${:catalina_base}/webapps
+catalina_work = $${:catalina_base}/work
+catalina_conf = $${:catalina_base}/conf
+
+[palo_web_server_xml]
+recipe = slapos.recipe.template:jinja2
+template = ${template_palo_web_server_xml:target}
+rendered = $${directory_palo_web:catalina_conf}/server.xml
+context =
+  section palo_web_parameter palo_web_parameter
+  raw palo_web_server_war ${palo_web_download:location}
+
+[palo_web_context_xml]
+recipe = slapos.recipe.template:jinja2
+rendered =  $${directory_palo_web:catalina_conf}/context.xml
+template = inline:
+  <Context>
+    <WatchedResource>WEB-INF/web.xml</WatchedResource>
+  </Context>
+
+[palo_web_web_xml]
+recipe = slapos.recipe.template:jinja2
+rendered =  $${directory_palo_web:catalina_conf}/web.xml
+template = ${tomcat:location}/conf/web.xml
+
+[palo_web]
+recipe = slapos.cookbook:wrapper
+command-line = ${tomcat:location}/bin/catalina.sh run
+wrapper-path = $${directory:service}/palo_web
+environment =
+  JAVA_HOME = ${java:location}
+  CATALINA_BASE = $${directory_palo_web:catalina_base}
+dependencies = 
+  $${palo_web_server_xml:rendered}
+  $${palo_web_context_xml:rendered}
+  $${palo_web_web_xml:rendered}
+  $${palo_web_palo_properties:rendered}
+
+[palo_web_war]
+recipe = hexagonit.recipe.download
+url = ${palo_web_download:location}/PaloWebClient_2.0.0.414_JPA_181/web-palo.war
+destination = $${directory_palo_web:catalina_webapps}/web-palo/
+
+[palo_web_palo_properties]
+; This "patches" the content of the war that was extracted by palo_web_war
+recipe = slapos.recipe.template:jinja2
+template = ${template_palo_web_palo_properties:target}
+rendered = $${palo_web_war:destination}/WEB-INF/classes/palo.properties
+context =
+  section palo_olap_parameter palo_olap_parameter
 

software/palo/palo.ini.in

 template-directory {{ palo_olap_repository_location }}/Api
 http {{ palo_olap_parameter.ipv4 }} {{ palo_olap_parameter.palo_olap_port }}
+{% if ssl %}https {{ palo_olap_parameter.palo_olap_ssl_port }}
+{% endif %}
+
+
 http {{ palo_olap_parameter.ipv6 }} {{ palo_olap_parameter.palo_olap_port }}
+{% if ssl %}https {{ palo_olap_parameter.palo_olap_ssl_port }}
+{% endif %}
+
+{% if ssl %}
+encryption optional
+key-files {{ palo_olap_parameter['key-file'] }} {{ palo_olap_parameter['key-file'] }} {{ palo_olap_parameter['dh1024-file'] }}
+extensions {{ palo_olap_parameter.extensions_dir }}
+{% endif %}
+
 log {{ directory_log }}/palo_olap.log
 verbose debug
 
 {% if erp5_url %}
-user-login
 workerlogin authorization
 worker {{ erp5_login_worker_path }} {{ directory_log }}/erp5_login_worker.log {{ erp5_url }}
-admin {{ palo_olap_parameter.ipv4 }} {{ palo_olap_parameter.palo_olap_admin_port }}
+#admin {{ palo_olap_parameter.ipv4 }} {{ palo_olap_parameter.palo_olap_admin_port }}
 {% else %}
 # no authorization enabled, as instance parameter erp5_url was not defined
 {% endif %}
-

software/palo/palo.properties.in

+# Web client login and passwords, which allows access to Web Palo Server
+user=guest
+password=pass
+
+server1.dispname=Palo
+server1.url={{ palo_olap_parameter.ipv4 }}
+server1.service={{ palo_olap_parameter.palo_olap_port }}
+server1.login=admin
+server1.password=admin
+server1.provider=palo
+
+
+
+
+# the rest is the original configuration
+
+
+
+
+# This option defines maximal number of opened connections between Web Palo Server and Palo Server
+
+# (Server-to-Server connections), so it determines the number of request,
+
+# which can be handled by Web Palo Server simultaneously.
+
+connection.pool.max=6
+
+# This option defines if Server-to-Server connections should be reload on every client login.
+# If it is set to true, it ensures, that newly logged in client has actual structures.
+
+# The drawback: other working clients will suffer speed penalty every time somebody logs in.
+
+client.behavior.database.reload-on-login=true
+
+# This option defines visibility of dimension folder under cube node in Palo Database Explorer.
+client.ui.show.cube.dimensions=false
+
+# This option defines visibility of dimension folder under database node in Palo Database Explorer.
+client.ui.show.database.dimensions=false
+
+# This option defines initial visibility of Navigation Panel.
+client.ui.show.navigationpanel=true
+
+# This option defines visibility of the Favorite Views page.
+client.ui.show.favoriteviews=true
+
+# This option defines visibility of the Database Explorer
+client.ui.show.dbexplorer=true
+
+# This option defines initial width of the Navigation Panel
+client.ui.navigationpanel.width=240
+
+# This option defines the number of decimal places displayed in cube table after rounding.
+client.ui.format.fraction.number_of_digits=2
+
+# This option defines if the whole hierarchy down to selected element should be loaded and shown to user,
+
+# when he clicks on dim selection section (POV=Point-of-view) to select another element.
+
+client.ui.views.pov.loadselectedpath=false
+
+# These two options define the number of levels, which should be expanded by default on show cube,
+
+# but it is not relevant for views open.
+# The first of them defines the number in POV, the second in cube table view headers
+client.ui.views.pov.showlevels=1
+client.ui.views.td.showlevels=1
+
+# These two options define strings, which determine minimal and maximal width of column in cube view table.
+client.ui.views.table.columns.min_visible_string=9.999,99
+client.ui.views.table.columns.max_visible_string=999.999.999.999,99
+
+# This option defines mouse pointer idle time in milliseconds, after which cell tooltip will be displayed.
+
+client.ui.views.table.hint_time=1000
+
+# Internal option. Do not change it
+right.manager.class=com.tensegrity.palowebviewer.server.DefaultRightManager
\ No newline at end of file

software/palo/palo_web_server.xml.in

+<?xml version='1.0' encoding='utf-8'?>
+<Server port="-1" shutdown="SHUTDOWN">
+  <Service name="Catalina">
+    <Connector
+      protocol="org.apache.coyote.http11.Http11Protocol"
+      address="{{ palo_web_parameter['tomcat_host'] }}" 
+      port="{{ palo_web_parameter['tomcat_port'] }}"
+      maxThreads="10"
+      scheme="https"
+      secure="true"
+      clientAuth="false"
+      sslProtocol="TLS"
+      SSLEnabled="true"
+      keypass="{{ palo_web_parameter['keystore_pass'] }}"
+      keystore="{{ palo_web_parameter['keystore_file'] }}"
+    />
+    
+    <Connector
+      protocol="org.apache.coyote.http11.Http11Protocol"
+      address="{{ palo_web_parameter['tomcat_host'] }}" 
+      port="8080"
+      maxThreads="10"
+    />
+    
+    
+<!--
+    <Context path="/" docBase="{{ palo_web_parameter['palo_web_war'] }}/palo-web.war" 
+       privileged="true">
+    </Context>
+    -->
+  
+    <Engine name="Catalina" defaultHost="localhost">
+      <Host name="localhost"  appBase="webapps"
+            unpackWARs="true" autoDeploy="true"
+            xmlValidation="false" xmlNamespaceAware="false">
+      </Host>
+    </Engine>
+  </Service>
+</Server>
\ No newline at end of file

software/palo/server.xml.in

+<?xml version='1.0' encoding='utf-8'?>
+<Server port="-1" shutdown="SHUTDOWN">
+  <Service name="Catalina">
+    <Connector
+      protocol="org.apache.coyote.http11.Http11Protocol"
+      address="{{ palo_etl_parameter['tomcat_host'] }}" 
+      port="{{ palo_etl_parameter['tomcat_port'] }}"
+      maxThreads="10"
+      scheme="https"
+      secure="true"
+      clientAuth="false"
+      sslProtocol="TLS"
+      SSLEnabled="true"
+      keypass="{{ palo_etl_parameter['keystore_pass'] }}"
+      keystore="{{ palo_etl_parameter['keystore_file'] }}"
+    />
+
+    <Context path="/" docBase="{{ palo_etl_parameter['palo_etl_war'] }}/etlserver.war" 
+       privileged="true">
+    </Context>
+    
+    <Engine name="Catalina" defaultHost="localhost">
+      <Host name="localhost"  appBase="webapps"
+            unpackWARs="true" autoDeploy="true"
+            xmlValidation="false" xmlNamespaceAware="false">
+      </Host>
+    </Engine>
+  </Service>
+</Server>
\ No newline at end of file

software/palo/software.cfg

@@ -6,27 +6,51 @@ extends =
 
 parts =
   palo_olap
-  palo_etl
+  palo_etl_download
+  palo_web_download
+  template_palo_olap_palo_ini
+  template_palo_etl_server_xml
+  template_palo_web_server_xml
   slapos-cookbook
   instance-profile
 
-
 [instance-profile]
 recipe = slapos.recipe.template
 url = ${:_profile_base_location_}/instance.cfg.in
 output = ${buildout:directory}/instance.cfg
-#md5sum = 
+#md5sum = b30db17333d11ce11f486a96e68ad9b1
 mode = 0644
 
 [palo_etl_download]
 recipe = hexagonit.recipe.download
 url = http://downloads.sourceforge.net/project/palo-etl-server/etlserver/Palo%20ETL%20Server%203.2/etlserver.war
 download-only = true
-;destination = ${buildout:directory}/var/tomcat/webapps/
 
-[palo_etl]
-recipe = plone.recipe.command
-command = echo "TODO"
-needs = ${palo_etl_download:url}
+[palo_web_download]
+recipe = hexagonit.recipe.download
+url = http://sourceforge.net/projects/jpalo/files/web-palo/Web-Palo%202.0/PaloWebClient_2.0.0.414_JPA_181.zip/download
+
+[download-base]
+recipe = slapos.recipe.build:download
+url = ${:_profile_base_location_}/${:filename}
+mode = 644
+
+[template_palo_etl_server_xml]
+< = download-base
+filename = server.xml.in
+#md5sum = 19daa17468de0940e0d10d4bd4adf86e
+
+[template_palo_web_server_xml]
+< = download-base
+filename = palo_web_server.xml.in
+#md5sum = 19daa17468de0940e0d10d4bd4adf86e
+
+[template_palo_web_palo_properties]
+< = download-base
+filename = palo.properties.in
+#md5sum = 19daa17468de0940e0d10d4bd4adf86e
 
-[versions]
+[template_palo_olap_palo_ini]
+< = download-base
+filename = palo.ini.in
+#md5sum = d85f449ce322eba4d75b6ad28c8bece8