diff --git a/software/erp5/README.rst b/software/erp5/README.rst index dd634f5d020ee330745974ebe7f2ccac210ebedc..d141967f4e46acbe456be179154cfc198fd3a96b 100644 --- a/software/erp5/README.rst +++ b/software/erp5/README.rst @@ -39,6 +39,7 @@ This software release assigns the following port ranges by default: zeo 2100-2149 balancer 2150-2199 zope 2200-* + jupyter 8888 ==================== ========== Non-zope partitions are unique in an ERP5 cluster, so you shouldn't have to diff --git a/software/erp5/instance-erp5-input-schema.json b/software/erp5/instance-erp5-input-schema.json index 50a5d5c13fecca8689f557b5a1cb8959c034a0a0..06949923a5719286d2e01abeac7dca709f935753 100644 --- a/software/erp5/instance-erp5-input-schema.json +++ b/software/erp5/instance-erp5-input-schema.json @@ -235,6 +235,22 @@ "type": "object" }, "type": "array" + }, + "jupyter": { + "description": "Jupyter slave instance parameters", + "properties": { + "enable": { + "description": "Whether to enable creation of associated slave Jupyter instance", + "default": false, + "type": "boolean" + }, + "zope-family": { + "description": "Zope family to connect Jupyter to by default", + "default": "<first instantiated Zope family>", + "type": "string" + } + }, + "type": "object" } } } diff --git a/software/erp5/instance-erp5-output-schema.json b/software/erp5/instance-erp5-output-schema.json index da358cc5397914af140d849980ee7f1235d640d5..e9b535977008751ef705e41b68ca5cfc08b0821c 100644 --- a/software/erp5/instance-erp5-output-schema.json +++ b/software/erp5/instance-erp5-output-schema.json @@ -40,6 +40,11 @@ "description": "Relational database access information", "type": "string" } + "jupyter-url": { + "description": "Jupyter notebook web UI access information", + "type": "string", + "optional": true + } }, "patternProperties": { "family-.*": { diff --git a/software/ipython_notebook/instance.cfg.in b/software/ipython_notebook/instance.cfg.in index 1bb64cde2373ef34fc8bb88bf6d44ff05e5420e3..39ab990df1d680a4da197a4ba0f0afc9d0f5cbc7 100644 --- a/software/ipython_notebook/instance.cfg.in +++ b/software/ipython_notebook/instance.cfg.in @@ -34,13 +34,17 @@ develop-eggs-directory = {{ develop_eggs_directory }} offline = true [slapconfiguration] -recipe = slapos.cookbook:slapconfiguration +recipe = slapos.cookbook:slapconfiguration.serialised computer = ${slap-connection:computer-id} partition = ${slap-connection:partition-id} url = ${slap-connection:server-url} key = ${slap-connection:key-file} cert = ${slap-connection:cert-file} +# ERP5 URL to use in Jupyter by default +# default value is empty - which means no default ERP5 URL +configuration.erp5-url = + [instance-parameter] port = 8888 host = ${slapconfiguration:ipv6-random} @@ -141,6 +145,7 @@ rendered = ${directory:erp5_kernel_dir}/ERP5kernel.py # Use ipython as executable python as we'll be needing requests library in kernel context = raw python_executable {{ bin_directory }}/ipython + key erp5_url slapconfiguration:configuration.erp5-url [kernel-json] <= dynamic-jinja2-template-base diff --git a/software/ipython_notebook/software.cfg b/software/ipython_notebook/software.cfg index 2c2d8405417990a9a5fa9c1476f1464cf288d8cc..54b747ed0f9a3c185090a7bf8c780653e0ac4def 100644 --- a/software/ipython_notebook/software.cfg +++ b/software/ipython_notebook/software.cfg @@ -43,7 +43,7 @@ md5sum = d7d4a7e19d55bf14007819258bf42100 [erp5-kernel] <= download-file-base filename = ERP5kernel.py.jinja -md5sum = da2f592075c414d4bb26cf7a7dfd147b +md5sum = 3dfc6a7c16828bff55dec4cf96b730d3 [kernel-json] <= download-file-base @@ -60,7 +60,7 @@ recipe = slapos.recipe.template:jinja2 template = ${:_profile_base_location_}/instance.cfg.in rendered = ${buildout:directory}/template.cfg mode = 0644 -md5sum = 1a993b1f8fa3f001c45075fe95a48332 +md5sum = c6b82a386a72ed72301302c3132ffb71 context = key bin_directory buildout:bin-directory key develop_eggs_directory buildout:develop-eggs-directory diff --git a/software/ipython_notebook/template/ERP5kernel.py.jinja b/software/ipython_notebook/template/ERP5kernel.py.jinja index 349651a3a7cb587bd7366c38fb1ca712efc32bc4..2aca5d4c9f8b3e58073675cb35cb2d512c743b1c 100644 --- a/software/ipython_notebook/template/ERP5kernel.py.jinja +++ b/software/ipython_notebook/template/ERP5kernel.py.jinja @@ -9,9 +9,11 @@ import requests import json # erp5_url from buildout -# TODO: Uncomment after adding automated installation of erp5-data-notebook bt5 -# url = "" -# url = "%s/erp5/Base_executeJupyter"%url +erp5_url = "{{ erp5_url }}" +if not erp5_url: + erp5_url = None +else: + erp5_url = "%s/erp5/Base_executeJupyter" % erp5_url class MagicInfo: """ @@ -69,9 +71,12 @@ class ERP5Kernel(Kernel): super(ERP5Kernel, self).__init__(*args, **kwargs) self.user = user self.password = password - # Use URL provided by buildout during initiation + # By default use URL provided by buildout during initiation # It can later be overridden - self.url = url + if url is None: + self.url = erp5_url + else: + self.url = url self.status_code = status_code self.reference = None self.title = None @@ -167,11 +172,11 @@ class ERP5Kernel(Kernel): """ try: - erp5_request = requests.get( + erp5_request = requests.post( self.url, verify=False, auth=(self.user, self.password), - params={ + data={ 'python_expression': code, 'reference': self.reference, 'title': self.title, diff --git a/software/wendelin/software.cfg b/software/wendelin/software.cfg index eb13123e3e36950c45dbd2f4e6ac24660d889a6b..48b3315e25a2f57c52d621e96176077a2cdc8289 100644 --- a/software/wendelin/software.cfg +++ b/software/wendelin/software.cfg @@ -1,25 +1,17 @@ [buildout] versions = versions extends = - ../../software/ipython_notebook/software.cfg ../../component/fluentd/buildout.cfg - ../../component/matplotlib/buildout.cfg - ../../component/ipython/buildout.cfg - ../../component/pandas/buildout.cfg ../../component/wendelin.core/buildout.cfg ../../component/msgpack-python/buildout.cfg ../../component/scipy/buildout.cfg - ../../component/scikit-learn/buildout.cfg ../../software/erp5/software.cfg parts += wendelin scipy - scikit-learn - pandas msgpack-python ipython wendelin.core - matplotlib fluentd ipython-notebook @@ -29,13 +21,10 @@ initialization = extra-paths += ${wendelin:location} eggs += - ${scikit-learn:egg} ${scipy:egg} - ${pandas:egg} ${msgpack-python:egg} ${wendelin.core:egg} ${ipython:egg} - ${matplotlib:egg} [erp5_repository_list] repository_id_list += wendelin @@ -44,15 +33,16 @@ repository_id_list += wendelin # we need to override it list = ${erp5:location}/bt5 ${erp5:location}/product/ERP5/bootstrap ${wendelin:location}/bt5/ +# Jupyter is by default enabled in Wendelin +[erp5-defaults] +jupyter-enable-default = true + [wendelin] <= erp5 repository = https://lab.nexedi.com/nexedi/wendelin.git branch = master [versions] -scikit-learn = 0.16.1 scipy = 0.15.1 -pandas = 0.16.1 msgpack-python = 0.4.6 wendelin.core = 0.5 -matplotlib = 1.4.3 diff --git a/stack/erp5/buildout.cfg b/stack/erp5/buildout.cfg index f45d6cccba843b29f53909f47e12d275519fd8d5..88832f89e454e1444f9112c462b04c3ce2d6531d 100644 --- a/stack/erp5/buildout.cfg +++ b/stack/erp5/buildout.cfg @@ -21,14 +21,17 @@ extends = ../../component/libffi/buildout.cfg ../../component/libpng/buildout.cfg ../../component/libreoffice-bin/buildout.cfg + ../../component/matplotlib/buildout.cfg ../../component/mesa/buildout.cfg ../../component/numpy/buildout.cfg + ../../component/pandas/buildout.cfg ../../component/percona-toolkit/buildout.cfg ../../component/patch/buildout.cfg ../../component/pillow/buildout.cfg ../../component/pysvn-python/buildout.cfg ../../component/python-ldap-python/buildout.cfg ../../component/rdiff-backup/buildout.cfg + ../../component/scikit-learn/buildout.cfg ../../component/stunnel/buildout.cfg ../../component/subversion/buildout.cfg ../../component/tesseract/buildout.cfg @@ -49,6 +52,7 @@ extends = ../../component/findutils/buildout.cfg ../../component/userhosts/buildout.cfg ../../component/postfix/buildout.cfg + ../../software/ipython_notebook/software.cfg ../../software/neoppod/software-common.cfg # keep neoppod extends last @@ -123,6 +127,15 @@ parts += # Create instance template template +# jupyter + ipython-notebook + instance-jupyter + monitor-eggs + +# override instance-jupyter not to render into default template.cfg +[instance-jupyter] +rendered = ${buildout:directory}/template-jupyter.cfg + [download-base] <= download-base-neo url = ${:_profile_base_location_}/${:filename} @@ -220,7 +233,7 @@ recipe = slapos.recipe.template:jinja2 # XXX: "template.cfg" is hardcoded in instanciation recipe rendered = ${buildout:directory}/template.cfg template = ${:_profile_base_location_}/instance.cfg.in -md5sum = 540956c635acc9707045510c11f80016 +md5sum = 98a4edfb18cfd810ea570f56d502a2cc mode = 640 context = key mariadb_link_binary template-mariadb:link-binary @@ -250,6 +263,7 @@ context = key haproxy_location haproxy:location key instance_common_cfg instance-common:rendered key jsl_location jsl:location + key jupyter_enable_default erp5-defaults:jupyter-enable-default key kumo_location kumo:location key libICE_location libICE:location key libSM_location libSM:location @@ -283,6 +297,7 @@ context = key template_create_erp5_site_real template-create-erp5-site-real:target key template_erp5 template-erp5:target key template_haproxy_cfg template-haproxy-cfg:target + key template_jupyter_cfg instance-jupyter:rendered key template_kumofs template-kumofs:target key template_mariadb template-mariadb:target key template_mariadb_initial_setup template-mariadb-initial-setup:target @@ -314,7 +329,7 @@ rendered = ${monitor-template-dummy:target} [template-erp5] <= download-base filename = instance-erp5.cfg.in -md5sum = 977119d0b876df827c97bb64e6e98273 +md5sum = 66edf64eeaecded8977459acb26f4424 [template-zeo] <= download-base @@ -384,6 +399,11 @@ update-command = ${:command} [erp5_repository_list] repository_id_list = erp5 +# ERP5 defaults, which can be overridden in inheriting recipes (e.g. wendelin) +[erp5-defaults] +# Jupyter is by default disabled in ERP5 +jupyter-enable-default = false + [erp5] recipe = slapos.recipe.build:gitclone repository = http://git.erp5.org/repos/erp5.git @@ -451,12 +471,15 @@ initialization = <= neoppod eggs = ${numpy:egg} + ${matplotlib:egg} ${mysql-python:egg} ${lxml-python:egg} + ${pandas:egg} ${pillow-python:egg} ${python-ldap-python:egg} ${pysvn-python:egg} ${pycrypto-python:egg} + ${scikit-learn:egg} lock_file PyStemmer PyXML @@ -673,8 +696,10 @@ interval = 1.0.0 ipdb = 0.8.1 ipython = 4.0.0 logilab-common = 1.1.0 +matplotlib = 1.4.3 numpy = 1.10.4 objgraph = 2.0.1 +pandas = 0.16.1 ply = 3.8 polib = 1.0.7 pprofile = 1.7.3 @@ -688,6 +713,7 @@ pytracemalloc = 1.2 qrcode = 5.2.2 restkit = 4.2.2 rtjp-eventlet = 0.3.2 +scikit-learn = 0.16.1 simplegeneric = 0.8.1 socketpool = 0.5.3 spyne = 2.12.11 diff --git a/stack/erp5/instance-erp5.cfg.in b/stack/erp5/instance-erp5.cfg.in index c9c453efe3bacb106e0fc3f2483e95dd06f559a1..b6d173fc64ef02d68caeb24357b3044fed05af72 100644 --- a/stack/erp5/instance-erp5.cfg.in +++ b/stack/erp5/instance-erp5.cfg.in @@ -5,6 +5,9 @@ {% set inituser_login = slapparameter_dict.get('inituser-login', 'zope') -%} {% set publish_dict = {'site-id': site_id, 'inituser-login': inituser_login} -%} {% set has_posftix = slapparameter_dict.get('smtp', {}).get('postmaster') -%} +{% set jupyter_dict = slapparameter_dict.get('jupyter', {}) -%} +{% set has_jupyter = jupyter_dict.get('enable', jupyter_enable_default).lower() in ('true', 'yes') -%} +{% set jupyter_zope_family = jupyter_dict.get('zope-family', '') -%} [request-common] <= request-common-base config-use-ipv6 = {{ dumps(slapparameter_dict.get('use-ipv6', False)) }} @@ -119,7 +122,11 @@ name = neo-${gen-neo-cluster-base:passwd} return = zope-address-list hosts-dict -config-bt5 = {{ dumps(slapparameter_dict.get('bt5', 'erp5_full_text_myisam_catalog erp5_configurator_standard erp5_configurator_maxma_demo erp5_configurator_ung erp5_configurator_run_my_doc')) }} +{% set bt5_default_list = 'erp5_full_text_myisam_catalog erp5_configurator_standard erp5_configurator_maxma_demo erp5_configurator_ung erp5_configurator_run_my_doc' -%} +{% if has_jupyter -%} +{% set bt5_default_list = bt5_default_list + ' erp5_data_notebook' -%} +{% endif -%} +config-bt5 = {{ dumps(slapparameter_dict.get('bt5', bt5_default_list)) }} config-bt5-repository-url = {{ dumps(slapparameter_dict.get('bt5-repository-url', local_bt5_repository)) }} config-cloudooo-url = ${request-cloudooo:connection-url} config-deadlock-debugger-password = ${publish-early:deadlock-debugger-password} @@ -150,10 +157,17 @@ config-tidstorage-port = ${request-zodb:connection-tidstorage-port} software-type = zope {% set zope_family_dict = {} -%} +{% set jupyter_zope_family_default = [] -%} {% for custom_name, zope_parameter_dict in slapparameter_dict.get('zope-partition-dict', {'1': {}}).items() -%} {% set partition_name = 'zope-' ~ custom_name -%} {% set section_name = 'request-' ~ partition_name -%} -{% do zope_family_dict.setdefault(zope_parameter_dict.get('family', 'default'), []).append(section_name) -%} +{% set zope_family = zope_parameter_dict.get('family', 'default') -%} +{# # default jupyter zope family is first zope family. -#} +{# # use list.append() to update it, because in jinja2 set changes only local scope. -#} +{% if not jupyter_zope_family_default -%} +{% do jupyter_zope_family_default.append(zope_family) -%} +{% endif -%} +{% do zope_family_dict.setdefault(zope_family, []).append(section_name) -%} [{{ section_name }}] <= request-zope-base name = {{ partition_name }} @@ -168,6 +182,12 @@ config-port-base = {{ dumps(zope_parameter_dict.get('port-base', 2200)) }} config-webdav = {{ dumps(zope_parameter_dict.get('webdav', False)) }} {% endfor -%} +{# if not explicitly configured, connect jupyter to first zope family, which -#} +{# will be 'default' if zope families are not configured also -#} +{% if not jupyter_zope_family and jupyter_zope_family_default -%} +{% set jupyter_zope_family = jupyter_zope_family_default[0] -%} +{% endif -%} + {# We need to concatenate lists that we cannot read as lists, so this gets hairy. -#} {% set zope_address_list_id_dict = {} -%} {% set zope_family_parameter_dict = {} -%} @@ -190,6 +210,20 @@ config-url = ${request-balancer:connection-{{ family_name }}-v6} {% endif -%} {% endfor -%} +{% if has_jupyter -%} +{# request jupyter connected to balancer of proper zope family -#} +{{ request('jupyter', 'jupyter', 'jupyter', {}, key_config={'erp5-url': 'request-balancer:connection-' ~ jupyter_zope_family}) }} + +{% if has_frontend -%} +[frontend-jupyter] +<= request-frontend-base +name = frontend-jupyter +config-url = ${request-jupyter:connection-url} +{# # override jupyter-url in publish_dict with frontend address -#} +{% do publish_dict.__setitem__('jupyter-url', '${frontend-jupyter:connection-site_url}') -%} +{% endif -%} +{%- endif %} + {% set balancer_dict = slapparameter_dict.get('balancer', {}) -%} [request-balancer] <= request-common diff --git a/stack/erp5/instance.cfg.in b/stack/erp5/instance.cfg.in index 9e915c41124d76961fc30df7447a07bdcdaa6fbb..b769852f4a2c605c7e32a1248102330ffc5d196a 100644 --- a/stack/erp5/instance.cfg.in +++ b/stack/erp5/instance.cfg.in @@ -64,6 +64,7 @@ extra-context = import urllib urllib [dynamic-template-erp5-parameters] +jupyter-enable-default = {{ jupyter_enable_default }} local-bt5-repository = {{ local_bt5_repository }} [dynamic-template-erp5] @@ -71,6 +72,7 @@ local-bt5-repository = {{ local_bt5_repository }} template = {{ template_erp5 }} filename = instance-erp5.cfg extra-context = + key jupyter_enable_default dynamic-template-erp5-parameters:jupyter-enable-default key local_bt5_repository dynamic-template-erp5-parameters:local-bt5-repository import urlparse urlparse import-list = @@ -177,6 +179,11 @@ filename = instance-create-erp5-site.cfg extra-context = section parameter_dict dynamic-template-create-erp5-site-parameters +# we need this value to be present in a section, +# for slapos.cookbook:switch-softwaretype to work +[dynamic-template-jupyter] +rendered = {{ template_jupyter_cfg }} + [switch-softwaretype] recipe = slapos.cookbook:switch-softwaretype override = {{ dumps(override_switch_softwaretype |default) }} @@ -195,3 +202,4 @@ postfix = dynamic-template-postfix:rendered zodb-zeo = dynamic-template-zeo:rendered zodb-neo = neo-storage-mysql:rendered zope = dynamic-template-zope:rendered +jupyter = dynamic-template-jupyter:rendered