software/jupyter: move to python 3

See merge request nexedi/slapos!868

component/jupyter-py2/ERP5kernel.py

+from ipykernel.kernelbase import Kernel
+from ipykernel.kernelapp import IPKernelApp
+from IPython.core.display import HTML
+import requests
+import json
+import sys
+
+erp5_url = None
+if len(sys.argv) > 1:
+    erp5_url = "%s/erp5/Base_executeJupyter" % (sys.argv[1],)
+
+class MagicInfo:
+  """
+  Magics definition structure.
+  Initializes a new MagicInfo class with specific paramters to identify a magic.
+  """
+  def __init__(self, magic_name, variable_name, send_request, request_reference, display_message):
+    self.magic_name = magic_name
+    self.variable_name = variable_name
+    self.send_request = send_request
+    self.request_reference = request_reference
+    self.display_message = display_message
+
+# XXX: New magics to be added here in the dictionary.
+# In this dictionary,
+# key = magic_name,
+# value = MagicInfo Structure corresponding to the magics
+# Different parameters of the structures are :-
+# magics_name(str) = Name which would be used on jupyter frontend
+# variable_name(str) = Name of variable on which magic would be set in kernel
+# send_request(boolean) = Magics for which requests to erp5 backend need to be made
+# request_reference(boolean) = Request for notebook references(and titles) from erp5
+# display_message(boolean) = If the magics need to display message after
+#                             making request. Useful for magics which do get some
+#                             useful content from erp5 backend and need to display
+
+MAGICS = {
+  'erp5_user': MagicInfo('erp5_user', 'user', True, False, True),
+  'erp5_password': MagicInfo('erp5_password', 'password', True, False, True),
+  'erp5_url': MagicInfo('erp5_url', 'url', True, False, True),
+  'notebook_set_reference': MagicInfo('notebook_set_reference', 'reference', True, False, True),
+  'notebook_set_title': MagicInfo('notebook_set_title', 'title', False, False, True),
+  'my_notebooks': MagicInfo('my_notebooks', '', True, True, False)}
+
+class ERP5Kernel(Kernel):
+  """
+  Jupyter Kernel class to interact with erp5 backend for code from frontend.
+  To use this kernel with erp5, user need to install 'erp5_data_notebook' bt5 
+  Also, handlers(aka magics) starting with '%' are predefined.
+
+  Each request to erp5 for code execution requires erp5_user, erp5_password
+  and reference of the notebook.
+  """
+
+  implementation = 'ERP5'
+  implementation_version = '1.0'
+  language = 'ERP5'
+  language_version = '0.1'
+  language_info = {'mimetype': 'text/plain', 'name':'python'}
+  banner = "ERP5 integration with jupyter notebook"
+
+  def __init__(self, user=None, password=None, url=None, status_code=None,
+              *args, **kwargs):
+    super(ERP5Kernel, self).__init__(*args, **kwargs)
+    self.user = user
+    self.password = password
+    # By default use URL provided by buildout during initiation
+    # It can later be overridden
+    if url is None:
+        self.url = erp5_url
+    else:
+        self.url = url
+    self.status_code = status_code
+    self.reference = None
+    self.title = None
+    # Allowed HTTP request code list for making request to erp5 from Kernel
+    # This list should be to used check status_code before making requests to erp5
+    self.allowed_HTTP_request_code_list = range(500, 511)
+    # Append request code 200 in the allowed HTTP status code list
+    self.allowed_HTTP_request_code_list.append(200)
+
+  def display_response(self, response=None):
+    """
+      Dispays the stream message response to jupyter frontend.
+    """
+    if response:
+      stream_content = {'name': 'stdout', 'text': response}
+      self.send_response(self.iopub_socket, 'stream', stream_content)
+
+  def set_magic_attribute(self, magic_info=None, code=None):
+    """
+      Set attribute for magic which are necessary for making requests to erp5.
+      Catch errors and display message. Since user is in contact with jupyter
+      frontend, so its better to catch exceptions and dispaly messages than to
+      let them fail in backend and stuck the kernel.
+      For a making a request to erp5, we need -
+      erp5_url, erp5_user, erp5_password, notebook_set_reference
+    """
+    # Set attributes only for magic who do have any varible to set value to
+    if magic_info.variable_name:
+      try:
+        # Get the magic value recived via code from frontend
+        magic_value = code.split()[1]
+        # Set magic_value to the required attribute
+        
+        if magic_info.magic_name == 'notebook_set_reference':
+          required_attributes = ['url', 'password', 'user']
+          missing_attributes = []
+          for attribute in required_attributes:
+            if not getattr(self, attribute):
+              missing_attributes.append(attribute)
+          
+          if missing_attributes != []:
+            self.response = "You still haven't entered all required magics. \
+Please do so before inputting your reference."
+          else:
+            if self.check_existing_reference(reference=magic_value):
+              self.response = 'WARNING: You already have a notebook with \
+reference %s. It might be a good idea to use different references for new \
+notebooks. \n' % magic_value
+            else:
+              self.response = ''
+            setattr(self, magic_info.variable_name , magic_value)
+            self.response = self.response + 'Your %s is %s. '%(magic_info.magic_name, magic_value)
+        elif magic_info.magic_name != 'erp5_password':
+          setattr(self, magic_info.variable_name , magic_value)
+          self.response = 'Your %s is %s. '%(magic_info.magic_name, magic_value)
+        else:
+          setattr(self, magic_info.variable_name , magic_value)
+          self.response = ""
+
+      # Catch exception while setting attribute and set message in response
+      except AttributeError:
+        self.response = 'Please enter %s magic value'%magic_info.variable_name
+
+      # Catch IndexError while getting magic_value and set message in response object
+      except IndexError:
+        self.response = 'Empty value for %s magic'%magic_info.variable_name
+
+      # Catch all other exceptions and set error_message in response object
+      # XXX: Might not be best way, but its better to display error to the user
+      # via notebook frontend than to fail in backend and stuck the Kernel without
+      # any failure message to user.
+      except Exception as e:
+        self.response = str(e)
+
+      # Display the message/response from this fucntion before moving forward so
+      # as to keep track of the status
+      if self.response != "":
+        self.display_response(response=(self.response + '\n'))
+
+  def check_required_attributes(self):
+    """
+      Check if the required attributes for making a request are already set or not.
+      Display message to frontend to provide with the values in case they aren't.
+      This function can be called anytime to check if the attributes are set. The
+      output result will be in Boolean form.
+      Also, in case any of attribute is not set, call to display_response would be
+      made to ask user to enter value.
+    """
+    result_list = []
+    required_attributes  = ['url', 'user', 'password', 'reference']
+    missing_attributes = []
+
+    # Set response to empty so as to flush the response set by some earlier fucntion call
+    self.response = ''
+
+    # Loop to check if the required attributes are set
+    for attribute in required_attributes:
+      if getattr(self, attribute):
+        result_list.append(True)
+      else:
+        # Set response/message for attributes which aren't set
+        missing_attributes.append(attribute)
+        result_list.append(False)
+    
+    # Compare result_list to get True for all True results and False for any False result 
+    check_attributes = all(result_list)
+    
+    if check_attributes:
+      self.response = 'You have entered all required magics. You may now use your notebook.'
+    else:
+      self.response = '''You have these required magics remaining: %s. \n''' % (
+      ', '.join(map(str, missing_attributes)))
+
+    # Display response to frontend before moving forward
+    self.display_response(response=(self.response + '\n'))
+
+    return check_attributes
+
+  def make_erp5_request(self, request_reference=False, display_message=True,
+                        code=None, message=None, title=None, *args, **kwargs):
+    """
+      Function to make request to erp5 as per the magics.
+      Should return the response json object.
+    """
+
+    try:
+      erp5_request = requests.post(
+        self.url,
+        verify=False,
+        auth=(self.user, self.password),
+        data={
+          'python_expression': code,
+          'reference': self.reference,
+          'title': self.title,
+          'request_reference': request_reference,
+	  'store_history': kwargs.get('store_history')
+          })
+
+      # Set value for status_code for self object which would later be used to
+      # dispaly response after statement check
+      self.status_code = erp5_request.status_code
+
+      # Dispaly error response in case the request give any other status
+      # except 200 and 5xx(which is for errors on server side)
+      if self.status_code not in self.allowed_HTTP_request_code_list:
+        self.response = '''Error code %s on request to ERP5,\n
+        check credentials or ERP5 family URL'''%self.status_code
+      else:
+        # Set value of self.response to the given value in case response from function
+        # call. In all other case, response should be the content from request
+        if display_message and message:
+          self.response = message
+        else:
+          self.response = erp5_request.content
+
+    except requests.exceptions.RequestException as e:
+      self.response = str(e)
+
+  def do_execute(self, code, silent, store_history=True, user_expressions=None,
+                  allow_stdin=False):
+    """
+      Validate magic and call functions to make request to erp5 backend where
+      the code is being executed and response is sent back which is then sent
+      to jupyter frontend.
+    """
+    # By default, take the status of response as 'ok' so as show the responses
+    # for erp5_url and erp5_user on notebook frontend as successful response.
+    status = 'ok'
+
+    if not silent:
+      # Remove spaces and newlines from both ends of code
+      code = code.strip()
+
+      extra_data_list = []
+      print_result = {}
+      displayhook_result = {}
+
+      if code.startswith('%'):
+          # No need to try-catch here as its already been taken that the code
+          # starts-with '%', so we'll get magic_name, no matter what be after '%'
+          magic_name = code.split()[0][1:]
+          magics_name_list = [magic.magic_name for magic in MAGICS.values()]
+
+          # Check validation of magic
+          if magic_name and magic_name in magics_name_list:
+
+            # Get MagicInfo object related to the magic
+            magic_info = MAGICS.get(magic_name)
+
+            # Function call to set the required magics
+            self.set_magic_attribute(magic_info=magic_info, code=code)
+
+            # Call to check if the required_attributes are set
+            checked_attribute = self.check_required_attributes()
+            if checked_attribute and magic_info.send_request:
+              # Call the function to send request to erp5 with the arguments given
+              self.make_erp5_request(message='Please proceed\n',
+              request_reference=magic_info.request_reference,
+              display_message=magic_info.display_message)
+
+              # Display response from erp5 request for magic
+              # Since this response would be either success message or failure
+              # error message, both of which are string type, so, we can simply
+              # display the stream response.
+              if self.response != 'Please proceed\n':
+                self.display_response(response=self.response)
+
+          else:
+            # Set response if there is no magic or the magic name is not in MAGICS
+            self.response = 'Invalid Magics'
+            self.display_response(response=self.response)
+
+      else:
+        # Check for status_code before making request to erp5 and make request in
+        # only if the status_code is in the allowed_HTTP_request_code_list
+        if self.status_code in self.allowed_HTTP_request_code_list:
+          self.make_erp5_request(code=code, store_history=store_history)
+
+          # For 200 status_code, Kernel will receive predefined format for data
+          # from erp5 which is either json of result or simple result string
+          if self.status_code == 200:
+            mime_type = 'text/plain'
+            try:
+              content = json.loads(self.response)
+
+              # Example format for the json result we are expecting is :
+              # content = {
+              #            "status": "ok",
+              #            "ename": null,
+              #            "evalue": null,
+              #            "traceback": null,
+              #            "code_result": "",
+              #            "print_result": {},
+              #            "displayhook_result": {},
+              #            "mime_type": "text/plain",
+              #            "extra_data_list": []
+              #            }
+              # So, we can easily use any of the key to update values as such.
+
+              # Getting code_result for succesfull execution of code
+              code_result = content['code_result']
+              print_result = content['print_result']
+              displayhook_result = content['displayhook_result']
+
+              # Update mime_type with the mime_type from the http response result
+              # Required in case the mime_type is anything other than 'text/plain'
+              mime_type = content['mime_type']
+
+              extra_data_list = content.get('extra_data_list', [])
+
+            # Display to frontend the error message for content status as 'error'
+              if content['status']=='error':
+                reply_content = {
+                  'status': 'error',
+                  'execution_count': self.execution_count,
+                  'ename': content['ename'],
+                  'evalue': content['evalue'],
+                  'traceback': content['traceback']}
+                self.send_response(self.iopub_socket, u'error', reply_content)
+                return reply_content
+            # Catch exception for content which isn't json
+            except ValueError:
+              content = self.response
+              code_result = content
+              print_result = {'data':{'text/plain':content}, 'metadata':{}}
+          # Display basic error message to frontend in case of error on server side
+          else:
+            self.make_erp5_request(code=code)
+            code_result = "Error at Server Side"
+            print_result = {'data':{'text/plain':'Error at Server Side'}, 'metadata':{}}
+            mime_type = 'text/plain'
+
+        # For all status_code except allowed_HTTP_response_code_list show unauthorized message
+        else:
+          code_result = 'Unauthorized access'
+          print_result = {'data':{'text/plain':'Unauthorized access'}, 'metadata':{}}
+          mime_type = 'text/plain'
+
+        if print_result.get('data'):
+          self.send_response(self.iopub_socket, 'display_data', print_result)
+
+        if displayhook_result.get('data'):
+          displayhook_result['execution_count'] = self.execution_count
+          self.send_response(self.iopub_socket, 'execute_result', displayhook_result)
+
+        for extra_data in extra_data_list:
+          self.send_response(self.iopub_socket, 'display_data', extra_data)
+
+    reply_content = {
+      'status': status,
+      # The base class increments the execution count
+      'execution_count': self.execution_count,
+      'payload': [],
+      'user_expressions': {}}
+
+    return reply_content
+  
+  # Checks the ERP5 site if there are existing notebooks with the same reference.
+  # Returns True if there are.
+  def check_existing_reference(self, reference):
+    if reference == None:
+      return False
+    
+    modified_url = self.url[:self.url.rfind('/')] + '/Base_checkExistingReference'
+    result = True
+    
+    try:
+      erp5_request = requests.post(
+        modified_url,
+        verify=False,
+        auth=(self.user, self.password),
+        data={
+          'reference': reference,
+          })
+      result = erp5_request.content
+      
+    except requests.exceptions.RequestException as e:
+      self.response = str(e)
+      
+    return result
+
+
+if __name__ == '__main__':
+  IPKernelApp.launch_instance(kernel_class=ERP5Kernel)

component/jupyter-py2/README.md

+This directory is only temporary (until ERP5 is Python3) and will not actively
+be maintained.
+
+It has to be removed once ERP5 uses `software/jupyter` or do not use jupyter at
+all.

component/jupyter-py2/buildout.cfg

+[buildout]
+extends = 
+  buildout.hash.cfg
+  ../../stack/slapos.cfg
+  ../openssl/buildout.cfg
+  ../jupyter/buildout.cfg
+  ../../stack/monitor/buildout.cfg
+parts +=
+  slapos-cookbook
+  jupyter
+  jupyter-notebook-initialized-scripts
+  instance-jupyter-notebook
+
+[gcc]
+# Always build GCC for Fortran (see openblas).
+max_version = 0
+
+[jupyter]
+python_executable = ${buildout:bin-directory}/${:interpreter}
+
+[download-file-base]
+recipe = slapos.recipe.build:download
+url = ${:_profile_base_location_}/${:filename}
+download-only = true
+mode = 0644
+
+[jupyter-notebook-config]
+<= download-file-base
+
+[jupyter-set-password]
+<= download-file-base
+
+[erp5-kernel]
+<= download-file-base
+
+[kernel-json]
+<= download-file-base
+
+[custom-js]
+<= download-file-base
+
+[instance-jupyter-notebook]
+recipe = slapos.recipe.template:jinja2
+template = ${:_profile_base_location_}/${:filename}
+rendered = ${buildout:directory}/template.cfg
+mode = 0644
+context =
+  key bin_directory buildout:bin-directory
+  key develop_eggs_directory buildout:develop-eggs-directory
+  key eggs_directory buildout:eggs-directory
+  key openssl_output openssl-output:openssl
+  key python_executable jupyter:python_executable
+  key jupyter_config_location jupyter-notebook-config:location
+  key jupyter_config_filename jupyter-notebook-config:filename
+  key jupyter_set_password_location jupyter-set-password:location
+  key jupyter_set_password_filename jupyter-set-password:filename
+  key erp5_kernel_location erp5-kernel:location
+  key erp5_kernel_filename erp5-kernel:filename
+  key kernel_json_location kernel-json:location
+  key kernel_json_filename kernel-json:filename
+  key custom_js_location custom-js:location
+  key custom_js_filename custom-js:filename
+  key monitor_template_rendered buildout:directory
+
+[versions]
+Pygments = 2.2.0
+astor = 0.5
+backports-abc = 0.5
+backports.functools-lru-cache = 1.6.1
+backports.shutil-get-terminal-size = 1.0.0
+cycler = 0.10.0
+ipykernel = 4.5.2
+ipython = 5.3.0
+ipython-genutils = 0.1.0
+ipywidgets = 6.0.0
+jupyter-client = 5.0.0
+jupyter-core = 4.3.0
+jupyterlab = 0.26.3
+jupyterlab-launcher = 0.3.1
+matplotlib = 2.1.2
+mistune = 0.7.3
+nbformat = 4.3.0
+notebook = 4.4.1
+pandas = 0.19.2
+plone.recipe.command = 1.1
+prompt-toolkit = 1.0.13
+ptyprocess = 0.5.1
+pyzmq = 16.0.2
+scikit-learn = 0.18.1
+seaborn = 0.7.1
+simplegeneric = 0.8.1
+slapos.recipe.template = 4.4
+statsmodels = 0.8.0
+terminado = 0.6
+tornado = 4.4.2
+traitlets = 4.3.2
+widgetsnbextension = 2.0.0
+# numpy >= 1.13.1 is required for numpy.core.multiarray
+numpy = 1.13.1
+
+
+# Required by:
+# tornado==4.4.2
+certifi = 2020.6.20
+
+# Required by:
+# notebook==4.3.2
+# nbconvert 4.2.0 depends on entrypoints egg that is not available as tar/zip source.
+nbconvert = 4.1.0
+
+# Required by:
+# ipython==5.3.0
+pathlib2 = 2.2.1
+
+# Required by:
+# statsmodels==0.8.0
+patsy = 0.4.1
+
+# Required by:
+# ipython==5.3.0
+pexpect = 4.2.1
+
+# Required by:
+# ipython==5.3.0
+pickleshare = 0.7.4
+
+# Required by:
+# matplotlib==2.1.2
+# pandas==0.19.2
+python-dateutil = 2.6.0
+
+# Required by:
+# pathlib2==2.2.1
+scandir = 1.5
+
+# Required by:
+# statsmodels==0.8.0
+scipy = 0.19.0
+
+# Required by:
+# tornado==4.4.2
+singledispatch = 3.4.0.3
+
+# Required by:
+# prompt-toolkit==1.0.13
+wcwidth = 0.1.7
+
+jupyter = 1.0.0
+jupyter-console = 5.1.0
+
+# Required by:
+# jupyter==1.0.0
+qtconsole = 4.3.0
+
+et-xmlfile = 1.0.1
+h5py = 2.7.1
+mpmath = 1.0.0
+openpyxl = 2.5.2
+sympy = 1.1.1
+xlrd = 1.1.0
+
+# Required by:
+# openpyxl==2.5.2
+jdcal = 1.4

component/jupyter-py2/buildout.hash.cfg

+# THIS IS NOT A BUILDOUT FILE, despite purposedly using a compatible syntax.
+# The only allowed lines here are (regexes):
+# - "^#" comments, copied verbatim
+# - "^[" section beginings, copied verbatim
+# - lines containing an "=" sign which must fit in the following categorie.
+#   - "^\s*filename\s*=\s*path\s*$" where "path" is relative to this file
+#     Copied verbatim.
+#   - "^\s*hashtype\s*=.*" where "hashtype" is one of the values supported
+#     by the re-generation script.
+#     Re-generated.
+# - other lines are copied verbatim
+# Substitution (${...:...}), extension ([buildout] extends = ...) and
+# section inheritance (< = ...) are NOT supported (but you should really
+# not need these here).
+
+[instance-jupyter-notebook]
+filename = instance.cfg.in
+md5sum = 1d5fe6cc4e48672ae7be1c223794a932
+
+[jupyter-notebook-config]
+filename = jupyter_notebook_config.py.jinja
+md5sum = 720e90a829c63371696bc3009917a743
+
+[jupyter-set-password]
+filename = jupyter_set_password.cgi.jinja
+md5sum = ac10fbcf790bd8e58750cfdd069812d2
+
+[erp5-kernel]
+filename = ERP5kernel.py
+md5sum = 7d5309fe79afbcb455c0d8181b42e56c
+
+[kernel-json]
+filename = kernel.json.jinja
+md5sum = 33547be93a67530165e079dc3ecfdac3
+
+[custom-js]
+filename = custom.js
+md5sum = 147ccce38f7d7cf10664975d0dd80e33

component/jupyter-py2/custom.js

+// leave at least 2 line with only a star on it below, or doc generation fails
+/**
+ *
+ *
+ * Placeholder for custom user javascript
+ * mainly to be overridden in profile/static/custom/custom.js
+ * This will always be an empty file in IPython
+ *
+ * User could add any javascript in the `profile/static/custom/custom.js` file.
+ * It will be executed by the ipython notebook at load time.
+ *
+ * Same thing with `profile/static/custom/custom.css` to inject custom css into the notebook.
+ *
+ *
+ * The object available at load time depend on the version of IPython in use.
+ * there is no guaranties of API stability.
+ *
+ * The example below explain the principle, and might not be valid.
+ *
+ * Instances are created after the loading of this file and might need to be accessed using events:
+ *     define([
+ *        'base/js/namespace',
+ *        'base/js/events'
+ *     ], function(IPython, events) {
+ *         events.on("app_initialized.NotebookApp", function () {
+ *             IPython.keyboard_manager....
+ *         });
+ *     });
+ *
+ * __Example 1:__
+ *
+ * Create a custom button in toolbar that execute `%qtconsole` in kernel
+ * and hence open a qtconsole attached to the same kernel as the current notebook
+ *
+ *    define([
+ *        'base/js/namespace',
+ *        'base/js/events'
+ *    ], function(IPython, events) {
+ *        events.on('app_initialized.NotebookApp', function(){
+ *            IPython.toolbar.add_buttons_group([
+ *                {
+ *                    'label'   : 'run qtconsole',
+ *                    'icon'    : 'icon-terminal', // select your icon from http://fortawesome.github.io/Font-Awesome/icons
+ *                    'callback': function () {
+ *                        IPython.notebook.kernel.execute('%qtconsole')
+ *                    }
+ *                }
+ *                // add more button here if needed.
+ *                ]);
+ *        });
+ *    });
+ *
+ * __Example 2:__
+ *
+ * At the completion of the dashboard loading, load an unofficial javascript extension
+ * that is installed in profile/static/custom/
+ *
+ *    define([
+ *        'base/js/events'
+ *    ], function(events) {
+ *        events.on('app_initialized.DashboardApp', function(){
+ *            require(['custom/unofficial_extension.js'])
+ *        });
+ *    });
+ *
+ * __Example 3:__
+ *
+ *  Use `jQuery.getScript(url [, success(script, textStatus, jqXHR)] );`
+ *  to load custom script into the notebook.
+ *
+ *    // to load the metadata ui extension example.
+ *    $.getScript('/static/notebook/js/celltoolbarpresets/example.js');
+ *    // or
+ *    // to load the metadata ui extension to control slideshow mode / reveal js for nbconvert
+ *    $.getScript('/static/notebook/js/celltoolbarpresets/slideshow.js');
+ *
+ *
+ * @module IPython
+ * @namespace IPython
+ * @class customjs
+ * @static
+ */
+
+$([Jupyter.events]).on('notebook_loaded.Notebook', function(){
+  var kernelname = Jupyter.notebook.kernel_selector.current_selection;
+  var display_text="<div class='output_subarea output_text output_result'>\
+  <pre>Follow these steps to customize your notebook with ERP5 kernel :-</br>\
+  1. Make sure you have 'erp5_data_notebook' business template installed in your ERP5</br>\
+  2. <b>%erp5_user &lt;your_erp5_username&gt;</b></br>\
+  3. <b>%erp5_password &lt;your_erp5_password&gt;</b></br>\
+  4. <b>%notebook_set_reference &lt;your_notebook_reference&gt;</b></br>\
+  It would be better to set the reference to match with erp5 reference pattern.</br>\
+  5. As soon as you see 'Please Proceed' message you can now access your erp5 using notebook.</br>\
+  <p><u>OTHER USEFUL MAGICS</u> -</br>\
+  <b>%my_notebooks</b> -This is used to display all the notebooks created by the specific user.</br>\
+  <b>%notebook_set_title</b> -This sets the title of the current notebook.</br>\
+  NOTE: Do not dynamically alter imported module objects as they are not being saved in DB, </br>\
+  so changes to them would be disregarded and would throw an error.</br>\
+  <p><u>About classes, functions and global state on modules:</u></p>\
+  Your code is going to be executed by ERP5, which can have many nodes </br>\
+  and there is no guarantee that your code is always going to be executed by the same server.</br>\
+  This means that objects which cannot be stored in the ZODB, like functions, classes and modules </br>\
+  won't be available across nodes. To solve this issue, you need to use a special object </br>\
+  called 'environment' to store your global setup. This object was designed to hold global </br>\
+  state and restore it for each code cell. Example:</br></br>\
+  <b>def my_setup():</br>\
+     # import modules, define functions and classes</br>\
+     # and set global state on modules</br>\
+     # return dict of variables to be available in code cells</br>\
+     {'my_var': 1}</br>\
+  environment.define(my_setup, 'my custom setup')</b></br></br>\
+  After you execute this cell, the <b>my_setup</b> function will run before each of the</br>\
+  following cells and the <b>my_var</b> variable will be created and set to 1.</br></br>\
+  <b>WARNING:</b> it is not recommended to have too many setup functions in the environment, </br>\
+  because they will be executed in every code cell and can cause a substantial slow down.\
+  </pre></div>";
+  if (kernelname=="erp5"){
+  $('div#notebook-container').prepend(display_text);
+  }
+  });

component/jupyter-py2/instance.cfg.in

+[buildout]
+parts =
+  instance
+  jupyter_notebook
+  read-knowledge0
+  publish-connection-parameter
+  jupyter-notebook-config
+  erp5-kernel
+  kernel-json
+  custom-js
+  monitor-base
+extends =
+  {{ monitor_template_rendered }}/template-monitor.cfg
+
+
+eggs-directory = {{ eggs_directory }}
+develop-eggs-directory = {{ develop_eggs_directory }}
+offline = true
+
+[slapconfiguration]
+recipe = slapos.cookbook:slapconfiguration.serialised
+computer = ${slap-connection:computer-id}
+partition = ${slap-connection:partition-id}
+url = ${slap-connection:server-url}
+key = ${slap-connection:key-file}
+cert = ${slap-connection:cert-file}
+
+# ERP5 URL to use in Jupyter by default
+# default value is empty - which means no default ERP5 URL
+configuration.erp5-url =
+
+[instance-parameter]
+port = 8888
+host = ${slapconfiguration:ipv6-random}
+cert_file = ${generate-certificate:cert_file}
+key_file = ${generate-certificate:key_file}
+logfile = ${directory:log}/jupyter_notebook.log
+notebook_dir = ${directory:notebook_dir}
+
+[dynamic-jinja2-template-base]
+recipe = slapos.recipe.template:jinja2
+mode = 0644
+
+[generate-certificate]
+; TODO: there is a slapos recipe to generate certificates. Use it instead
+recipe = plone.recipe.command
+command =
+  if [ ! -e ${instance-parameter:key_file} ]
+  then
+    {{ openssl_output }} req -x509 -nodes -days 3650 \
+      -subj "/C=AA/ST=X/L=X/O=Dis/CN=${instance-parameter:host}" \
+      -newkey rsa:1024 -keyout ${instance-parameter:key_file} \
+      -out ${instance-parameter:cert_file}
+  fi
+update-command = ${:command}
+cert_file = ${directory:etc}/jupyter_cert.crt
+key_file = ${directory:etc}/jupyter_cert.key
+
+[instance]
+recipe = slapos.cookbook:wrapper
+command-line =
+  {{ bin_directory }}/jupyter-lab
+  --no-browser
+  --ip=${instance-parameter:host}
+  --port=${instance-parameter:port}
+  --port-retries=0
+  --certfile=${instance-parameter:cert_file}
+  --keyfile=${instance-parameter:key_file}
+  --notebook-dir=${instance-parameter:notebook_dir}
+  --log-level="DEBUG"
+wrapper-path = ${directory:service}/jupyter-lab
+environment =
+  JUPYTER_PATH=${directory:jupyter_dir}
+  JUPYTER_CONFIG_DIR=${directory:jupyter_config_dir}
+  JUPYTER_RUNTIME_DIR=${directory:jupyter_runtime_dir}
+  LANG=C.UTF-8
+
+[jupyter-notebook-config]
+<= dynamic-jinja2-template-base
+template = {{ jupyter_config_location }}/{{ jupyter_config_filename }}
+rendered = ${directory:jupyter_config_dir}/jupyter_notebook_config.py
+mode = 0744
+context =
+  raw config_cfg ${buildout:directory}/knowledge0.cfg
+
+[directory]
+recipe = slapos.cookbook:mkdirectory
+home = ${buildout:directory}
+etc = ${:home}/etc
+var = ${:home}/var
+script = ${:etc}/run/
+service = ${:etc}/service
+log = ${:var}/log
+notebook_dir = ${:var}/notebooks
+# Add folders to explicitly define jupyter directory
+jupyter_dir = ${:home}/jupyter
+jupyter_config_dir = ${:jupyter_dir}/etc
+jupyter_kernel_dir = ${:jupyter_dir}/kernels
+jupyter_runtime_dir = ${:jupyter_dir}/runtime
+jupyter_custom_dir = ${:jupyter_config_dir}/custom
+jupyter_nbextensions_dir = ${:jupyter_dir}/nbextensions
+erp5_kernel_dir = ${:jupyter_kernel_dir}/ERP5
+
+[jupyter_notebook]
+# This part is called like this because knowledge0.write uses the part name for
+# the section name in the config file.
+recipe = slapos.cookbook:zero-knowledge.write
+password = 
+filename = knowledge0.cfg
+
+[read-knowledge0]
+recipe = slapos.cookbook:zero-knowledge.read
+filename = knowledge0.cfg
+password =
+
+[monitor-instance-parameter]
+monitor-base-url = ${monitor-frontend-promise:url}
+# In case you're using a developer instance you should edit these in:
+# monitor-base-url = ${monitor-httpd-conf-parameter:url}
+# cors-domains = softinstXXXXX.host.vifib.net (or equivalent)
+# interface-url = https://softinstXXXXX.host.vifib.net/erp5/web_site_module/monitoring_rjs_unsafe
+instance-configuration =
+  raw jupyter-password ${read-knowledge0:password}
+
+[publish-connection-parameter]
+recipe = slapos.cookbook:publish.serialised
+jupyter-classic-url = https://[${instance-parameter:host}]:${instance-parameter:port}/tree
+url = ${:jupyter-classic-url}
+jupyterlab-url = https://[${instance-parameter:host}]:${instance-parameter:port}/lab
+
+[erp5-kernel]
+recipe = slapos.cookbook:symbolic.link
+link-binary = {{ erp5_kernel_location }}/{{ erp5_kernel_filename }}
+target-directory = ${directory:erp5_kernel_dir}
+
+[kernel-json]
+<= dynamic-jinja2-template-base
+template = {{ kernel_json_location }}/{{ kernel_json_filename }}
+rendered = ${directory:erp5_kernel_dir}/kernel.json
+# Use python2.7 executable bin file for kernel config
+context =
+  raw python_executable {{ python_executable }}
+  raw kernel_dir ${erp5-kernel:target-directory}/{{ erp5_kernel_filename }}
+  key erp5_url slapconfiguration:configuration.erp5-url 
+  raw display_name ERP5
+  raw language_name python
+
+[custom-js]
+recipe = slapos.cookbook:symbolic.link
+target-directory = ${directory:jupyter_custom_dir}
+link-binary = {{ custom_js_location }}/custom.js

component/jupyter-py2/jupyter_notebook_config.py.jinja

+'''
+  This script initializes Jupyter's configuration such as passwords and other
+  things. It is run by IPython hence why it can use functions like get_config().
+'''
+import ConfigParser
+import random
+from notebook.auth import passwd
+import os
+
+def random_password(length = 10):
+  result = ""
+  for i in range(0, length):
+    result = result + chr(random.randint(0, 25) + ord('a'))
+  return result
+
+knowledge_0 = '{{ config_cfg }}'
+
+if not os.path.exists(knowledge_0):
+  print "Your software does <b>not</b> embed 0-knowledge. \
+  This interface is useless in this case</body></html>"
+  exit(0)
+
+c = get_config()
+
+parser = ConfigParser.ConfigParser()
+parser.read(knowledge_0)
+
+if not parser.has_section("jupyter_notebook"):
+  parser.add_section("jupyter_notebook")
+
+if not parser.has_option("jupyter_notebook", "password") or \
+  parser.get("jupyter_notebook", "password") == "":
+  parser.set("jupyter_notebook", "password", random_password())
+
+c.NotebookApp.password = passwd(parser.get("jupyter_notebook", "password"))
+
+with open(knowledge_0, 'w') as file:
+  parser.write(file)

component/jupyter-py2/jupyter_set_password.cgi.jinja

+#!{{ python_executable }}
+
+import cgi
+import cgitb
+import ConfigParser
+import os
+import re
+import subprocess
+
+from IPython.lib import passwd
+
+#cgitb.enable(display=0, logdir="/tmp/cgi.log")
+cgitb.enable()
+form = cgi.FieldStorage()
+
+config_file = "{{ config_cfg }}"
+
+
+if not os.path.exists(config_file):
+  print "Your software does <b>not</b> embed 0-knowledge. \
+  This interface is useless in this case</body></html>"
+  exit(0)
+
+
+parser = ConfigParser.ConfigParser()
+parser.read(config_file)
+if not parser.has_section("jupyter_notebook"):
+  parser.add_section("jupyter_notebook")
+
+if not parser.has_option("jupyter_notebook", "password"):
+  parser.set("jupyter_notebook", "password", "")
+
+if "password" in form:
+  parser.set("jupyter_notebook", "password", passwd(form["password"].value))
+  # subprocess.call('{{ httpd_graceful }}')
+  # TODO: we should restart jupyter
+
+with open(config_file, 'w') as file:
+  parser.write(file)
+
+# TODO cleanup
+print "<html><head>"
+print "<link rel=\"stylesheet\" href=\"static/pure-min.css\">"
+print "<link rel=\"stylesheet\" href=\"static/style.css\">"
+print "</head><body>"
+print "<h1>Jupyter Notebook Password :</h1>"
+print "<form action=\"/index.cgi\" method=\"post\" class=\"pure-form-aligned\">"
+print "<input type=\"hidden\" name=\"posting-script\" value=\"{{ pwd }}/{{ this_file }}\">"
+
+print """<div class="pure-control-group">
+<label for="password">Password*:</label>
+<input placeholder="Set your password" type="password" name="password" id="password"></br>
+</div><div class="pure-control-group">
+<label for="password">Verify Password*:</label>
+<input placeholder="Verify password" type="password" name="password_2" id="password_2"></br>
+</div><p id="validate-status" style="color:red"></p>
+<div class="pure-controls">
+<button id="register-button" type="submit" class="pure-button pure-button-primary" disabled>Access</button></div>
+</form>
+<script type="text/javascript" src="static/jquery-1.10.2.min.js"></script>
+<script type="text/javascript" src="static/monitor-register.js"></script>
+</body></html>
+"""

component/jupyter-py2/kernel.json.jinja

+{
+  "argv": [
+    "{{python_executable}}",
+    "{{kernel_dir}}",
+    "{{erp5_url}}",
+    "-f",
+    "{connection_file}"
+  ],
+  "display_name": "{{display_name}}",
+  "language": "{{language_name}}",
+  "language_info": {"name": "python"}
+}

software/erp5/test/test/test_erp5.py

@@ -119,6 +119,29 @@ class TestMedusa(ERP5InstanceTestCase, TestPublishedURLIsReachableMixin):
   def getInstanceParameterDict(cls):
     return {'_': json.dumps({'wsgi': False})}
 
+class TestJupyter(ERP5InstanceTestCase, TestPublishedURLIsReachableMixin):
+  """Test ERP5 Jupyter notebook
+  """
+  __partition_reference__ = 'jupyter'
+
+  @classmethod
+  def getInstanceParameterDict(cls):
+    return {'_': json.dumps({'jupyter': {'enable': True}})}
+
+  def test_jupyter_notebook_is_reachable(self):
+    param_dict = self.getRootPartitionConnectionParameterDict()
+
+    self.assertEqual(
+      'https://[%s]:8888/tree' % self._ipv6_address,
+      param_dict['jupyter-url']
+    )
+
+    result = requests.get(
+      param_dict['jupyter-url'], verify=False, allow_redirects=False)
+    self.assertEqual(
+      [requests.codes.found, True, '/login?next=%2Ftree'],
+      [result.status_code, result.is_redirect, result.headers['Location']]
+    )
 
 class TestBalancerPorts(ERP5InstanceTestCase):
   """Instantiate with two zope families, this should create for each family:

software/jupyter/ERP5kernel.py

@@ -75,7 +75,7 @@ class ERP5Kernel(Kernel):
     self.title = None
     # Allowed HTTP request code list for making request to erp5 from Kernel
     # This list should be to used check status_code before making requests to erp5
-    self.allowed_HTTP_request_code_list = range(500, 511)
+    self.allowed_HTTP_request_code_list = list(range(500, 511))
     # Append request code 200 in the allowed HTTP status code list
     self.allowed_HTTP_request_code_list.append(200)
 

software/jupyter/buildout.hash.cfg

@@ -19,7 +19,7 @@ md5sum = 1d5fe6cc4e48672ae7be1c223794a932
 
 [jupyter-notebook-config]
 filename = jupyter_notebook_config.py.jinja
-md5sum = 720e90a829c63371696bc3009917a743
+md5sum = 3da50c37760a42e42ae3f9d78d9ca449
 
 [jupyter-set-password]
 filename = jupyter_set_password.cgi.jinja
@@ -27,7 +27,7 @@ md5sum = b8d31441780b524a7e52d1710dd78385
 
 [erp5-kernel]
 filename = ERP5kernel.py
-md5sum = 7d5309fe79afbcb455c0d8181b42e56c
+md5sum = da04b99b70b2e327c9e9b4cdd056098e
 
 [kernel-json]
 filename = kernel.json.jinja
@@ -35,4 +35,4 @@ md5sum = 33547be93a67530165e079dc3ecfdac3
 
 [custom-js]
 filename = custom.js
-md5sum = 40d938bb09261c65421a7725b40f87dc
+md5sum = 295aeee765bc4d09bf0686021f32f72c

software/jupyter/custom.js

@@ -81,16 +81,17 @@
  * @static
  */
 
-$([Jupyter.events]).on('notebook_loaded.Notebook', function(){
+function prependERP5Help() {
   var kernelname = Jupyter.notebook.kernel_selector.current_selection;
-  var display_text="<div class='output_subarea output_text output_result'>\
+  var display_text = "<div class='output_subarea output_text output_result'>\
     <pre>Follow these steps to customize your notebook with ERP5 kernel :-</br>\
     1. Make sure you have 'erp5_data_notebook' business template installed in your ERP5</br>\
-  2. <b>%erp5_user &lt;your_erp5_username&gt;</b></br>\
-  3. <b>%erp5_password &lt;your_erp5_password&gt;</b></br>\
-  4. <b>%notebook_set_reference &lt;your_notebook_reference&gt;</b></br>\
+    2. <b>%erp5_url &lt;your_erp5_url&gt;/Base_executeJupyter</b></br>\
+    3. <b>%erp5_user &lt;your_erp5_username&gt;</b></br>\
+    4. <b>%erp5_password &lt;your_erp5_password&gt;</b></br>\
+    5. <b>%notebook_set_reference &lt;your_notebook_reference&gt;</b></br>\
     It would be better to set the reference to match with erp5 reference pattern.</br>\
-  5. As soon as you see 'Please Proceed' message you can now access your erp5 using notebook.</br>\
+    6. As soon as you see 'Please Proceed' message you can now access your erp5 using notebook.</br>\
     <p><u>OTHER USEFUL MAGICS</u> -</br>\
     <b>%my_notebooks</b> -This is used to display all the notebooks created by the specific user.</br>\
     <b>%notebook_set_title</b> -This sets the title of the current notebook.</br>\
@@ -114,8 +115,17 @@ $([Jupyter.events]).on('notebook_loaded.Notebook', function(){
     <b>WARNING:</b> it is not recommended to have too many setup functions in the environment, </br>\
     because they will be executed in every code cell and can cause a substantial slow down.\
     </pre></div>";
+
     if (kernelname=="erp5"){
       $('div#notebook-container').prepend(display_text);
     }
-  });
+}
 
+define([
+  'base/js/namespace',
+  'base/js/promises'
+], function(Jupyter, promises) {
+  promises.notebook_loaded.then(function() {
+    prependERP5Help();
+  });
+});

software/jupyter/jupyter_notebook_config.py.jinja

@@ -2,7 +2,7 @@
   This script initializes Jupyter's configuration such as passwords and other
   things. It is run by IPython hence why it can use functions like get_config().
 '''
-import ConfigParser
+import configparser
 import random
 from notebook.auth import passwd
 import os
@@ -16,13 +16,13 @@ def random_password(length = 10):
 knowledge_0 = '{{ config_cfg }}'
 
 if not os.path.exists(knowledge_0):
-  print "Your software does <b>not</b> embed 0-knowledge. \
-  This interface is useless in this case</body></html>"
+  print("Your software does <b>not</b> embed 0-knowledge. \
+  This interface is useless in this case</body></html>")
   exit(0)
 
 c = get_config()
 
-parser = ConfigParser.ConfigParser()
+parser = configparser.ConfigParser()
 parser.read(knowledge_0)
 
 if not parser.has_section("jupyter_notebook"):

software/jupyter/software.cfg

@@ -11,6 +11,9 @@ parts +=
   jupyter-notebook-initialized-scripts
   instance-jupyter-notebook
 
+[python]
+part = python3
+
 [gcc]
 # Always build GCC for Fortran (see openblas).
 max_version = 0
@@ -63,40 +66,49 @@ context =
   key monitor_template_rendered buildout:directory
 
 [versions]
-Pygments = 2.2.0
+Pygments = 2.7.2
 astor = 0.5
+async-generator = 1.10
 backports-abc = 0.5
 backports.functools-lru-cache = 1.6.1
 backports.shutil-get-terminal-size = 1.0.0
+bleach = 3.2.1
 cycler = 0.10.0
-ipykernel = 4.5.2
+defusedxml = 0.6.0
+entrypoints = 0.3
+ipykernel = 5.3.4
 ipython = 5.3.0
 ipython-genutils = 0.1.0
 ipywidgets = 6.0.0
-jupyter-client = 5.0.0
-jupyter-core = 4.3.0
+jupyter-client = 6.1.7
+jupyter-core = 4.7.0
 jupyterlab = 0.26.3
 jupyterlab-launcher = 0.3.1
+jupyterlab-pygments = 0.1.2
 matplotlib = 2.1.2
-mistune = 0.7.3
-nbformat = 4.3.0
-notebook = 4.4.1
-pandas = 0.19.2
+mistune = 0.8.4
+nest-asyncio = 1.4.3
+nbclient = 0.5.1
+nbformat = 5.0.8
+notebook = 6.1.5
+numpy = 1.14.6
+pandas = 0.25.3
+pandocfilters = 1.4.3
 plone.recipe.command = 1.1
 prompt-toolkit = 1.0.13
 ptyprocess = 0.5.1
-pyzmq = 16.0.2
-scikit-learn = 0.18.1
+pyzmq = 20.0.0
+scikit-learn = 0.20.4
 seaborn = 0.7.1
 simplegeneric = 0.8.1
 slapos.recipe.template = 4.4
-statsmodels = 0.8.0
-terminado = 0.6
-tornado = 4.4.2
-traitlets = 4.3.2
+statsmodels = 0.11.1
+testpath = 0.4.4
+terminado = 0.9.1
+tornado = 6.1
+traitlets = 5.0.5
+webencodings = 0.5.1
 widgetsnbextension = 2.0.0
-# numpy >= 1.13.1 is required for numpy.core.multiarray
-numpy = 1.13.1
 
 
 # Required by:
@@ -104,41 +116,55 @@ numpy = 1.13.1
 certifi = 2020.6.20
 
 # Required by:
-# notebook==4.3.2
-# nbconvert 4.2.0 depends on entrypoints egg that is not available as tar/zip source.
-nbconvert = 4.1.0
+# notebook==6.1.5
+Send2Trash = 1.5.0
+
+# Required by:
+# notebook==6.1.5
+argon2-cffi = 20.1.0
+
+# Required by:
+# notebook==6.1.5
+nbconvert = 6.0.7
 
 # Required by:
 # ipython==5.3.0
 pathlib2 = 2.2.1
 
 # Required by:
-# statsmodels==0.8.0
-patsy = 0.4.1
+# statsmodels==0.11.1
+patsy = 0.5.1
 
 # Required by:
 # ipython==5.3.0
-pexpect = 4.2.1
+pexpect = 4.8.0
 
 # Required by:
 # ipython==5.3.0
 pickleshare = 0.7.4
 
 # Required by:
-# matplotlib==2.1.2
-# pandas==0.19.2
-python-dateutil = 2.6.0
+# notebook==6.1.5
+prometheus-client = 0.9.0
+
+# Required by:
+# statsmodels==0.11.1
+python-dateutil = 2.8.1
 
 # Required by:
 # pathlib2==2.2.1
 scandir = 1.5
 
 # Required by:
-# statsmodels==0.8.0
-scipy = 0.19.0
+# statsmodels==0.11.1
+pytz = 2020.4
 
 # Required by:
-# tornado==4.4.2
+# statsmodels==0.11.1
+scipy = 1.0.1
+
+# Required by:
+# tornado==6.1
 singledispatch = 3.4.0.3
 
 # Required by:

software/jupyter/test/test.py

@@ -26,7 +26,7 @@
 ##############################################################################
 
 
-import httplib
+import http.client
 import json
 import os
 import requests
@@ -60,7 +60,7 @@ class TestJupyter(InstanceTestCase):
     result = requests.get(
       connection_dict['url'], verify=False, allow_redirects=False)
     self.assertEqual(
-      [httplib.FOUND, True, '/login?next=%2Ftree'],
+      [http.client.FOUND, True, '/login?next=%2Ftree'],
       [result.status_code, result.is_redirect, result.headers['Location']]
     )
 
@@ -68,7 +68,7 @@ class TestJupyter(InstanceTestCase):
       connection_dict['jupyter-classic-url'],
       verify=False, allow_redirects=False)
     self.assertEqual(
-      [httplib.FOUND, True, '/login?next=%2Ftree'],
+      [http.client.FOUND, True, '/login?next=%2Ftree'],
       [result.status_code, result.is_redirect, result.headers['Location']]
     )
 
@@ -76,6 +76,6 @@ class TestJupyter(InstanceTestCase):
       connection_dict['jupyterlab-url'],
       verify=False, allow_redirects=False)
     self.assertEqual(
-      [httplib.FOUND, True, '/login?next=%2Flab'],
+      [http.client.FOUND, True, '/login?next=%2Flab'],
       [result.status_code, result.is_redirect, result.headers['Location']]
     )

software/slapos-sr-testing/software-py3.cfg

@@ -13,6 +13,7 @@ extra-eggs +=
 [template]
 extra =
   helloworld ${slapos.test.helloworld-setup:setup}
+  jupyter ${slapos.test.jupyter-setup:setup}
   monitor ${slapos.test.monitor-setup:setup}
   plantuml ${slapos.test.plantuml-setup:setup}
   powerdns ${slapos.test.powerdns-setup:setup}

software/slapos-sr-testing/software.cfg

@@ -271,7 +271,6 @@ extra =
   re6stnet ${slapos.test.re6stnet-setup:setup}
   seleniumserver ${slapos.test.seleniumserver-setup:setup}
   jstestnode ${slapos.test.jstestnode-setup:setup}
-  jupyter ${slapos.test.jupyter-setup:setup}
   nextcloud ${slapos.test.nextcloud-setup:setup}
   turnserver ${slapos.test.turnserver-setup:setup}
   theia ${slapos.test.theia-setup:setup}

stack/erp5/buildout.cfg

@@ -56,8 +56,8 @@ extends =
   ../../component/pylint/buildout.cfg
   ../../component/perl-Image-ExifTool/buildout.cfg
   ../../component/wendelin.core/buildout.cfg
+  ../../component/jupyter-py2/buildout.cfg
   ../../stack/caucase/buildout.cfg
-  ../../software/jupyter/software.cfg
   ../../software/neoppod/software-common.cfg
 # keep neoppod extends last
 

stack/slapos.cfg

@@ -191,7 +191,7 @@ scandir = 1.10.0
 setuptools-dso = 1.7
 rubygemsrecipe  = 0.3.0
 six = 1.12.0
-slapos.cookbook = 1.0.167
+slapos.cookbook = 1.0.171
 slapos.core = 1.6.3
 slapos.extension.strip = 0.4
 slapos.extension.shared = 1.0