Commit 33954c66 authored by Ayush Tiwari's avatar Ayush Tiwari Committed by Kirill Smelkov

Jupyter: Hook ERP5 Kernel

ERP5 kernel basic info/workflow:

1. User enters code on notebook cell and executes
2. Code is sent to kernel via websockets
3. Kernel sends request to ERP5
4. Code is executed by ERP5 and the result is returned back via request.
5. Result is received and rendered on the notebook frontend.
6. Other message formats such as error and status are also conveyed by the Kernel.

[ kirr: in IPython notebook speak kernel is something that allows IPython
  notebook server side to talk to execution backend. ERP5 kernel is a thing that
  allows ipython notbook to talk to ERP5 (with help on-ERP5-server special bt5
  installed which accepts and executes commands).

  The bt5 to handle notebook calls on ERP5 side - erp5-data-notebook - is
  proposed to be merged into erp5.git on erp5!29 ]

/initially-reviewed-by @kirr, @Tyagov  (in a lot of places, last time on !33)
parent 77ffa61e
...@@ -23,6 +23,9 @@ parts = ...@@ -23,6 +23,9 @@ parts =
## Monitor for ipython ## Monitor for ipython
monitor-current-log-access monitor-current-log-access
monitor-deploy-set-password-cgi monitor-deploy-set-password-cgi
erp5-kernel
kernel-json
custom-js
extends = {{ monitor_template }} extends = {{ monitor_template }}
...@@ -122,8 +125,34 @@ notebook_dir = ${:var}/notebooks ...@@ -122,8 +125,34 @@ notebook_dir = ${:var}/notebooks
# Add folders to explicitly define ipython directory # Add folders to explicitly define ipython directory
ipython_dir = ${:home}/ipython ipython_dir = ${:home}/ipython
ipython_kernel_dir = ${:ipython_dir}/kernels ipython_kernel_dir = ${:ipython_dir}/kernels
erp5_kernel_dir = ${:ipython_kernel_dir}/ERP5
[publish-connection-parameter] [publish-connection-parameter]
recipe = slapos.cookbook:publish recipe = slapos.cookbook:publish
url = https://[${instance-parameter:host}]:${instance-parameter:port} url = https://[${instance-parameter:host}]:${instance-parameter:port}
monitor_url = ${monitor-parameters:url} monitor_url = ${monitor-parameters:url}
[erp5-kernel]
<= dynamic-jinja2-template-base
template = {{ erp5_kernel_location }}/{{ erp5_kernel_filename }}
rendered = ${directory:erp5_kernel_dir}/ERP5kernel.py
# Use ipython as executable python as we'll be needing requests library in kernel
context =
raw python_executable {{ bin_directory }}/ipython
[kernel-json]
<= dynamic-jinja2-template-base
template = {{ kernel_json_location }}/{{ kernel_json_filename }}
rendered = ${directory:erp5_kernel_dir}/kernel.json
# Use python2.7 executable bin file for kernel config
context =
raw python_executable {{ python_executable }}
key kernel_dir erp5-kernel:rendered
raw display_name ERP5
raw language_name python
[custom-js]
<= dynamic-jinja2-template-base
template = {{ custom_js_location }}/{{ custom_js_filename }}
rendered = ${directory:ipython_dir}/profile_default/static/custom/custom.js
mode = 0744
...@@ -18,6 +18,7 @@ parts = ...@@ -18,6 +18,7 @@ parts =
; In the ipython notebook software, we use more eggs than in the minimal ; In the ipython notebook software, we use more eggs than in the minimal
; ipython notebook component ; ipython notebook component
eggs += eggs +=
requests
${scipy:egg} ${scipy:egg}
${pandas:egg} ${pandas:egg}
${scikit-learn:egg} ${scikit-learn:egg}
...@@ -39,22 +40,44 @@ md5sum = a5bc4ee8539109d1de7ab33b4c2c97ea ...@@ -39,22 +40,44 @@ md5sum = a5bc4ee8539109d1de7ab33b4c2c97ea
filename = ipython_set_password.cgi.jinja filename = ipython_set_password.cgi.jinja
md5sum = d7d4a7e19d55bf14007819258bf42100 md5sum = d7d4a7e19d55bf14007819258bf42100
[erp5-kernel]
<= download-file-base
filename = ERP5kernel.py.jinja
md5sum = a6a0ad790d100167c42c354fe5744e11
[kernel-json]
<= download-file-base
filename = kernel.json.jinja
md5sum = ab6e78ea20855e07d388b5b86d1770fe
[custom-js]
<= download-file-base
filename = custom.js.jinja
md5sum = 0e8262d04a6dafbc1b77d95aea2192bc
[instance] [instance]
recipe = slapos.recipe.template:jinja2 recipe = slapos.recipe.template:jinja2
template = ${:_profile_base_location_}/instance.cfg.in template = ${:_profile_base_location_}/instance.cfg.in
rendered = ${buildout:directory}/template.cfg rendered = ${buildout:directory}/template.cfg
mode = 0644 mode = 0644
md5sum = e67b3cc33db32993a70895173e7b0150 md5sum = 6bc65c138215dc995e4721cc74bdf7ef
context = context =
key bin_directory buildout:bin-directory key bin_directory buildout:bin-directory
key develop_eggs_directory buildout:develop-eggs-directory key develop_eggs_directory buildout:develop-eggs-directory
key eggs_directory buildout:eggs-directory key eggs_directory buildout:eggs-directory
key monitor_template monitor-template:output key monitor_template monitor-template:output
key openssl_output openssl-output:openssl key openssl_output openssl-output:openssl
key python_executable python2.7:executable
key ipython_notebook_config_location ipython-notebook-config:location key ipython_notebook_config_location ipython-notebook-config:location
key ipython_notebook_config_filename ipython-notebook-config:filename key ipython_notebook_config_filename ipython-notebook-config:filename
key ipython_notebook_set_password_location ipython-notebook-set-password:location key ipython_notebook_set_password_location ipython-notebook-set-password:location
key ipython_notebook_set_password_filename ipython-notebook-set-password:filename key ipython_notebook_set_password_filename ipython-notebook-set-password:filename
key erp5_kernel_location erp5-kernel:location
key erp5_kernel_filename erp5-kernel:filename
key kernel_json_location kernel-json:location
key kernel_json_filename kernel-json:filename
key custom_js_location custom-js:location
key custom_js_filename custom-js:filename
[versions] [versions]
PyRSS2Gen = 1.1 PyRSS2Gen = 1.1
...@@ -73,6 +96,7 @@ scipy = 0.15.1 ...@@ -73,6 +96,7 @@ scipy = 0.15.1
slapos.recipe.template = 2.8 slapos.recipe.template = 2.8
terminado = 0.5 terminado = 0.5
tornado = 4.2 tornado = 4.2
requests = 2.7.0
# Required by: # Required by:
# tornado==4.2 # tornado==4.2
......
#!{{ python_executable }}
from IPython.kernel.zmq.kernelbase import Kernel
from IPython.kernel.zmq.kernelapp import IPKernelApp
from IPython.core.display import HTML
import requests
import json
# erp5_url from buildout
# TODO: Uncomment after adding automated installation of erp5-data-notebook bt5
# url = ""
# url = "%s/erp5/Base_executeJupyter"%url
class MagicInfo:
"""
Magics definition structure.
Initializes a new MagicInfo class with specific paramters to identify a magic.
"""
def __init__(self, magic_name, variable_name, send_request, request_reference, display_message):
self.magic_name = magic_name
self.variable_name = variable_name
self.send_request = send_request
self.request_reference = request_reference
self.display_message = display_message
# XXX: New magics to be added here in the dictionary.
# In this dictionary,
# key = magic_name,
# value = MagicInfo Structure corresponding to the magics
# Different parameters of the structures are :-
# magics_name(str) = Name which would be used on jupyter frontend
# variable_name(str) = Name of variable on which magic would be set in kernel
# send_request(boolean) = Magics for which requests to erp5 backend need to be made
# request_reference(boolean) = Request for notebook references(and titles) from erp5
# display_message(boolean) = If the magics need to display message after
# making request. Useful for magics which do get some
# useful content from erp5 backend and need to display
MAGICS = {
'erp5_user': MagicInfo('erp5_user', 'user', True, False, True),
'erp5_password': MagicInfo('erp5_password', 'password', True, False, True),
'erp5_url': MagicInfo('erp5_url', 'url', True, False, True),
'notebook_set_reference': MagicInfo('notebook_set_reference', 'reference', True, False, True),
'notebook_set_title': MagicInfo('notebook_set_title', 'title', False, False, True),
'my_notebooks': MagicInfo('my_notebooks', '', True, True, False)
}
class ERP5Kernel(Kernel):
"""
Jupyter Kernel class to interact with erp5 backend for code from frontend.
To use this kernel with erp5, user need to install 'erp5_data_notebook' bt5
Also, handlers(aka magics) starting with '%' are predefined.
Each request to erp5 for code execution requires erp5_user, erp5_password
and reference of the notebook.
"""
implementation = 'ERP5'
implementation_version = '1.0'
language = 'ERP5'
language_version = '0.1'
language_info = {'mimetype': 'text/plain', 'name':'python'}
banner = "ERP5 integration with ipython notebook"
def __init__(self, user=None, password=None, url=None, status_code=None,
*args, **kwargs):
super(ERP5Kernel, self).__init__(*args, **kwargs)
self.user = user
self.password = password
# Use URL provided by buildout during initiation
# It can later be overridden
self.url = url
self.status_code = status_code
self.reference = None
self.title = None
# Allowed HTTP request code list for making request to erp5 from Kernel
# This list should be to used check status_code before making requests to erp5
self.allowed_HTTP_request_code_list = range(500, 511)
# Append request code 200 in the allowed HTTP status code list
self.allowed_HTTP_request_code_list.append(200)
def display_response(self, response=None):
"""
Dispays the stream message response to jupyter frontend.
"""
if response:
stream_content = {'name': 'stdout', 'text': response}
self.send_response(self.iopub_socket, 'stream', stream_content)
def set_magic_attribute(self, magic_info=None, code=None):
"""
Set attribute for magic which are necessary for making requests to erp5.
Catch errors and display message. Since user is in contact with jupyter
frontend, so its better to catch exceptions and dispaly messages than to
let them fail in backend and stuck the kernel.
For a making a request to erp5, we need -
erp5_url, erp5_user, erp5_password, notebook_set_reference
"""
# Set attributes only for magic who do have any varible to set value to
if magic_info.variable_name:
try:
# Get the magic value recived via code from frontend
magic_value = code.split()[1]
# Set magic_value to the required attribute
setattr(self, magic_info.variable_name , magic_value)
self.response = 'Your %s is %s. '%(magic_info.magic_name, magic_value)
# Catch exception while setting attribute and set message in response
except AttributeError:
self.response = 'Please enter %s magic value'%magic_info.variable_name
# Catch IndexError while getting magic_value and set message in response object
except IndexError:
self.response = 'Empty value for %s magic'%magic_info.variable_name
# Catch all other exceptions and set error_message in response object
# XXX: Might not be best way, but its better to display error to the user
# via notebook frontend than to fail in backend and stuck the Kernel without
# any failure message to user.
except Exception as e:
self.response = str(e)
# Display the message/response from this fucntion before moving forward so
# as to keep track of the status
self.display_response(response=self.response)
def check_required_attributes(self):
"""
Check if the required attributes for making a request are already set or not.
Display message to frontend to provide with the values in case they aren't.
This function can be called anytime to check if the attributes are set. The
output result will be in Boolean form.
Also, in case any of attribute is not set, call to display_response would be
made to ask user to enter value.
"""
result_list = []
required_attributes = ['url', 'password', 'user', 'reference']
# Set response to empty so as to flush the response set by some earlier fucntion call
self.response = ''
# Loop to check if the attributes are set
for attribute in required_attributes:
if getattr(self, attribute):
result_list.append(True)
else:
# Set response/message for attributes which aren't set
self.response = '\nPlease enter %s in next cell. '%attribute
result_list.append(False)
# Compare result_list to get True for all True results and False for any False result
check_attributes = all(result_list)
# Display response to frontend before moving forward
self.display_response(response=self.response)
return check_attributes
def make_erp5_request(self, request_reference=False, display_message=True,
code=None, message=None, title=None, *args, **kwargs):
"""
Function to make request to erp5 as per the magics.
Should return the response json object.
"""
try:
erp5_request = requests.get(
self.url,
verify=False,
auth=(self.user, self.password),
params={
'python_expression': code,
'reference': self.reference,
'title': self.title,
'request_reference': request_reference,
},
)
# Set value for status_code for self object which would later be used to
# dispaly response after statement check
self.status_code = erp5_request.status_code
# Dispaly error response in case the request give any other status
# except 200 and 5xx(which is for errors on server side)
if self.status_code not in self.allowed_HTTP_request_code_list:
self.response = '''Error code %s on request to ERP5,\n
check credentials or ERP5 family URL'''%self.status_code
else:
# Set value of self.response to the given value in case response from function
# call. In all other case, response should be the content from request
if display_message and message:
self.response = message
else:
self.response = erp5_request.content
except requests.exceptions.RequestException as e:
self.response = str(e)
def do_execute(self, code, silent, store_history=True, user_expressions=None,
allow_stdin=False):
"""
Validate magic and call functions to make request to erp5 backend where
the code is being executed and response is sent back which is then sent
to jupyter frontend.
"""
# By default, take the status of response as 'ok' so as show the responses
# for erp5_url and erp5_user on notebook frontend as successful response.
status = 'ok'
if not silent:
# Remove spaces and newlines from both ends of code
code = code.strip()
if code.startswith('%'):
# No need to try-catch here as its already been taken that the code
# starts-with '%', so we'll get magic_name, no matter what be after '%'
magic_name = code.split()[0][1:]
magics_name_list = [magic.magic_name for magic in MAGICS.values()]
# Check validation of magic
if magic_name and magic_name in magics_name_list:
# Get MagicInfo object related to the magic
magic_info = MAGICS.get(magic_name)
# Function call to set the required magics
self.set_magic_attribute(magic_info=magic_info, code=code)
# Call to check if the required_attributes are set
checked_attribute = self.check_required_attributes()
if checked_attribute and magic_info.send_request:
# Call the function to send request to erp5 with the arguments given
self.make_erp5_request(message='\nPlease proceed',
request_reference=magic_info.request_reference,
display_message=magic_info.display_message)
# Display response from erp5 request for magic
# Since this response would be either success message or failure
# error message, both of which are string type, so, we can simply
# display the stream response.
self.display_response(response=self.response)
else:
# Set response if there is no magic or the magic name is not in MAGICS
self.response = 'Invalid Magics'
self.display_response(response=self.response)
else:
# Check for status_code before making request to erp5 and make request in
# only if the status_code is in the allowed_HTTP_request_code_list
if self.status_code in self.allowed_HTTP_request_code_list:
self.make_erp5_request(code=code)
# For 200 status_code, Kernel will receive predefined format for data
# from erp5 which is either json of result or simple result string
if self.status_code == 200:
mime_type = 'text/plain'
try:
content = json.loads(self.response)
code_result = content['code_result']
# Display to frontend the error message for content status as 'error'
if content['status']=='error':
reply_content = {
'status': 'error',
'execution_count': self.execution_count,
'ename': content['ename'],
'evalue': content['evalue'],
'traceback': content['traceback']}
self.send_response(self.iopub_socket, u'error', reply_content)
return reply_content
# Catch exception for content which isn't json
except ValueError:
content = self.response
code_result = content
# Display basic error message to frontend in case of error on server side
else:
self.make_erp5_request(code=code)
code_result = "Error at Server Side"
mime_type = 'text/plain'
# For all status_code except allowed_HTTP_response_code_list show unauthorized message
else:
code_result = 'Unauthorized access'
mime_type = 'text/plain'
data = {
'data': {mime_type: code_result},
'metadata': {}
}
self.send_response(self.iopub_socket, 'display_data', data)
reply_content = {
'status': status,
# The base class increments the execution count
'execution_count': self.execution_count,
'payload': [],
'user_expressions': {},
}
return reply_content
if __name__ == '__main__':
IPKernelApp.launch_instance(kernel_class=ERP5Kernel)
// leave at least 2 line with only a star on it below, or doc generation fails
/**
*
*
* Placeholder for custom user javascript
* mainly to be overridden in profile/static/custom/custom.js
* This will always be an empty file in IPython
*
* User could add any javascript in the `profile/static/custom/custom.js` file.
* It will be executed by the ipython notebook at load time.
*
* Same thing with `profile/static/custom/custom.css` to inject custom css into the notebook.
*
*
* The object available at load time depend on the version of IPython in use.
* there is no guaranties of API stability.
*
* The example below explain the principle, and might not be valid.
*
* Instances are created after the loading of this file and might need to be accessed using events:
* define([
* 'base/js/namespace',
* 'base/js/events'
* ], function(IPython, events) {
* events.on("app_initialized.NotebookApp", function () {
* IPython.keyboard_manager....
* });
* });
*
* __Example 1:__
*
* Create a custom button in toolbar that execute `%qtconsole` in kernel
* and hence open a qtconsole attached to the same kernel as the current notebook
*
* define([
* 'base/js/namespace',
* 'base/js/events'
* ], function(IPython, events) {
* events.on('app_initialized.NotebookApp', function(){
* IPython.toolbar.add_buttons_group([
* {
* 'label' : 'run qtconsole',
* 'icon' : 'icon-terminal', // select your icon from http://fortawesome.github.io/Font-Awesome/icons
* 'callback': function () {
* IPython.notebook.kernel.execute('%qtconsole')
* }
* }
* // add more button here if needed.
* ]);
* });
* });
*
* __Example 2:__
*
* At the completion of the dashboard loading, load an unofficial javascript extension
* that is installed in profile/static/custom/
*
* define([
* 'base/js/events'
* ], function(events) {
* events.on('app_initialized.DashboardApp', function(){
* require(['custom/unofficial_extension.js'])
* });
* });
*
* __Example 3:__
*
* Use `jQuery.getScript(url [, success(script, textStatus, jqXHR)] );`
* to load custom script into the notebook.
*
* // to load the metadata ui extension example.
* $.getScript('/static/notebook/js/celltoolbarpresets/example.js');
* // or
* // to load the metadata ui extension to control slideshow mode / reveal js for nbconvert
* $.getScript('/static/notebook/js/celltoolbarpresets/slideshow.js');
*
*
* @module IPython
* @namespace IPython
* @class customjs
* @static
*/
$([IPython.events]).on('notebook_loaded.Notebook', function(){
var kernelname = IPython.notebook.kernel_selector.current_selection;
var display_text="<div class='output_subarea output_text output_result'>\
<pre>Follow these steps to customize your notebook with ERP5 kernel :-</br>\
1. Add <b>%erp5_url &lt;your_erp5_url&gt;/erp5/Base_executeJupyter</b>.</br>\
Make sure you have 'erp5_data_notebook' business template installed in your erp5</br>\
2. <b>%notebook_set_reference &lt;your_notebook_refernce&gt;</b></br>\
It would be better to set the reference to match with erp5 reference pattern.</br>\
3. <b>%erp5_user &lt;your_erp5_username&gt;</b></br>\
4. <b>%erp5_password &lt;your_erp5_password&gt;</b></br>\
5. As soon as you see 'Please Proceed' message you can now access your erp5 using notebook.</br>\
<p><u>OTHER USEFUL MAGICS</u> -</br>\
<b>%my_notebooks</b> -This is used to display all the notebooks created by the specific user.</br>\
NOTE: Do not dynamically alter imported module objects as they are not being saved in DB, </br>\
so changes to them would be disregarded and would throw an error.</br>\
</pre></div>";
if (kernelname=="erp5"){
$('div#notebook-container').prepend(display_text);
}
});
\ No newline at end of file
{
"argv": [
"{{python_executable}}",
"{{kernel_dir}}",
"-f",
"{connection_file}"
],
"display_name": "{{display_name}}",
"language": "{{language_name}}",
"language_info": {"name": "python"}
}
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment