extension.erp5.JupyterCompile.py 8.45 KB
Newer Older
1 2
# -*- coding: utf-8 -*-

3
from cStringIO import StringIO
4
from Products.ERP5Type.Globals import  PersistentMapping
5
from OFS.Image import Image as OFSImage
6 7

import sys
8
import ast
9
import types
10

11 12
mime_type = 'text/plain'

13 14 15
def Base_compileJupyterCode(self, jupyter_code, old_local_variable_dict):
  """
    Function to execute jupyter code and update the local_varibale dictionary.
16 17
    Code execution depends on 'interactivity', a.k.a , if the ast.node object has
    ast.Expr instance(valid for expressions) or not.
18
    
19 20
    old_local_variable_dict should contain both variables dict and modules imports.
    Here, imports dict is key, value pair of modules and their name in sys.path,
21 22 23 24 25
    executed separately everytime before execution of jupyter_code to populate
    sys modules beforehand.

    For example :
    old_local_variable_dict = {
26
                                'imports': {'numpy': 'np', 'sys': 'sys'},
27 28
                                'variables': {'np.split': <function split at 0x7f4e6eb48b90>}
                                }
29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45

    The behaviour would be similar to that of jupyter notebook:-
    ( https://github.com/ipython/ipython/blob/master/IPython/core/interactiveshell.py#L2954 )
    Example:

      code1 = '''
      23
      print 23 #Last node not an expression, interactivity = 'last'
      '''
      out1 = '23'

      code2 = '''
      123
      12 #Last node an expression, interactivity = 'none'
      '''
      out2 = '12'

46
  """
47
  # Updating global variable mime_type to its original value
48
  # Required when call to Base_displayImage is made which is changing
49 50 51 52
  # the value of gloabl mime_type
  global mime_type
  mime_type = 'text/plain'

53 54 55
  # Other way would be to use all the globals variables instead of just an empty
  # dictionary, but that might hamper the speed of exec or eval.
  # Something like -- g = globals(); g['context'] = self;
56 57 58 59
  g = {}

  # Saving the initial globals dict so as to compare it after code execution
  globals_dict = globals()
60 61 62
  g['context'] = self
  result_string = None
  ename, evalue, tb_list = None, None, None
63
  # Update globals dict and use it while running exec command
64
  g.update(old_local_variable_dict['variables'])
65 66

  # IPython expects 2 status message - 'ok', 'error'
67 68 69 70
  # XXX: The focus is on 'ok' status only, we're letting errors to be raised on
  # erp5 for now, so as not to hinder the transactions while catching them.
  # TODO: This can be refactored by using client side error handling instead of
  # catching errors on server/erp5.
71 72
  status = u'ok'

73 74
  # Execute only if jupyter_code is not empty
  if jupyter_code:
75
    # Import all the modules from local_variable_dict['imports']
76 77 78 79 80 81 82
    # While any execution, in locals() dict, a module is saved as:
    # code : 'from os import path'
    # {'path': <module 'posixpath'>}
    # So, here we would try to get the name 'posixpath' and import it as 'path'
    for k, v in old_local_variable_dict['imports'].iteritems():
      import_statement_code = 'import %s as %s'%(v, k)
      exec(import_statement_code, g, g)
83
  
84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102
    # Create ast parse tree
    ast_node = ast.parse(jupyter_code)
    # Get the node list from the parsed tree
    nodelist = ast_node.body

    # If the last node is instance of ast.Expr, set its interactivity as 'last'
    # This would be the case if the last node is expression
    if isinstance(nodelist[-1], ast.Expr):
      interactivity = "last"
    else:
      interactivity = "none"

    # Here, we define which nodes to execute with 'single' and which to execute
    # with 'exec' mode.
    if interactivity == 'none':
      to_run_exec, to_run_interactive = nodelist, []
    elif interactivity == 'last':
      to_run_exec, to_run_interactive = nodelist[:-1], nodelist[-1:]

103 104 105 106
    old_stdout = sys.stdout
    result = StringIO()
    sys.stdout = result

107 108 109 110 111 112 113 114 115 116 117 118
    # Execute the nodes with 'exec' mode
    for node in to_run_exec:
      mod = ast.Module([node])
      code = compile(mod, '<string>', "exec")
      exec(code, g, g)

    # Execute the interactive nodes with 'single' mode
    for node in to_run_interactive:
      mod = ast.Interactive([node])
      code = compile(mod, '<string>', "single")
      exec(code, g, g)

119 120 121 122 123 124
    # Letting the code fail in case of error while executing the python script/code
    # XXX: Need to be refactored so to acclimitize transactions failure as well as
    # normal python code failure and show it to user on jupyter frontend.
    # Decided to let this fail silently in backend without letting the frontend
    # user know the error so as to let tranasction or its error be handled by ZODB
    # in uniform way instead of just using half transactions.
125

126 127
    sys.stdout = old_stdout
    result_string = result.getvalue()
128 129
  else:
    result_string = jupyter_code
130

131 132 133 134 135
  # Difference between the globals variable before and after exec/eval so that
  # we don't have to save unnecessary variables in database which might or might
  # not be picklabale
  local_variable_dict = old_local_variable_dict
  local_variable_dict_new = {key: val for key, val in g.items() if key not in globals_dict.keys()}
136 137 138
  local_variable_dict['variables'].update(local_variable_dict_new)

  # Differentiate 'module' objects from local_variable_dict and save them as
139 140
  # string in the dict as {'imports': {'numpy': 'np', 'matplotlib': 'mp']}
  if 'variables' and 'imports' in local_variable_dict:
141 142 143
    for key, val in local_variable_dict['variables'].items():
      # Check if the val in the dict is ModuleType and remove it in case it is
      if isinstance(val, types.ModuleType):
144 145 146 147 148 149
        # Update local_variable_dict['imports'] dictionary with key, value pairs
        # with key corresponding to module name as its imported and value as the
        # module name being stored in sys.path
        # For example : 'np': <numpy module at ...> -- {'np': numpy}
        local_variable_dict['imports'][key] = val.__name__

150 151 152
        # XXX: The next line is mutating the dict, beware in case any reference
        # is made later on to local_variable_dict['variables'] dictionary
        local_variable_dict['variables'].pop(key)
153

154 155
  result = {
    'result_string': result_string,
156
    'local_variable_dict': local_variable_dict,
157
    'status': status,
158
    'mime_type': mime_type,
159 160
    'evalue': evalue,
    'ename': ename,
161
    'traceback': tb_list,
162
  }
163

164
  return result
165

166
def AddNewLocalVariableDict(self):
167
  """
168
  Function to add a new Local Variable for a Data Notebook
169 170
  """
  new_dict = PersistentMapping()
171
  variable_dict = PersistentMapping()
172
  module_dict = PersistentMapping()
173
  new_dict['variables'] = variable_dict
174
  new_dict['imports'] = module_dict
175 176
  return new_dict

177
def UpdateLocalVariableDict(self, existing_dict):
178
  """
179
  Function to update local_varibale_dict for a Data Notebook
180
  """
181
  new_dict = self.Base_addLocalVariableDict()
182 183
  for key, val in existing_dict['variables'].iteritems():
    new_dict['variables'][key] = val
184 185
  for key, val in existing_dict['imports'].iteritems():
    new_dict['imports'][key] = val
186
  return new_dict
187

188
def Base_displayImage(self, image_object=None):
189
  """
190
  External function to display Image objects to jupyter frontend.
191 192 193 194
  
  Parameters
  ----------
  
195 196
  image_object :Any image object from ERP5 
                Any matplotlib object from which we can create a plot.
197 198 199 200 201 202 203 204
                Can be <matplotlib.lines.Line2D>, <matplotlib.text.Text>, etc.
  
  Output
  -----
  
  Returns base64 encoded string of the plot on which it has been called.

  """
205
  if image_object:
206

207
    import base64
208 209 210
    # Chanage global variable 'mime_type' to 'image/png'
    global mime_type

211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233
    # Image object in ERP5 is instance of OFS.Image object
    if isinstance(image_object, OFSImage):
      figdata = base64.b64encode(image_object.getData())
      mime_type = image_object.getContentType()
    else:
      # For matplotlib objects
      # XXX: Needs refactoring to handle cases

      # Create a ByteFile on the server which would be used to save the plot
      figfile = StringIO()
      # Save plot as 'png' format in the ByteFile
      image_object.savefig(figfile, format='png')
      figfile.seek(0)
      # Encode the value in figfile to base64 string so as to serve it jupyter frontend
      figdata = base64.b64encode(figfile.getvalue())
      mime_type = 'image/png'

    # XXX: We are not returning anything because we want this function to be called
    # by Base_executeJupyter , inside exec(), and its better to get the printed string
    # instead of returned string from this function as after exec, we are getting
    # value from stdout and using return we would get that value as string inside
    # an string which is unfavourable.
    print figdata