Implement the load python extension feature

bt5/erp5_notebook/SkinTemplateItem/portal_skins/erp5_notebook/gadget_jsmd_eval.html.html

@@ -12,6 +12,7 @@
 
     <script src="rsvp.js" type="text/javascript"></script>
     <script src="marked.js" type="text/javascript"></script>
+    <script src="iodide_utils.js" type="text/javascript"></script>
     <script src="gadget_jsmd_eval.js" type="text/javascript"></script>
   </head>
 

bt5/erp5_notebook/SkinTemplateItem/portal_skins/erp5_notebook/gadget_jsmd_eval.js.js

-/*global window, console, RSVP, document, URL, eval, XMLHttpRequest, marked, WebAssembly */
+/*global window, console, RSVP, document, URL, eval, XMLHttpRequest, marked, WebAssembly, loadSharedlib */
 /*jslint nomen: true, indent: 2, maxerr: 3 */
 (function (window) {
   "use strict";
@@ -294,6 +294,80 @@
     return true;
   }
 
+  // Pyodide package loading
+  function pyodideLoadPackage(names) {
+    var queue, toLoad, package_name, k,
+      subpackage, packages, loadedPackages, defer;
+    packages = pyodide_instance.packages.dependencies;
+    loadedPackages = pyodide_instance.loadedPackages;
+    // packages is dependency list of each package
+    // e.g.: [pandas: ['numpy', 'python-dateutil', 'pytz'], numpy: []]
+    queue = new Array(names);
+
+    toLoad = [];
+    defer = RSVP.defer();
+    // Find all packages that need to load, store in the "toLoad" list
+    while (queue.length) {
+      package_name = queue.pop();
+
+      if (loadedPackages.indexOf(package_name) === -1) {
+        toLoad.push(package_name);
+        // packages is the dependency list
+        if (packages.hasOwnProperty(package_name)) {
+          for (k = 0; k < packages[package_name].length; k += 1) {
+            subpackage = packages[package_name][k];
+            if (loadedPackages.indexOf(subpackage) === -1 && toLoad.indexOf(subpackage) === -1) {
+              queue.push(subpackage);
+            }
+          }
+        } else {
+          throw new Error("Unknown package " + package_name);
+        }
+      }
+    }
+
+    if (toLoad.length === 0) {
+      // No new packages to load
+      return;
+    }
+
+    // pyodide built with option "--use-preload-plugins"
+    // So emscripten load the pyodide itself and its extension using createPreloadedFile.
+    // See createPreloadedFile section in:
+    // https://github.com/emscripten-core/emscripten/blob/master/site/source/docs/api_reference/Filesystem-API.rst
+
+    // And createPreloadedFile use addRunDependency or removeRunDependency to handle the dependencies,
+    // it will call monitorRunDependencies.
+    // https://github.com/emscripten-core/emscripten/blob/master/src/preamble.js#L1842
+
+    // The logic in here is if we loaded all wasm module, aka the "left" is 0.
+    // We load all shared lib to the wasm file system through loadSharedlib.
+
+    pyodide_instance.monitorRunDependencies = function (left) {
+      if (left === 0) {
+        for (k = 0; k < toLoad.length; k += 1) {
+          package_name = toLoad[k];
+          loadedPackages.push(package_name);
+        }
+        delete pyodide_instance.monitorRunDependencies;
+        return loadSharedlib(pyodide_instance._module.FS)
+          .push(function () {
+            defer.resolve();
+            return;
+          });
+      }
+    };
+
+    for (k = 0; k < toLoad.length; k += 1) {
+      package_name = toLoad[k];
+      loadJSResource(package_name + ".js");
+    }
+
+    pyodide_instance.runPython('import importlib as _importlib\n' +
+      '_importlib.invalidate_caches()\n');
+    return defer.promise;
+  }
+
   function initPyodide() {
     var Module = {};
 
@@ -322,6 +396,27 @@
         // https://emscripten.org/docs/api_reference/preamble.js.html
         Module.postRun = defer.resolve;
         return defer.promise;
+      })
+      .push(function () {
+        return ajax({
+          url: 'packages.json'
+        });
+      })
+      .push(function (evt) {
+        return JSON.parse(evt.target.response);
+      })
+      .push(function (json) {
+        pyodide_instance._module = Module;
+        pyodide_instance.loadedPackages = [];
+        pyodide_instance._module.packages = json;
+        // Make pyodide happy...
+        // pyodide.asm.js need window.pyodide:
+        // https://github.com/iodide-project/pyodide/blob/master/src/runpython.c#L91
+        // var packageNames = self.pyodide._module.packages.import_name_to_package_name;
+        // import_name_to_package_name was defined when reading package.json
+        // https://github.com/iodide-project/pyodide/blob/master/pyodide_build/buildall.py#L58
+        window.pyodide = pyodide_instance;
+        return;
       });
   }
 
@@ -338,7 +433,7 @@
       document.body.appendChild(div);
     }
   }
-  
+
   function executePyCell(line_list) {
     var result, code_text = line_list.join('\n');
     result = pyodide_instance.runPython(code_text);
@@ -355,6 +450,9 @@
       return initPyodide()
         .push(function () {
           return executePyCell(line_list);
+        })
+        .push(function () {
+          return pyodideLoadPackage('matplotlib');
         });
     }
     return executePyCell(line_list);

bt5/erp5_notebook/SkinTemplateItem/portal_skins/erp5_notebook/iodide_utils.js.js

+/*global window, console, document, RSVP */
+/*jslint nomen: true, indent: 2, maxerr: 3 */
+
+//  The code within this file is copied from iodide/pyodide project with modification,
+//  which subject to the terms of the Mozilla Public License, v. 2.0.
+//  If a copy of the MPL was not distributed with this file,
+//  You can obtain one at http://mozilla.org/MPL/2.0/.
+
+function loadSharedlib(file_system) {
+  "use strict";
+  // On Chrome, we have to instantiate wasm asynchronously. Since that
+  // can't be done synchronously within the call to dlopen, we instantiate
+  // every .so that comes our way up front, caching it in the
+  // `preloadedWasm` dictionary.
+  var queue, path;
+  queue = new RSVP.Queue();
+
+  function recurseDir(rootpath) {
+    var i, entry, dirs;
+    try {
+      dirs = file_system.readdir(rootpath);
+    } catch (e) {
+      if (e instanceof file_system.ErrnoError) {
+        return;
+      }
+      throw e;
+    }
+
+    function readModuleFile(path) {
+      return function () {
+        return window.Module.loadWebAssemblyModule(
+          file_system.readFile(path),
+          {loadAsync: true}
+        );
+      };
+    }
+    function setModule(path) {
+      return function (module) {
+        window.Module.preloadedWasm[path] = module;
+      };
+    }
+
+    for (i = 0; i < dirs.length; i += 1) {
+      entry = dirs[i];
+      if (!entry.startsWith('.')) {
+        path = rootpath + entry;
+
+        if (entry.endsWith('.so')) {
+          if (window.Module.preloadedWasm[path] === undefined) {
+            queue.push(readModuleFile(path))
+              .push(setModule(path));
+          }
+        } else if (file_system.isDir(file_system.lookupPath(path).node.mode)) {
+          recurseDir(path + '/');
+        }
+      }
+    }
+  }
+
+  recurseDir('/');
+
+  return queue;
+}
\ No newline at end of file

bt5/erp5_notebook/SkinTemplateItem/portal_skins/erp5_notebook/iodide_utils.js.xml

+<?xml version="1.0"?>
+<ZopeData>
+  <record id="1" aka="AAAAAAAAAAE=">
+    <pickle>
+      <global name="File" module="OFS.Image"/>
+    </pickle>
+    <pickle>
+      <dictionary>
+        <item>
+            <key> <string>_Cacheable__manager_id</string> </key>
+            <value> <string>must_revalidate_http_cache</string> </value>
+        </item>
+        <item>
+            <key> <string>__name__</string> </key>
+            <value> <string>iodide_utils.js</string> </value>
+        </item>
+        <item>
+            <key> <string>content_type</string> </key>
+            <value> <string>application/javascript</string> </value>
+        </item>
+        <item>
+            <key> <string>precondition</string> </key>
+            <value> <string></string> </value>
+        </item>
+        <item>
+            <key> <string>title</string> </key>
+            <value> <string></string> </value>
+        </item>
+      </dictionary>
+    </pickle>
+  </record>
+</ZopeData>