Commit 459cd18b authored by Boxiang Sun's avatar Boxiang Sun

Implement the load python extension feature

parent 18a04087
...@@ -12,6 +12,7 @@ ...@@ -12,6 +12,7 @@
<script src="rsvp.js" type="text/javascript"></script> <script src="rsvp.js" type="text/javascript"></script>
<script src="marked.js" type="text/javascript"></script> <script src="marked.js" type="text/javascript"></script>
<script src="iodide_utils.js" type="text/javascript"></script>
<script src="gadget_jsmd_eval.js" type="text/javascript"></script> <script src="gadget_jsmd_eval.js" type="text/javascript"></script>
</head> </head>
......
/*global window, console, RSVP, document, URL, eval, XMLHttpRequest, marked, WebAssembly */ /*global window, console, RSVP, document, URL, eval, XMLHttpRequest, marked, WebAssembly, loadSharedlib */
/*jslint nomen: true, indent: 2, maxerr: 3 */ /*jslint nomen: true, indent: 2, maxerr: 3 */
(function (window) { (function (window) {
"use strict"; "use strict";
...@@ -294,6 +294,80 @@ ...@@ -294,6 +294,80 @@
return true; return true;
} }
// Pyodide package loading
function pyodideLoadPackage(names) {
var queue, toLoad, package_name, k,
subpackage, packages, loadedPackages, defer;
packages = pyodide_instance.packages.dependencies;
loadedPackages = pyodide_instance.loadedPackages;
// packages is dependency list of each package
// e.g.: [pandas: ['numpy', 'python-dateutil', 'pytz'], numpy: []]
queue = new Array(names);
toLoad = [];
defer = RSVP.defer();
// Find all packages that need to load, store in the "toLoad" list
while (queue.length) {
package_name = queue.pop();
if (loadedPackages.indexOf(package_name) === -1) {
toLoad.push(package_name);
// packages is the dependency list
if (packages.hasOwnProperty(package_name)) {
for (k = 0; k < packages[package_name].length; k += 1) {
subpackage = packages[package_name][k];
if (loadedPackages.indexOf(subpackage) === -1 && toLoad.indexOf(subpackage) === -1) {
queue.push(subpackage);
}
}
} else {
throw new Error("Unknown package " + package_name);
}
}
}
if (toLoad.length === 0) {
// No new packages to load
return;
}
// pyodide built with option "--use-preload-plugins"
// So emscripten load the pyodide itself and its extension using createPreloadedFile.
// See createPreloadedFile section in:
// https://github.com/emscripten-core/emscripten/blob/master/site/source/docs/api_reference/Filesystem-API.rst
// And createPreloadedFile use addRunDependency or removeRunDependency to handle the dependencies,
// it will call monitorRunDependencies.
// https://github.com/emscripten-core/emscripten/blob/master/src/preamble.js#L1842
// The logic in here is if we loaded all wasm module, aka the "left" is 0.
// We load all shared lib to the wasm file system through loadSharedlib.
pyodide_instance.monitorRunDependencies = function (left) {
if (left === 0) {
for (k = 0; k < toLoad.length; k += 1) {
package_name = toLoad[k];
loadedPackages.push(package_name);
}
delete pyodide_instance.monitorRunDependencies;
return loadSharedlib(pyodide_instance._module.FS)
.push(function () {
defer.resolve();
return;
});
}
};
for (k = 0; k < toLoad.length; k += 1) {
package_name = toLoad[k];
loadJSResource(package_name + ".js");
}
pyodide_instance.runPython('import importlib as _importlib\n' +
'_importlib.invalidate_caches()\n');
return defer.promise;
}
function initPyodide() { function initPyodide() {
var Module = {}; var Module = {};
...@@ -322,6 +396,27 @@ ...@@ -322,6 +396,27 @@
// https://emscripten.org/docs/api_reference/preamble.js.html // https://emscripten.org/docs/api_reference/preamble.js.html
Module.postRun = defer.resolve; Module.postRun = defer.resolve;
return defer.promise; return defer.promise;
})
.push(function () {
return ajax({
url: 'packages.json'
});
})
.push(function (evt) {
return JSON.parse(evt.target.response);
})
.push(function (json) {
pyodide_instance._module = Module;
pyodide_instance.loadedPackages = [];
pyodide_instance._module.packages = json;
// Make pyodide happy...
// pyodide.asm.js need window.pyodide:
// https://github.com/iodide-project/pyodide/blob/master/src/runpython.c#L91
// var packageNames = self.pyodide._module.packages.import_name_to_package_name;
// import_name_to_package_name was defined when reading package.json
// https://github.com/iodide-project/pyodide/blob/master/pyodide_build/buildall.py#L58
window.pyodide = pyodide_instance;
return;
}); });
} }
...@@ -338,7 +433,7 @@ ...@@ -338,7 +433,7 @@
document.body.appendChild(div); document.body.appendChild(div);
} }
} }
function executePyCell(line_list) { function executePyCell(line_list) {
var result, code_text = line_list.join('\n'); var result, code_text = line_list.join('\n');
result = pyodide_instance.runPython(code_text); result = pyodide_instance.runPython(code_text);
...@@ -355,6 +450,9 @@ ...@@ -355,6 +450,9 @@
return initPyodide() return initPyodide()
.push(function () { .push(function () {
return executePyCell(line_list); return executePyCell(line_list);
})
.push(function () {
return pyodideLoadPackage('matplotlib');
}); });
} }
return executePyCell(line_list); return executePyCell(line_list);
......
/*global window, console, document, RSVP */
/*jslint nomen: true, indent: 2, maxerr: 3 */
// The code within this file is copied from iodide/pyodide project with modification,
// which subject to the terms of the Mozilla Public License, v. 2.0.
// If a copy of the MPL was not distributed with this file,
// You can obtain one at http://mozilla.org/MPL/2.0/.
function loadSharedlib(file_system) {
"use strict";
// On Chrome, we have to instantiate wasm asynchronously. Since that
// can't be done synchronously within the call to dlopen, we instantiate
// every .so that comes our way up front, caching it in the
// `preloadedWasm` dictionary.
var queue, path;
queue = new RSVP.Queue();
function recurseDir(rootpath) {
var i, entry, dirs;
try {
dirs = file_system.readdir(rootpath);
} catch (e) {
if (e instanceof file_system.ErrnoError) {
return;
}
throw e;
}
function readModuleFile(path) {
return function () {
return window.Module.loadWebAssemblyModule(
file_system.readFile(path),
{loadAsync: true}
);
};
}
function setModule(path) {
return function (module) {
window.Module.preloadedWasm[path] = module;
};
}
for (i = 0; i < dirs.length; i += 1) {
entry = dirs[i];
if (!entry.startsWith('.')) {
path = rootpath + entry;
if (entry.endsWith('.so')) {
if (window.Module.preloadedWasm[path] === undefined) {
queue.push(readModuleFile(path))
.push(setModule(path));
}
} else if (file_system.isDir(file_system.lookupPath(path).node.mode)) {
recurseDir(path + '/');
}
}
}
}
recurseDir('/');
return queue;
}
\ No newline at end of file
<?xml version="1.0"?>
<ZopeData>
<record id="1" aka="AAAAAAAAAAE=">
<pickle>
<global name="File" module="OFS.Image"/>
</pickle>
<pickle>
<dictionary>
<item>
<key> <string>_Cacheable__manager_id</string> </key>
<value> <string>must_revalidate_http_cache</string> </value>
</item>
<item>
<key> <string>__name__</string> </key>
<value> <string>iodide_utils.js</string> </value>
</item>
<item>
<key> <string>content_type</string> </key>
<value> <string>application/javascript</string> </value>
</item>
<item>
<key> <string>precondition</string> </key>
<value> <string></string> </value>
</item>
<item>
<key> <string>title</string> </key>
<value> <string></string> </value>
</item>
</dictionary>
</pickle>
</record>
</ZopeData>
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment