Jupyter with Cython+

See merge request nexedi/slapos!880

software/cython-test/software.cfg → component/cythonplus/buildout.cfg

 [buildout]
 extends =
-  ../../stack/slapos.cfg
-  ../../component/numpy/buildout.cfg
+  ../defaults.cfg
+  ../git/buildout.cfg
+  ../numpy/buildout.cfg
 
-parts =
-  slapos-cookbook
-  template
+parts = cythonplus_env.sh
+
+[gcc]
+min_version = 8.4
 
 [python]
 part = python3
 
-[template]
-recipe = slapos.recipe.template:jinja2
-rendered = ${buildout:directory}/template.cfg
-template =
-  inline:[buildout]
-  eggs-directory = ${buildout:eggs-directory}
-  develop-eggs-directory = ${buildout:develop-eggs-directory}
-  parts = runTestSuite
-
-  [slap-configuration]
-  recipe = slapos.cookbook:slapconfiguration.serialised
-  computer = $${slap-connection:computer-id}
-  partition = $${slap-connection:partition-id}
-  url = $${slap-connection:server-url}
-  key = $${slap-connection:key-file}
-  cert = $${slap-connection:cert-file}
-
-  [directory]
-  recipe = slapos.cookbook:mkdirectory
-  bin = $${buildout:directory}/bin
-  tmp = $${buildout:directory}/tmp
-
-  [runTestSuite]
-  recipe = slapos.recipe.template:jinja2
-  rendered = $${directory:bin}/$${:_buildout_section_name_}
-  template = ${runTestSuite.in:target}
-  mode = 0755
-  context =
-    key tmpdir directory:tmp
-    key slapparameter_dict slap-configuration:configuration
-    key cython_repository cython-repository:location
-    raw runTestSuite_py ${runTestSuite_py:bin-directory}/${runTestSuite_py:interpreter}
-    raw cython_env_sh ${cython_env.sh:rendered}
-
-  [cython-repository]
-  recipe = slapos.recipe.build:gitclone
-  repository = ${cython-repository:location}
-  git-executable = ${git:location}/bin/git
-  shared = true
-
-[cython-repository]
-recipe = slapos.recipe.build:gitclone
-repository = https://lab.nexedi.com/nexedi/cython.git
-git-executable = ${git:location}/bin/git
-sparse-checkout = /.gitignore
-
-[runTestSuite.in]
-recipe = slapos.recipe.build:download
-url = ${:_profile_base_location_}/${:_buildout_section_name_}
-md5sum = 02094e80cde9631081077fc96b401065
-
-[runTestSuite_py]
-recipe = zc.recipe.egg
-eggs = erp5.util
-interpreter = ${:_buildout_section_name_}
-
+# Dependencies for the Cython+ test suite
 [eggs]
 recipe = zc.recipe.egg
 eggs =
@@ -73,7 +20,13 @@ eggs =
   coverage
   pycodestyle
 
-[cython_env.sh]
+[cythonplus-repository]
+recipe = slapos.recipe.build:gitclone
+repository = https://lab.nexedi.com/nexedi/cython.git
+revision = cythonplus-0.1
+git-executable = ${git:location}/bin/git
+
+[cythonplus_env.sh]
 recipe = slapos.recipe.template:jinja2
 rendered = ${buildout:directory}/${:_buildout_section_name_}
 template =
@@ -81,19 +34,20 @@ template =
   {% if 'part' in gcc -%}
   {%   set path = path + ':' + gcc.prefix + '/bin' -%}
   {% endif -%}
-  export PATH={{ path }}:$PATH
+  export PATH={{ path }}$${PATH:+:$PATH}
   export PYTHON={{ python }}
+  export PYTHONPATH={{ cythonplus_repository }}$${PYTHONPATH:+:$PYTHONPATH}
   export PYTHONPATH={{ ':'.join(easy_install.working_set(eggs['eggs'].split(), [
       eggs['develop-eggs-directory'],
       eggs['eggs-directory'],
     ]).entries) }}$${PYTHONPATH:+:$PYTHONPATH}
-  # EmbedTest needs libintl.
+  {# Set path to libintl needed for cython EmbedTest #}
   export LD_RUN_PATH={{ gettext }}/lib$${LD_RUN_PATH:+:$LD_RUN_PATH}
   export LIBRARY_PATH={{ gettext }}/lib$${LIBRARY_PATH:+:$LIBRARY_PATH}
-  {##}
 context =
   section eggs eggs
   section gcc gcc
+  key cythonplus_repository cythonplus-repository:location
   key gettext gettext:location
   key python python:executable
   import os os

component/gcc/buildout.cfg

@@ -71,6 +71,11 @@ patches =
 version = 8.4.0
 md5sum = bb815a8e3b7be43c4a26fa89dbbd9795
 
+[gcc-10.2]
+<= gcc-common
+version = 10.2.0
+md5sum = e9fd9b1789155ad09bcf3ae747596b50
+
 [gcc-minimal]
 <= gcc-5.5
 configure-options =

software/cythonplus-dev/buildout.hash.cfg

+[instance.cfg]
+filename = instance.cfg.in
+md5sum = a24384487467a07e827edab17a0b7206
+
+[runTestSuite.in]
+_update_hash_filename_ = runTestSuite.in
+md5sum = 21a8a202b14475707c414056ba393b3d

software/cythonplus-dev/instance.cfg.in

+[buildout]
+
+parts =
+  publish-env-path
+  runTestSuite
+
+eggs-directory = ${buildout:eggs-directory}
+develop-eggs-directory = ${buildout:develop-eggs-directory}
+
+[publish-env-path]
+recipe = slapos.cookbook:publish
+readme = Source the script to set up the environment.
+script = ${cythonplus_env.sh:rendered}
+repository = ${cythonplus-repository:repository}
+
+[slap-configuration]
+recipe = slapos.cookbook:slapconfiguration.serialised
+computer = $${slap-connection:computer-id}
+partition = $${slap-connection:partition-id}
+url = $${slap-connection:server-url}
+key = $${slap-connection:key-file}
+cert = $${slap-connection:cert-file}
+
+[directory]
+recipe = slapos.cookbook:mkdirectory
+bin = $${buildout:directory}/bin
+tmp = $${buildout:directory}/tmp
+
+[cythonplus-repository]
+recipe = slapos.recipe.build:gitclone
+repository = ${cythonplus-repository:location}
+git-executable = ${git:location}/bin/git
+shared = true
+
+[runTestSuite]
+recipe = slapos.recipe.template:jinja2
+rendered = $${directory:bin}/$${:_buildout_section_name_}
+template = ${runTestSuite.in:target}
+mode = 0755
+context =
+  key tmpdir directory:tmp
+  key slapparameter_dict slap-configuration:configuration
+  key cythonplus_repository cythonplus-repository:location
+  raw runTestSuite_interpreter ${runTestSuite_interpreter:bin-directory}/${runTestSuite_interpreter:interpreter}
+  raw cythonplus_env_sh ${cythonplus_env.sh:rendered}

software/cython-test/runTestSuite.in → software/cythonplus-dev/runTestSuite.in

-#!{{ runTestSuite_py }}
+#!{{ runTestSuite_interpreter }}
 """
   Script to run the Cython test suite using Nexedi's test node framework.
 """
@@ -9,7 +9,7 @@ from erp5.util import taskdistribution, testsuite
 
 os.environ['TEMP'] = {{ repr(tmpdir) }}
 
-command = """. {{ cython_env_sh }}
+command = """. {{ cythonplus_env_sh }}
 make all test
 """
 
@@ -96,7 +96,7 @@ def main():
       with open(os.devnull) as stdin:
         p = subprocess.Popen(command, shell=True, stdin=stdin,
                              stdout=subprocess.PIPE, stderr=subprocess.PIPE,
-                             cwd={{repr(cython_repository)}})
+                             cwd={{ repr(cythonplus_repository) }})
     except Exception:
       end = time()
       stderr = traceback.format_exc()

software/cythonplus-dev/software.cfg

+[buildout]
+extends =
+  buildout.hash.cfg
+  ../../stack/slapos.cfg
+  ../../component/cythonplus/buildout.cfg
+
+parts =
+  slapos-cookbook
+  instance.cfg
+
+[instance.cfg]
+recipe = slapos.recipe.template
+output = ${buildout:directory}/${:_buildout_section_name_}
+url = ${:_profile_base_location_}/${:filename}
+
+[runTestSuite.in]
+recipe = slapos.recipe.build:download
+url = ${:_profile_base_location_}/${:_update_hash_filename_}
+
+[runTestSuite_interpreter]
+recipe = zc.recipe.egg
+eggs = erp5.util
+interpreter = ${:_buildout_section_name_}
+
+[versions]
+slapos.recipe.template = 4.4

software/jupyter/buildout.hash.cfg

@@ -15,11 +15,11 @@
 
 [instance-jupyter-notebook]
 filename = instance.cfg.in
-md5sum = 1d5fe6cc4e48672ae7be1c223794a932
+md5sum = 659e0d583acd70ebb8d815c22735d91b
 
 [jupyter-notebook-config]
 filename = jupyter_notebook_config.py.jinja
-md5sum = 3da50c37760a42e42ae3f9d78d9ca449
+md5sum = 4514df152740dbd7990da9705f40e978
 
 [jupyter-set-password]
 filename = jupyter_set_password.cgi.jinja

software/jupyter/instance.cfg.in

@@ -82,6 +82,8 @@ rendered = ${directory:jupyter_config_dir}/jupyter_notebook_config.py
 mode = 0744
 context =
   raw config_cfg ${buildout:directory}/knowledge0.cfg
+  raw gcc_location {{ gcc_location }}
+  raw cythonplus_repository {{ cythonplus_repository }}
 
 [directory]
 recipe = slapos.cookbook:mkdirectory

software/jupyter/jupyter_notebook_config.py.jinja

@@ -36,3 +36,6 @@ c.NotebookApp.password = passwd(parser.get("jupyter_notebook", "password"))
 
 with open(knowledge_0, 'w') as file:
   parser.write(file)
+
+os.environ['PATH'] = "{{ gcc_location }}/bin" + os.pathsep + os.environ['PATH']
+os.environ['PYTHONPATH'] = "{{ cythonplus_repository }}" + os.pathsep + os.environ['PYTHONPATH']

software/jupyter/software.cfg

@@ -2,6 +2,7 @@
 extends = 
   buildout.hash.cfg
   ../../stack/slapos.cfg
+  ../../component/cythonplus/buildout.cfg
   ../../component/openssl/buildout.cfg
   ../../component/jupyter/buildout.cfg
   ../../stack/monitor/buildout.cfg
@@ -64,6 +65,8 @@ context =
   key custom_js_location custom-js:location
   key custom_js_filename custom-js:filename
   key monitor_template_rendered buildout:directory
+  key cythonplus_repository cythonplus-repository:location
+  key gcc_location gcc:prefix
 
 [versions]
 Pygments = 2.7.2