nayuos: Remove git/curl dependencies on SlapOS hosts

/reviewed-by @vpelletier

software/nayuos/README.md

@@ -8,18 +8,23 @@ GPL v2 or later
 
 ## Requirements
 
-*   sudo and git on the host (for now)
+*   sudo on the host (for now)
+*   some environment variables need to be authorized to be propagated when cros_sdk calls sudo:
+
+: Defaults        env_keep += "CROS_CACHEDIR DEPOT_TOOLS"
+
 *   slapuser with sudo rights to execute the cros_sdk scripts (needed to access the chroot environment provided by Chromium OS)
-in /etc/sudoers:
-<pre><code>
-\<slapuser\>       ALL= NOPASSWD: /srv/slapgrid/\<part\>/srv/runner/instance/\<inst_part\>/parts/clone-depot-tools/cros_sdk, /srv/slapgrid/\<part\>/srv/runner/instance/\<inst_part\>/parts/chromiumos/\<release\>/chromite/bin/cros_sdk, /srv/slapgrid/\<part\>/srv/runner/instance/\<inst_part\>/parts/chromiumos/\<release\>/chromite/bootstrap/cros_sdk, /srv/slapgrid/\<part\>/srv/runner/instance/\<inst_part\>/parts/chromiumos/\<release\>/chromite/scripts/cros_sdk.py, /srv/slapgrid/\<part\>/srv/runner/instance/\<inst_part\>/parts/chromiumos/\<release\>/chromite/scripts/cros_sdk.pyc, /srv/slapgrid/\<part\>/srv/runner/instance/\<inst_part\>/parts/chromiumos/\<release\>/chromium/tools/depot_tools/cros_sdk
-</code></pre>
+in /etc/sudoers (replace slapuser9 by your user, and release-R48-7647.B by the release you have chosen):
+
+: slapuser9    ALL=NOPASSWD: /srv/slapgrid/slappart9/srv/runner/instance/slappart0/parts/chromiumos/release-R48-7647.B/chromite/bin/cros_sdk, /srv/slapgrid/slappart9/srv/runner/instance/slappart0/wrapper_bin/wrapper_cros_sdk, /bin/kill
+
+It's useful to have the right to kill cros_sdk processes, when needed. ;)
 
 ## Input
 In the vifib parameters (softinst\<nb\>.host.vifib.net \> Services \> Parameters):
 
 *   board / ex: peppy, swanky, ... (choosing daisy will accept all licenses for the daisy board build only, in order to use Mali drivers, see [chromium mailing list](https://groups.google.com/a/chromium.org/forum/#!topic/chromium-os-dev/Pf9ZG2itxWM))
-*   branch / ex: release-R46-7390.B
+*   branch / ex: release-R46-7390.B (you can find the release in the [Chromium OS source tree](https://chromium.googlesource.com/chromiumos/manifest/+refs))
 *   keep_cache / yes|no (choosing "no" saves about 15Go of disk space per board, choosing "yes" will makes next build faster and less expensive in term of needed ressources because of not rebuilding everything)
 
 
@@ -35,10 +40,10 @@ The script that download the sources and build is located in
 ## External documents
 
 *   [ <code>repo</code> command reference ](https://source.android.com/source/using-repo.html)
-*   [ NayuOS official website ](https://www.nayuos.org)
+*   [ NayuOS official website ](https://www.nayuos.com)
 *   [ crouton for chroot ](https://github.com/dnschneid/crouton) ([warning about verified boot](https://github.com/dnschneid/crouton/blob/2a1fc9da380650f47e2bcf37d00962bfb68c4830/installer/main.sh#L517-L536))
 
 ## Notes for possible improvements
 *   [ Running virtual machines on your chromebook ](https://www.chromium.org/chromium-os/developer-information-for-chrome-os-devices/running-virtual-machines-on-your-chromebook)
-
 *   to have a more common User Agent (the one of ChromiumOS/NayuOS is quite rare and identifies the user, see [studies of the EFF](https://panopticlick.eff.org/static/browser-uniqueness.pdf)), it seems possible to change the User-Agent flag for guest mode in the getOffTheRecord function, and adding a line (key "kUserAgent" , value "some common user agent" string). Then rebuild Chromium and [add it to NayuOS](https://www.chromium.org/chromium-os/developer-guide#TOC-Making-changes-to-the-Chromium-web-).
+*   remove need of root priviledge for entering the chroot, maybe by using fakeroot in 'scripts/wrapper_sudo.in'?

software/nayuos/instance.cfg

@@ -2,6 +2,8 @@
 parts =
   parameters
   template-full-build-script
+  template-sudo-wrapper
+  template-cros-sdk-wrapper
 
 # eggs given by software.cfg
 eggs-directory = {{ eggs_directory }}
@@ -15,9 +17,10 @@ recipe = slapos.recipe.build:gitclone
 repository = https://chromium.googlesource.com/chromium/tools/depot_tools.git
 branch = master
 
-[use-depot-tools]
-export-path =
-  export PATH=${depot-tools:location}:"$PATH";
+[customize-path]
+# add depot tools directory (for cros_sdk binary among others) and git directory to the path
+command =
+  export PATH="${directory:wrapper_dir}":"{{ git_path }}":"{{ curl_path }}":"${depot-tools:location}":"$PATH";
 
 [nayuos-ebuilds]
 recipe = slapos.recipe.build:gitclone
@@ -41,24 +44,53 @@ cert = ${slap_connection:cert_file}
 recipe = slapos.cookbook:mkdirectory
 log = ${buildout:directory}/var/log
 run = ${buildout:directory}/etc/run
+wrapper_dir = ${buildout:directory}/wrapper_bin
 cros_location = ${buildout:directory}/parts/chromiumos
 ebuilds_dir = ${nayuos-ebuilds:location}
 scripts_dir = {{ scripts_dir }}
 logo_dir = {{ logo_dir }}
 
+[bin]
+wrapper_cros_sdk=${directory:wrapper_dir}/wrapper_cros_sdk
+bash=/bin/bash
+sudo=/usr/bin/sudo
+
+[template-sudo-wrapper]
+recipe = slapos.recipe.template:jinja2
+template = {{ scripts_dir }}/wrapper_sudo.in
+rendered = ${directory:wrapper_dir}/sudo
+md5sum = ded5a92be4e37ec32eb9d3087d3e19bd
+mode = 0700
+context =
+  key fake_cros_sdk_path bin:wrapper_cros_sdk
+  key bash_path bin:bash
+  key host_sudo_path bin:sudo
+
+[template-cros-sdk-wrapper]
+recipe = slapos.recipe.template:jinja2
+template = {{ scripts_dir }}/wrapper_cros_sdk.in
+rendered = ${bin:wrapper_cros_sdk}
+md5sum = 7159fe3d5b85a283733cf686c4ee0a74
+mode = 0700
+context =
+# XXX bash path is the one from the host
+  key bash_path bin:bash
+  raw git_path {{ git_path }}
+  raw curl_path {{ curl_path }}
+
 [template-full-build-script]
-# create the instance.cfg file in the buildout directory
+# create the rendered script in the buildout-directory/etc/run
 # (use jinja for templating)
 recipe = slapos.recipe.template:jinja2
 template = {{ scripts_dir }}/cros_full_build.in
 rendered = ${directory:run}/cros_full_build
-md5sum = 76942f32e682b36397b41db28297625d
-mode = 0770
+md5sum = d2c41ded046c75473eda0f923ee215e2
+mode = 0700
 context =
-# for access to the eggs from the instance
+  key bash_path bin:bash
   key instance_log_dir directory:log
   key cros_location directory:cros_location
-  key depot_tools_export_path_cmd use-depot-tools:export-path
+  key export_path_cmd customize-path:command
   key branch parameters:configuration.branch
   key boards_list parameters:configuration.boards
   key keep_cache parameters:configuration.keep_cache

software/nayuos/scripts/cros_full_build.in

-#!/bin/bash
-#XXX: should not depend on the host's bash ^
+#! {{ bash_path }}
 
 ######################## Download of sources using the "repo" command ########################
-DL_LOG={{ instance_log_dir }}/cros_sources_dl.log
-BRANCH={{ branch }}
-CHROMIUM_OVERLAY={{ cros_location }}/{{ branch }}/src/third_party/chromiumos-overlay
+DL_LOG="{{ instance_log_dir }}/cros_sources_dl.log"
+BRANCH="{{ branch }}"
+CHROMIUM_OVERLAY="{{ cros_location }}/{{ branch }}/src/third_party/chromiumos-overlay"
 
+{{ export_path_cmd }}
 echo "getting Chromium OS sources..." >> $DL_LOG
-{{ depot_tools_export_path_cmd }}
 echo "{{ cros_location }}/{{ branch }}"
 install -d "{{ cros_location }}/{{ branch }}"
 cd "{{ cros_location }}/{{ branch }}"
 
-repo init -u https://chromium.googlesource.com/chromiumos/manifest.git -b {{ branch }} >> $DL_LOG
+# git setup
+if ! git config user.name || ! git config user.email ; then
+  git config --global user.name "Your Name"
+  git config --global user.email "you@example.com" 
+fi
+
+repo init -u https://chromium.googlesource.com/chromiumos/manifest.git -b {{ branch }} >> $DL_LOG || { echo "Problem while initiating Chromium OS repo (repo init). PATH is: '${PATH}'. Exiting." ; exit 1 ; }
 
 # in case changes were already made, stash changes to be able to pull
 cd ${CHROMIUM_OVERLAY}
 git stash
 
-repo sync >> $DL_LOG
+repo sync >> $DL_LOG || { echo "Problem while downloading Chromium OS sources (repo sync). Exiting." ; exit 1 ; }
+
 
 ############################## Prepare chroot environment ###################################
 BUILD_LOG={{ instance_log_dir }}/cros_build.log
 TEST_LOG={{ instance_log_dir }}/tests_nayuos_image.log
-{{ depot_tools_export_path_cmd }}
 cd {{ cros_location }}/{{ branch }}
 
 # create chroot environment (exit on failure)
-cros_sdk --download || { echo "Problem entering chroot or downloading chroot environment. Exiting." ; exit 1 ; }
+cros_sdk --download || { echo "Problem while entering chroot or downloading chroot environment. PATH is: '${PATH}' Exiting." ; exit 1 ; }
 
 # compile Python with sqlite support (-> change USE flag) for dev-lang/python
 # it is needed by re6st
@@ -104,9 +109,10 @@ KEEP_CACHE="{{ keep_cache }}"
 for board in ${BOARDS} ; do
   echo ${board}
   if [ ${board} == daisy ] ; then
+    # XXX: broken by sucessive wrapping
     echo "daisy board: accepting license for Mali drivers..."
-    cros_sdk -- sudo sh -c "cp /etc/make.conf.user /etc/make.conf.user.save"
-    cros_sdk -- sudo sh -c "echo 'ACCEPT_LICENSE=\"*\"' >> /etc/make.conf.user"
+    cros_sdk -- "sudo cp /etc/make.conf.user /etc/make.conf.user.save"
+    cros_sdk -- "echo 'ACCEPT_LICENSE=\"*\"' | sudo tee --append /etc/make.conf.user"
   fi
 
   # preparing packages (for chroot and image)
@@ -142,8 +148,9 @@ for board in ${BOARDS} ; do
   fi
 
   if [ ${board} == daisy ]; then
+    # XXX: broken by sucessive wrapping
     echo "daisy board: removing accepted license for the next builds..."
-    cros_sdk -- sudo sh -c "mv /etc/make.conf.user.save /etc/make.conf.user"
+    cros_sdk -- "sudo mv /etc/make.conf.user.save /etc/make.conf.user"
   fi
 done
 

software/nayuos/scripts/wrapper_cros_sdk.in

+#! {{ bash_path }}
+
+# XXX: not secure: normal users need this script to have root priviledge
+# but have write permission on it
+
+# add curl/git to the path, required by cros_sdk, but not available after sudo                                                                   
+export PATH="{{ git_path }}:{{ curl_path }}:$PATH"
+exec "${@:1}"

software/nayuos/scripts/wrapper_sudo.in

+#! {{ bash_path }}
+
+COMMAND=()
+
+# add wrapper_cros_sdk before cros_sdk command
+pos=0
+for option in "$@"; do
+    if [[ "$option" == *"cros_sdk"* ]]; then
+        COMMAND=("${@:1:$pos}" "{{ fake_cros_sdk_path }}" "${@:(($pos+1))}")
+        break;
+    fi
+    ((pos++))
+done
+
+exec {{ host_sudo_path }} "${COMMAND[@]}"

software/nayuos/software.cfg

@@ -6,7 +6,7 @@ extends =
 parts +=
 # use stack/slapos.cfg
   slapos-cookbook
-  verify-sudo-exists-on-host
+  verify-packages-exist-on-host
   template-instance
   install-eggs-for-the-instance
 
@@ -22,7 +22,7 @@ plone.recipe.command = 1.1
 slapos.recipe.template = 2.8
 slapos.recipe.build = 0.21
 
-[verify-sudo-exists-on-host]
+[verify-packages-exist-on-host]
 recipe = plone.recipe.command
 stop-on-error = true
 # run the same command when installing and on updates
@@ -35,16 +35,20 @@ command = sudo -V
 recipe = slapos.recipe.template:jinja2
 template = ${:_profile_base_location_}/instance.cfg
 rendered = ${buildout:directory}/instance.cfg
-md5sum = d5a12b1ea814b4bb6792dfebaa655f90
+md5sum = 875d7fd5990a8645e761849193b09630
 mode = 0644
 scripts_dir = ${:_profile_base_location_}/scripts
 logo_dir = ${:_profile_base_location_}/logo
+curl_path = ${curl:location}/bin
+git_path = ${git:location}/bin
 context =
 # for access to the eggs from the instance
   key eggs_directory buildout:eggs-directory
   key develop_eggs_directory buildout:develop-eggs-directory
   key scripts_dir :scripts_dir
   key logo_dir :logo_dir
+  key curl_path :curl_path
+  key git_path :git_path
 
 [install-eggs-for-the-instance]
 # after installation of the eggs, the recipes