librecipe/execute: fallback to polling when inotify fails

When user ran out of inotify watch, starting wrappers using
wait-for-files is not possible:

    AssertionError: Traceback (most recent call last):
      File "/srv/slapgrid/slappart15/tmp/tmpzurwmg4q/wrapper", line 13, in <module>
        sys.exit(slapos.recipe.librecipe.execute.generic_exec(['/bin/echo', 'done'], wait_list=['/srv/slapgrid/slappart15/tmp/tmpzurwmg4q/wait']))
      File "/srv/slapgrid/slappart15/srv/project/slapos-clean/slapos/recipe/librecipe/execute.py", line 100, in generic_exec
        _wait_files_creation(wait_list)
      File "/srv/slapgrid/slappart15/srv/project/slapos-clean/slapos/recipe/librecipe/execute.py", line 30, in _wait_files_creation
        watchdescriptors = {inotify.add_watch(dirname,
      File "/srv/slapgrid/slappart15/srv/project/slapos-clean/slapos/recipe/librecipe/execute.py", line 30, in <dictcomp>
        watchdescriptors = {inotify.add_watch(dirname,
      File "/srv/slapgrid/slappart15/srv/project/venv/lib/python3.9/site-packages/inotify_simple.py", line 110, in add_watch
        return _libc_call(_libc.inotify_add_watch, self.fileno(), fsencode(path), mask)
      File "/srv/slapgrid/slappart15/srv/project/venv/lib/python3.9/site-packages/inotify_simple.py", line 39, in _libc_call
        raise OSError(errno, os.strerror(errno))
    OSError: [Errno 28] No space left on device

This catches inotify errors and fallback to simple polling in that case.

slapos/recipe/librecipe/execute.py

@@ -5,6 +5,7 @@ import sys
 import os
 import signal
 import subprocess
+import time
 from collections import defaultdict
 from inotify_simple import INotify, flags
 
@@ -14,25 +15,38 @@ def _wait_files_creation(file_list):
   # Establish a list of directory and subfiles.
   # and test existence before watching, so that we don't miss an event.
   directories = defaultdict(dict)
-  for f in file_list:
-    dirname, filename = os.path.split(f)
-    directories[dirname][filename] = os.path.lexists(f)
+  def check_if_files_exists():
+    for f in file_list:
+      dirname, filename = os.path.split(f)
+      directories[dirname][filename] = os.path.lexists(f)
+  check_if_files_exists()
 
   def all_files_exists():
     return all(all(six.itervalues(files)) for files in six.itervalues(directories))
 
+  inotify_available = True
   with INotify() as inotify:
-    watchdescriptors = {inotify.add_watch(dirname,
-        flags.CREATE | flags.DELETE | flags.MOVED_TO | flags.MOVED_FROM
-        ): dirname
-      for dirname in directories}
-
-    while not all_files_exists():
+    try:
+      watchdescriptors = {inotify.add_watch(dirname,
+          flags.CREATE | flags.DELETE | flags.MOVED_TO | flags.MOVED_FROM
+          ): dirname
+        for dirname in directories}
+    except OSError as e:
+      if e.errno not in (errno.ENOSPC, errno.EMFILE):
+        raise
+      print('Error using inotify, falling back to polling')
+      inotify_available = False
+
+    while inotify_available and not all_files_exists():
       for event in inotify.read():
         directory = directories[watchdescriptors[event.wd]]
         if event.name in directory:
           directory[event.name] = event.mask & (flags.CREATE | flags.MOVED_TO)
 
+  while not all_files_exists():
+    time.sleep(0.1)
+    check_if_files_exists()
+
 def _libc():
   from ctypes import CDLL, get_errno, c_char_p, c_int, c_ulong, util
   libc = CDLL(util.find_library('c'), use_errno=True)