librecipe/execute: fallback to polling when inotify fails

When user ran out of inotify watch, starting wrappers using
wait-for-files is not possible:

    AssertionError: Traceback (most recent call last):
      File "/srv/slapgrid/slappart15/tmp/tmpzurwmg4q/wrapper", line 13, in <module>
        sys.exit(slapos.recipe.librecipe.execute.generic_exec(['/bin/echo', 'done'], wait_list=['/srv/slapgrid/slappart15/tmp/tmpzurwmg4q/wait']))
      File "/srv/slapgrid/slappart15/srv/project/slapos-clean/slapos/recipe/librecipe/execute.py", line 100, in generic_exec
        _wait_files_creation(wait_list)
      File "/srv/slapgrid/slappart15/srv/project/slapos-clean/slapos/recipe/librecipe/execute.py", line 30, in _wait_files_creation
        watchdescriptors = {inotify.add_watch(dirname,
      File "/srv/slapgrid/slappart15/srv/project/slapos-clean/slapos/recipe/librecipe/execute.py", line 30, in <dictcomp>
        watchdescriptors = {inotify.add_watch(dirname,
      File "/srv/slapgrid/slappart15/srv/project/venv/lib/python3.9/site-packages/inotify_simple.py", line 110, in add_watch
        return _libc_call(_libc.inotify_add_watch, self.fileno(), fsencode(path), mask)
      File "/srv/slapgrid/slappart15/srv/project/venv/lib/python3.9/site-packages/inotify_simple.py", line 39, in _libc_call
        raise OSError(errno, os.strerror(errno))
    OSError: [Errno 28] No space left on device

This catches inotify errors and fallback to simple polling in that case.

slapos/recipe/librecipe/execute.py

@@ -5,6 +5,7 @@ import sys
 import os
 import signal
 import subprocess
+import time
 from collections import defaultdict
 from inotify_simple import INotify, flags
 
@@ -14,19 +15,29 @@ def _wait_files_creation(file_list):
   # Establish a list of directory and subfiles.
   # and test existence before watching, so that we don't miss an event.
   directories = defaultdict(dict)
+  def check_if_files_exists():
     for f in file_list:
       dirname, filename = os.path.split(f)
       directories[dirname][filename] = os.path.lexists(f)
+  check_if_files_exists()
 
   def all_files_exists():
     return all(all(six.itervalues(files)) for files in six.itervalues(directories))
 
   with INotify() as inotify:
+    try:
       watchdescriptors = {inotify.add_watch(dirname,
           flags.CREATE | flags.DELETE | flags.MOVED_TO | flags.MOVED_FROM
           ): dirname
         for dirname in directories}
-
+    except OSError as e:
+      if e.errno not in (errno.ENOSPC, errno.EMFILE):
+        raise
+      print('Error using inotify, falling back to polling')
+      while not all_files_exists():
+        time.sleep(0.1)
+        check_if_files_exists()
+    else:
       while not all_files_exists():
         for event in inotify.read():
           directory = directories[watchdescriptors[event.wd]]

slapos/test/recipe/test_wrapper.py

@@ -175,6 +175,12 @@ class TestPidFile(WrapperTestCase):
 
 
 class TestWaitForFiles(WrapperTestCase):
+  env = None
+  if sys.platform.startswith("linux"):
+    expected_output = 'done\n'
+  else:
+    expected_output = 'Error using inotify, falling back to polling\ndone\n'
+
   def getOptions(self):
     self.waitfile = self.getTempPath('wait')
     return {
@@ -192,6 +198,7 @@ class TestWaitForFiles(WrapperTestCase):
       stdout=subprocess.PIPE,
       stderr=subprocess.STDOUT,
       universal_newlines=True,
+      env=self.env,
     )
     self.addCleanup(self.terminate_process, process)
     if process.poll():
@@ -207,13 +214,57 @@ class TestWaitForFiles(WrapperTestCase):
     for _ in range(20):
       time.sleep(0.1)
       if process.poll() is not None:
-        self.assertEqual(process.stdout.read(), 'done\n')
+        self.assertEqual(process.stdout.read(), self.expected_output)
         self.assertEqual(process.returncode, 0)
         break
     else:
       self.fail('process did not start after file was created')
 
 
+@unittest.skipUnless(sys.platform.startswith("linux"), "Inotify is linux only")
+class TestWaitForFilesInotifyError(TestWaitForFiles):
+
+  def setUp(self):
+    super(TestWaitForFilesInotifyError, self).setUp()
+    # use LD_PRELOAD to inject errors into inotify_add_watch calls
+    inotify_mock_c = self.getTempPath('inotify_mock.c')
+    inotify_mock_o = self.getTempPath('inotify_mock.o')
+    inotify_mock_so = self.getTempPath('inotify_mock.so')
+    with open(inotify_mock_c, 'w') as f:
+      f.write('''
+        #include <sys/inotify.h>
+        #include <string.h>
+        #include <errno.h>
+
+        int inotify_add_watch(int fd, const char *pathname, uint32_t mask) {
+          errno = ENOSPC;
+          return -1;
+        }
+
+        /* This is a bit tricky because inotify_simple calls
+        inotify_add_watch with ctypes.CDLL("libc.so"), which uses
+        dlopen("libc.so") and dlsym("inotify_add_watch"), so we first
+        override dlsym to return our own inotify_add_watch.
+        https://github.com/chrisjbillington/inotify_simple/blob/55737898/inotify_simple.py#L110
+        */
+        extern void *__libc_dlsym (void *, const char *);
+        void *dlsym(void *handle, const char *symbol) {
+          if (strcmp(symbol, "inotify_add_watch") == 0) {
+            return (void *)inotify_add_watch;
+          }
+          return (void *)__libc_dlsym(handle, symbol);
+        }
+        ''')
+    subprocess.check_call(['gcc', '-c', '-fPIC', '-o', inotify_mock_o, inotify_mock_c])
+    subprocess.check_call(['gcc', '-shared', '-o', inotify_mock_so, inotify_mock_o])
+    self.env = dict(
+      os.environ,
+      PYTHONUNBUFFERED='1',
+      LD_PRELOAD=inotify_mock_so)
+
+  expected_output = 'Error using inotify, falling back to polling\ndone\n'
+
+
 class TestPrivateTmpFS(WrapperTestCase):
   def getOptions(self):
     self.tmpdir = self.getTempPath('tmpdir')