Merge remote-tracking branch 'origin/master' into zope4py2

setup.py

@@ -28,7 +28,7 @@ from setuptools import setup, find_packages
 import glob
 import os
 
-version = '1.0.297'
+version = '1.0.305'
 name = 'slapos.cookbook'
 long_description = open("README.rst").read()
 

slapos/recipe/librecipe/execute.py

@@ -5,6 +5,7 @@ import sys
 import os
 import signal
 import subprocess
+import time
 from collections import defaultdict
 from inotify_simple import INotify, flags
 
@@ -14,24 +15,34 @@ def _wait_files_creation(file_list):
   # Establish a list of directory and subfiles.
   # and test existence before watching, so that we don't miss an event.
   directories = defaultdict(dict)
-  for f in file_list:
-    dirname, filename = os.path.split(f)
-    directories[dirname][filename] = os.path.lexists(f)
+  def check_if_files_exists():
+    for f in file_list:
+      dirname, filename = os.path.split(f)
+      directories[dirname][filename] = os.path.lexists(f)
+  check_if_files_exists()
 
   def all_files_exists():
     return all(all(six.itervalues(files)) for files in six.itervalues(directories))
 
   with INotify() as inotify:
-    watchdescriptors = {inotify.add_watch(dirname,
-        flags.CREATE | flags.DELETE | flags.MOVED_TO | flags.MOVED_FROM
-        ): dirname
-      for dirname in directories}
-
-    while not all_files_exists():
-      for event in inotify.read():
-        directory = directories[watchdescriptors[event.wd]]
-        if event.name in directory:
-          directory[event.name] = event.mask & (flags.CREATE | flags.MOVED_TO)
+    try:
+      watchdescriptors = {inotify.add_watch(dirname,
+          flags.CREATE | flags.DELETE | flags.MOVED_TO | flags.MOVED_FROM
+          ): dirname
+        for dirname in directories}
+    except OSError as e:
+      if e.errno not in (errno.ENOSPC, errno.EMFILE):
+        raise
+      print('Error using inotify, falling back to polling')
+      while not all_files_exists():
+        time.sleep(0.1)
+        check_if_files_exists()
+    else:
+      while not all_files_exists():
+        for event in inotify.read():
+          directory = directories[watchdescriptors[event.wd]]
+          if event.name in directory:
+            directory[event.name] = event.mask & (flags.CREATE | flags.MOVED_TO)
 
 def _libc():
   from ctypes import CDLL, get_errno, c_char_p, c_int, c_ulong, util

slapos/test/recipe/test_wrapper.py

@@ -175,6 +175,12 @@ class TestPidFile(WrapperTestCase):
 
 
 class TestWaitForFiles(WrapperTestCase):
+  env = None
+  if sys.platform.startswith("linux"):
+    expected_output = 'done\n'
+  else:
+    expected_output = 'Error using inotify, falling back to polling\ndone\n'
+
   def getOptions(self):
     self.waitfile = self.getTempPath('wait')
     return {
@@ -192,6 +198,7 @@ class TestWaitForFiles(WrapperTestCase):
       stdout=subprocess.PIPE,
       stderr=subprocess.STDOUT,
       universal_newlines=True,
+      env=self.env,
     )
     self.addCleanup(self.terminate_process, process)
     if process.poll():
@@ -207,13 +214,57 @@ class TestWaitForFiles(WrapperTestCase):
     for _ in range(20):
       time.sleep(0.1)
       if process.poll() is not None:
-        self.assertEqual(process.stdout.read(), 'done\n')
+        self.assertEqual(process.stdout.read(), self.expected_output)
         self.assertEqual(process.returncode, 0)
         break
     else:
       self.fail('process did not start after file was created')
 
 
+@unittest.skipUnless(sys.platform.startswith("linux"), "Inotify is linux only")
+class TestWaitForFilesInotifyError(TestWaitForFiles):
+
+  def setUp(self):
+    super(TestWaitForFilesInotifyError, self).setUp()
+    # use LD_PRELOAD to inject errors into inotify_add_watch calls
+    inotify_mock_c = self.getTempPath('inotify_mock.c')
+    inotify_mock_o = self.getTempPath('inotify_mock.o')
+    inotify_mock_so = self.getTempPath('inotify_mock.so')
+    with open(inotify_mock_c, 'w') as f:
+      f.write('''
+        #include <sys/inotify.h>
+        #include <string.h>
+        #include <errno.h>
+
+        int inotify_add_watch(int fd, const char *pathname, uint32_t mask) {
+          errno = ENOSPC;
+          return -1;
+        }
+
+        /* This is a bit tricky because inotify_simple calls
+        inotify_add_watch with ctypes.CDLL("libc.so"), which uses
+        dlopen("libc.so") and dlsym("inotify_add_watch"), so we first
+        override dlsym to return our own inotify_add_watch.
+        https://github.com/chrisjbillington/inotify_simple/blob/55737898/inotify_simple.py#L110
+        */
+        extern void *__libc_dlsym (void *, const char *);
+        void *dlsym(void *handle, const char *symbol) {
+          if (strcmp(symbol, "inotify_add_watch") == 0) {
+            return (void *)inotify_add_watch;
+          }
+          return (void *)__libc_dlsym(handle, symbol);
+        }
+        ''')
+    subprocess.check_call(['gcc', '-c', '-fPIC', '-o', inotify_mock_o, inotify_mock_c])
+    subprocess.check_call(['gcc', '-shared', '-o', inotify_mock_so, inotify_mock_o])
+    self.env = dict(
+      os.environ,
+      PYTHONUNBUFFERED='1',
+      LD_PRELOAD=inotify_mock_so)
+
+  expected_output = 'Error using inotify, falling back to polling\ndone\n'
+
+
 class TestPrivateTmpFS(WrapperTestCase):
   def getOptions(self):
     self.tmpdir = self.getTempPath('tmpdir')

stack/slapos.cfg

@@ -290,7 +290,7 @@ sgmllib3k = 1.0.0
 simplegeneric = 0.8.1
 singledispatch = 3.4.0.3
 six = 1.16.0
-slapos.cookbook = 1.0.297
+slapos.cookbook = 1.0.305
 slapos.core = 1.8.6
 slapos.extension.shared = 1.0
 slapos.libnetworkcache = 0.25