Try to reuse existing file to avoid excessive IO on update + other minor optimisations

slapos/recipe/librecipe/generic.py

@@ -26,6 +26,7 @@
 # Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.
 #
 ##############################################################################
+import errno
 import io
 import logging
 import os
@@ -33,6 +34,7 @@ import sys
 import inspect
 import re
 import shutil
+import stat
 import urllib
 import urlparse
 
@@ -92,9 +94,27 @@ class GenericBaseRecipe(object):
     """Create a file with content
 
     The parent directory should exists, else it would raise IOError"""
-    with open(name, 'w') as fileobject:
-      fileobject.write(content)
-      os.chmod(fileobject.name, mode)
+    if not isinstance(content, bytes):
+      content = content.encode('utf-8')
+    # Try to reuse existing file. This is particularly
+    # important to avoid excessive IO during update.
+    try:
+      with open(name, 'rb') as f:
+        if f.read(len(content)+1) == content:
+          if None is not mode != stat.S_IMODE(os.fstat(f.fileno()).st_mode):
+            os.fchmod(f.fileno(), mode)
+          return os.path.abspath(name)
+    except (IOError, OSError) as e:
+      pass
+    try:
+      os.unlink(name)
+    except OSError as e:
+      if e.errno != errno.ENOENT:
+        raise
+    with open(name, 'wb') as f:
+      if mode is not None:
+        os.fchmod(f.fileno(), mode)
+      f.write(content)
     return os.path.abspath(name)
 
   def createExecutable(self, name, content, mode=0700):

slapos/recipe/promise_plugin.py

@@ -24,18 +24,17 @@
 # Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.
 #
 ##############################################################################
-import json
 import re
 import logging, os
 import zc.buildout.easy_install
+from pprint import pformat
 from slapos.recipe.librecipe import GenericBaseRecipe
 
 script_template = '''# This script is auto generated by slapgrid, do not edit!
-import json
 import sys
 sys.path[0:0] = %(path)s
 
-extra_config_dict = json.loads("""%(config)s""", strict=False)
+extra_config_dict = %(config)s
 
 # We want to cleanup all imported modules from slapos namespace, because
 # they will conflict with slapos.core.
@@ -69,12 +68,9 @@ class Recipe(GenericBaseRecipe):
     """Return a mapping where to store generated working sets.
     from https://github.com/buildout/buildout/blob/master/zc.recipe.egg_/src/zc/recipe/egg/egg.py#L170
     """
-    cache_storage = getattr(
-      self.buildout,
-      self._WORKING_SET_CACHE_NAME,
-      None
-    )
-    if cache_storage is None:
+    try:
+      return getattr(self.buildout, self._WORKING_SET_CACHE_NAME)
+    except AttributeError:
       cache_storage = {}
       try:
         setattr(
@@ -83,29 +79,29 @@ class Recipe(GenericBaseRecipe):
           cache_storage
         )
       except AttributeError:
-        if type(self.buildout) == type({}):
-          # failed to set attribute in test mode, cache not used
-          pass
-        else:
+        if not isinstance(self.buildout, dict):
           raise
+        # failed to set attribute in test mode, cache not used
     return cache_storage
 
   def install(self):
     develop_eggs_dir = self.options['develop-eggs-directory']
     eggs_dir = self.options['eggs-directory']
-    egg_list = [
+    egg_list = tuple(
       egg.strip()
-      for egg in self.options['eggs'].split('\n')
+      for egg in self.options['eggs'].splitlines()
       if egg.strip()
-    ]
+    )
 
     cache_storage = self._get_cache_storage()
     cache_key = (
-      tuple(egg_list),
+      egg_list,
       eggs_dir,
       develop_eggs_dir,
     )
-    if cache_key not in cache_storage:
+    try:
+      working_set = cache_storage[cache_key]
+    except KeyError:
       if develop_eggs_dir and eggs_dir:
         working_set = zc.buildout.easy_install.working_set(
           egg_list,
@@ -114,8 +110,6 @@ class Recipe(GenericBaseRecipe):
         cache_storage[cache_key] = working_set
       else:
         working_set = set()
-    else:
-      working_set = cache_storage[cache_key]
 
     regex = r"^[\w_\-\.\s]+$"
     import_path = self.options.get('import', '').strip()
@@ -129,24 +123,14 @@ class Recipe(GenericBaseRecipe):
       if not re.search(regex, content_string):
         raise ValueError("Promise content %r is not valid" % content_string)
 
-    output = self.options['output']
-    mode = self.options.get('mode', '0644')
-    path_list = []
-    for dist in working_set:
-      path_list.append(dist.location)
-
-    config_dict = dict()
-    for key in self.options:
-      if key.startswith('config-'):
-        config_dict[key[7:]] = self.options[key]
-
-    option_dict = dict(path=json.dumps(path_list, indent=2),
-                       content=content_string,
-                       config=json.dumps(config_dict, indent=2, sort_keys=True))
-    with open(output, 'w') as f:
-      f.write(script_template % option_dict)
+    config_dict = {key[7:]: self.options[key]
+      for key in self.options
+      if key.startswith('config-')}
 
-    os.chmod(output, int(mode, 8))
-    return (output,)
+    return self.createFile(self.options['output'], script_template % {
+      'path': pformat([dist.location for dist in working_set], indent=2),
+      'content': content_string,
+      'config': pformat(config_dict, indent=2),
+      }, int(self.options.get('mode', '0644'), 8)),
 
   update = install