Commit fa8d8cc6 authored by Julien Muchembled's avatar Julien Muchembled Committed by Matevz Golob

Try to reuse existing file to avoid excessive IO on update + other minor optimisations

parent 1cc92abe
...@@ -26,6 +26,7 @@ ...@@ -26,6 +26,7 @@
# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. # Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
# #
############################################################################## ##############################################################################
import errno
import io import io
import logging import logging
import os import os
...@@ -33,6 +34,7 @@ import sys ...@@ -33,6 +34,7 @@ import sys
import inspect import inspect
import re import re
import shutil import shutil
import stat
import urllib import urllib
import urlparse import urlparse
...@@ -92,9 +94,27 @@ class GenericBaseRecipe(object): ...@@ -92,9 +94,27 @@ class GenericBaseRecipe(object):
"""Create a file with content """Create a file with content
The parent directory should exists, else it would raise IOError""" The parent directory should exists, else it would raise IOError"""
with open(name, 'w') as fileobject: if not isinstance(content, bytes):
fileobject.write(content) content = content.encode('utf-8')
os.chmod(fileobject.name, mode) # Try to reuse existing file. This is particularly
# important to avoid excessive IO during update.
try:
with open(name, 'rb') as f:
if f.read(len(content)+1) == content:
if None is not mode != stat.S_IMODE(os.fstat(f.fileno()).st_mode):
os.fchmod(f.fileno(), mode)
return os.path.abspath(name)
except (IOError, OSError) as e:
pass
try:
os.unlink(name)
except OSError as e:
if e.errno != errno.ENOENT:
raise
with open(name, 'wb') as f:
if mode is not None:
os.fchmod(f.fileno(), mode)
f.write(content)
return os.path.abspath(name) return os.path.abspath(name)
def createExecutable(self, name, content, mode=0700): def createExecutable(self, name, content, mode=0700):
......
...@@ -24,18 +24,17 @@ ...@@ -24,18 +24,17 @@
# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. # Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
# #
############################################################################## ##############################################################################
import json
import re import re
import logging, os import logging, os
import zc.buildout.easy_install import zc.buildout.easy_install
from pprint import pformat
from slapos.recipe.librecipe import GenericBaseRecipe from slapos.recipe.librecipe import GenericBaseRecipe
script_template = '''# This script is auto generated by slapgrid, do not edit! script_template = '''# This script is auto generated by slapgrid, do not edit!
import json
import sys import sys
sys.path[0:0] = %(path)s sys.path[0:0] = %(path)s
extra_config_dict = json.loads("""%(config)s""", strict=False) extra_config_dict = %(config)s
# We want to cleanup all imported modules from slapos namespace, because # We want to cleanup all imported modules from slapos namespace, because
# they will conflict with slapos.core. # they will conflict with slapos.core.
...@@ -69,12 +68,9 @@ class Recipe(GenericBaseRecipe): ...@@ -69,12 +68,9 @@ class Recipe(GenericBaseRecipe):
"""Return a mapping where to store generated working sets. """Return a mapping where to store generated working sets.
from https://github.com/buildout/buildout/blob/master/zc.recipe.egg_/src/zc/recipe/egg/egg.py#L170 from https://github.com/buildout/buildout/blob/master/zc.recipe.egg_/src/zc/recipe/egg/egg.py#L170
""" """
cache_storage = getattr( try:
self.buildout, return getattr(self.buildout, self._WORKING_SET_CACHE_NAME)
self._WORKING_SET_CACHE_NAME, except AttributeError:
None
)
if cache_storage is None:
cache_storage = {} cache_storage = {}
try: try:
setattr( setattr(
...@@ -83,29 +79,29 @@ class Recipe(GenericBaseRecipe): ...@@ -83,29 +79,29 @@ class Recipe(GenericBaseRecipe):
cache_storage cache_storage
) )
except AttributeError: except AttributeError:
if type(self.buildout) == type({}): if not isinstance(self.buildout, dict):
# failed to set attribute in test mode, cache not used
pass
else:
raise raise
# failed to set attribute in test mode, cache not used
return cache_storage return cache_storage
def install(self): def install(self):
develop_eggs_dir = self.options['develop-eggs-directory'] develop_eggs_dir = self.options['develop-eggs-directory']
eggs_dir = self.options['eggs-directory'] eggs_dir = self.options['eggs-directory']
egg_list = [ egg_list = tuple(
egg.strip() egg.strip()
for egg in self.options['eggs'].split('\n') for egg in self.options['eggs'].splitlines()
if egg.strip() if egg.strip()
] )
cache_storage = self._get_cache_storage() cache_storage = self._get_cache_storage()
cache_key = ( cache_key = (
tuple(egg_list), egg_list,
eggs_dir, eggs_dir,
develop_eggs_dir, develop_eggs_dir,
) )
if cache_key not in cache_storage: try:
working_set = cache_storage[cache_key]
except KeyError:
if develop_eggs_dir and eggs_dir: if develop_eggs_dir and eggs_dir:
working_set = zc.buildout.easy_install.working_set( working_set = zc.buildout.easy_install.working_set(
egg_list, egg_list,
...@@ -114,8 +110,6 @@ class Recipe(GenericBaseRecipe): ...@@ -114,8 +110,6 @@ class Recipe(GenericBaseRecipe):
cache_storage[cache_key] = working_set cache_storage[cache_key] = working_set
else: else:
working_set = set() working_set = set()
else:
working_set = cache_storage[cache_key]
regex = r"^[\w_\-\.\s]+$" regex = r"^[\w_\-\.\s]+$"
import_path = self.options.get('import', '').strip() import_path = self.options.get('import', '').strip()
...@@ -129,24 +123,14 @@ class Recipe(GenericBaseRecipe): ...@@ -129,24 +123,14 @@ class Recipe(GenericBaseRecipe):
if not re.search(regex, content_string): if not re.search(regex, content_string):
raise ValueError("Promise content %r is not valid" % content_string) raise ValueError("Promise content %r is not valid" % content_string)
output = self.options['output'] config_dict = {key[7:]: self.options[key]
mode = self.options.get('mode', '0644') for key in self.options
path_list = [] if key.startswith('config-')}
for dist in working_set:
path_list.append(dist.location)
config_dict = dict()
for key in self.options:
if key.startswith('config-'):
config_dict[key[7:]] = self.options[key]
option_dict = dict(path=json.dumps(path_list, indent=2),
content=content_string,
config=json.dumps(config_dict, indent=2, sort_keys=True))
with open(output, 'w') as f:
f.write(script_template % option_dict)
os.chmod(output, int(mode, 8)) return self.createFile(self.options['output'], script_template % {
return (output,) 'path': pformat([dist.location for dist in working_set], indent=2),
'content': content_string,
'config': pformat(config_dict, indent=2),
}, int(self.options.get('mode', '0644'), 8)),
update = install update = install
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment