Cache downloaded data in zc/buildout/buildout.py:_open() in memory to accelerate remote extends.

CHANGES.SlapOS.txt

+1.7.1-dev-SlapOS-005
+--------------------
+
+- Cache downloaded data in zc/buildout/buildout.py:_open() in memory to accelerate remote extends.
+
 1.7.1-dev-SlapOS-004
 --------------------
 

setup.py

@@ -12,7 +12,7 @@
 #
 ##############################################################################
 name = "zc.buildout"
-version = "1.7.1-dev-SlapOS-004"
+version = "1.7.1-dev-SlapOS-005"
 
 import os
 from setuptools import setup

src/zc/buildout/buildout.py

@@ -22,6 +22,7 @@ except ImportError:
     from md5 import md5
 
 import ConfigParser
+from cStringIO import StringIO
 import copy
 import distutils.errors
 import glob
@@ -1691,6 +1692,7 @@ def _save_options(section, options, f):
     for option in sorted(options.keys()):
         _save_option(option, options.get(option), f)
 
+_open_download_cache = {}
 _open_parser_cache = {}
 def _open(base, filename, seen, dl_options, override, downloaded):
     """Open a configuration file and return the result as a dictionary,
@@ -1722,8 +1724,14 @@ def _open(base, filename, seen, dl_options, override, downloaded):
             base = os.path.dirname(filename)
         else:
             filename = base + '/' + filename
-            path, is_temp = download(filename)
-            fp = open(path)
+            data = _open_download_cache.get(filename)
+            if data is None:
+                path, is_temp = download(filename)
+                data = file(path).read()
+                _open_download_cache[filename] = data
+            else:
+                is_temp = False
+            fp = StringIO(data)
             base = filename[:filename.rfind('/')]
     else:
         filename = os.path.join(base, filename)