Create a directive for configuring the large file threshold where temp files...

Create a directive for configuring the large file threshold where temp files are created and use it for NullResource.PUT.

lib/python/ZServer/HTTPServer.py

@@ -53,7 +53,7 @@ from medusa.default_handler import unquote
 from asyncore import compact_traceback, dispatcher
 
 from ZServer import CONNECTION_LIMIT, ZOPE_VERSION, ZSERVER_VERSION
-from ZServer import requestCloseOnExec
+from ZServer import requestCloseOnExec, LARGE_FILE_THRESHOLD
 from zLOG import LOG, register_subsystem, BLATHER, INFO, WARNING, ERROR
 import DebugLogger
 from medusa import logger
@@ -76,7 +76,7 @@ class zhttp_collector:
     def __init__(self, handler, request, size):
         self.handler = handler
         self.request = request
-        if size > 524288:
+        if size > LARGE_FILE_THRESHOLD:
             # write large upload data to a file
             from tempfile import TemporaryFile
             self.data = TemporaryFile('w+b')

lib/python/ZServer/__init__.py

@@ -21,13 +21,18 @@ import utils
 exit_code = 0
 
 # the ZServer version number
-ZSERVER_VERSION='1.1'
+ZSERVER_VERSION = '1.1'
 
 # the maximum number of incoming connections to ZServer
-CONNECTION_LIMIT=1000 # may be reset by max_listen_sockets handler in Zope
+CONNECTION_LIMIT = 1000 # may be reset by max_listen_sockets handler in Zope
+
+# request bigger than this size get saved into a
+# temporary file instead of being read completely into memory
+LARGE_FILE_THRESHOLD = 1 << 19 # may be reset by large_file_threshold
+                               # handler in Zope
 
 # the Zope version string
-ZOPE_VERSION=utils.getZopeVersion()
+ZOPE_VERSION = utils.getZopeVersion()
 
 # backwards compatibility aliases
 from utils import requestCloseOnExec

lib/python/Zope/Startup/handlers.py

@@ -91,6 +91,10 @@ def rest_language_code(value):
     value and _setenv('REST_LANGUAGE_CODE' , value)
     return value
 
+def large_file_threshold(value):
+    import ZServer
+    ZServer.LARGE_FILE_THRESHOLD = value
+
 # server handlers
 
 def root_handler(config):

lib/python/Zope/Startup/zopeschema.xml

@@ -734,8 +734,9 @@
     </description>
   </section>
 
-  <!-- max-listen-sockets should really go into the ZServer package, but
-       I can't quite figure out how to put it there -->
+  <!-- max-listen-sockets and large-file-threshold should really go
+       into the ZServer package, but I can't quite figure out how to
+       put it there -->
 
   <key name="max-listen-sockets" datatype="integer"
        default="1000">
@@ -745,6 +746,14 @@
      </description>
   </key>
 
+  <key name="large-file-threshold" datatype="byte-size"
+       handler="large_file_threshold" default="512KB">
+     <description>
+       Requests bigger than this size get saved into a temporary file
+       instead of being read completely into memory.
+     </description>
+  </key>
+
   <multisection type="ZServer.server" name="*" attribute="servers"/>
   <key name="port-base" datatype="integer" default="0">
     <description>

lib/python/webdav/NullResource.py

@@ -81,13 +81,16 @@ class NullResource(Persistent, Acquisition.Implicit, Resource):
             ob=File(name, '', body, content_type=typ)
         return ob
 
-    PUT__roles__=('Anonymous',)
+    PUT__roles__ = ('Anonymous',)
     def PUT(self, REQUEST, RESPONSE):
-        """Create a new non-collection resource."""
+        """Create a new non-collection resource.
+        """
+        from ZServer import LARGE_FILE_THRESHOLD
+
         self.dav__init(REQUEST, RESPONSE)
 
-        name=self.__name__
-        parent=self.__parent__
+        name = self.__name__
+        parent = self.__parent__
 
         ifhdr = REQUEST.get_header('If', '')
         if WriteLockInterface.isImplementedBy(parent) and parent.wl_isLocked():
@@ -101,17 +104,40 @@ class NullResource(Persistent, Acquisition.Implicit, Resource):
             # There was an If header, but the parent is not locked
             raise PreconditionFailed
 
-        body=REQUEST.get('BODY', '')
+        # SDS: Only use BODY if the file size is smaller than
+        # LARGE_FILE_THRESHOLD, otherwise read LARGE_FILE_THRESHOLD
+        # bytes from the file which should be enough to trigger
+        # content_type detection, and possibly enough for CMF's
+        # content_type_registry too.
+        #
+        # Note that body here is really just used for detecting the
+        # content type and figuring out the correct factory. The correct
+        # file content will be uploaded on ob.PUT(REQUEST, RESPONSE) after
+        # the object has been created.
+        #
+        # A problem I could see is content_type_registry predicates
+        # that do depend on the whole file being passed here as an
+        # argument. There's none by default that does this though. If
+        # they really do want to look at the file, they should use
+        # REQUEST['BODYFILE'] directly and try as much as possible not
+        # to read the whole file into memory.
+
+        if int(REQUEST.get('CONTENT_LENGTH') or 0) > LARGE_FILE_THRESHOLD:
+            file = REQUEST['BODYFILE']
+            body = file.read(LARGE_FILE_THRESHOLD)
+            file.seek(0)
+        else:
+            body = REQUEST.get('BODY', '')
+
         typ=REQUEST.get_header('content-type', None)
         if typ is None:
             typ, enc=OFS.content_types.guess_content_type(name, body)
 
         factory = getattr(parent, 'PUT_factory', self._default_PUT_factory )
         ob = factory(name, typ, body)
-        ob = (ob is None and
-              self._default_PUT_factory(name, typ, body) or
-              ob
-              )
+        if ob is None:
+            ob = self._default_PUT_factory(name, typ, body)
+
         # We call _verifyObjectPaste with verify_src=0, to see if the
         # user can create this type of object (and we don't need to
         # check the clipboard.
@@ -122,9 +148,11 @@ class NullResource(Persistent, Acquisition.Implicit, Resource):
         except:
             raise Forbidden, sys.exc_info()[1]
 
-        # Delegate actual PUT handling to the new object.
-        ob.PUT(REQUEST, RESPONSE)
+        # Delegate actual PUT handling to the new object,
+        # SDS: But just *after* it has been stored.
         self.__parent__._setObject(name, ob)
+        ob = self.__parent__._getOb(name)
+        ob.PUT(REQUEST, RESPONSE)
 
         RESPONSE.setStatus(201)
         RESPONSE.setBody('')

skel/etc/zope.conf.in

@@ -781,6 +781,18 @@ instancehome $INSTANCE
 #    max-listen-sockets 500
 
 
+# Directive: large-file-threshold
+#
+# Description:
+#     Requests bigger than this size get saved into a temporary file
+#     instead of being read completely into memory.
+#
+# Default: 512K
+#
+# Example:
+#
+#    large-file-threshold 1Mb
+
 
 # Directives: servers
 #