Merge remote-tracking branch 'origin/master' into lazy_simulation_causality

bt5/erp5_dms/SkinTemplateItem/portal_skins/erp5_dms/ERP5Type_getSecurityCategoryMapping.xml

-<?xml version="1.0"?>
-<ZopeData>
-  <record id="1" aka="AAAAAAAAAAE=">
-    <pickle>
-      <global name="PythonScript" module="Products.PythonScripts.PythonScript"/>
-    </pickle>
-    <pickle>
-      <dictionary>
-        <item>
-            <key> <string>Script_magic</string> </key>
-            <value> <int>3</int> </value>
-        </item>
-        <item>
-            <key> <string>_bind_names</string> </key>
-            <value>
-              <object>
-                <klass>
-                  <global name="NameAssignments" module="Shared.DC.Scripts.Bindings"/>
-                </klass>
-                <tuple/>
-                <state>
-                  <dictionary>
-                    <item>
-                        <key> <string>_asgns</string> </key>
-                        <value>
-                          <dictionary>
-                            <item>
-                                <key> <string>name_container</string> </key>
-                                <value> <string>container</string> </value>
-                            </item>
-                            <item>
-                                <key> <string>name_context</string> </key>
-                                <value> <string>context</string> </value>
-                            </item>
-                            <item>
-                                <key> <string>name_m_self</string> </key>
-                                <value> <string>script</string> </value>
-                            </item>
-                            <item>
-                                <key> <string>name_subpath</string> </key>
-                                <value> <string>traverse_subpath</string> </value>
-                            </item>
-                          </dictionary>
-                        </value>
-                    </item>
-                  </dictionary>
-                </state>
-              </object>
-            </value>
-        </item>
-        <item>
-            <key> <string>_body</string> </key>
-            <value> <string>"""\n
-Core security script - defines the way to get security groups of the current user.\n
-\n
-WARNING: providing such script in erp5_dms could be dangerous\n
-if this conflicts with an existing production site which uses\n
-deprecated ERP5Type_asSecurityGroupIdList\n
-"""\n
-\n
-return (\n
-(\'ERP5Type_getSecurityCategoryFromAssignmentStrict\', [\'function\'] ),\n
-(\'ERP5Type_getSecurityCategoryFromAssignmentStrict\', [\'follow_up\'] ),\n
-(\'ERP5Type_getSecurityCategoryFromAssignmentStrict\', [\'function\', \'follow_up\'] ),\n
-(\'ERP5Type_getSecurityCategoryFromAssignmentStrict\', [\'group\'] ),\n
-(\'ERP5Type_getSecurityCategoryRoot\', [\'group\']),\n
-)\n
-</string> </value>
-        </item>
-        <item>
-            <key> <string>_params</string> </key>
-            <value> <string></string> </value>
-        </item>
-        <item>
-            <key> <string>id</string> </key>
-            <value> <string>ERP5Type_getSecurityCategoryMapping</string> </value>
-        </item>
-      </dictionary>
-    </pickle>
-  </record>
-</ZopeData>

bt5/erp5_dms/bt/revision

-1261
+1262
\ No newline at end of file

bt5/erp5_web/SkinTemplateItem/portal_skins/erp5_web/WebPage_getStandardSubstitutionMappingDict.xml

@@ -59,9 +59,16 @@
 # your own script instead.\n
 \n
 website = context.getWebSiteValue()\n
-websection = context.getWebSectionValue()\n
-\n
-return dict(website_url=website.absolute_url(),\n
+if website is None:\n
+  # handle the case when substitution happens on web page module context (i.e. reindex, or mass pre conversion)\n
+  # then fall back to ERP5 site as a Web Site\n
+  website = context.getPortalObject()\n
+  return dict(website_url=website.absolute_url(),\n
+              websection_url=website.absolute_url(),\n
+              webpage_url=context.absolute_url())\n
+else:\n
+  websection = context.getWebSectionValue()\n
+  return dict(website_url=website.absolute_url(),\n
               websection_url=websection.absolute_url(),\n
               webpage_url=websection.getPermanentURL(context))\n
 </string> </value>

bt5/erp5_web/bt/revision

-1074
+1075
\ No newline at end of file

erp5/util/README.webchecker.txt

@@ -21,6 +21,17 @@ Vary = Accept-Language, Cookie, Accept-Encoding
        Accept-Language,Cookie
 Expires = True
 
+[header url=.*/sitemap]
+Last-Modified = True
+
+[header content-type=.*/javascript]
+Last-Modified = True
+Cache-Control = max-age=3600
+Expires = True
+
+[no_header content-type=(image/.*|text/css)]
+Vary = None
+
 [erp5_extension_list]
 prohibited_file_name_list = WebSection_viewAsWeb
                             Base_viewHistory

erp5/util/webchecker/__init__.py

@@ -75,7 +75,8 @@ class HTTPCacheCheckerTestSuite(object):
 
   def __init__(self, root_url, working_directory, varnishlog_binary_path,
                wget_binary_path, header_list, email_address, smtp_host,
-               debug_level, file_log_path):
+               debug_level, file_log_path, conditional_header_dict,
+               no_header_dict):
     """
       root_url : website to check
       working_directory : where fetched data will be downloaded
@@ -92,12 +93,19 @@ class HTTPCacheCheckerTestSuite(object):
                                                info=>normal,
                                                warning=> nothing)
       file_log_path: path to log file
+      conditional_header_dict : Key  == Section id (like 'header url='.*/login')
+                                value: the configuration lines in this sction
+                                       (config format is same as header_list)
+      no_header_dict : Key == Section id (like 'no_header url=.*/sitemap')
+                       value: = not exsiting headers
     """
     self.root_url = root_url
     self.working_directory = working_directory
     self.varnishlog_binary_path = varnishlog_binary_path
     self.wget_binary_path = wget_binary_path
     self.header_list = header_list
+    self.conditional_header_dict = conditional_header_dict
+    self.no_header_dict = no_header_dict
     self.email_address = email_address
     self.smtp_host = smtp_host
     level = self.LOG_LEVEL_DICT.get(debug_level, logging.INFO)
@@ -242,6 +250,119 @@ class HTTPCacheCheckerTestSuite(object):
     stdout, stderr = wget_process.communicate()
     return stdout
 
+  def _getHeaderPolicyList(self, url, fetched_data):
+    """
+    create the header checking policy list by the url, and the header
+
+    [header_list]
+    Last-Modified = True
+    Vary = Accept-Language, Cookie, Accept-Encoding
+    Cache-Control = max-age=300
+    max-age=3600
+    [header url=.*/contact_form]
+    Last-Modified = True
+
+    [no-header content-type=(image/.*|.*/javascript)]
+    Vary = False
+
+    [erp5_extension_list]
+    prohibited_folder_name_list = web_page_module
+    document_module
+    prohibited_file_name_list = WebSection_viewAsWeb
+    Base_viewHistory
+    list
+    """
+    def getNoCheckHeaderList(url, fetched_data):
+      """
+        create no check header list
+      """
+      pick_content_type = re.compile('^no_header\s*content-type=(.*)')
+      pick_url = re.compile('^no_header\s*url=(.*)')
+      section_list = self.no_header_dict.keys()
+      no_check_header_list = []
+      for section in section_list:
+        content_type_regex_str_match = pick_content_type.match(section)
+        url_regex_str_match = pick_url.match(section)
+        if content_type_regex_str_match is not None:
+          content_type_regex_str = content_type_regex_str_match.group(1)
+          content_type_regex = \
+            re.compile('Content-Type:\s%s' % content_type_regex_str,
+                       re.MULTILINE | re.IGNORECASE)
+          if content_type_regex.search(fetched_data) is not None:
+            for header, value in self.no_header_dict[section]:
+              no_check_header_list.append(header)
+          continue
+        if url_regex_str_match is not None:
+          url_regex_str = url_regex_str_match.group(1)
+          if re.compile(url_regex_str).match(url) is not None:
+            for header, value in self.no_header_dict[section]:
+              no_check_header_list.append(header)
+      return no_check_header_list
+
+    def getConditionalHeaderDict(url, fetched_data):
+      """ create header policy by the url and header"""
+      conditional_header_dict = {}
+      section_list = self.conditional_header_dict.keys()
+      pick_content_type = re.compile('header\s*content-type=(.*)')
+      pick_url = re.compile('header\s*url=(.*)')
+      for section in section_list:
+        content_type_regex_str_match = pick_content_type.match(section)
+        url_regex_str_match = pick_url.match(section)
+        if content_type_regex_str_match is not None:
+          content_type_regex_str = content_type_regex_str_match.group(1)
+          content_type_regex = \
+            re.compile('Content-Type:\s%s' % content_type_regex_str,
+                       re.MULTILINE | re.IGNORECASE)
+          if content_type_regex.search(fetched_data) is not None:
+            for header, value in self.conditional_header_dict[section]:
+              conditional_header_dict.setdefault(header, []).append(value)
+          continue
+        if url_regex_str_match is not None:
+          url_regex_str = url_regex_str_match.group(1)
+          if re.compile(url_regex_str).match(url) is not None:
+            for header, value in self.conditional_header_dict[section]:
+              conditional_header_dict.setdefault(header, []).append(value)
+      return conditional_header_dict
+
+    validator_dict = {}
+    no_header_list = getNoCheckHeaderList(url, fetched_data)
+    conditional_header_dict = getConditionalHeaderDict(url, fetched_data)
+    conditional_header_list = conditional_header_dict.keys()
+    global_header_list = self.header_list.keys()
+    header_policy_list = []
+    if conditional_header_list:
+      conditional_check_header_set = (set(conditional_header_list)
+                                      - set(no_header_list))
+      for header in conditional_check_header_set:
+        header_policy_list.append((header, conditional_header_dict[header]))
+    else:
+      global_check_header_set = (set(global_header_list)
+                                 - set(no_header_list))
+      for header in global_check_header_set:
+        header_policy_list.append((header, self.header_list[header]))
+    # return items
+    return header_policy_list
+
+  def _validateHeader(self, url, header, reference_value, fetched_data):
+    """validate header with the header policy"""
+    re_compiled = re.compile(self.generic_header_search_regex % header,
+                             re.MULTILINE | re.IGNORECASE)
+    match_object = re_compiled.search(fetched_data)
+    if match_object is None:
+      message = 'header:%r not found' % (header)
+      self.report_dict.setdefault(url, []).append(message)
+    else:
+      read_value = match_object.group(1)
+      if reference_value is True and not read_value:
+        message = 'value of header:%r not found' % (header)
+        self.report_dict.setdefault(url, []).append(message)
+      elif isinstance(reference_value, (tuple,list)):
+        if read_value not in reference_value:
+          message = 'value of header:%r does not match'\
+                    ' (%r not in %r)' %\
+                                    (header, read_value, reference_value)
+          self.report_dict.setdefault(url, []).append(message)
+
   def _parseWgetLogs(self, wget_log_file, discarded_url_list=_MARKER,
                      prohibited_file_name_list=None,
                      prohibited_folder_name_list=None):
@@ -332,24 +453,11 @@ class HTTPCacheCheckerTestSuite(object):
             discarded_url_list.append(url)
           else:
             x_varnish_reference_list.append((x_varnish_reference, True, url))
-        for header, reference_value in self.header_list.iteritems():
-          re_compiled = re.compile(self.generic_header_search_regex % header,
-                                   re.MULTILINE | re.IGNORECASE)
-          match_object = re_compiled.search(fetched_data)
-          if match_object is None:
-            message = 'header:%r not found' % (header)
-            self.report_dict.setdefault(url, []).append(message)
-          else:
-            read_value = match_object.group(1)
-            if reference_value is True and not read_value:
-              message = 'value of header:%r not found' % (header)
-              self.report_dict.setdefault(url, []).append(message)
-            elif isinstance(reference_value, (tuple,list)):
-              if read_value not in reference_value:
-                message = 'value of header:%r does not match'\
-                          ' (%r not in %r)' %\
-                                          (header, read_value, reference_value)
-                self.report_dict.setdefault(url, []).append(message)
+        # parse the web checker configuration file and run the header
+        # validation method
+        for header, reference_value in self._getHeaderPolicyList(
+                                              url, fetched_data):
+          self._validateHeader(url, header, reference_value, fetched_data)
     return x_varnish_reference_list, discarded_url_list[:]
 
   def start(self, prohibited_file_name_list=None,
@@ -428,16 +536,19 @@ def web_checker_utility():
     parser.error('incorrect number of arguments')
   config_path = args[0]
 
-  config = ConfigParser.RawConfigParser(defaults=dict(wget_binary_path='wget',
-                                              file_log_path='web_checker.log'))
+  config = ConfigParser.RawConfigParser()
   config.read(config_path)
   working_directory = config.get('web_checker', 'working_directory')
   url = config.get('web_checker', 'url')
   varnishlog_binary_path = config.get('web_checker', 'varnishlog_binary_path')
+  wget_binary_path = 'wget'
+  if config.has_option('web_checker', 'wget_binary_path'):
     wget_binary_path = config.get('web_checker', 'wget_binary_path')
   email_address = config.get('web_checker', 'email_address')
   smtp_host = config.get('web_checker', 'smtp_host')
   debug_level = config.get('web_checker', 'debug_level')
+  file_log_path = 'web_checker.log'
+  if config.has_option('web_checker', 'file_log_path'):
     file_log_path = config.get('web_checker', 'file_log_path')
   header_list = {}
   for header, configuration in config.items('header_list'):
@@ -450,6 +561,14 @@ def web_checker_utility():
     else:
       value = configuration.splitlines()
     header_list[header] = value
+  conditional_header_dict = {}
+  no_header_dict = {}
+  for section in config.sections():
+    item_list = config.items(section)
+    if re.compile("^header\s.*").match(section) is not None:
+      conditional_header_dict.setdefault(section, []).extend(item_list)
+    if re.compile("^no_header\s.*").match(section) is not None:
+      no_header_dict.setdefault(section, []).extend(item_list)
   if config.has_section('erp5_extension_list'):
     prohibited_file_name_list = config.get('erp5_extension_list',
                                       'prohibited_file_name_list').splitlines()
@@ -465,7 +584,9 @@ def web_checker_utility():
                                        email_address,
                                        smtp_host,
                                        debug_level,
-                                       file_log_path)
+                                       file_log_path,
+                                       conditional_header_dict,
+                                       no_header_dict)
 
   result = instance.start(prohibited_file_name_list=prohibited_file_name_list,
                        prohibited_folder_name_list=prohibited_folder_name_list)

product/ERP5/Document/TextDocument.py

@@ -36,7 +36,7 @@ from Products.ERP5.Document.Document import Document, ConversionError, _MARKER,
 from Products.ERP5.Document.File import File
 from Products.ERP5Type.WebDAVSupport import TextContent
 import re
-from Products.ERP5.Document.Document import VALID_IMAGE_FORMAT_LIST
+from Products.ERP5.Document.Document import VALID_IMAGE_FORMAT_LIST, VALID_TEXT_FORMAT_LIST
 import cStringIO
 from string import Template
 
@@ -179,6 +179,8 @@ class TextDocument(CachedConvertableMixin, BaseConvertableFileMixin,
           self.setConversion(result, original_mime_type, **kw)
         else:
           mime_type, result = self.getConversion(**kw)
+        if format in VALID_TEXT_FORMAT_LIST:
+          # only textual content can be sustituted 
           if substitution_method_parameter_dict is None:
             substitution_method_parameter_dict = {}
           result = self._substituteTextContent(result, safe_substitute=safe_substitute,

product/ERP5OOo/tests/testDms.py

@@ -2021,7 +2021,8 @@ return 1
   
   def test_convertWebPageWithEmbeddedZODBImageToImageOnTraversal(self):
     """
-    Test Web Page using embedded Images into ZODB case (in its HTML body)
+    Test Web Page conversion to image using embedded Images into its HTML body.
+    Test various dumb ways to include an image (relative to instance or external ones).
     """
     display= 'thumbnail'
     convert_kw = {'display':display, 
@@ -2037,36 +2038,39 @@ return 1
     web_page_image_size, web_page_file_size = self.getURLSizeList(web_page_document_url, **convert_kw)
     self.assertTrue(max(preffered_size_for_display) - max(web_page_image_size) <= 1)
 
-    # XXX: how to simulate the case when web page contains (through reference) link to document for which based conversion failed?
-    # XXX: how to fix case when web page contains (through reference) link to itself (causes infinite recursion)
-
-    # images from same instance accessed by reference and wrong arguments (dispay NOT display)
+    # images from same instance accessed by reference and wrong conversion arguments (dispay NOT display)
     # code should be more resilient
     upload_file = makeFileUpload('cmyk_sample.jpg')
     image = self.portal.image_module.newContent(portal_type='Image',
                                                reference='Embedded-XXX',
                                                version='001',
                                                language='en')
+    image.setData(upload_file.read())
     image.publish()
     convert_kw['quality'] = 99 # to not get cached
     web_page_document = self.portal.web_page_module.newContent(portal_type="Web Page")
-    web_page_document.setTextContent('''<b> test </b><img src="Embedded-XXX?format=jpeg&amp;dispay=medium"/>''')
+    web_page_document.setTextContent('''<b> test </b><img src="Embedded-XXX?format=jpeg&amp;dispay=medium&amp;quality=50"/>''')
     self.stepTic()
     web_page_document_url = '%s/%s' %(self.portal.absolute_url(), web_page_document.getRelativeUrl())
     web_page_image_size, web_page_file_size = self.getURLSizeList(web_page_document_url, **convert_kw)
     self.assertTrue(max(preffered_size_for_display) - max(web_page_image_size) <= 1)
 
     # external images
-    convert_kw['quality'] = 98 # to not get cached
+    convert_kw['quality'] = 98
     web_page_document = self.portal.web_page_module.newContent(portal_type="Web Page")
     web_page_document.setTextContent('''<b> test </b><img src="http://www.erp5.com/images/favourite.png"/>
 <img style="width: 26px; height: 26px;" src="http://www.erp5.com//images/save2.png" />
+<img style="width: 26px; height: 26px;" src="http:////www.erp5.com//images/save2.png" />
+<img style="width: 26px; height: 26px;" src="http://www.erp5.com/./images/save2.png" />
 ''')
     self.stepTic()
     web_page_document_url = '%s/%s' %(self.portal.absolute_url(), web_page_document.getRelativeUrl())
     web_page_image_size, web_page_file_size = self.getURLSizeList(web_page_document_url, **convert_kw)
     self.assertTrue(max(preffered_size_for_display) - max(web_page_image_size) <= 1)
 
+    # XXX: how to simulate the case when web page contains (through reference) link to document for which based conversion failed?
+    # XXX: how to fix case when web page contains (through reference) link to itself (causes infinite recursion)
+
 
   def test_convertToImageOnTraversal(self):
     """
@@ -2091,7 +2095,9 @@ return 1
     image_document.edit(file=upload_file)
     
     web_page_document = self.portal.web_page_module.newContent(portal_type="Web Page")
-    web_page_document.setTextContent('<b> test </b>')    
+    web_page_document.setTextContent('<b> test </b> $website_url $website_url')
+    # a Web Page can generate dynamic text so test is as well
+    web_page_document.setTextContentSubstitutionMappingMethodId('WebPage_getStandardSubstitutionMappingDict')    
     self.stepTic()
 
     ooo_document_url = '%s/%s' %(self.portal.absolute_url(), ooo_document.getRelativeUrl())
@@ -2704,8 +2710,8 @@ class TestDocumentPerformance(TestDocumentMixin):
 def test_suite():
   suite = unittest.TestSuite()
   suite.addTest(unittest.makeSuite(TestDocument))
-  suite.addTest(unittest.makeSuite(TestDocumentWithSecurity))
-  suite.addTest(unittest.makeSuite(TestDocumentPerformance))
+  #suite.addTest(unittest.makeSuite(TestDocumentWithSecurity))
+  #suite.addTest(unittest.makeSuite(TestDocumentPerformance))
   return suite
 
 

product/ERP5OOo/transforms/oood_commandtransform.py

@@ -110,8 +110,10 @@ class OOOdCommandTransform(commandtransform):
       url = href_attribute_list[0]
       parse_result = urlparse(unquote(url))
       # urlparse return a 6-tuple: scheme, netloc, path, params, query, fragment
+      netloc = parse_result[1]
       path = parse_result[2]
-      if path:
+      if path and netloc in ('', None):
+        # it makes sense to include only relative to current site images not remote ones which can be taken by OOo
         # OOo corrupt relative Links inside HTML content during odt conversion
         # <img src="REF.TO.IMAGE" ... /> become <draw:image xlink:href="../REF.TO.IMAGE" ... />
         # So remove "../" added by OOo
@@ -119,6 +121,9 @@ class OOOdCommandTransform(commandtransform):
         # in some cases like Web Page content "/../" can be contained in image URL which will break 
         # restrictedTraverse calls, our best guess is to remove it
         path = path.replace('/../', '')
+        # remove occurencies of '//' or '///' in path (happens with web pages) and leave 
+        # a traversable relative URL
+        path = '/'.join([x for x in path.split('/') if x.strip()!=''])
         # retrieve http parameters and use them to convert image
         query_parameter_string = parse_result[4]
         image_parameter_dict = dict(parse_qsl(query_parameter_string))
@@ -131,6 +136,10 @@ class OOOdCommandTransform(commandtransform):
           odt_content_modified = True
           content_type = image.getContentType()
           format = image_parameter_dict.pop('format', None)
+          # convert API accepts only a certail range of arguments
+          for key, value in image_parameter_dict.items():
+            if key not in ('format', 'display', 'quality', 'resolution',):
+              image_parameter_dict.pop(key)
           if getattr(image, 'convert', None) is not None:
             # The document support conversion so perform conversion
             # according given parameters

product/ERP5SyncML/Document/SyncMLPublication.py

@@ -71,14 +71,8 @@ class SyncMLPublication(SyncMLSubscription):
     """
       Reset all subscribers
     """
-    id_list = []
-    for subscriber in self.contentValues(portal_type='SyncML Subscription'):
-      subscriber.resetSignatureList()
-      id_list.append(subscriber.getId())
     self.activate(activity='SQLQueue',
-                  tag=self.getId(),
-                  after_tag=id_list,
-                  priority=ACTIVITY_PRIORITY).manage_delObjects(id_list)
+                  priority=ACTIVITY_PRIORITY).manage_delObjects(ids=list(self.getObjectIds()))
 
   security.declareProtected(Permissions.AccessContentsInformation,
                             'getConflictList')

product/ERP5SyncML/Document/SyncMLSubscription.py

@@ -403,7 +403,6 @@ class SyncMLSubscription(XMLObject):
     for i in xrange(0, object_list_len, MAX_OBJECTS):
       current_id_list = object_id_list[i:i+MAX_OBJECTS]
       self.activate(activity='SQLQueue',
-                    tag=self.getId(),
                     priority=ACTIVITY_PRIORITY).manage_delObjects(current_id_list)
 
   security.declareProtected(Permissions.AccessContentsInformation,
@@ -449,7 +448,7 @@ class SyncMLSubscription(XMLObject):
                               'conflict_resolved_with_merge',
                               'conflict_resolved_with_client_command_winning'):
         if self.getIsActivityEnabled():
-          signature.activate(tag=self.getId(), activity='SQLQueue',
+          signature.activate(activity='SQLQueue',
                              priority=ACTIVITY_PRIORITY).reset()
         else:
           signature.reset()