LP #246983: Unicode conflict resolution on variables inside 'string:' expressions

doc/CHANGES.rst

@@ -11,6 +11,9 @@ Zope 2.12.3 (unreleased)
 Bugs Fixed
 ++++++++++
 
+- LP #246983: Enabled unicode conflict resolution on variables inside "string:"
+  expressions in TALES.
+
 - Fixed possible TypeError while sending multipart emails.
 
 - Also look for ZEXP imports within the clienthome directory. This

src/Products/Five/browser/tests/test_pagetemplatefile.py

@@ -37,13 +37,13 @@ class ViewPageTemplateFileTests(unittest.TestCase):
         from zope.tales.expressions import DeferExpr
         from zope.tales.expressions import NotExpr
         from zope.tales.expressions import PathExpr
-        from zope.tales.expressions import StringExpr
         from zope.tales.expressions import Undefs
         from zope.tales.pythonexpr import PythonExpr
         from zope.contentprovider.tales import TALESProviderExpression
         from Products.PageTemplates.DeferExpr import LazyExpr
         from Products.PageTemplates.Expressions import TrustedZopePathExpr
         from Products.PageTemplates.Expressions import SecureModuleImporter
+        from Products.PageTemplates.Expressions import UnicodeAwareStringExpr
 
         vptf = self._makeOne('seagull.pt')
         engine = vptf.pt_getEngine()
@@ -51,7 +51,7 @@ class ViewPageTemplateFileTests(unittest.TestCase):
         self.assertEqual(engine.types['path'], TrustedZopePathExpr)
         self.assertEqual(engine.types['exists'], TrustedZopePathExpr)
         self.assertEqual(engine.types['nocall'], TrustedZopePathExpr)
-        self.assertEqual(engine.types['string'], StringExpr)
+        self.assertEqual(engine.types['string'], UnicodeAwareStringExpr)
         self.assertEqual(engine.types['python'], PythonExpr)
         self.assertEqual(engine.types['not'], NotExpr)
         self.assertEqual(engine.types['defer'], DeferExpr)

src/Products/PageTemplates/Expressions.py

@@ -372,12 +372,26 @@ class PathIterator(ZopeIterator):
             return False
         return ob1 == ob2
 
+class UnicodeAwareStringExpr(StringExpr):
+
+    def __call__(self, econtext):
+        vvals = []
+        if isinstance(self._expr, unicode):
+            # coerce values through the Unicode Conflict Resolver
+            evaluate = econtext.evaluateText
+        else:
+            evaluate = econtext.evaluate
+        for var in self._vars:
+            v = evaluate(var)
+            vvals.append(v)
+        return self._expr % tuple(vvals)
+
 def createZopeEngine(zpe=ZopePathExpr):
     e = ZopeEngine()
     e.iteratorFactory = PathIterator
     for pt in zpe._default_type_names:
         e.registerType(pt, zpe)
-    e.registerType('string', StringExpr)
+    e.registerType('string', UnicodeAwareStringExpr)
     e.registerType('python', ZRPythonExpr.PythonExpr)
     e.registerType('not', NotExpr)
     e.registerType('defer', DeferExpr)

src/Products/PageTemplates/tests/testExpressions.py

@@ -25,12 +25,20 @@ class EngineTestsBase(PlacelessSetup):
             __allow_access_to_unprotected_subobjects__ = 1
             def __call__(self):
                 return 'dummy'
+            
+            management_page_charset = 'iso-8859-15'
 
         class DummyDocumentTemplate:
             __allow_access_to_unprotected_subobjects__ = 1
             isDocTemp = True
             def __call__(self, client=None, REQUEST={}, RESPONSE=None, **kw):
                 return 'dummy'
+            
+            def absolute_url(self, relative=0):
+                url = 'dummy'
+                if not relative:
+                    url = "http://server/" + url
+                return url
 
         _DEFAULT_BINDINGS = dict(
             one = 1,
@@ -38,6 +46,12 @@ class EngineTestsBase(PlacelessSetup):
             blank = '',
             dummy = Dummy(),
             dummy2 = DummyDocumentTemplate(),
+            eightbit = '',
+            # ZopeContext needs 'context' and 'template' keys for unicode
+            # conflict resolution, and 'context' needs a 
+            # 'management_page_charset'
+            context = Dummy(),
+            template = DummyDocumentTemplate(),
             )
 
         if bindings is None:
@@ -158,6 +172,34 @@ class EngineTestsBase(PlacelessSetup):
         ec = self._makeContext()
         self.assertEquals(ec.evaluate('  \n'), None)
 
+    def test_unicode(self):
+        # All our string expressions are unicode now
+        eng = self._makeEngine()
+        ec = self._makeContext()
+        # XXX: can't do ec.evaluate(u'string:x') directly because ZopeContext
+        # only bothers compiling true strings, not unicode strings
+        result = ec.evaluate(eng.compile(u'string:x'))
+        self.assertEqual(result, u'x')
+        self.failUnless(isinstance(result, unicode))
+
+    def test_mixed(self):
+        # 8-bit strings in unicode string expressions cause UnicodeDecodeErrors
+        eng = self._makeEngine()
+        ec = self._makeContext()
+        expr = eng.compile(u'string:$eightbit')
+        self.assertRaises(UnicodeDecodeError,
+                          ec.evaluate, expr)
+        # But registering an appropriate IUnicodeEncodingConflictResolver
+        # should fix it
+        from zope.component import provideUtility
+        from Products.PageTemplates.unicodeconflictresolver \
+            import StrictUnicodeEncodingConflictResolver
+        from Products.PageTemplates.interfaces \
+            import IUnicodeEncodingConflictResolver
+        provideUtility(StrictUnicodeEncodingConflictResolver, 
+                                      IUnicodeEncodingConflictResolver)        
+        self.assertEqual(ec.evaluate(expr), u'')
+
 class UntrustedEngineTests(EngineTestsBase, unittest.TestCase):
 
     def _makeEngine(self):

src/Products/PageTemplates/tests/testZopePageTemplate.py

@@ -202,6 +202,26 @@ class ZPTUnicodeEncodingConflictResolution(ZopeTestCase):
         state = cPickle.dumps(empty, protocol=1)
         clone = cPickle.loads(state)
 
+    def testBug246983(self):
+        # See https://bugs.launchpad.net/bugs/246983
+        self.app.REQUEST.set('HTTP_ACCEPT_CHARSET', 'utf-8')
+        self.app.REQUEST.set('data', u''.encode('utf-8'))
+        # Direct inclusion of encoded strings is hadled normally by the unicode
+        # conflict resolver
+        textDirect = """
+        <tal:block content="request/data" />
+        """.strip()
+        manage_addPageTemplate(self.app, 'test', text=textDirect)
+        zpt = self.app['test']
+        self.assertEquals(zpt.pt_render(), u'')
+        # Indirect inclusion of encoded strings through String Expressions
+        # should be resolved as well.
+        textIndirect = """
+        <tal:block content="string:x ${request/data}" />
+        """.strip()
+        zpt.pt_edit(textIndirect, zpt.content_type)
+        self.assertEquals(zpt.pt_render(), u'x ')
+
     def testDebugFlags(self):
         # Test for bug 229549
         manage_addPageTemplate(self.app, 'test',