added regression test to test ZCatalogs unicode functionality

lib/python/Products/ZCatalog/regressiontests/framework.py

+##############################################################################
+# 
+# Zope Public License (ZPL) Version 1.0
+# -------------------------------------
+# 
+# Copyright (c) Digital Creations.  All rights reserved.
+# 
+# This license has been certified as Open Source(tm).
+# 
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met:
+# 
+# 1. Redistributions in source code must retain the above copyright
+#    notice, this list of conditions, and the following disclaimer.
+# 
+# 2. Redistributions in binary form must reproduce the above copyright
+#    notice, this list of conditions, and the following disclaimer in
+#    the documentation and/or other materials provided with the
+#    distribution.
+# 
+# 3. Digital Creations requests that attribution be given to Zope
+#    in any manner possible. Zope includes a "Powered by Zope"
+#    button that is installed by default. While it is not a license
+#    violation to remove this button, it is requested that the
+#    attribution remain. A significant investment has been put
+#    into Zope, and this effort will continue if the Zope community
+#    continues to grow. This is one way to assure that growth.
+# 
+# 4. All advertising materials and documentation mentioning
+#    features derived from or use of this software must display
+#    the following acknowledgement:
+# 
+#      "This product includes software developed by Digital Creations
+#      for use in the Z Object Publishing Environment
+#      (http://www.zope.org/)."
+# 
+#    In the event that the product being advertised includes an
+#    intact Zope distribution (with copyright and license included)
+#    then this clause is waived.
+# 
+# 5. Names associated with Zope or Digital Creations must not be used to
+#    endorse or promote products derived from this software without
+#    prior written permission from Digital Creations.
+# 
+# 6. Modified redistributions of any form whatsoever must retain
+#    the following acknowledgment:
+# 
+#      "This product includes software developed by Digital Creations
+#      for use in the Z Object Publishing Environment
+#      (http://www.zope.org/)."
+# 
+#    Intact (re-)distributions of any official Zope release do not
+#    require an external acknowledgement.
+# 
+# 7. Modifications are encouraged but must be packaged separately as
+#    patches to official Zope releases.  Distributions that do not
+#    clearly separate the patches from the original work must be clearly
+#    labeled as unofficial distributions.  Modifications which do not
+#    carry the name Zope may be packaged in any form, as long as they
+#    conform to all of the clauses above.
+# 
+# 
+# Disclaimer
+# 
+#   THIS SOFTWARE IS PROVIDED BY DIGITAL CREATIONS ``AS IS'' AND ANY
+#   EXPRESSED OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+#   IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+#   PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL DIGITAL CREATIONS OR ITS
+#   CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+#   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+#   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF
+#   USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+#   ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+#   OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
+#   OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+#   SUCH DAMAGE.
+# 
+# 
+# This software consists of contributions made by Digital Creations and
+# many individuals on behalf of Digital Creations.  Specific
+# attributions are listed in the accompanying credits file.
+# 
+##############################################################################
+
+######################################################################
+# Set up unit testing framework
+#
+# The following code should be at the top of every test module:
+#
+# import os, sys
+# execfile(os.path.join(sys.path[0], 'framework.py'))
+#
+# ...and the following at the bottom:
+#
+# framework()
+
+
+# Find the Testing package
+if not sys.modules.has_key('Testing'):
+    p0 = sys.path[0]
+    if p0 and __name__ == '__main__':
+        os.chdir(p0)
+        p0 = ''
+    p = d = os.path.abspath(os.curdir)
+    while d:
+        if os.path.isdir(os.path.join(p, 'Testing')):
+            sys.path[:1] = [p0, os.pardir, p]
+            break
+        p, d = os.path.split(p)
+    else:
+        print 'Unable to locate Testing package.'
+        sys.exit(1)
+
+import Testing, unittest
+execfile(os.path.join(os.path.split(Testing.__file__)[0], 'common.py'))
+
+

lib/python/Products/ZCatalog/regressiontests/regressionUnicode.py

+import os,sys
+execfile(os.path.join(sys.path[0],'framework.py'))
+import unittest
+
+import Zope
+from Products.ZCatalog.ZCatalog import ZCatalog
+
+
+from Products.PluginIndexes.TextIndex import Splitter
+
+# This patch pretends the ZCatalog is using the Unicode Splitter
+# but by default the ZCatalog/TextIndexes uses the standard
+# non-unicode-aware ZopeSplitter
+
+Splitter.availableSplitters =    [ ("UnicodeSplitter" , "Unicode-aware splitter")]
+Splitter.splitterNames =    [ "UnicodeSplitter" ]
+
+
+
+class TO:
+
+    def __init__(self,txt,kw=''):
+        self.text = txt
+        self.kw   = kw
+
+
+class UnicodeTextIndexCatalogTest(unittest.TestCase):
+
+    def setUp(self):
+
+        self.cat = ZCatalog("catalog")
+        self.cat.addIndex('text',"TextIndex")
+        self.cat.addColumn('text')
+        self.cat.addIndex('kw','KeywordIndex')
+        self.cat.addColumn('kw')
+
+        t1 = TO('the quick brown fox jumps over the lazy dog',['quick','fox'])
+        t2 = TO('i am the nice alien from the future',['alien','future'])
+        t3 = TO('i am a brown fox dancing with a future alien',['zerstrt','knnten'])
+        t4 = TO('i am a brown ' + unicode('fox') + ' dancing with a future alien',[])
+        t5 = TO("""
+        Die USA und Grobritannien knnen nach der Zerstrung der
+        afghanischen Luftabwehr nun rund um die Uhr Angriffe fliegen. Das gab 
+        Verteidigungsminister Donald Rumsfeld bekannt. Bei den dreitgigen Angriffen
+        seien auch bis auf einen alle Flugpltze der Taliban zerstrt worden. Rumsfeld
+        erklrte weiter, er knne die Berichte nicht besttigen, wonach bei den 
+        amerikanischen Angriffen vier afghanische Mitarbeiter einer von den UN
+        finanzierten Hilfsorganisation gettet wurden. Diese knnten auch durch
+        Gegenfeuer der Taliban gettet worden sein.
+        """,[unicode('dreitgigen','latin1')])
+                                                                                                            
+
+        self.cat.catalog_object(t1,"o1")
+        self.cat.catalog_object(t2,"o2")
+        self.cat.catalog_object(t3,"o3")
+        self.cat.catalog_object(t4,"o4")
+        self.cat.catalog_object(t5,"o5")
+
+        self.tests = [('quick',('o1',)),
+              ('fox',('o1','o2','o3','o4')),
+              ('afghanischen', ('o5',)),
+              ('dreitgigen',('o5',))
+            ]
+
+        
+        self.kw_tests = [ ('quick',('o1',) ),
+                          ('zerstrt',('o3',)),
+                          ('dreitgigen',('o5',))
+                        ]
+
+
+    def testAsciiQuery(self):
+        """ simple query test """
+
+        for q,objs in self.tests:
+            res=self.cat.searchResults({'text':{'query':q}})
+
+            for r in res:
+                assert r.getURL() in objs,\
+                    "%s: %s vs %s" % (q,str(r.getURL()),str(objs)) 
+
+
+    def testUnicodeQuery(self):
+        """ unicode query test """
+
+        for q,objs in self.tests:
+            res=self.cat.searchResults({'text':{'query':unicode(q,'latin1')}})
+
+            for r in res:
+                assert r.getURL() in objs, \
+                    "%s: %s vs %s" % (q,str(r.getURL()),str(objs)) 
+
+
+    def testAsciiKeywords(self):
+        """ test keyword index """
+
+        for q,objs in self.kw_tests:
+            res=self.cat.searchResults({'kw':{'query':q}})
+
+            for r in res:
+                assert r.getURL() in objs, \
+                    "%s: %s vs %s" % (q,str(r.getURL()),str(objs)) 
+
+
+    def testUnicodeKeywords(self):
+        """ test unicode keyword index """
+
+        for q,objs in self.kw_tests:
+            res=self.cat.searchResults({'kw':{'query':unicode(q,'latin1')}})
+
+            for r in res:
+                assert r.getURL() in objs, \
+                    "%s: %s vs %s" % (q,str(r.getURL()),str(objs)) 
+
+
+framework()
+