Skip to content

Z
Zope
Commit 4a9b8497 authored by Tres Seaver's avatar Tres Seaver
Browse files
Options

parent c64741f4
Hide whitespace changes
Inline Side-by-side
Showing with 0 additions and 624 deletions
lib/python/SearchIndex/tests/testSplitter.py deleted 100644 → 0
View file @ c64741f4
##############################################################################
#
# Copyright (c) 2001 Zope Corporation and Contributors. All Rights Reserved.
#
# This software is subject to the provisions of the Zope Public License,
# Version 2.0 (ZPL). A copy of the ZPL should accompany this distribution.
# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
# FOR A PARTICULAR PURPOSE
#
##############################################################################
import unittest
from SearchIndex.Splitter import Splitter
class Tests(unittest.TestCase):
def testSplitNormalText(self):
text = 'this is a long string of words'
a = Splitter(text)
r = map(None, a)
assert r == ['this', 'is', 'long', 'string', 'of', 'words']
def testDropNumeric(self):
text = '123 456 789 foobar without you nothing'
a = Splitter(text)
r = map(None, a)
assert r == ['foobar', 'without', 'you', 'nothing'], r
def testDropSingleLetterWords(self):
text = 'without you I nothing'
a = Splitter(text)
r = map(None, a)
assert r == ['without', 'you', 'nothing'], r
def testSplitOnNonAlpha(self):
text = 'without you I\'m nothing'
a = Splitter(text)
r = map(None, a)
assert r == ['without', 'you', 'nothing'], r
def test_suite():
return unittest.TestSuite((
unittest.makeSuite(Tests),
))
if __name__=='__main__':
unittest.main(defaultTest='test_suite')
lib/python/SearchIndex/tests/testUnKeywordIndex.py deleted 100644 → 0
View file @ c64741f4
##############################################################################
#
# Copyright (c) 2001 Zope Corporation and Contributors. All Rights Reserved.
#
# This software is subject to the provisions of the Zope Public License,
# Version 2.0 (ZPL). A copy of the ZPL should accompany this distribution.
# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
# FOR A PARTICULAR PURPOSE
#
##############################################################################
import unittest
import ZODB
from SearchIndex.UnKeywordIndex import UnKeywordIndex
class Dummy:
def __init__( self, foo ):
self._foo = foo
def foo( self ):
return self._foo
def __str__( self ):
return '<Dummy: %s>' % self._foo
__repr__ = __str__
class Tests( unittest.TestCase ):
"""
Test KeywordIndex objects.
"""
def setUp( self ):
"""
"""
self._index = UnKeywordIndex( 'foo' )
self._marker = []
self._values = [ ( 0, Dummy( ['a'] ) )
, ( 1, Dummy( ['a','b'] ) )
, ( 2, Dummy( ['a','b','c'] ) )
, ( 3, Dummy( ['a','b','c', 'a'] ) )
, ( 4, Dummy( ['a', 'b', 'c', 'd'] ) )
, ( 5, Dummy( ['a', 'b', 'c', 'e'] ) )
, ( 6, Dummy( ['a', 'b', 'c', 'e', 'f'] ))
, ( 7, Dummy( [0] ) )
]
self._noop_req = { 'bar': 123 }
self._all_req = { 'foo': ['a'] }
self._some_req = { 'foo': ['e'] }
self._overlap_req = { 'foo': ['c', 'e'] }
self._string_req = {'foo': 'a'}
self._zero_req = { 'foo': [0] }
def tearDown( self ):
"""
"""
def _populateIndex( self ):
for k, v in self._values:
self._index.index_object( k, v )
def _checkApply( self, req, expectedValues ):
result, used = self._index._apply_index( req )
assert used == ( 'foo', )
assert len(result) == len( expectedValues ), \
'%s | %s' % ( map( None, result ),
map(lambda x: x[0], expectedValues ))
if hasattr(result, 'keys'): result=result.keys()
for k, v in expectedValues:
assert k in result
def testAddObjectWOKeywords(self):
import zLOG
def log_write(subsystem, severity, summary, detail, error,
PROBLEM=zLOG.PROBLEM):
if severity >= PROBLEM:
assert 0, "%s(%s): %s" % (subsystem, severity, summary)
old_log_write=zLOG.log_write
zLOG.log_write=log_write
try:
self._populateIndex()
self._index.index_object(999, None)
finally:
zLOG.log_write=old_log_write
def testEmpty( self ):
assert len( self._index ) == 0
assert len( self._index.referencedObjects() ) == 0
assert self._index.getEntryForObject( 1234 ) is None
assert ( self._index.getEntryForObject( 1234, self._marker )
is self._marker ), self._index.getEntryForObject(1234)
self._index.unindex_object( 1234 ) # nothrow
assert self._index.hasUniqueValuesFor( 'foo' )
assert not self._index.hasUniqueValuesFor( 'bar' )
assert len( self._index.uniqueValues( 'foo' ) ) == 0
assert self._index._apply_index( self._noop_req ) is None
self._checkApply( self._all_req, [] )
self._checkApply( self._some_req, [] )
self._checkApply( self._overlap_req, [] )
self._checkApply( self._string_req, [] )
def testPopulated( self ):
self._populateIndex()
values = self._values
#assert len( self._index ) == len( values )
assert len( self._index.referencedObjects() ) == len( values )
assert self._index.getEntryForObject( 1234 ) is None
assert ( self._index.getEntryForObject( 1234, self._marker )
is self._marker )
self._index.unindex_object( 1234 ) # nothrow
for k, v in values:
assert self._index.getEntryForObject( k ) == v.foo()
assert (len( self._index.uniqueValues( 'foo' ) ) == len( values )-1,
len(values)-1)
assert self._index._apply_index( self._noop_req ) is None
self._checkApply( self._all_req, values[:-1])
self._checkApply( self._some_req, values[ 5:7 ] )
self._checkApply( self._overlap_req, values[2:7] )
self._checkApply( self._string_req, values[:-1] )
def testZero( self ):
self._populateIndex()
values = self._values
self._checkApply( self._zero_req, values[ -1: ] )
assert 0 in self._index.uniqueValues( 'foo' )
def testReindexChange(self):
self._populateIndex()
expected = Dummy(['x', 'y'])
self._index.index_object(6, expected)
result, used = self._index._apply_index({'foo': ['x', 'y']})
result=result.keys()
assert len(result) == 1
assert result[0] == 6
result, used = self._index._apply_index(
{'foo': ['a', 'b', 'c', 'e', 'f']}
)
result = result.keys()
assert 6 not in result
def testReindexNoChange(self):
self._populateIndex()
expected = Dummy(['foo', 'bar'])
self._index.index_object(8, expected)
result, used = self._index._apply_index(
{'foo': ['foo', 'bar']})
result = result.keys()
assert len(result) == 1
assert result[0] == 8
self._index.index_object(8, expected)
result, used = self._index._apply_index(
{'foo': ['foo', 'bar']})
result = result.keys()
assert len(result) == 1
assert result[0] == 8
def test_suite():
return unittest.TestSuite((
unittest.makeSuite(Tests),
))
if __name__=='__main__':
unittest.main(defaultTest='test_suite')
lib/python/SearchIndex/tests/testUnTextIndex.py deleted 100644 → 0
View file @ c64741f4
##############################################################################
#
# Copyright (c) 2001 Zope Corporation and Contributors. All Rights Reserved.
#
# This software is subject to the provisions of the Zope Public License,
# Version 2.0 (ZPL). A copy of the ZPL should accompany this distribution.
# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
# FOR A PARTICULAR PURPOSE
#
##############################################################################
import unittest
from Testing.ZODButil import makeDB, cleanDB
import SearchIndex.UnTextIndex
import SearchIndex.GlobbingLexicon
class Dummy:
def __init__( self, text ):
self.text = text
class Tests(unittest.TestCase):
def setUp(self):
self.index=SearchIndex.UnTextIndex.UnTextIndex('text')
self.doc=Dummy(text='this is the time, when all good zopes')
def dbopen(self):
db = self.db = makeDB()
self.jar=db.open()
if not self.jar.root().has_key('index'):
self.jar.root()['index']=SearchIndex.UnTextIndex.UnTextIndex('text')
get_transaction().commit()
return self.jar.root()['index']
def dbclose(self):
self.jar.close()
self.db.close()
del self.jar
del self.db
def tearDown(self):
get_transaction().abort()
if hasattr(self, 'jar'):
self.dbclose()
cleanDB()
self.__dict__.clear()
def testSimpleAddDelete(self):
"Test that we can add and delete an object without error"
self.index.index_object(0, self.doc)
self.index.index_object(1, self.doc)
self.doc.text='spam is good, spam is fine, span span span'
self.index.index_object(0, self.doc)
self.index.unindex_object(0)
def testPersistentUpdate1(self):
"Test simple persistent indexing"
index=self.dbopen()
self.doc.text='this is the time, when all good zopes'
index.index_object(0, self.doc)
get_transaction().commit()
self.doc.text='time waits for no one'
index.index_object(1, self.doc)
get_transaction().commit()
self.dbclose()
index=self.dbopen()
r = index._apply_index({})
assert r==None
r = index._apply_index({'text': 'python'})
assert len(r) == 2 and r[1]==('text',), 'incorrectly not used'
assert not r[0], "should have no results"
r = index._apply_index({'text': 'time'})
r=list(r[0].keys())
assert r == [0,1], r
def testPersistentUpdate2(self):
"Test less simple persistent indexing"
index=self.dbopen()
self.doc.text='this is the time, when all good zopes'
index.index_object(0, self.doc)
get_transaction().commit()
self.doc.text='time waits for no one'
index.index_object(1, self.doc)
get_transaction().commit()
self.doc.text='the next task is to test'
index.index_object(3, self.doc)
get_transaction().commit()
self.doc.text='time time'
index.index_object(2, self.doc)
get_transaction().commit()
self.dbclose()
index=self.dbopen()
r = index._apply_index({})
assert r==None
r = index._apply_index({'text': 'python'})
assert len(r) == 2 and r[1]==('text',), 'incorrectly not used'
assert not r[0], "should have no results"
r = index._apply_index({'text': 'time'})
r=list(r[0].keys())
assert r == [0,1,2], r
sample_texts = [
"""This is the time for all good men to come to
the aid of their country""",
"""ask not what your country can do for you,
ask what you can do for your country""",
"""Man, I can't wait to get to Montross!""",
"""Zope Public License (ZPL) Version 1.0""",
"""Copyright (c) Digital Creations. All rights reserved.""",
"""This license has been certified as Open Source(tm).""",
"""I hope I get to work on time""",
]
def globTest(self, qmap, rlist):
"Test a glob query"
index = getattr(self, '_v_index', None)
if index is None:
index=self.dbopen()
index._lexicon = SearchIndex.GlobbingLexicon.GlobbingLexicon()
for i in range(len(self.sample_texts)):
self.doc.text=self.sample_texts[i]
index.index_object(i, self.doc)
get_transaction().commit()
self.dbclose()
index = self._v_index = self.dbopen()
r = list(index._apply_index(qmap)[0].keys())
assert r == rlist, r
def testStarQuery(self):
"Test a star query"
self.globTest({'text':'m*n'}, [0,2])
def testAndQuery(self):
"Test an AND query"
self.globTest({'text':'time and country'}, [0,])
def testOrQuery(self):
"Test an OR query"
self.globTest({'text':'time or country'}, [0,1,6])
def testDefOrQuery(self):
"Test a default OR query"
self.globTest({'text':'time country'}, [0,1,6])
self.globTest({'text':'time good country'}, [0,1,6])
def testNearQuery(self):
"""Test a NEAR query.. (NOTE:ACTUALLY AN 'AND' TEST!!)"""
# NEAR never worked, so Zopes post-2.3.1b3 define near to mean AND
self.globTest({'text':'time ... country'}, [0,])
def testQuotesQuery(self):
"""Test a quoted query"""
self.globTest({'text':'"This is the time"'}, [0,])
self.globTest({'text':'"now is the time"'}, [])
def testAndNotQuery(self):
"Test an ANDNOT query"
self.globTest({'text':'time and not country'}, [6,])
def testParenMatchingQuery(self):
"Test a query with parens"
self.globTest({'text':'(time and country) men'}, [0,])
self.globTest({'text':'(time and not country) or men'}, [0, 6])
def testTextIndexOperatorQuery(self):
"Test a query with 'textindex_operator' in the request"
self.globTest({'text':'time men', 'textindex_operator':'and'}, [0,])
def testNonExistentWord(self):
""" Test for nonexistent word """
self.globTest({'text':'zop'}, [])
def testShortWord(self):
""" Test for short word """
self.globTest({'text':'to'}, [0, 2, 6])
self.globTest({'text':'*to'}, [0, 2, 6])
self.globTest({'text':'to*'}, [0, 2, 6])
self.globTest({'text':'*to*'}, [0, 2, 6])
def testComplexQuery1(self):
""" Test complex query 1 """
self.globTest({'text':'((?ount* or get) and not wait) '
'"been *ert*"'}, [0, 1, 5, 6])
def test_suite():
return unittest.TestSuite((
unittest.makeSuite(Tests),
))
if __name__=='__main__':
unittest.main(defaultTest='test_suite')
lib/python/SearchIndex/tests/test_UnIndex.py deleted 100644 → 0
View file @ c64741f4
##############################################################################
#
# Copyright (c) 2001 Zope Corporation and Contributors. All Rights Reserved.
#
# This software is subject to the provisions of the Zope Public License,
# Version 2.0 (ZPL). A copy of the ZPL should accompany this distribution.
# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
# FOR A PARTICULAR PURPOSE
#
##############################################################################
import unittest
import ZODB
from SearchIndex.UnIndex import UnIndex
class Dummy:
def __init__( self, foo ):
self._foo = foo
def foo( self ):
return self._foo
def __str__( self ):
return '<Dummy: %s>' % self._foo
__repr__ = __str__
class Tests( unittest.TestCase ):
"""
Test FieldIndex objects.
"""
def setUp( self ):
"""
"""
self._index = UnIndex( 'foo' )
self._marker = []
self._values = [ ( 0, Dummy( 'a' ) )
, ( 1, Dummy( 'ab' ) )
, ( 2, Dummy( 'abc' ) )
, ( 3, Dummy( 'abca' ) )
, ( 4, Dummy( 'abcd' ) )
, ( 5, Dummy( 'abce' ) )
, ( 6, Dummy( 'abce' ) )
, ( 7, Dummy( 0 ) ) # Collector #1959
, ( 8, Dummy(None) )]
self._forward = {}
self._backward = {}
for k, v in self._values:
self._backward[k] = v
keys = self._forward.get( v, [] )
self._forward[v] = keys
self._noop_req = { 'bar': 123 }
self._request = { 'foo': 'abce' }
self._min_req = { 'foo': 'abc'
, 'foo_usage': 'range:min'
}
self._max_req = { 'foo': 'abc'
, 'foo_usage': 'range:max'
}
self._range_req = { 'foo': ( 'abc', 'abcd' )
, 'foo_usage': 'range:min:max'
}
self._zero_req = { 'foo': 0 }
self._none_req = { 'foo': None }
def tearDown( self ):
"""
"""
def _populateIndex( self ):
for k, v in self._values:
self._index.index_object( k, v )
def _checkApply( self, req, expectedValues ):
result, used = self._index._apply_index( req )
if hasattr(result, 'keys'):
result = result.keys()
assert used == ( 'foo', )
assert len( result ) == len( expectedValues ), \
'%s | %s' % ( map( None, result ), expectedValues )
for k, v in expectedValues:
assert k in result
def testEmpty( self ):
"Test an empty FieldIndex."
assert len( self._index ) == 0
assert len( self._index.referencedObjects() ) == 0
assert self._index.getEntryForObject( 1234 ) is None
assert ( self._index.getEntryForObject( 1234, self._marker )
is self._marker )
self._index.unindex_object( 1234 ) # nothrow
assert self._index.hasUniqueValuesFor( 'foo' )
assert not self._index.hasUniqueValuesFor( 'bar' )
assert len( self._index.uniqueValues( 'foo' ) ) == 0
assert self._index._apply_index( self._noop_req ) is None
self._checkApply( self._request, [] )
self._checkApply( self._min_req, [] )
self._checkApply( self._max_req, [] )
self._checkApply( self._range_req, [] )
def testPopulated( self ):
""" Test a populated FieldIndex """
self._populateIndex()
values = self._values
assert len( self._index ) == len( values )-1 #'abce' is duplicate
assert len( self._index.referencedObjects() ) == len( values )
assert self._index.getEntryForObject( 1234 ) is None
assert ( self._index.getEntryForObject( 1234, self._marker )
is self._marker )
self._index.unindex_object( 1234 ) # nothrow
for k, v in values:
assert self._index.getEntryForObject( k ) == v.foo()
assert len( self._index.uniqueValues( 'foo' ) ) == len( values )-1
assert self._index._apply_index( self._noop_req ) is None
self._checkApply( self._request, values[ -4:-2 ] )
self._checkApply( self._min_req, values[ 2:-2 ] )
self._checkApply( self._max_req, values[ :3 ] + values[ -2: ] )
self._checkApply( self._range_req, values[ 2:5 ] )
def testZero( self ):
""" Make sure 0 gets indexed """
self._populateIndex()
values = self._values
self._checkApply( self._zero_req, values[ -2:-1 ] )
assert 0 in self._index.uniqueValues( 'foo' )
def testNone(self):
""" make sure None gets indexed """
self._populateIndex()
values = self._values
self._checkApply(self._none_req, values[-1:])
assert None in self._index.uniqueValues('foo')
def testRange(self):
"""Test a range search"""
index = UnIndex( 'foo' )
for i in range(100):
index.index_object(i, Dummy(i%10))
r=index._apply_index({
'foo_usage': 'range:min:max',
'foo': [-99, 3]})
assert tuple(r[1])==('foo',), r[1]
r=list(r[0].keys())
expect=[
0, 1, 2, 3, 10, 11, 12, 13, 20, 21, 22, 23, 30, 31, 32, 33,
40, 41, 42, 43, 50, 51, 52, 53, 60, 61, 62, 63, 70, 71, 72, 73,
80, 81, 82, 83, 90, 91, 92, 93
]
assert r==expect, r
def test_suite():
return unittest.TestSuite((
unittest.makeSuite(Tests),
))
if __name__=='__main__':
unittest.main(defaultTest='test_suite')
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment
GitLab Nexedi Edition | About GitLab | About Nexedi | 沪ICP备2021021310号-2 | 沪ICP备2021021310号-7