Commit 76ac35fa authored by Guido van Rossum's avatar Guido van Rossum

Add PersistentList.py and fsIndex.py and their tests (merge)

parent 09d96587
##############################################################################
#
# Copyright (c) 2001, 2002 Zope Corporation and Contributors.
# All Rights Reserved.
#
# This software is subject to the provisions of the Zope Public License,
# Version 2.0 (ZPL). A copy of the ZPL should accompany this distribution.
# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
# FOR A PARTICULAR PURPOSE
#
##############################################################################
"""Python implementation of persistent list.
$Id: PersistentList.py,v 1.2 2002/02/11 23:49:07 gvanrossum Exp $"""
__version__='$Revision: 1.2 $'[11:-2]
import Persistence
from UserList import UserList
class PersistentList(UserList, Persistence.Persistent):
__super_setitem = UserList.__setitem__
__super_delitem = UserList.__delitem__
__super_setslice = UserList.__setslice__
__super_delslice = UserList.__delslice__
__super_iadd = UserList.__iadd__
__super_imul = UserList.__imul__
__super_append = UserList.append
__super_insert = UserList.insert
__super_pop = UserList.pop
__super_remove = UserList.remove
__super_reverse = UserList.reverse
__super_sort = UserList.sort
__super_extend = UserList.extend
def __setitem__(self, i, item):
self.__super_setitem(i, item)
self._p_changed = 1
def __delitem__(self, i):
self.__super_delitem(i)
self._p_changed = 1
def __setslice__(self, i, j, other):
self.__super_setslice(i, j, other)
self._p_changed = 1
def __delslice__(self, i, j):
self.__super_delslice(i, j)
self._p_changed = 1
def __iadd__(self, other):
self.__super_iadd(other)
self._p_changed = 1
def __imul__(self, n):
self.__super_imul(n)
self._p_changed = 1
def append(self, item):
self.__super_append(item)
self._p_changed = 1
def insert(self, i, item):
self.__super_insert(i, item)
self._p_changed = 1
def pop(self, i=-1):
rtn = self.__super_pop(i)
self._p_changed = 1
return rtn
def remove(self, item):
self.__super_remove(item)
self._p_changed = 1
def reverse(self):
self.__super_reverse()
self._p_changed = 1
def sort(self, *args):
self.__super_sort(*args)
self._p_changed = 1
def extend(self, other):
self.__super_extend(other)
self._p_changed = 1
# This works around a bug in Python 2.1.x (up to 2.1.2 at least) where the
# __cmp__ bogusly raises a RuntimeError, and because this is an extension
# class, none of the rich comparison stuff works anyway.
def __cmp__(self, other):
return cmp(self.data, self._UserList__cast(other))
##############################################################################
#
# Copyright (c) 2001, 2002 Zope Corporation and Contributors.
# All Rights Reserved.
#
# This software is subject to the provisions of the Zope Public License,
# Version 2.0 (ZPL). A copy of the ZPL should accompany this distribution.
# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
# FOR A PARTICULAR PURPOSE
#
##############################################################################
"""Python implementation of persistent list.
$Id: PersistentList.py,v 1.2 2002/02/11 23:49:07 gvanrossum Exp $"""
__version__='$Revision: 1.2 $'[11:-2]
import Persistence
from UserList import UserList
class PersistentList(UserList, Persistence.Persistent):
__super_setitem = UserList.__setitem__
__super_delitem = UserList.__delitem__
__super_setslice = UserList.__setslice__
__super_delslice = UserList.__delslice__
__super_iadd = UserList.__iadd__
__super_imul = UserList.__imul__
__super_append = UserList.append
__super_insert = UserList.insert
__super_pop = UserList.pop
__super_remove = UserList.remove
__super_reverse = UserList.reverse
__super_sort = UserList.sort
__super_extend = UserList.extend
def __setitem__(self, i, item):
self.__super_setitem(i, item)
self._p_changed = 1
def __delitem__(self, i):
self.__super_delitem(i)
self._p_changed = 1
def __setslice__(self, i, j, other):
self.__super_setslice(i, j, other)
self._p_changed = 1
def __delslice__(self, i, j):
self.__super_delslice(i, j)
self._p_changed = 1
def __iadd__(self, other):
self.__super_iadd(other)
self._p_changed = 1
def __imul__(self, n):
self.__super_imul(n)
self._p_changed = 1
def append(self, item):
self.__super_append(item)
self._p_changed = 1
def insert(self, i, item):
self.__super_insert(i, item)
self._p_changed = 1
def pop(self, i=-1):
rtn = self.__super_pop(i)
self._p_changed = 1
return rtn
def remove(self, item):
self.__super_remove(item)
self._p_changed = 1
def reverse(self):
self.__super_reverse()
self._p_changed = 1
def sort(self, *args):
self.__super_sort(*args)
self._p_changed = 1
def extend(self, other):
self.__super_extend(other)
self._p_changed = 1
# This works around a bug in Python 2.1.x (up to 2.1.2 at least) where the
# __cmp__ bogusly raises a RuntimeError, and because this is an extension
# class, none of the rich comparison stuff works anyway.
def __cmp__(self, other):
return cmp(self.data, self._UserList__cast(other))
##############################################################################
#
# Copyright (c) 2001, 2002 Zope Corporation and Contributors.
# All Rights Reserved.
#
# This software is subject to the provisions of the Zope Public License,
# Version 2.0 (ZPL). A copy of the ZPL should accompany this distribution.
# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
# FOR A PARTICULAR PURPOSE
#
##############################################################################
"""Implement an OID to File-position (long integer) mapping
"""
#
# To save space, we do two things:
#
# 1. We split the keys (OIDS) into 6-byte prefixes and 2-byte suffixes.
# We use the prefixes as keys in a mapping from prefix to mappings
# of suffix to data:
#
# data is {prefix -> {suffix -> data}}
#
# 2. We limit the data size to 48 bits. This should allow databases
# as large as 256 terabytes.
#
# Mostof the space is consumed by items in the mappings from 2-byte
# suffix to 6-byte data. This should reduce the overall memory usage to
# 8-16 bytes per OID.
#
# We use p64 to convert integers to 8-byte strings and lop off the two
# high-order bytes when saving. On loading data, we add the leading
# bytes back before using U64 to convert the data back to (long)
# integers.
from BTrees._fsBTree import fsBTree as _fsBTree
import struct
# convert between numbers and six-byte strings
_t32 = 1L<< 32
def num2str(n):
h, l = divmod(long(n), _t32)
return struct.pack(">HI", h, l)
def str2num(s):
h, l = struct.unpack(">HI", s)
if h:
return (long(h) << 32) + l
else:
return l
class fsIndex:
def __init__(self):
self._data = {}
def __getitem__(self, key):
return str2num(self._data[key[:6]][key[6:]])
def get(self, key, default=None):
tree = self._data.get(key[:6], default)
if tree is default:
return default
v = tree.get(key[6:], default)
if v is default:
return default
return str2num(v)
def __setitem__(self, key, value):
value = num2str(value)
treekey = key[:6]
tree = self._data.get(treekey)
if tree is None:
tree = _fsBTree()
self._data[treekey] = tree
tree[key[6:]] = value
def __len__(self):
r = 0
for tree in self._data.values():
r += len(tree)
return r
def update(self, mapping):
for k, v in mapping.items():
self[k] = v
def has_key(self, key):
v=self.get(key, self)
return v is not self
def clear(self):
self._data.clear()
def keys(self):
r = []
for prefix, tree in self._data.items():
for suffix in tree.keys():
r.append(prefix + suffix)
return r
def items(self):
r = []
for prefix, tree in self._data.items():
for suffix, v in tree.items():
r.append(((prefix + suffix), str2num(v)))
return r
def values(self):
r = []
for prefix, tree in self._data.items():
for v in tree.values():
r.append(str2num(v))
return r
"""Test the list interface to PersistentList
"""
import unittest
from ZODB.PersistentList import PersistentList
l0 = []
l1 = [0]
l2 = [0, 1]
class TestPList(unittest.TestCase):
def checkTheWorld(self):
# Test constructors
u = PersistentList()
u0 = PersistentList(l0)
u1 = PersistentList(l1)
u2 = PersistentList(l2)
uu = PersistentList(u)
uu0 = PersistentList(u0)
uu1 = PersistentList(u1)
uu2 = PersistentList(u2)
v = PersistentList(tuple(u))
class OtherList:
def __init__(self, initlist):
self.__data = initlist
def __len__(self):
return len(self.__data)
def __getitem__(self, i):
return self.__data[i]
v0 = PersistentList(OtherList(u0))
vv = PersistentList("this is also a sequence")
# Test __repr__
eq = self.assertEqual
eq(str(u0), str(l0), "str(u0) == str(l0)")
eq(repr(u1), repr(l1), "repr(u1) == repr(l1)")
eq(`u2`, `l2`, "`u2` == `l2`")
# Test __cmp__ and __len__
def mycmp(a, b):
r = cmp(a, b)
if r < 0: return -1
if r > 0: return 1
return r
all = [l0, l1, l2, u, u0, u1, u2, uu, uu0, uu1, uu2]
for a in all:
for b in all:
eq(mycmp(a, b), mycmp(len(a), len(b)),
"mycmp(a, b) == mycmp(len(a), len(b))")
# Test __getitem__
for i in range(len(u2)):
eq(u2[i], i, "u2[i] == i")
# Test __setitem__
uu2[0] = 0
uu2[1] = 100
try:
uu2[2] = 200
except IndexError:
pass
else:
raise TestFailed("uu2[2] shouldn't be assignable")
# Test __delitem__
del uu2[1]
del uu2[0]
try:
del uu2[0]
except IndexError:
pass
else:
raise TestFailed("uu2[0] shouldn't be deletable")
# Test __getslice__
for i in range(-3, 4):
eq(u2[:i], l2[:i], "u2[:i] == l2[:i]")
eq(u2[i:], l2[i:], "u2[i:] == l2[i:]")
for j in range(-3, 4):
eq(u2[i:j], l2[i:j], "u2[i:j] == l2[i:j]")
# Test __setslice__
for i in range(-3, 4):
u2[:i] = l2[:i]
eq(u2, l2, "u2 == l2")
u2[i:] = l2[i:]
eq(u2, l2, "u2 == l2")
for j in range(-3, 4):
u2[i:j] = l2[i:j]
eq(u2, l2, "u2 == l2")
uu2 = u2[:]
uu2[:0] = [-2, -1]
eq(uu2, [-2, -1, 0, 1], "uu2 == [-2, -1, 0, 1]")
uu2[0:] = []
eq(uu2, [], "uu2 == []")
# Test __contains__
for i in u2:
self.failUnless(i in u2, "i in u2")
for i in min(u2)-1, max(u2)+1:
self.failUnless(i not in u2, "i not in u2")
# Test __delslice__
uu2 = u2[:]
del uu2[1:2]
del uu2[0:1]
eq(uu2, [], "uu2 == []")
uu2 = u2[:]
del uu2[1:]
del uu2[:1]
eq(uu2, [], "uu2 == []")
# Test __add__, __radd__, __mul__ and __rmul__
#self.failUnless(u1 + [] == [] + u1 == u1, "u1 + [] == [] + u1 == u1")
self.failUnless(u1 + [1] == u2, "u1 + [1] == u2")
#self.failUnless([-1] + u1 == [-1, 0], "[-1] + u1 == [-1, 0]")
self.failUnless(u2 == u2*1 == 1*u2, "u2 == u2*1 == 1*u2")
self.failUnless(u2+u2 == u2*2 == 2*u2, "u2+u2 == u2*2 == 2*u2")
self.failUnless(u2+u2+u2 == u2*3 == 3*u2, "u2+u2+u2 == u2*3 == 3*u2")
# Test append
u = u1[:]
u.append(1)
eq(u, u2, "u == u2")
# Test insert
u = u2[:]
u.insert(0, -1)
eq(u, [-1, 0, 1], "u == [-1, 0, 1]")
# Test pop
u = PersistentList([0, -1, 1])
u.pop()
eq(u, [0, -1], "u == [0, -1]")
u.pop(0)
eq(u, [-1], "u == [-1]")
# Test remove
u = u2[:]
u.remove(1)
eq(u, u1, "u == u1")
# Test count
u = u2*3
eq(u.count(0), 3, "u.count(0) == 3")
eq(u.count(1), 3, "u.count(1) == 3")
eq(u.count(2), 0, "u.count(2) == 0")
# Test index
eq(u2.index(0), 0, "u2.index(0) == 0")
eq(u2.index(1), 1, "u2.index(1) == 1")
try:
u2.index(2)
except ValueError:
pass
else:
raise TestFailed("expected ValueError")
# Test reverse
u = u2[:]
u.reverse()
eq(u, [1, 0], "u == [1, 0]")
u.reverse()
eq(u, u2, "u == u2")
# Test sort
u = PersistentList([1, 0])
u.sort()
eq(u, u2, "u == u2")
# Test extend
u = u1[:]
u.extend(u2)
eq(u, u1 + u2, "u == u1 + u2")
def test_suite():
return unittest.makeSuite(TestPList, 'check')
if __name__ == "__main__":
loader = unittest.TestLoader()
loader.testMethodPrefix = "check"
unittest.main(testLoader=loader)
import unittest, sys
from ZODB.fsIndex import fsIndex
from ZODB.utils import p64
class Test(unittest.TestCase):
def testInserts(self):
index=fsIndex()
for i in range(200):
index[p64(i*1000)]=(i*1000L+1)
for i in range(0,200):
self.assertEqual((i,index[p64(i*1000)]), (i,(i*1000L+1)))
self.assertEqual(len(index), 200)
key=p64(2000)
self.assertEqual(index.get(key), 2001)
key=p64(2001)
self.assertEqual(index.get(key), None)
self.assertEqual(index.get(key, ''), '')
# self.failUnless(len(index._data) > 1)
def testUpdate(self):
index=fsIndex()
d={}
for i in range(200):
d[p64(i*1000)]=(i*1000L+1)
index.update(d)
for i in range(400,600):
d[p64(i*1000)]=(i*1000L+1)
index.update(d)
for i in range(100, 500):
d[p64(i*1000)]=(i*1000L+2)
index.update(d)
self.assertEqual(index.get(p64(2000)), 2001)
self.assertEqual(index.get(p64(599000)), 599001)
self.assertEqual(index.get(p64(399000)), 399002)
self.assertEqual(len(index), 600)
def test_suite():
loader=unittest.TestLoader()
return loader.loadTestsFromTestCase(Test)
if __name__=='__main__':
unittest.TextTestRunner().run(test_suite())
##############################################################################
#
# Copyright (c) 2001, 2002 Zope Corporation and Contributors.
# All Rights Reserved.
#
# This software is subject to the provisions of the Zope Public License,
# Version 2.0 (ZPL). A copy of the ZPL should accompany this distribution.
# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
# FOR A PARTICULAR PURPOSE
#
##############################################################################
"""Python implementation of persistent list.
$Id: list.py,v 1.2 2002/02/11 23:49:07 gvanrossum Exp $"""
__version__='$Revision: 1.2 $'[11:-2]
import Persistence
from UserList import UserList
class PersistentList(UserList, Persistence.Persistent):
__super_setitem = UserList.__setitem__
__super_delitem = UserList.__delitem__
__super_setslice = UserList.__setslice__
__super_delslice = UserList.__delslice__
__super_iadd = UserList.__iadd__
__super_imul = UserList.__imul__
__super_append = UserList.append
__super_insert = UserList.insert
__super_pop = UserList.pop
__super_remove = UserList.remove
__super_reverse = UserList.reverse
__super_sort = UserList.sort
__super_extend = UserList.extend
def __setitem__(self, i, item):
self.__super_setitem(i, item)
self._p_changed = 1
def __delitem__(self, i):
self.__super_delitem(i)
self._p_changed = 1
def __setslice__(self, i, j, other):
self.__super_setslice(i, j, other)
self._p_changed = 1
def __delslice__(self, i, j):
self.__super_delslice(i, j)
self._p_changed = 1
def __iadd__(self, other):
self.__super_iadd(other)
self._p_changed = 1
def __imul__(self, n):
self.__super_imul(n)
self._p_changed = 1
def append(self, item):
self.__super_append(item)
self._p_changed = 1
def insert(self, i, item):
self.__super_insert(i, item)
self._p_changed = 1
def pop(self, i=-1):
rtn = self.__super_pop(i)
self._p_changed = 1
return rtn
def remove(self, item):
self.__super_remove(item)
self._p_changed = 1
def reverse(self):
self.__super_reverse()
self._p_changed = 1
def sort(self, *args):
self.__super_sort(*args)
self._p_changed = 1
def extend(self, other):
self.__super_extend(other)
self._p_changed = 1
# This works around a bug in Python 2.1.x (up to 2.1.2 at least) where the
# __cmp__ bogusly raises a RuntimeError, and because this is an extension
# class, none of the rich comparison stuff works anyway.
def __cmp__(self, other):
return cmp(self.data, self._UserList__cast(other))
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment