Skip to content

Z
ZODB
Commit 1c519e6e authored by Shane Hathaway's avatar Shane Hathaway
Browse files
Options

Merged shane-oid-length-branch. 

The ZEO client cache now works with OIDs of length up to 65535 bytes.
Also added some sanity checks that check the size of the data being written,
since there are some hard limits.
parent a2381f21
Expand all Hide whitespace changes
Inline Side-by-side
Showing with 120 additions and 72 deletions
src/ZEO/ClientCache.py
View file @ 1c519e6e
This diff is collapsed.
src/ZEO/stats.py
View file @ 1c519e6e
......@@ -35,8 +35,9 @@ Offset Size Contents
0 4 timestamp (seconds since 1/1/1970)
4 3 data size, in 256-byte increments, rounded up
7 1 code (see below)
8 8 object id
16 8 serial number
8 2 object id length
10 8 serial number
18 variable object id
The code at offset 7 packs three fields:
......@@ -56,6 +57,7 @@ import sys
import time
import getopt
import struct
from types import StringType
def usage(msg):
print >>sys.stderr, msg
......@@ -155,12 +157,16 @@ def main():
if not quiet:
print "Skipping 8 bytes at offset", offset-8
continue
r = f_read(16)
if len(r) < 16:
r = f_read(10)
if len(r) < 10:
break
offset += 16
offset += 10
records += 1
oid, serial = struct_unpack(">8s8s", r)
oidlen, serial = struct_unpack(">H8s", r)
oid = f_read(oidlen)
if len(oid) != oidlen:
break
offset += oidlen
if t0 is None:
t0 = ts
thisinterval = t0 / interval
......@@ -197,11 +203,11 @@ def main():
bysizew[dlen] = d = bysizew.get(dlen) or {}
d[oid] = d.get(oid, 0) + 1
if verbose:
print "%s %d %02x %016x %016x %1s %s" % (
print "%s %d %02x %s %016x %1s %s" % (
time.ctime(ts)[4:-5],
current,
code,
U64(oid),
oid_repr(oid),
U64(serial),
version,
dlen and str(dlen) or "")
......@@ -346,6 +352,12 @@ def U64(s):
h, v = struct.unpack(">II", s)
return (long(h) << 32) + v
def oid_repr(oid):
if isinstance(oid, StringType) and len(oid) == 8:
return '%16x' % U64(oid)
else:
return repr(oid)
def addcommas(n):
sign, s = '', str(n)
if s[0] == '-':
......
src/ZEO/tests/testClientCache.py
View file @ 1c519e6e
......@@ -27,6 +27,10 @@ from ZEO.ClientCache import ClientCache
class ClientCacheTests(unittest.TestCase):
_oid = 'abcdefgh'
_oid2 = 'bcdefghi'
_oid3 = 'cdefghij'
def setUp(self):
unittest.TestCase.setUp(self)
self.cachesize = 10*1000*1000
......@@ -42,7 +46,7 @@ class ClientCacheTests(unittest.TestCase):
def testStoreLoad(self):
cache = self.cache
oid = 'abcdefgh'
oid = self._oid
data = '1234'*100
serial = 'ABCDEFGH'
cache.store(oid, data, serial, '', '', '')
......@@ -51,7 +55,7 @@ class ClientCacheTests(unittest.TestCase):
def testMissingLoad(self):
cache = self.cache
oid = 'abcdefgh'
oid = self._oid
data = '1234'*100
serial = 'ABCDEFGH'
cache.store(oid, data, serial, '', '', '')
......@@ -60,7 +64,7 @@ class ClientCacheTests(unittest.TestCase):
def testInvalidate(self):
cache = self.cache
oid = 'abcdefgh'
oid = self._oid
data = '1234'*100
serial = 'ABCDEFGH'
cache.store(oid, data, serial, '', '', '')
......@@ -72,7 +76,7 @@ class ClientCacheTests(unittest.TestCase):
def testVersion(self):
cache = self.cache
oid = 'abcdefgh'
oid = self._oid
data = '1234'*100
serial = 'ABCDEFGH'
vname = 'myversion'
......@@ -86,7 +90,7 @@ class ClientCacheTests(unittest.TestCase):
def testVersionOnly(self):
cache = self.cache
oid = 'abcdefgh'
oid = self._oid
data = ''
serial = ''
vname = 'myversion'
......@@ -100,7 +104,7 @@ class ClientCacheTests(unittest.TestCase):
def testInvalidateNonVersion(self):
cache = self.cache
oid = 'abcdefgh'
oid = self._oid
data = '1234'*100
serial = 'ABCDEFGH'
vname = 'myversion'
......@@ -122,7 +126,7 @@ class ClientCacheTests(unittest.TestCase):
# Invalidating a version should not invalidate the non-version data.
# (This tests for the same bug as testInvalidatePersists below.)
cache = self.cache
oid = 'abcdefgh'
oid = self._oid
data = '1234'*100
serial = 'ABCDEFGH'
cache.store(oid, data, serial, '', '', '')
......@@ -139,7 +143,7 @@ class ClientCacheTests(unittest.TestCase):
results.append((oid, serial, vserial))
cache.verify(verifier)
self.assertEqual(results, [])
oid = 'abcdefgh'
oid = self._oid
data = '1234'*100
serial = 'ABCDEFGH'
cache.store(oid, data, serial, '', '', '')
......@@ -151,12 +155,12 @@ class ClientCacheTests(unittest.TestCase):
# Make sure that cache._index[oid] is erased for oids that are
# stored in the cache file that's rewritten after a flip.
cache = self.cache
oid = 'abcdefgh'
oid = self._oid
data = '1234'*100
serial = 'ABCDEFGH'
cache.store(oid, data, serial, '', '', '')
cache.checkSize(10*self.cachesize) # Force a file flip
oid2 = 'abcdefgz'
oid2 = self._oid2
data2 = '1234'*10
serial2 = 'ABCDEFGZ'
cache.store(oid2, data2, serial2, '', '', '')
......@@ -178,17 +182,17 @@ class ClientCacheTests(unittest.TestCase):
cache = self.cache
# Create some objects
oid1 = 'abcdefgh'
oid1 = self._oid
data1 = '1234' * 100
serial1 = 'ABCDEFGH'
oid2 = 'bcdefghi'
oid2 = self._oid2
data2 = '2345' * 200
serial2 = 'BCDEFGHI'
version2 = 'myversion'
nonversion = 'nada'
vdata2 = '5432' * 250
vserial2 = 'IHGFEDCB'
oid3 = 'cdefghij'
oid3 = self._oid3
data3 = '3456' * 300
serial3 = 'CDEFGHIJ'
......@@ -276,6 +280,8 @@ class ClientCacheTests(unittest.TestCase):
class PersistentClientCacheTests(unittest.TestCase):
_oid = 'abcdefgh'
def setUp(self):
unittest.TestCase.setUp(self)
self.vardir = os.getcwd() # Don't use /tmp, it's a security risk
......@@ -323,13 +329,12 @@ class PersistentClientCacheTests(unittest.TestCase):
# 'current' file when a persistent cache was opened.
cache = self.cache
self.assertEqual(cache._current, 0) # Check that file 0 is current
oid = 'abcdefgh'
oid = self._oid
data = '1234'
serial = 'ABCDEFGH'
cache.store(oid, data, serial, '', '', '')
cache.checkSize(10*self.cachesize) # Force a file flip
self.assertEqual(cache._current, 1) # Check that the flip worked
oid = 'abcdefgh'
data = '123'
serial = 'ABCDEFGZ'
cache.store(oid, data, serial, '', '', '')
......@@ -348,7 +353,7 @@ class PersistentClientCacheTests(unittest.TestCase):
cache = self.cache
magicsize = (ord('i') + 1) << 16
cache = self.cache
oid = 'abcdefgh'
oid = self._oid
data = '!'*magicsize
serial = 'ABCDEFGH'
cache.store(oid, data, serial, '', '', '')
......@@ -367,7 +372,7 @@ class PersistentClientCacheTests(unittest.TestCase):
ltid = 'pqrstuvw'
cache.setLastTid(ltid)
self.assertEqual(cache.getLastTid(), ltid)
oid = 'abcdefgh'
oid = self._oid
data = '1234'
serial = 'ABCDEFGH'
cache.store(oid, data, serial, '', '', '')
......@@ -381,10 +386,23 @@ class PersistentClientCacheTests(unittest.TestCase):
cache.checkSize(10*self.cachesize) # Force a file flip
self.failUnless(cache.getLastTid() is None)
class ClientCacheLongOIDTests(ClientCacheTests):
_oid = 'abcdefghijklmnop' * 2
_oid2 = 'bcdefghijklmnopq' * 2
_oid3 = 'cdefghijklmnopqr' * 2
class PersistentClientCacheLongOIDTests(PersistentClientCacheTests):
_oid = 'abcdefghijklmnop' * 2
def test_suite():
suite = unittest.TestSuite()
suite.addTest(unittest.makeSuite(ClientCacheTests))
suite.addTest(unittest.makeSuite(ClientCacheLongOIDTests))
suite.addTest(unittest.makeSuite(PersistentClientCacheTests))
suite.addTest(unittest.makeSuite(PersistentClientCacheLongOIDTests))
return suite
if __name__ == '__main__':
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment
GitLab Nexedi Edition | About GitLab | About Nexedi | 沪ICP备2021021310号-2 | 沪ICP备2021021310号-7