Commit c36d4c48 authored by Jim Fulton's avatar Jim Fulton

Renamed IStorageDB to IStorageWrapper and expanded it to provide

methods for transforming and untransforming data records. This was
needed to make storage wrappers that transform data work with
conflict resolution.
parent 09a987c1
...@@ -32,6 +32,11 @@ New Features ...@@ -32,6 +32,11 @@ New Features
- You can now pass None (rather than a storage or file name) to get - You can now pass None (rather than a storage or file name) to get
a database with a mapping storage. a database with a mapping storage.
- Renamed IStorageDB to IStorageWrapper and expanded it to provide
methods for transforming and untransforming data records. This was
needed to make storage wrappers that transform data work with
conflict resolution.
Bugs Fixed Bugs Fixed
---------- ----------
......
...@@ -778,6 +778,7 @@ class StorageServerDB: ...@@ -778,6 +778,7 @@ class StorageServerDB:
def invalidateCache(self): def invalidateCache(self):
self.server._invalidateCache(self.storage_id) self.server._invalidateCache(self.storage_id)
transform_record_data = untransform_record_data = lambda self, data: data
class StorageServer: class StorageServer:
......
...@@ -53,6 +53,7 @@ def find_global(*args): ...@@ -53,6 +53,7 @@ def find_global(*args):
def state(self, oid, serial, prfactory, p=''): def state(self, oid, serial, prfactory, p=''):
p = p or self.loadSerial(oid, serial) p = p or self.loadSerial(oid, serial)
p = self._crs_wrapper.untransform_record_data(p)
file = StringIO(p) file = StringIO(p)
unpickler = Unpickler(file) unpickler = Unpickler(file)
unpickler.find_global = find_global unpickler.find_global = find_global
...@@ -80,13 +81,13 @@ class IPersistentReference(zope.interface.Interface): ...@@ -80,13 +81,13 @@ class IPersistentReference(zope.interface.Interface):
def __cmp__(other): def __cmp__(other):
'''if other is equivalent reference, return 0; else raise ValueError. '''if other is equivalent reference, return 0; else raise ValueError.
Equivalent in this case means that oid and database_name are the same. Equivalent in this case means that oid and database_name are the same.
If either is a weak reference, we only support `is` equivalence, and If either is a weak reference, we only support `is` equivalence, and
otherwise raise a ValueError even if the datbase_names and oids are otherwise raise a ValueError even if the datbase_names and oids are
the same, rather than guess at the correct semantics. the same, rather than guess at the correct semantics.
It is impossible to sort reliably, since the actual persistent It is impossible to sort reliably, since the actual persistent
class may have its own comparison, and we have no idea what it is. class may have its own comparison, and we have no idea what it is.
We assert that it is reasonably safe to assume that an object is We assert that it is reasonably safe to assume that an object is
...@@ -135,7 +136,7 @@ class PersistentReference(object): ...@@ -135,7 +136,7 @@ class PersistentReference(object):
def __cmp__(self, other): def __cmp__(self, other):
if self is other or ( if self is other or (
isinstance(other, PersistentReference) and isinstance(other, PersistentReference) and
self.oid == other.oid and self.oid == other.oid and
self.database_name == other.database_name and self.database_name == other.database_name and
not self.weak and not self.weak and
...@@ -179,6 +180,7 @@ def tryToResolveConflict(self, oid, committedSerial, oldSerial, newpickle, ...@@ -179,6 +180,7 @@ def tryToResolveConflict(self, oid, committedSerial, oldSerial, newpickle,
# class_tuple, old, committed, newstate = ('',''), 0, 0, 0 # class_tuple, old, committed, newstate = ('',''), 0, 0, 0
try: try:
prfactory = PersistentReferenceFactory() prfactory = PersistentReferenceFactory()
newpickle = self._crs_wrapper.untransform_record_data(newpickle)
file = StringIO(newpickle) file = StringIO(newpickle)
unpickler = Unpickler(file) unpickler = Unpickler(file)
unpickler.find_global = find_global unpickler.find_global = find_global
...@@ -215,7 +217,7 @@ def tryToResolveConflict(self, oid, committedSerial, oldSerial, newpickle, ...@@ -215,7 +217,7 @@ def tryToResolveConflict(self, oid, committedSerial, oldSerial, newpickle,
pickler.inst_persistent_id = persistent_id pickler.inst_persistent_id = persistent_id
pickler.dump(meta) pickler.dump(meta)
pickler.dump(resolved) pickler.dump(resolved)
return file.getvalue(1) return self._crs_wrapper.transform_record_data(file.getvalue(1))
except (ConflictError, BadClassName): except (ConflictError, BadClassName):
return None return None
except: except:
...@@ -227,7 +229,11 @@ def tryToResolveConflict(self, oid, committedSerial, oldSerial, newpickle, ...@@ -227,7 +229,11 @@ def tryToResolveConflict(self, oid, committedSerial, oldSerial, newpickle,
logger.error("Unexpected error", exc_info=True) logger.error("Unexpected error", exc_info=True)
return None return None
class ConflictResolvingStorage: class ConflictResolvingStorage(object):
"Mix-in class that provides conflict resolution handling for storages" "Mix-in class that provides conflict resolution handling for storages"
tryToResolveConflict = tryToResolveConflict tryToResolveConflict = tryToResolveConflict
def registerDB(self, wrapper):
self._crs_wrapper = wrapper
super(ConflictResolvingStorage, self).registerDB(wrapper)
...@@ -705,6 +705,8 @@ class DB(object): ...@@ -705,6 +705,8 @@ class DB(object):
""" """
self._connectionMap(lambda c: c.invalidateCache()) self._connectionMap(lambda c: c.invalidateCache())
transform_record_data = untransform_record_data = lambda self, data: data
def objectCount(self): def objectCount(self):
return len(self.storage) return len(self.storage)
......
...@@ -287,23 +287,32 @@ class IConnection(Interface): ...@@ -287,23 +287,32 @@ class IConnection(Interface):
""" """
class IStorageDB(Interface): class IStorageWrapper(Interface):
"""Database interface exposed to storages """Storage wrapper interface
This interface provides 2 facilities: This interface provides 3 facilities:
- Out-of-band invalidation support - Out-of-band invalidation support
A storage can notify it's database of object invalidations that A storage can notify it's wrapper of object invalidations that
don't occur due to direct operations on the storage. Currently don't occur due to direct operations on the storage. Currently
this is only used by ZEO client storages to pass invalidation this is only used by ZEO client storages to pass invalidation
messages sent from a server. messages sent from a server.
- Record-reference extraction. - Record-reference extraction
The references method can be used to extract referenced object The references method can be used to extract referenced object
IDs from a database record. This can be used by storages to IDs from a database record. This can be used by storages to
provide more advanced garbage collection. provide more advanced garbage collection. A wrapper storage
that transforms data will provide a references method that
untransforms data passed to it and then pass the data to the
layer above it.
- Record transformation
A storage wrapper may transform data, for example for
compression or encryption. Methods are provided to transform or
untransform data.
This interface may be implemented by storage adapters or other This interface may be implemented by storage adapters or other
intermediaries. For example, a storage adapter that provides intermediaries. For example, a storage adapter that provides
...@@ -337,6 +346,16 @@ class IStorageDB(Interface): ...@@ -337,6 +346,16 @@ class IStorageDB(Interface):
be created and returned. be created and returned.
""" """
def transform_record_data(data):
"""Return transformed data
"""
def untransform_record_data(data):
"""Return untransformed data
"""
IStorageDB = IStorageWrapper # for backward compatibility
class IDatabase(IStorageDB): class IDatabase(IStorageDB):
"""ZODB DB. """ZODB DB.
...@@ -595,12 +614,18 @@ class IStorage(Interface): ...@@ -595,12 +614,18 @@ class IStorage(Interface):
revisions. revisions.
""" """
def registerDB(db): def registerDB(wrapper):
"""Register an IStorageDB. """Register a storage wrapper IStorageWrapper.
The passed object is a wrapper object that provides an upcall
interface to support composition.
Note that, for historical reasons, an implementation may Note that, for historical reasons, an implementation may
require a second argument, however, if required, the None will require a second argument, however, if required, the None will
be passed as the second argument. be passed as the second argument.
Also, for historical reasons, this is called registerDB rather
than register_wrapper.
""" """
def sortKey(): def sortKey():
......
...@@ -53,9 +53,15 @@ class PCounter4(PCounter): ...@@ -53,9 +53,15 @@ class PCounter4(PCounter):
def _p_resolveConflict(self, oldState, savedState): def _p_resolveConflict(self, oldState, savedState):
raise RuntimeError("Can't get here; not enough args") raise RuntimeError("Can't get here; not enough args")
class StorageWrapper:
transform_record_data = untransform_record_data = lambda self, data: data
class ConflictResolvingStorage: class ConflictResolvingStorage:
def checkResolve(self): def checkResolve(self):
self._storage.registerDB(StorageWrapper())
obj = PCounter() obj = PCounter()
obj.inc() obj.inc()
...@@ -76,6 +82,8 @@ class ConflictResolvingStorage: ...@@ -76,6 +82,8 @@ class ConflictResolvingStorage:
self.assertEqual(inst._value, 5) self.assertEqual(inst._value, 5)
def checkUnresolvable(self): def checkUnresolvable(self):
self._storage.registerDB(StorageWrapper())
obj = PCounter2() obj = PCounter2()
obj.inc() obj.inc()
...@@ -97,11 +105,15 @@ class ConflictResolvingStorage: ...@@ -97,11 +105,15 @@ class ConflictResolvingStorage:
self.fail("Expected ConflictError") self.fail("Expected ConflictError")
def checkZClassesArentResolved(self): def checkZClassesArentResolved(self):
self._storage.registerDB(StorageWrapper())
from ZODB.ConflictResolution import find_global, BadClassName from ZODB.ConflictResolution import find_global, BadClassName
dummy_class_tuple = ('*foobar', ()) dummy_class_tuple = ('*foobar', ())
self.assertRaises(BadClassName, find_global, '*foobar', ()) self.assertRaises(BadClassName, find_global, '*foobar', ())
def checkBuggyResolve1(self): def checkBuggyResolve1(self):
self._storage.registerDB(StorageWrapper())
obj = PCounter3() obj = PCounter3()
obj.inc() obj.inc()
...@@ -120,6 +132,8 @@ class ConflictResolvingStorage: ...@@ -120,6 +132,8 @@ class ConflictResolvingStorage:
oid, revid=revid1, data=zodb_pickle(obj)) oid, revid=revid1, data=zodb_pickle(obj))
def checkBuggyResolve2(self): def checkBuggyResolve2(self):
self._storage.registerDB(StorageWrapper())
obj = PCounter4() obj = PCounter4()
obj.inc() obj.inc()
...@@ -144,6 +158,8 @@ class ConflictResolvingTransUndoStorage: ...@@ -144,6 +158,8 @@ class ConflictResolvingTransUndoStorage:
# TransactionalUndoStorage test suite. Except here, conflict # TransactionalUndoStorage test suite. Except here, conflict
# resolution should allow us to undo the transaction anyway. # resolution should allow us to undo the transaction anyway.
self._storage.registerDB(StorageWrapper())
obj = PCounter() obj = PCounter()
obj.inc() obj.inc()
oid = self._storage.new_oid() oid = self._storage.new_oid()
...@@ -166,6 +182,8 @@ class ConflictResolvingTransUndoStorage: ...@@ -166,6 +182,8 @@ class ConflictResolvingTransUndoStorage:
# TransactionalUndoStorage test suite. Except here, conflict # TransactionalUndoStorage test suite. Except here, conflict
# resolution should allow us to undo the transaction anyway. # resolution should allow us to undo the transaction anyway.
self._storage.registerDB(StorageWrapper())
obj = PCounter2() obj = PCounter2()
obj.inc() obj.inc()
oid = self._storage.new_oid() oid = self._storage.new_oid()
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment