Commit c48621b1 authored by Jim Fulton's avatar Jim Fulton

Removed the transaction id cache.

parent f401a0b7
...@@ -43,8 +43,6 @@ from ZODB.FileStorage.format import CorruptedDataError ...@@ -43,8 +43,6 @@ from ZODB.FileStorage.format import CorruptedDataError
from ZODB.loglevels import BLATHER from ZODB.loglevels import BLATHER
from ZODB.fsIndex import fsIndex from ZODB.fsIndex import fsIndex
import BTrees.OOBTree
packed_version = "FS21" packed_version = "FS21"
logger = logging.getLogger('ZODB.FileStorage') logger = logging.getLogger('ZODB.FileStorage')
...@@ -122,10 +120,8 @@ class FileStorage(BaseStorage.BaseStorage, ...@@ -122,10 +120,8 @@ class FileStorage(BaseStorage.BaseStorage,
BaseStorage.BaseStorage.__init__(self, file_name) BaseStorage.BaseStorage.__init__(self, file_name)
(index, vindex, tindex, tvindex, index, vindex, tindex, tvindex = self._newIndexes()
oid2tid, toid2tid, toid2tid_delete) = self._newIndexes() self._initIndex(index, vindex, tindex, tvindex)
self._initIndex(index, vindex, tindex, tvindex,
oid2tid, toid2tid, toid2tid_delete)
# Now open the file # Now open the file
...@@ -159,8 +155,7 @@ class FileStorage(BaseStorage.BaseStorage, ...@@ -159,8 +155,7 @@ class FileStorage(BaseStorage.BaseStorage,
self._used_index = 1 # Marker for testing self._used_index = 1 # Marker for testing
index, vindex, start, ltid = r index, vindex, start, ltid = r
self._initIndex(index, vindex, tindex, tvindex, self._initIndex(index, vindex, tindex, tvindex)
oid2tid, toid2tid, toid2tid_delete)
self._pos, self._oid, tid = read_index( self._pos, self._oid, tid = read_index(
self._file, file_name, index, vindex, tindex, stop, self._file, file_name, index, vindex, tindex, stop,
ltid=ltid, start=start, read_only=read_only, ltid=ltid, start=start, read_only=read_only,
...@@ -194,11 +189,7 @@ class FileStorage(BaseStorage.BaseStorage, ...@@ -194,11 +189,7 @@ class FileStorage(BaseStorage.BaseStorage,
self._quota = quota self._quota = quota
# tid cache statistics. def _initIndex(self, index, vindex, tindex, tvindex):
self._oid2tid_nlookups = self._oid2tid_nhits = 0
def _initIndex(self, index, vindex, tindex, tvindex,
oid2tid, toid2tid, toid2tid_delete):
self._index=index self._index=index
self._vindex=vindex self._vindex=vindex
self._tindex=tindex self._tindex=tindex
...@@ -206,32 +197,12 @@ class FileStorage(BaseStorage.BaseStorage, ...@@ -206,32 +197,12 @@ class FileStorage(BaseStorage.BaseStorage,
self._index_get=index.get self._index_get=index.get
self._vindex_get=vindex.get self._vindex_get=vindex.get
# .store() needs to compare the passed-in serial to the
# current tid in the database. _oid2tid caches the oid ->
# current tid mapping for non-version data (if the current
# record for oid is version data, the oid is not a key in
# _oid2tid). The point is that otherwise seeking into the
# storage is needed to extract the current tid, and that's
# an expensive operation. For example, if a transaction
# stores 4000 objects, and each random seek + read takes 7ms
# (that was approximately true on Linux and Windows tests in
# mid-2003), that's 28 seconds just to find the old tids.
# TODO: Probably better to junk this and redefine _index as mapping
# oid to (offset, tid) pair, via a new memory-efficient BTree type.
self._oid2tid = oid2tid
# oid->tid map to transactionally add to _oid2tid.
self._toid2tid = toid2tid
# Set of oids to transactionally delete from _oid2tid (e.g.,
# oids reverted by undo, or for which the most recent record
# becomes version data).
self._toid2tid_delete = toid2tid_delete
def __len__(self): def __len__(self):
return len(self._index) return len(self._index)
def _newIndexes(self): def _newIndexes(self):
# hook to use something other than builtin dict # hook to use something other than builtin dict
return fsIndex(), {}, {}, {}, BTrees.OOBTree.OOBTree(), {}, {} return fsIndex(), {}, {}, {}
_saved = 0 _saved = 0
def _save_index(self): def _save_index(self):
...@@ -409,27 +380,6 @@ class FileStorage(BaseStorage.BaseStorage, ...@@ -409,27 +380,6 @@ class FileStorage(BaseStorage.BaseStorage,
# Log the error and continue # Log the error and continue
logger.error("Error saving index on close()", exc_info=True) logger.error("Error saving index on close()", exc_info=True)
# Return tid of most recent record for oid if that's in the
# _oid2tid cache. Else return None. It's important to use this
# instead of indexing _oid2tid directly so that cache statistics
# can be logged.
def _get_cached_tid(self, oid):
self._oid2tid_nlookups += 1
result = self._oid2tid.get(oid)
if result is not None:
self._oid2tid_nhits += 1
# Log a msg every ~8000 tries.
if self._oid2tid_nlookups & 0x1fff == 0:
logger.log(BLATHER,
"_oid2tid size %s lookups %s hits %s rate %.1f%%",
len(self._oid2tid),
self._oid2tid_nlookups,
self._oid2tid_nhits,
100.0 * self._oid2tid_nhits / self._oid2tid_nlookups)
return result
def abortVersion(self, src, transaction): def abortVersion(self, src, transaction):
return self.commitVersion(src, '', transaction, abort=True) return self.commitVersion(src, '', transaction, abort=True)
...@@ -504,7 +454,6 @@ class FileStorage(BaseStorage.BaseStorage, ...@@ -504,7 +454,6 @@ class FileStorage(BaseStorage.BaseStorage,
srcpos = h.vprev srcpos = h.vprev
spos = p64(srcpos) spos = p64(srcpos)
self._toid2tid_delete.update(current_oids)
return self._tid, oids return self._tid, oids
def getSize(self): def getSize(self):
...@@ -616,24 +565,23 @@ class FileStorage(BaseStorage.BaseStorage, ...@@ -616,24 +565,23 @@ class FileStorage(BaseStorage.BaseStorage,
if oid > self._oid: if oid > self._oid:
self.set_max_oid(oid) self.set_max_oid(oid)
old = self._index_get(oid, 0) old = self._index_get(oid, 0)
cached_tid = None committed_tid = None
pnv = None pnv = None
if old: if old:
cached_tid = self._get_cached_tid(oid) h = self._read_data_header(old, oid)
if cached_tid is None: if h.version:
h = self._read_data_header(old, oid) if h.version != version:
if h.version: raise VersionLockError(oid, h.version)
if h.version != version: pnv = h.pnv
raise VersionLockError(oid, h.version) committed_tid = h.tid
pnv = h.pnv
cached_tid = h.tid if oldserial != committed_tid:
rdata = self.tryToResolveConflict(oid, committed_tid,
if oldserial != cached_tid:
rdata = self.tryToResolveConflict(oid, cached_tid,
oldserial, data) oldserial, data)
if rdata is None: if rdata is None:
raise POSException.ConflictError( raise POSException.ConflictError(
oid=oid, serials=(cached_tid, oldserial), data=data) oid=oid, serials=(committed_tid, oldserial),
data=data)
else: else:
data = rdata data = rdata
...@@ -651,9 +599,6 @@ class FileStorage(BaseStorage.BaseStorage, ...@@ -651,9 +599,6 @@ class FileStorage(BaseStorage.BaseStorage,
pnv = old pnv = old
new.setVersion(version, pnv, pv) new.setVersion(version, pnv, pv)
self._tvindex[version] = here self._tvindex[version] = here
self._toid2tid_delete[oid] = 1
else:
self._toid2tid[oid] = self._tid
self._tfile.write(new.asString()) self._tfile.write(new.asString())
self._tfile.write(data) self._tfile.write(data)
...@@ -663,7 +608,7 @@ class FileStorage(BaseStorage.BaseStorage, ...@@ -663,7 +608,7 @@ class FileStorage(BaseStorage.BaseStorage,
raise FileStorageQuotaError( raise FileStorageQuotaError(
"The storage quota has been exceeded.") "The storage quota has been exceeded.")
if old and oldserial != cached_tid: if old and oldserial != committed_tid:
return ConflictResolution.ResolvedSerial return ConflictResolution.ResolvedSerial
else: else:
return self._tid return self._tid
...@@ -771,9 +716,6 @@ class FileStorage(BaseStorage.BaseStorage, ...@@ -771,9 +716,6 @@ class FileStorage(BaseStorage.BaseStorage,
vprev = self._vindex.get(version, 0) vprev = self._vindex.get(version, 0)
new.setVersion(version, pnv, vprev) new.setVersion(version, pnv, vprev)
self._tvindex[version] = here self._tvindex[version] = here
self._toid2tid_delete[oid] = 1
else:
self._toid2tid[oid] = serial
self._tfile.write(new.asString()) self._tfile.write(new.asString())
...@@ -822,8 +764,6 @@ class FileStorage(BaseStorage.BaseStorage, ...@@ -822,8 +764,6 @@ class FileStorage(BaseStorage.BaseStorage,
def _clear_temp(self): def _clear_temp(self):
self._tindex.clear() self._tindex.clear()
self._tvindex.clear() self._tvindex.clear()
self._toid2tid.clear()
self._toid2tid_delete.clear()
if self._tfile is not None: if self._tfile is not None:
self._tfile.seek(0) self._tfile.seek(0)
...@@ -894,13 +834,7 @@ class FileStorage(BaseStorage.BaseStorage, ...@@ -894,13 +834,7 @@ class FileStorage(BaseStorage.BaseStorage,
self._index.update(self._tindex) self._index.update(self._tindex)
self._vindex.update(self._tvindex) self._vindex.update(self._tvindex)
self._oid2tid.update(self._toid2tid)
for oid in self._toid2tid_delete.keys():
try:
del self._oid2tid[oid]
except KeyError:
pass
# Update the number of records that we've written # Update the number of records that we've written
# +1 for the transaction record # +1 for the transaction record
self._records_written += len(self._tindex) + 1 self._records_written += len(self._tindex) + 1
...@@ -945,17 +879,12 @@ class FileStorage(BaseStorage.BaseStorage, ...@@ -945,17 +879,12 @@ class FileStorage(BaseStorage.BaseStorage,
def getTid(self, oid): def getTid(self, oid):
self._lock_acquire() self._lock_acquire()
try: try:
result = self._get_cached_tid(oid) pos = self._lookup_pos(oid)
if result is None: h = self._read_data_header(pos, oid)
pos = self._lookup_pos(oid) if h.plen == 0 and h.back == 0:
h = self._read_data_header(pos, oid) # Undone creation
if h.plen == 0 and h.back == 0: raise POSKeyError(oid)
# Undone creation return h.tid
raise POSKeyError(oid)
else:
result = h.tid
self._oid2tid[oid] = result
return result
finally: finally:
self._lock_release() self._lock_release()
...@@ -1103,10 +1032,6 @@ class FileStorage(BaseStorage.BaseStorage, ...@@ -1103,10 +1032,6 @@ class FileStorage(BaseStorage.BaseStorage,
tpos = self._txn_find(tid, 1) tpos = self._txn_find(tid, 1)
tindex = self._txn_undo_write(tpos) tindex = self._txn_undo_write(tpos)
self._tindex.update(tindex) self._tindex.update(tindex)
# Arrange to clear the affected oids from the oid2tid cache.
# It's too painful to try to update them to correct current
# values instead.
self._toid2tid_delete.update(tindex)
return self._tid, tindex.keys() return self._tid, tindex.keys()
def _txn_find(self, tid, stop_at_pack): def _txn_find(self, tid, stop_at_pack):
...@@ -1343,9 +1268,7 @@ class FileStorage(BaseStorage.BaseStorage, ...@@ -1343,9 +1268,7 @@ class FileStorage(BaseStorage.BaseStorage,
# OK, we're beyond the point of no return # OK, we're beyond the point of no return
os.rename(self._file_name + '.pack', self._file_name) os.rename(self._file_name + '.pack', self._file_name)
self._file = open(self._file_name, 'r+b') self._file = open(self._file_name, 'r+b')
self._initIndex(p.index, p.vindex, p.tindex, p.tvindex, self._initIndex(p.index, p.vindex, p.tindex, p.tvindex)
p.oid2tid, p.toid2tid,
p.toid2tid_delete)
self._pos = opos self._pos = opos
self._save_index() self._save_index()
finally: finally:
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment