Commit 60e508aa authored by Tim Peters's avatar Tim Peters

Merge rev 28673 from 3.3 branch.

More cleanup.  Notable:

- Object.fromFile():  renamed `header_only` arg to `skip_data`.
  Skipping the data is what it does, while there are at least 3
  distinct notions of what "a header" means in this module.
  Object has two notions of "header" all by itself (& I'm still
  not sure why Object.serialize_header() considers the version
  string to be part of "the header").

- FileCache.__init__():  log a warning if reuse=True but the
  given file path doesn't exist.  The code ignores `reuse` then
  (before, and now).  Not sure that's the best thing to do.
parent 6f12261d
...@@ -117,7 +117,7 @@ class ClientCache: ...@@ -117,7 +117,7 @@ class ClientCache:
# `ent` is an Entry giving the object's key ((oid, start_tid) pair). # `ent` is an Entry giving the object's key ((oid, start_tid) pair).
def install(self, f, ent): def install(self, f, ent):
# Called by cache storage layer to insert object. # Called by cache storage layer to insert object.
o = Object.fromFile(f, ent.key, header_only=True) o = Object.fromFile(f, ent.key, skip_data=True)
if o is None: if o is None:
return return
oid = o.key[0] oid = o.key[0]
...@@ -126,8 +126,12 @@ class ClientCache: ...@@ -126,8 +126,12 @@ class ClientCache:
elif o.end_tid is None: elif o.end_tid is None:
self.current[oid] = o.start_tid self.current[oid] = o.start_tid
else: else:
L = self.noncurrent.setdefault(oid, []) this_span = o.start_tid, o.end_tid
bisect.insort_left(L, (o.start_tid, o.end_tid)) span_list = self.noncurrent.get(oid)
if span_list:
bisect.insort_left(span_list, this_span)
else:
self.noncurrent[oid] = [this_span]
def close(self): def close(self):
self.fc.close() self.fc.close()
...@@ -422,7 +426,7 @@ class ClientCache: ...@@ -422,7 +426,7 @@ class ClientCache:
# data and whether it is in a version. # data and whether it is in a version.
# <p> # <p>
# The serialized format does not include the key, because it is stored # The serialized format does not include the key, because it is stored
# in the header used by the cache's storage format. # in the header used by the cache file's storage format.
# <p> # <p>
# Instances of Object are generally short-lived -- they're really a way to # Instances of Object are generally short-lived -- they're really a way to
# package data on the way to or from the disk file. # package data on the way to or from the disk file.
...@@ -506,9 +510,12 @@ class Object(object): ...@@ -506,9 +510,12 @@ class Object(object):
# fromFile is a class constructor, unserializing an Object from the # fromFile is a class constructor, unserializing an Object from the
# current position in file f. Exclusive access to f for the duration # current position in file f. Exclusive access to f for the duration
# is assumed. The key is a (oid, start_tid) pair, and the oid must # is assumed. The key is a (oid, start_tid) pair, and the oid must
# match the serialized oid. If header_only is true, .data is left # match the serialized oid. If `skip_data` is true, .data is left
# None in the Object returned. # None in the Object returned, but all the other fields are populated.
def fromFile(cls, f, key, header_only=False): # Else (`skip_data` is false, the default), all fields including .data
# are populated. .data can be big, so it's prudent to skip it when it
# isn't needed.
def fromFile(cls, f, key, skip_data=False):
s = f.read(cls.FIXED_HEADER_SIZE) s = f.read(cls.FIXED_HEADER_SIZE)
if not s: if not s:
return None return None
...@@ -522,7 +529,7 @@ class Object(object): ...@@ -522,7 +529,7 @@ class Object(object):
if vlen != len(version): if vlen != len(version):
raise ValueError("corrupted record, version") raise ValueError("corrupted record, version")
if header_only: if skip_data:
data = None data = None
f.seek(dlen, 1) f.seek(dlen, 1)
else: else:
...@@ -675,9 +682,11 @@ class FileCache(object): ...@@ -675,9 +682,11 @@ class FileCache(object):
self.fpath = fpath self.fpath = fpath
if reuse and fpath and os.path.exists(fpath): if reuse and fpath and os.path.exists(fpath):
# Reuse an existing file. scan() will open & read it. # Reuse an existing file. scan() will open & read it.
assert fpath
self.f = None self.f = None
else: else:
if reuse:
logger.warning("reuse=True but the given file path %r "
"doesn't exist; ignoring reuse=True", fpath)
if fpath: if fpath:
self.f = open(fpath, 'wb+') self.f = open(fpath, 'wb+')
else: else:
...@@ -897,7 +906,7 @@ class FileCache(object): ...@@ -897,7 +906,7 @@ class FileCache(object):
# Load the object header into memory so we know how to # Load the object header into memory so we know how to
# update the parent's in-memory data structures. # update the parent's in-memory data structures.
self.f.seek(e.offset + OBJECT_HEADER_SIZE) self.f.seek(e.offset + OBJECT_HEADER_SIZE)
o = Object.fromFile(self.f, e.key, header_only=True) o = Object.fromFile(self.f, e.key, skip_data=True)
self.parent._evicted(o) self.parent._evicted(o)
## ##
...@@ -932,7 +941,7 @@ class FileCache(object): ...@@ -932,7 +941,7 @@ class FileCache(object):
size, e2 = self.filemap[offset] size, e2 = self.filemap[offset]
assert size >= 5 # only free blocks are tiny assert size >= 5 # only free blocks are tiny
self.f.seek(offset + OBJECT_HEADER_SIZE) self.f.seek(offset + OBJECT_HEADER_SIZE)
o = Object.fromFile(self.f, key, header_only=True) o = Object.fromFile(self.f, key, skip_data=True)
# Because `size` >= 5, we can change an allocated block to a free # Because `size` >= 5, we can change an allocated block to a free
# block just by overwriting the 'a' status byte with 'f' -- the # block just by overwriting the 'a' status byte with 'f' -- the
# size field stays the same. # size field stays the same.
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment