Commit 3bbb84bc authored by bescoto's avatar bescoto

Fixed metadata diff regressing


git-svn-id: http://svn.savannah.nongnu.org/svn/rdiff-backup@675 2b77aa54-bcbc-44c9-a7ec-4f6cf2b41109
parent 3bea0e45
......@@ -17,6 +17,8 @@ Alec Berryman's fs_abilities patch is supposed to help with AFS.
Fixed filename-too-long crash when quoting.
Patched carbonfile support, re-enabled it by default.
New in v1.1.0 (2005/10/24)
--------------------------
......
......@@ -2,8 +2,6 @@ For comparing, check source filesystem's abilities
Clean up connection dropped message
Make sure regress handles metadata diffs
Clean up compare reports
Test comparing of single files, and files/directories specified by
......@@ -22,4 +20,3 @@ Think about adding Gaudet's idea for keeping track of renamed files.
Look into different inode generation techniques (see treescan, Dean
Gaudet's other post).
......@@ -345,17 +345,19 @@ class FlatFile:
_extractor = FlatExtractor # Override to class that iterates objects
_object_to_record = None # Set to function converting object to record
_prefix = None # Set to required prefix
def __init__(self, rp, mode):
def __init__(self, rp, mode, check_path = 1, compress = 1):
"""Open rp for reading ('r') or writing ('w')"""
self.rp = rp
self.mode = mode
self._record_buffer = []
assert rp.isincfile() and rp.getincbase_str() == self._prefix, rp
if check_path:
assert rp.isincfile() and rp.getincbase_str() == self._prefix, rp
compress = rp.isinccompressed()
if mode == 'r':
self.fileobj = self.rp.open("rb", rp.isinccompressed())
self.fileobj = self.rp.open("rb", compress)
else:
assert mode == 'w' and not self.rp.lstat(), (mode, rp)
self.fileobj = self.rp.open("wb", rp.isinccompressed())
self.fileobj = self.rp.open("wb", compress)
def write_record(self, record):
"""Write a (text) record into the file"""
......@@ -556,18 +558,17 @@ class PatchDiffMan(Manager):
# exact compare here, can't use == on rorps
yield old_rorp
def sorted_meta_inclist(self, min_time = 0):
"""Return list of mirror_metadata incs, reverse sorted by time"""
if not self.prefixmap.has_key('mirror_metadata'): return []
sortlist = [(rp.getinctime(), rp)
for rp in self.prefixmap['mirror_metadata']]
def sorted_prefix_inclist(self, prefix, min_time = 0):
"""Return reverse sorted (by time) list of incs with given prefix"""
if not self.prefixmap.has_key(prefix): return []
sortlist = [(rp.getinctime(), rp) for rp in self.prefixmap[prefix]]
sortlist.sort()
sortlist.reverse()
return [rp for (time, rp) in sortlist if time >= min_time]
def check_needs_diff(self):
"""Check if we should diff, returns (new, old) rps, or (None, None)"""
inclist = self.sorted_meta_inclist()
inclist = self.sorted_prefix_inclist('mirror_metadata')
assert len(inclist) >= 1
if len(inclist) == 1: return (None, None)
newrp, oldrp = inclist[:2]
......@@ -604,7 +605,7 @@ class PatchDiffMan(Manager):
def relevant_meta_incs(self, time):
"""Return list [snapshotrp, diffrps ...] time sorted"""
inclist = self.sorted_meta_inclist(min_time = time)
inclist = self.sorted_prefix_inclist('mirror_metadata', min_time=time)
if not inclist: return inclist
assert inclist[-1].getinctime() == time, inclist[-1]
for i in range(len(inclist)-1, -1, -1):
......@@ -631,6 +632,7 @@ ManagerObj = None # Set this later to Manager instance
def SetManager():
global ManagerObj
ManagerObj = PatchDiffMan()
return ManagerObj
import eas_acls # put at bottom to avoid python circularity bug
......@@ -63,9 +63,9 @@ def Regress(mirror_rp):
assert mirror_rp.index == () and inc_rpath.index == ()
assert mirror_rp.isdir() and inc_rpath.isdir()
assert mirror_rp.conn is inc_rpath.conn is Globals.local_connection
set_regress_time()
manager, former_current_mirror_rp = set_regress_time()
set_restore_times()
former_current_mirror_rp = remove_rbdir_increments()
regress_rbdir(manager)
ITR = rorpiter.IterTreeReducer(RegressITRB, [])
for rf in iterate_meta_rfs(mirror_rp, inc_rpath): ITR(rf.index, rf)
ITR.Finish()
......@@ -81,14 +81,15 @@ def set_regress_time():
"""
global regress_time, unsuccessful_backup_time
curmir_incs = restore.get_inclist(Globals.rbdir.append("current_mirror"))
manager = metadata.SetManager()
curmir_incs = manager.sorted_prefix_inclist('current_mirror')
assert len(curmir_incs) == 2, \
"Found %s current_mirror flags, expected 2" % len(curmir_incs)
inctimes = [inc.getinctime() for inc in curmir_incs]
inctimes.sort()
regress_time = inctimes[0]
unsuccessful_backup_time = inctimes[-1]
mirror_rp_to_delete = curmir_incs[0]
regress_time = curmir_incs[1].getinctime()
unsuccessful_backup_time = mirror_rp_to_delete.getinctime()
log.Log("Regressing to " + Time.timetopretty(regress_time), 4)
return manager, mirror_rp_to_delete
def set_restore_times():
"""Set _rest_time and _mirror_time in the restore module
......@@ -100,24 +101,51 @@ def set_restore_times():
restore.MirrorStruct._mirror_time = unsuccessful_backup_time
restore.MirrorStruct._rest_time = regress_time
def remove_rbdir_increments():
def regress_rbdir(meta_manager):
"""Delete the increments in the rdiff-backup-data directory
Returns the former current mirror rp so we can delete it later.
All of the other rp's should be deleted before the actual regress,
to clear up disk space the rest of the procedure may need.
Also, in case the previous session failed while diffing the
metadata file, either recreate the mirror_metadata snapshot, or
delete the extra regress_time diff.
"""
former_current_mirror = None
for filename in Globals.rbdir.listdir():
rp = Globals.rbdir.append(filename)
if rp.isincfile() and rp.getinctime() == unsuccessful_backup_time:
if rp.getincbase_str() == "current_mirror":
former_current_mirror = rp
has_meta_diff, has_meta_snap = 0, 0
for old_rp in meta_manager.timerpmap[regress_time]:
if old_rp.getincbase_str() == 'mirror_metadata':
if old_rp.getinctype() == 'snapshot': has_meta_snap = 1
else:
log.Log("Removing rdiff-backup-data increment " + rp.path, 5)
rp.delete()
return former_current_mirror
assert old_rp.getinctype() == 'diff', old_rp
has_meta_diff = 1
if has_meta_diff and not has_meta_snap: recreate_meta(meta_manager)
for new_rp in meta_manager.timerpmap[unsuccessful_backup_time]:
if new_rp.getincbase_str() != 'current_mirror':
log.Log("Deleting old diff at " + new_rp.path, 5)
new_rp.delete()
def recreate_meta(meta_manager):
"""Make regress_time mirror_metadata snapshot by patching
We write to a tempfile first. Otherwise, in case of a crash, it
would seem we would have an intact snapshot and partial diff, not
the reverse.
"""
temprp = TempFile.new_in_dir(Globals.rbdir)
writer = metadata.MetadataFile(temprp, 'w', check_path = 0)
for rorp in meta_manager.get_meta_at_time(regress_time, None):
writer.write_object(rorp)
writer.close()
finalrp = Globals.rbdir.append("mirror_metadata.%s.snapshot.gz" %
Time.timetostring(regress_time))
assert not finalrp.lstat(), finalrp
rpath.rename(temprp, finalrp)
if Globals.fsync_directories: Globals.rbdir.fsync()
def iterate_raw_rfs(mirror_rp, inc_rp):
"""Iterate all RegressFile objects in mirror/inc directory
......@@ -146,8 +174,8 @@ def yield_metadata():
metadata.SetManager()
metadata_iter = metadata.ManagerObj.GetAtTime(regress_time)
if metadata_iter: return metadata_iter
log.Log.FatalError("No metadata for time %s found, cannot regress"
% Time.timetopretty(regress_time))
log.Log.FatalError("No metadata for time %s (%s) found,\ncannot regress"
% (Time.timetopretty(regress_time), regress_time))
def iterate_meta_rfs(mirror_rp, inc_rp):
"""Yield RegressFile objects with extra metadata information added
......
......@@ -199,7 +199,7 @@ class MetadataTest(unittest.TestCase):
man.ConvertMetaToDiff()
man = PatchDiffMan()
l = man.sorted_meta_inclist()
l = man.sorted_prefix_inclist('mirror_metadata')
assert l[0].getinctype() == 'snapshot'
assert l[0].getinctime() == 40000
assert l[1].getinctype() == 'snapshot'
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment