Commit 3bbb84bc authored by bescoto's avatar bescoto

Fixed metadata diff regressing


git-svn-id: http://svn.savannah.nongnu.org/svn/rdiff-backup@675 2b77aa54-bcbc-44c9-a7ec-4f6cf2b41109
parent 3bea0e45
...@@ -17,6 +17,8 @@ Alec Berryman's fs_abilities patch is supposed to help with AFS. ...@@ -17,6 +17,8 @@ Alec Berryman's fs_abilities patch is supposed to help with AFS.
Fixed filename-too-long crash when quoting. Fixed filename-too-long crash when quoting.
Patched carbonfile support, re-enabled it by default.
New in v1.1.0 (2005/10/24) New in v1.1.0 (2005/10/24)
-------------------------- --------------------------
......
...@@ -2,8 +2,6 @@ For comparing, check source filesystem's abilities ...@@ -2,8 +2,6 @@ For comparing, check source filesystem's abilities
Clean up connection dropped message Clean up connection dropped message
Make sure regress handles metadata diffs
Clean up compare reports Clean up compare reports
Test comparing of single files, and files/directories specified by Test comparing of single files, and files/directories specified by
...@@ -22,4 +20,3 @@ Think about adding Gaudet's idea for keeping track of renamed files. ...@@ -22,4 +20,3 @@ Think about adding Gaudet's idea for keeping track of renamed files.
Look into different inode generation techniques (see treescan, Dean Look into different inode generation techniques (see treescan, Dean
Gaudet's other post). Gaudet's other post).
...@@ -345,17 +345,19 @@ class FlatFile: ...@@ -345,17 +345,19 @@ class FlatFile:
_extractor = FlatExtractor # Override to class that iterates objects _extractor = FlatExtractor # Override to class that iterates objects
_object_to_record = None # Set to function converting object to record _object_to_record = None # Set to function converting object to record
_prefix = None # Set to required prefix _prefix = None # Set to required prefix
def __init__(self, rp, mode): def __init__(self, rp, mode, check_path = 1, compress = 1):
"""Open rp for reading ('r') or writing ('w')""" """Open rp for reading ('r') or writing ('w')"""
self.rp = rp self.rp = rp
self.mode = mode self.mode = mode
self._record_buffer = [] self._record_buffer = []
assert rp.isincfile() and rp.getincbase_str() == self._prefix, rp if check_path:
assert rp.isincfile() and rp.getincbase_str() == self._prefix, rp
compress = rp.isinccompressed()
if mode == 'r': if mode == 'r':
self.fileobj = self.rp.open("rb", rp.isinccompressed()) self.fileobj = self.rp.open("rb", compress)
else: else:
assert mode == 'w' and not self.rp.lstat(), (mode, rp) assert mode == 'w' and not self.rp.lstat(), (mode, rp)
self.fileobj = self.rp.open("wb", rp.isinccompressed()) self.fileobj = self.rp.open("wb", compress)
def write_record(self, record): def write_record(self, record):
"""Write a (text) record into the file""" """Write a (text) record into the file"""
...@@ -556,18 +558,17 @@ class PatchDiffMan(Manager): ...@@ -556,18 +558,17 @@ class PatchDiffMan(Manager):
# exact compare here, can't use == on rorps # exact compare here, can't use == on rorps
yield old_rorp yield old_rorp
def sorted_meta_inclist(self, min_time = 0): def sorted_prefix_inclist(self, prefix, min_time = 0):
"""Return list of mirror_metadata incs, reverse sorted by time""" """Return reverse sorted (by time) list of incs with given prefix"""
if not self.prefixmap.has_key('mirror_metadata'): return [] if not self.prefixmap.has_key(prefix): return []
sortlist = [(rp.getinctime(), rp) sortlist = [(rp.getinctime(), rp) for rp in self.prefixmap[prefix]]
for rp in self.prefixmap['mirror_metadata']]
sortlist.sort() sortlist.sort()
sortlist.reverse() sortlist.reverse()
return [rp for (time, rp) in sortlist if time >= min_time] return [rp for (time, rp) in sortlist if time >= min_time]
def check_needs_diff(self): def check_needs_diff(self):
"""Check if we should diff, returns (new, old) rps, or (None, None)""" """Check if we should diff, returns (new, old) rps, or (None, None)"""
inclist = self.sorted_meta_inclist() inclist = self.sorted_prefix_inclist('mirror_metadata')
assert len(inclist) >= 1 assert len(inclist) >= 1
if len(inclist) == 1: return (None, None) if len(inclist) == 1: return (None, None)
newrp, oldrp = inclist[:2] newrp, oldrp = inclist[:2]
...@@ -604,7 +605,7 @@ class PatchDiffMan(Manager): ...@@ -604,7 +605,7 @@ class PatchDiffMan(Manager):
def relevant_meta_incs(self, time): def relevant_meta_incs(self, time):
"""Return list [snapshotrp, diffrps ...] time sorted""" """Return list [snapshotrp, diffrps ...] time sorted"""
inclist = self.sorted_meta_inclist(min_time = time) inclist = self.sorted_prefix_inclist('mirror_metadata', min_time=time)
if not inclist: return inclist if not inclist: return inclist
assert inclist[-1].getinctime() == time, inclist[-1] assert inclist[-1].getinctime() == time, inclist[-1]
for i in range(len(inclist)-1, -1, -1): for i in range(len(inclist)-1, -1, -1):
...@@ -631,6 +632,7 @@ ManagerObj = None # Set this later to Manager instance ...@@ -631,6 +632,7 @@ ManagerObj = None # Set this later to Manager instance
def SetManager(): def SetManager():
global ManagerObj global ManagerObj
ManagerObj = PatchDiffMan() ManagerObj = PatchDiffMan()
return ManagerObj
import eas_acls # put at bottom to avoid python circularity bug import eas_acls # put at bottom to avoid python circularity bug
...@@ -63,9 +63,9 @@ def Regress(mirror_rp): ...@@ -63,9 +63,9 @@ def Regress(mirror_rp):
assert mirror_rp.index == () and inc_rpath.index == () assert mirror_rp.index == () and inc_rpath.index == ()
assert mirror_rp.isdir() and inc_rpath.isdir() assert mirror_rp.isdir() and inc_rpath.isdir()
assert mirror_rp.conn is inc_rpath.conn is Globals.local_connection assert mirror_rp.conn is inc_rpath.conn is Globals.local_connection
set_regress_time() manager, former_current_mirror_rp = set_regress_time()
set_restore_times() set_restore_times()
former_current_mirror_rp = remove_rbdir_increments() regress_rbdir(manager)
ITR = rorpiter.IterTreeReducer(RegressITRB, []) ITR = rorpiter.IterTreeReducer(RegressITRB, [])
for rf in iterate_meta_rfs(mirror_rp, inc_rpath): ITR(rf.index, rf) for rf in iterate_meta_rfs(mirror_rp, inc_rpath): ITR(rf.index, rf)
ITR.Finish() ITR.Finish()
...@@ -81,14 +81,15 @@ def set_regress_time(): ...@@ -81,14 +81,15 @@ def set_regress_time():
""" """
global regress_time, unsuccessful_backup_time global regress_time, unsuccessful_backup_time
curmir_incs = restore.get_inclist(Globals.rbdir.append("current_mirror")) manager = metadata.SetManager()
curmir_incs = manager.sorted_prefix_inclist('current_mirror')
assert len(curmir_incs) == 2, \ assert len(curmir_incs) == 2, \
"Found %s current_mirror flags, expected 2" % len(curmir_incs) "Found %s current_mirror flags, expected 2" % len(curmir_incs)
inctimes = [inc.getinctime() for inc in curmir_incs] mirror_rp_to_delete = curmir_incs[0]
inctimes.sort() regress_time = curmir_incs[1].getinctime()
regress_time = inctimes[0] unsuccessful_backup_time = mirror_rp_to_delete.getinctime()
unsuccessful_backup_time = inctimes[-1]
log.Log("Regressing to " + Time.timetopretty(regress_time), 4) log.Log("Regressing to " + Time.timetopretty(regress_time), 4)
return manager, mirror_rp_to_delete
def set_restore_times(): def set_restore_times():
"""Set _rest_time and _mirror_time in the restore module """Set _rest_time and _mirror_time in the restore module
...@@ -100,24 +101,51 @@ def set_restore_times(): ...@@ -100,24 +101,51 @@ def set_restore_times():
restore.MirrorStruct._mirror_time = unsuccessful_backup_time restore.MirrorStruct._mirror_time = unsuccessful_backup_time
restore.MirrorStruct._rest_time = regress_time restore.MirrorStruct._rest_time = regress_time
def remove_rbdir_increments(): def regress_rbdir(meta_manager):
"""Delete the increments in the rdiff-backup-data directory """Delete the increments in the rdiff-backup-data directory
Returns the former current mirror rp so we can delete it later. Returns the former current mirror rp so we can delete it later.
All of the other rp's should be deleted before the actual regress, All of the other rp's should be deleted before the actual regress,
to clear up disk space the rest of the procedure may need. to clear up disk space the rest of the procedure may need.
Also, in case the previous session failed while diffing the
metadata file, either recreate the mirror_metadata snapshot, or
delete the extra regress_time diff.
""" """
former_current_mirror = None has_meta_diff, has_meta_snap = 0, 0
for filename in Globals.rbdir.listdir(): for old_rp in meta_manager.timerpmap[regress_time]:
rp = Globals.rbdir.append(filename) if old_rp.getincbase_str() == 'mirror_metadata':
if rp.isincfile() and rp.getinctime() == unsuccessful_backup_time: if old_rp.getinctype() == 'snapshot': has_meta_snap = 1
if rp.getincbase_str() == "current_mirror":
former_current_mirror = rp
else: else:
log.Log("Removing rdiff-backup-data increment " + rp.path, 5) assert old_rp.getinctype() == 'diff', old_rp
rp.delete() has_meta_diff = 1
return former_current_mirror if has_meta_diff and not has_meta_snap: recreate_meta(meta_manager)
for new_rp in meta_manager.timerpmap[unsuccessful_backup_time]:
if new_rp.getincbase_str() != 'current_mirror':
log.Log("Deleting old diff at " + new_rp.path, 5)
new_rp.delete()
def recreate_meta(meta_manager):
"""Make regress_time mirror_metadata snapshot by patching
We write to a tempfile first. Otherwise, in case of a crash, it
would seem we would have an intact snapshot and partial diff, not
the reverse.
"""
temprp = TempFile.new_in_dir(Globals.rbdir)
writer = metadata.MetadataFile(temprp, 'w', check_path = 0)
for rorp in meta_manager.get_meta_at_time(regress_time, None):
writer.write_object(rorp)
writer.close()
finalrp = Globals.rbdir.append("mirror_metadata.%s.snapshot.gz" %
Time.timetostring(regress_time))
assert not finalrp.lstat(), finalrp
rpath.rename(temprp, finalrp)
if Globals.fsync_directories: Globals.rbdir.fsync()
def iterate_raw_rfs(mirror_rp, inc_rp): def iterate_raw_rfs(mirror_rp, inc_rp):
"""Iterate all RegressFile objects in mirror/inc directory """Iterate all RegressFile objects in mirror/inc directory
...@@ -146,8 +174,8 @@ def yield_metadata(): ...@@ -146,8 +174,8 @@ def yield_metadata():
metadata.SetManager() metadata.SetManager()
metadata_iter = metadata.ManagerObj.GetAtTime(regress_time) metadata_iter = metadata.ManagerObj.GetAtTime(regress_time)
if metadata_iter: return metadata_iter if metadata_iter: return metadata_iter
log.Log.FatalError("No metadata for time %s found, cannot regress" log.Log.FatalError("No metadata for time %s (%s) found,\ncannot regress"
% Time.timetopretty(regress_time)) % (Time.timetopretty(regress_time), regress_time))
def iterate_meta_rfs(mirror_rp, inc_rp): def iterate_meta_rfs(mirror_rp, inc_rp):
"""Yield RegressFile objects with extra metadata information added """Yield RegressFile objects with extra metadata information added
......
...@@ -199,7 +199,7 @@ class MetadataTest(unittest.TestCase): ...@@ -199,7 +199,7 @@ class MetadataTest(unittest.TestCase):
man.ConvertMetaToDiff() man.ConvertMetaToDiff()
man = PatchDiffMan() man = PatchDiffMan()
l = man.sorted_meta_inclist() l = man.sorted_prefix_inclist('mirror_metadata')
assert l[0].getinctype() == 'snapshot' assert l[0].getinctype() == 'snapshot'
assert l[0].getinctime() == 40000 assert l[0].getinctime() == 40000
assert l[1].getinctype() == 'snapshot' assert l[1].getinctype() == 'snapshot'
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment