Various modifications to backup, restore, and regress systems.

This version passes many tests but not all of them.  The backup patch
system was copied to restore.py.


git-svn-id: http://svn.savannah.nongnu.org/svn/rdiff-backup@281 2b77aa54-bcbc-44c9-a7ec-4f6cf2b41109

trunk/rdiff-backup/rdiff_backup/log.py

@@ -194,21 +194,21 @@ class ErrorLog:
 	created.  See the error policy file for more info.
 
 	"""
-	log_fileobj = None
-	log_inc_rp = None
+	_log_fileobj = None
+	_log_inc_rp = None
 	def open(cls, compress = 1):
 		"""Open the error log, prepare for writing"""
-		assert not cls.log_fileobj and not cls.log_inc_rp, "log already open"
+		assert not cls._log_fileobj and not cls._log_inc_rp, "log already open"
 		if compress: typestr = 'data.gz'
 		else: typestr = 'data'
-		cls.log_inc_rp = Global.rbdir.append("error_log.%s.%s" %
-											 (Time.curtimestr, typestr))
-		assert not cls.log_inc_rp.lstat(), "Error file already exists"
-		cls.log_fileobj = cls.log_inc_rp.open("wb", compress = compress)
+		cls._log_inc_rp = Global.rbdir.append("error_log.%s.%s" %
+											  (Time.curtimestr, typestr))
+		assert not cls._log_inc_rp.lstat(), "Error file already exists"
+		cls._log_fileobj = cls._log_inc_rp.open("wb", compress = compress)
 
 	def isopen(cls):
 		"""True if the error log file is currently open"""
-		return cls.log_fileobj is not None
+		return cls._log_fileobj is not None
 
 	def write(cls, error_type, rp, exc):
 		"""Add line to log file indicating error exc with file rp"""
@@ -218,7 +218,7 @@ class ErrorLog:
 		else:
 			s = re.sub("\n", " ", s)
 			s += "\n"
-		cls.log_fileobj.write(s)
+		cls._log_fileobj.write(s)
 
 	def get_indexpath(cls, rp):
 		"""Return filename for logging.  rp is a rpath, string, or tuple"""
@@ -240,8 +240,8 @@ class ErrorLog:
 
 	def close(cls):
 		"""Close the error log file"""
-		assert not cls.log_fileobj.close()
-		cls.log_fileobj = cls.log_inc_rp = None
+		assert not cls._log_fileobj.close()
+		cls._log_fileobj = cls._log_inc_rp = None
 
 static.MakeClass(ErrorLog)
 

trunk/rdiff-backup/rdiff_backup/metadata.py

@@ -280,6 +280,7 @@ def WriteMetadata(rorp):
 def CloseMetadata():
 	"""Close the metadata file"""
 	global metadata_rp, metadata_fileobj
+	assert metadata_fileobj, "Metadata file not open"
 	try: fileno = metadata_fileobj.fileno() # will not work if GzipFile
 	except AttributeError: fileno = metadata_fileobj.fileobj.fileno()
 	os.fsync(fileno)

trunk/rdiff-backup/rdiff_backup/regress.py

@@ -34,7 +34,7 @@ recovered.
 """
 
 from __future__ import generators
-import Globals, restore, log, rorpiter, journal, TempFile
+import Globals, restore, log, rorpiter, journal, TempFile, metadata, rpath
 
 # regress_time should be set to the time we want to regress back to
 # (usually the time of the last successful backup)
@@ -43,6 +43,10 @@ regress_time = None
 # This should be set to the latest unsuccessful backup time
 unsuccessful_backup_time = None
 
+# This is set by certain tests and allows overriding of global time
+# variables.
+time_override_mode = None
+
 
 class RegressException(Exception):
 	"""Raised on any exception in regress process"""
@@ -64,6 +68,9 @@ def Regress(mirror_rp):
 	assert mirror_rp.conn is inc_rpath.conn is Globals.local_connection
 	set_regress_time()
 	set_restore_times()
+	ITR = rorpiter.IterTreeReducer(RegressITRB, [])
+	for rf in iterate_meta_rfs(mirror_rp, inc_rpath): ITR(rf.index, rf)
+	ITR.Finish()
 
 def set_regress_time():
 	"""Set global regress_time to previous sucessful backup
@@ -73,6 +80,10 @@ def set_regress_time():
 
 	"""
 	global regress_time, unsuccessful_backup_time
+	if time_override_mode:
+		assert regress_time and unsuccessful_backup_time
+		return
+
 	curmir_incs = restore.get_inclist(Globals.rbdir.append("current_mirror"))
 	assert len(curmir_incs) == 2, \
 		   "Found %s current_mirror flags, expected 2" % len(curmir_incs)
@@ -134,10 +145,10 @@ class RegressFile(restore.RestoreFile):
 
 	"""
 	def __init__(self, mirror_rp, inc_rp, inc_list):
-		restore.RestoreFile._init__(self, mirror_rp, inc_rp, inclist)
+		restore.RestoreFile.__init__(self, mirror_rp, inc_rp, inc_list)
 		assert len(self.relevant_incs) <= 2, "Too many incs"
 		if len(self.relevant_incs) == 2:
-			self.regress_inc = self.relevant.incs[-1]
+			self.regress_inc = self.relevant_incs[-1]
 		else: self.regress_inc = None
 
 	def set_metadata_rorp(self, metadata_rorp):
@@ -178,7 +189,8 @@ class RegressITRB(rorpiter.ITRBranch):
 
 	def fast_process(self, index, rf):
 		"""Process when nothing is a directory"""
-		if not rpath.cmp_attribs(rf.metadata_rorp, rf.mirror_rp):
+		if (not rf.metadata_rorp.lstat() or not rf.mirror_rp.lstat() or
+			not rpath.cmp_attribs(rf.metadata_rorp, rf.mirror_rp)):
 			if rf.metadata_rorp.isreg(): self.restore_orig_regfile(rf)
 			else:
 				if rf.mirror_rp.lstat(): rf.mirror_rp.delete()
@@ -242,8 +254,8 @@ class RegressITRB(rorpiter.ITRBranch):
 
 		"""
 		if args and args[0] and isinstance(args[0], tuple):
-			filename = os.path.join(*args[0])
-		elif self.index: filename = os.path.join(*self.index)
+			filename = "/".join(args[0])
+		elif self.index: filename = "/".join(*self.index)
 		else: filename = "."
 		log.Log("Error '%s' processing %s" % (exc, filename), 2)
 		raise RegressException("Error during Regress")

trunk/rdiff-backup/rdiff_backup/restore.py

@@ -22,7 +22,7 @@
 from __future__ import generators
 import tempfile, os
 import Globals, Time, Rdiff, Hardlink, rorpiter, selection, rpath, \
-	   log, backup, static, robust, metadata
+	   log, static, robust, metadata, statistics, TempFile
 
 
 # This should be set to selection.Select objects over the source and
@@ -220,11 +220,18 @@ class TargetStruct:
 	def patch(cls, target, diff_iter):
 		"""Patch target with the diffs from the mirror side
 
-		This function was already written for use when backing up, so
-		just use that.
+		This function and the associated ITRB is similar to the
+		patching code in backup.py, but they have different error
+		correction requirements, so it seemed easier to just repeat it
+		all in this module.
 
 		"""
-		backup.DestinationStruct.patch(target, diff_iter)
+		ITR = rorpiter.IterTreeReducer(PatchITRB, [target])
+		for diff in rorpiter.FillInIter(diff_iter, target):
+			log.Log("Processing changed file " + diff.get_indexpath(), 5)
+			ITR(diff.index, diff)
+		ITR.Finish()
+		target.setdata()
 
 static.MakeClass(TargetStruct)
 
@@ -279,6 +286,7 @@ class CachedRF:
 		assert new_rfs, "No RFs added for index %s" % index
 		self.rf_list[0:0] = new_rfs
 
+
 class RestoreFile:
 	"""Hold data about a single mirror file and its related increments
 
@@ -390,8 +398,12 @@ class RestoreFile:
 	def yield_sub_rfs(self):
 		"""Return RestoreFiles under current RestoreFile (which is dir)"""
 		assert self.mirror_rp.isdir() or self.inc_rp.isdir()
-		mirror_iter = self.yield_mirrorrps(self.mirror_rp)
-		inc_pair_iter = self.yield_inc_complexes(self.inc_rp)
+		if self.mirror_rp.isdir():
+			mirror_iter = self.yield_mirrorrps(self.mirror_rp)
+		else: mirror_iter = iter([])
+		if self.inc_rp.isdir():
+			inc_pair_iter = self.yield_inc_complexes(self.inc_rp)
+		else: inc_pair_iter = iter([])
 		collated = rorpiter.Collate2Iters(mirror_iter, inc_pair_iter)
 
 		for mirror_rp, inc_pair in collated:
@@ -405,6 +417,7 @@ class RestoreFile:
 
 	def yield_mirrorrps(self, mirrorrp):
 		"""Yield mirrorrps underneath given mirrorrp"""
+		assert mirrorrp.isdir()
 		for filename in robust.listrp(mirrorrp):
 			rp = mirrorrp.append(filename)
 			if rp.index != ('rdiff-backup-data',): yield rp
@@ -440,3 +453,92 @@ class RestoreFile:
 		keys = inc_dict.keys()
 		keys.sort()
 		for key in keys: yield inc_dict[key]
+
+
+class PatchITRB(rorpiter.ITRBranch):
+	"""Patch an rpath with the given diff iters (use with IterTreeReducer)
+
+	The main complication here involves directories.  We have to
+	finish processing the directory after what's in the directory, as
+	the directory may have inappropriate permissions to alter the
+	contents or the dir's mtime could change as we change the
+	contents.
+
+	This code was originally taken from backup.py.  However, because
+	of different error correction requirements, it is repeated here.
+
+	"""
+	def __init__(self, basis_root_rp):
+		"""Set basis_root_rp, the base of the tree to be incremented"""
+		self.basis_root_rp = basis_root_rp
+		assert basis_root_rp.conn is Globals.local_connection
+		self.statfileobj = (statistics.get_active_statfileobj() or
+							statistics.StatFileObj())
+		self.dir_replacement, self.dir_update = None, None
+		self.cached_rp = None
+
+	def get_rp_from_root(self, index):
+		"""Return RPath by adding index to self.basis_root_rp"""
+		if not self.cached_rp or self.cached_rp.index != index:
+			self.cached_rp = self.basis_root_rp.new_index(index)
+		return self.cached_rp
+
+	def can_fast_process(self, index, diff_rorp):
+		"""True if diff_rorp and mirror are not directories"""
+		rp = self.get_rp_from_root(index)
+		return not diff_rorp.isdir() and not rp.isdir()
+
+	def fast_process(self, index, diff_rorp):
+		"""Patch base_rp with diff_rorp (case where neither is directory)"""
+		rp = self.get_rp_from_root(index)
+		tf = TempFile.new(rp)
+		self.patch_to_temp(rp, diff_rorp, tf)
+		rpath.rename(tf, rp)
+
+	def patch_to_temp(self, basis_rp, diff_rorp, new):
+		"""Patch basis_rp, writing output in new, which doesn't exist yet"""
+		if diff_rorp.isflaglinked():
+			Hardlink.link_rp(diff_rorp, new, self.basis_root_rp)
+		elif diff_rorp.get_attached_filetype() == 'snapshot':
+			rpath.copy(diff_rorp, new)
+		else:
+			assert diff_rorp.get_attached_filetype() == 'diff'
+			Rdiff.patch_local(basis_rp, diff_rorp, new)
+		if new.lstat(): rpath.copy_attribs(diff_rorp, new)
+
+	def start_process(self, index, diff_rorp):
+		"""Start processing directory - record information for later"""
+		base_rp = self.base_rp = self.get_rp_from_root(index)
+		assert diff_rorp.isdir() or base_rp.isdir() or not base_rp.index
+		if diff_rorp.isdir(): self.prepare_dir(diff_rorp, base_rp)
+		else: self.set_dir_replacement(diff_rorp, base_rp)
+
+	def set_dir_replacement(self, diff_rorp, base_rp):
+		"""Set self.dir_replacement, which holds data until done with dir
+
+		This is used when base_rp is a dir, and diff_rorp is not.
+
+		"""
+		assert diff_rorp.get_attached_filetype() == 'snapshot'
+		self.dir_replacement = TempFile.new(base_rp)
+		rpath.copy_with_attribs(diff_rorp, self.dir_replacement)
+		if base_rp.isdir(): base_rp.chmod(0700)
+
+	def prepare_dir(self, diff_rorp, base_rp):
+		"""Prepare base_rp to turn into a directory"""
+		self.dir_update = diff_rorp.getRORPath() # make copy in case changes
+		if not base_rp.isdir():
+			if base_rp.lstat(): base_rp.delete()
+			base_rp.mkdir()
+		base_rp.chmod(0700)
+
+	def end_process(self):
+		"""Finish processing directory"""
+		if self.dir_update:
+			assert self.base_rp.isdir()
+			rpath.copy_attribs(self.dir_update, self.base_rp)
+		else:
+			assert self.dir_replacement
+			self.base_rp.rmdir()
+			if self.dir_replacement.lstat():
+				rpath.rename(self.dir_replacement, self.base_rp)

trunk/rdiff-backup/rdiff_backup/rorpiter.py

@@ -149,24 +149,6 @@ def Collate2Iters(riter1, riter2):
 			yield (None, relem2)
 			relem2 = None
 
-def get_dissimilar_indicies(src_init_iter, dest_init_iter, statfileobj = None):
-	"""Get dissimilar indicies given two rorpiters
-
-	Returns an iterator which enumerates the indicies of the rorps
-	which are different on the source and destination ends.  If
-	statfileobj is given, call add_changed on each pair of different
-	indicies.
-
-	"""
-	collated = Collate2Iters(src_init_iter, dest_init_iter)
-	for src_rorp, dest_rorp in collated:
-		if (src_rorp and dest_rorp and src_rorp == dest_rorp and
-			(not Globals.preserve_hardlinks or
-			 Hardlink.rorp_eq(src_rorp, dest_rorp))): continue
-		if statfileobj: statfileobj.add_changed(src_rorp, dest_rorp)
-		if not dest_rorp: yield src_rorp.index
-		else: yield dest_rorp.index
-
 
 class IndexedTuple(UserList.UserList):
 	"""Like a tuple, but has .index

trunk/rdiff-backup/rdiff_backup/rpath.py

@@ -90,7 +90,7 @@ def copy(rpin, rpout, compress = 0):
 
 	if rpout.lstat():
 		if rpin.isreg() or not cmp(rpin, rpout):
-			rpout.delete()   # easier to write that compare
+			rpout.delete()   # easier to write than compare
 		else: return
 
 	if rpin.isreg(): copy_reg_file(rpin, rpout, compress)
@@ -177,7 +177,7 @@ def cmp_attribs(rp1, rp2):
 	elif rp1.ischardev() and rp2.ischardev(): result = 1
 	else: result = (rp1.getmtime() == rp2.getmtime())
 	log.Log("Compare attribs of %s and %s: %s" %
-			(rp1.path, rp2.path, result), 7)
+			(rp1.get_indexpath(), rp2.get_indexpath(), result), 7)
 	return result
 
 def copy_with_attribs(rpin, rpout, compress = 0):
@@ -694,11 +694,7 @@ class RPath(RORPath):
 	def delete(self):
 		"""Delete file at self.path.  Recursively deletes directories."""
 		log.Log("Deleting %s" % self.path, 7)
-		self.setdata()
-		if not self.lstat():
-			log.Log("Warning: %s does not exist---deleted in meantime?"
-				% (self.path,), 2)
-		elif self.isdir():
+		if self.isdir():
 			try: self.rmdir()
 			except os.error: shutil.rmtree(self.path)
 		else: self.conn.os.unlink(self.path)

trunk/rdiff-backup/testing/commontest.py

@@ -5,6 +5,7 @@ from rdiff_backup.rpath import RPath
 from rdiff_backup import Globals, Hardlink, SetConnections, Main, \
 	 selection, lazy, Time, rpath
 
+RBBin = "../rdiff-backup"
 SourceDir = "../rdiff_backup"
 AbsCurdir = os.getcwd() # Absolute path name of current directory
 AbsTFdir = AbsCurdir+"/testfiles"
@@ -56,12 +57,14 @@ def rdiff_backup(source_local, dest_local, src_dir, dest_dir,
 		dest_dir = ("test2/tmp; ../../%s/rdiff-backup --server::../../%s" %
 					(SourceDir, dest_dir))
 
-	cmdargs = [SourceDir + "/rdiff-backup", extra_options]
+	cmdargs = [RBBin, extra_options]
 	if not (source_local and dest_local): cmdargs.append("--remote-schema %s")
 
 	if current_time: cmdargs.append("--current-time %s" % current_time)
-
-	os.system(" ".join(cmdargs))	
+	cmdargs.extend([src_dir, dest_dir])
+	cmdline = " ".join(cmdargs)
+	print "Executing: ", cmdline
+	assert not os.system(cmdline)
 
 def cmd_schemas2rps(schema_list, remote_schema):
 	"""Input list of file descriptions and the remote schema, return rps

trunk/rdiff-backup/testing/regresstest.py

-"""regresstest - test the regress module.  Not to be confused with the
-regression tests."""
+"""regresstest - test the regress module.
+
+Not to be confused with the regression tests.
+
+"""
 
 import unittest
 from commontest import *
+from rdiff_backup import regress
+
+Log.setverbosity(7)
 
 class RegressTest(unittest.TestCase):
-	XXX
+	regress_rp1 = rpath.RPath(Globals.local_connection,
+							  "testfiles/regress_output1")
+	regress_rp2 = rpath.RPath(Globals.local_connection,
+							  "testfiles/regress_output2")
+	
+	def make_output(self, level):
+		"""Set up two rdiff-backup destination dir of level and level+1
+
+		testfiles/regress_output1 will be a copy of
+		testfiles/increment1 through testfiles/increment{level}
+
+		testfiles/regress_output2 will have all everything backed up
+		in testfiles/regress_output1 + testfiles/increment{level+1}.
+
+		The time of each increment will be 10000*level.
+
+		"""
+		assert 1 <= level <= 3
+		if self.regress_rp1.lstat(): Myrm(self.regress_rp1.path)
+		if self.regress_rp2.lstat(): Myrm(self.regress_rp2.path)
+
+		# Make regress_output1
+		Log("Making %s" % (self.regress_rp1.path,), 4)
+		for i in range(1, level+1):
+			rdiff_backup(1, 1,
+						 "testfiles/increment%s" % (i,),
+						 self.regress_rp1.path,
+						 current_time = 10000*i)
+
+		# Now make regress_output2
+		Log("Making %s" % (self.regress_rp2.path,), 4)
+		assert not os.system("cp -a %s %s" %
+							 (self.regress_rp1.path, self.regress_rp2.path))
+		rdiff_backup(1, 1,
+					 "testfiles/increment%s" % (level+1),
+					 self.regress_rp2.path,
+					 current_time = 10000*(level+1))
+		self.regress_rp1.setdata()
+		self.regress_rp2.setdata()
+
+	def test_full(self):
+		"""Test regressing a full directory to older state
+
+		Make two directories, one with one more backup in it.  Then
+		regress the bigger one, and then make sure they compare the
+		same.
+
+		"""
+		for level in range(1, 4):
+			self.make_output(level)
+			regress.regress_time = 10000*level
+			regress.unsuccessful_backup_time = 10000*(level+1)
+			regress.time_override_mode = 1
+			Globals.rbdir = self.regress_rp2.append_path("rdiff-backup-data")
+			Log("######### Beginning regress ###########", 5)
+			regress.Regress(self.regress_rp2)
+
+			assert CompareRecursive(self.regress_rp1, self.regress_rp2,
+									exclude_rbdir = 0)
 
 
 if __name__ == "__main__": unittest.main()