Added --verify and --verify-at-time switches

git-svn-id: http://svn.savannah.nongnu.org/svn/rdiff-backup@665 2b77aa54-bcbc-44c9-a7ec-4f6cf2b41109

trunk/rdiff-backup/TODO

---verify switch for checking hashs, and hash check on restore
+Do hash check on restore
+
+Don't copy metadata onto a hardlink twice
 
 For comparing, check source filesystem
 

trunk/rdiff-backup/rdiff_backup/Main.py

@@ -83,7 +83,8 @@ def parse_cmdlineoptions(arglist):
 		  "remove-older-than=", "restore-as-of=", "restrict=",
 		  "restrict-read-only=", "restrict-update-only=", "server",
 		  "ssh-no-compression", "terminal-verbosity=", "test-server",
-		  "user-mapping-file=", "verbosity=", "version"])
+		  "user-mapping-file=", "verbosity=", "verify",
+		  "verify-at-time=", "version"])
 	except getopt.error, e:
 		commandline_error("Bad commandline options: %s" % str(e))
 
@@ -190,10 +191,12 @@ def parse_cmdlineoptions(arglist):
 		elif opt == "--terminal-verbosity": Log.setterm_verbosity(arg)
 		elif opt == "--test-server": action = "test-server"
 		elif opt == "--user-mapping-file": user_mapping_filename = arg
+		elif opt == "-v" or opt == "--verbosity": Log.setverbosity(arg)
+		elif opt == "--verify": action, restore_timestr = "verify", "now"
+		elif opt == "--verify-at-time": action, restore_timestr = "verify", arg
 		elif opt == "-V" or opt == "--version":
 			print "rdiff-backup " + Globals.version
 			sys.exit(0)
-		elif opt == "-v" or opt == "--verbosity": Log.setverbosity(arg)
 		else: Log.FatalError("Unknown option %s" % opt)
 
 def check_action():
@@ -202,7 +205,8 @@ def check_action():
 	arg_action_dict = {0: ['server'],
 					   1: ['list-increments', 'list-increment-sizes',
 						   'remove-older-than', 'list-at-time',
-						   'list-changed-since', 'check-destination-dir'],
+						   'list-changed-since', 'check-destination-dir',
+						   'verify'],
 					   2: ['backup', 'restore', 'restore-as-of',
 						   'compare', 'compare-hash', 'compare-full']}
 	l = len(args)
@@ -276,6 +280,7 @@ def take_action(rps):
 	elif action == "restore": Restore(*rps)
 	elif action == "restore-as-of": Restore(rps[0], rps[1], 1)
 	elif action == "test-server": SetConnections.TestConnections()
+	elif action == "verify": Verify(rps[0])
 	else: raise AssertionError("Unknown action " + action)
 
 def cleanup():
@@ -722,6 +727,18 @@ def Compare(compare_type, src_rp, dest_rp, compare_time = None):
 		compare_func = compare.Compare_full
 	return_val = compare_func(src_rp, mirror_rp, inc_rp, compare_time)
 
+def Verify(dest_rp, verify_time = None):
+	"""Check the hashs of the regular files against mirror_metadata"""
+	global return_val
+	dest_rp = require_root_set(dest_rp, 1)
+	if not verify_time:
+		try: verify_time = Time.genstrtotime(restore_timestr)
+		except Time.TimeException, exc: Log.FatalError(str(exc))
+
+	mirror_rp = restore_root.new_index(restore_index)
+	inc_rp = Globals.rbdir.append_path("increments", restore_index)
+	return_val = dest_rp.conn.compare.Verify(mirror_rp, inc_rp, verify_time)
+
 
 def CheckDest(dest_rp):
 	"""Check the destination directory, """

trunk/rdiff-backup/rdiff_backup/Security.py

@@ -115,7 +115,7 @@ def set_security_level(action, cmdpairs):
 	elif action in ["test-server", "list-increments", 'list-increment-sizes',
 					"list-at-time", "list-changed-since",
 					"calculate-average", "remove-older-than", "compare",
-					"compare-hash", "compare-full"]:
+					"compare-hash", "compare-full", "verify"]:
 		sec_level = "minimal"
 		rdir = tempfile.gettempdir()
 	else: assert 0, "Unknown action %s" % action
@@ -159,7 +159,8 @@ def set_allowed_requests(sec_level):
 				  "compare.DataSide.get_source_select",
 				  "compare.DataSide.compare_fast",
 				  "compare.DataSide.compare_hash",
-				  "compare.DataSide.compare_full"])
+				  "compare.DataSide.compare_full",
+				  "compare.Verify"])
 	if sec_level == "update-only" or sec_level == "all":
 		l.extend(["log.Log.open_logfile_local", "log.Log.close_logfile_local",
 				  "log.ErrorLog.open", "log.ErrorLog.isopen",

trunk/rdiff-backup/rdiff_backup/compare.py

@@ -70,6 +70,35 @@ def Compare_full(src_rp, mirror_rp, inc_rp, compare_time):
 	repo_side.close_rf_cache()
 	return return_val
 
+def Verify(mirror_rp, inc_rp, verify_time):
+	"""Compute SHA1 sums of repository files and check against metadata"""
+	assert mirror_rp.conn is Globals.local_connection
+	repo_iter = RepoSide.init_and_get_iter(mirror_rp, inc_rp, verify_time)
+	base_index = RepoSide.mirror_base.index
+
+	bad_files = 0
+	for repo_rorp in repo_iter:
+		if not repo_rorp.isreg(): continue
+		if not repo_rorp.has_sha1():
+			log.Log("Warning: Cannot find SHA1 digest for file %s,\n"
+					"perhaps because these feature was added in v1.1.1"
+					% (repo_rorp.get_indexpath(),), 2)
+			continue
+		fp = RepoSide.rf_cache.get_fp(base_index + repo_rorp.index)
+		computed_hash = hash.compute_sha1_fp(fp)
+		if computed_hash == repo_rorp.get_sha1():
+			log.Log("Verified SHA1 digest of " + repo_rorp.get_indexpath(), 5)
+		else:
+			bad_files += 1
+			log.Log("Warning: Computed SHA1 digest of %s\n   %s\n"
+					"doesn't match recorded digest of\n   %s\n"
+					"Your backup repository may be corrupted!" %
+					(repo_rorp.get_indexpath(), computed_hash,
+					 repo_rorp.get_sha1()), 2)
+	RepoSide.close_rf_cache()
+	if not bad_files: log.Log("Every file verified successfully.", 3)
+	return bad_files
+
 def print_reports(report_iter):
 	"""Given an iter of CompareReport objects, print them to screen"""
 	assert not Globals.server
@@ -80,7 +109,7 @@ def print_reports(report_iter):
 		print "%s: %s" % (report.reason, indexpath)
 
 	if not changed_files_found:
-		log.Log("No changes found.  Directory matches archive data.", 2)
+		log.Log("No changes found.  Directory matches archive data.", 3)
 	return changed_files_found
 
 def get_basic_report(src_rp, repo_rorp, comp_data_func = None):
@@ -112,6 +141,11 @@ def get_basic_report(src_rp, repo_rorp, comp_data_func = None):
 	elif src_rp == repo_rorp: return None
 	else: return CompareReport(index, "changed")
 
+def log_success(src_rorp, mir_rorp = None):
+	"""Log that src_rorp and mir_rorp compare successfully"""
+	path = src_rorp and src_rorp.get_indexpath() or mir_rorp.get_indexpath()
+	log.Log("Successful compare: %s" % (path,), 5)
+
 
 class RepoSide(restore.MirrorStruct):
 	"""On the repository side, comparing is like restoring"""
@@ -132,13 +166,14 @@ class RepoSide(restore.MirrorStruct):
 		"""
 		repo_iter = cls.init_and_get_iter(mirror_rp, inc_rp, compare_time)
 		base_index = cls.mirror_base.index
-		for src_rp, mir_rorp in rorpiter.Collate2Iters(src_iter, repo_iter):
-			index = src_rp and src_rp.index or mir_rorp.index
-			if src_rp and mir_rorp:
-				if not src_rp.isreg() and src_rp == mir_rorp:
+		for src_rorp, mir_rorp in rorpiter.Collate2Iters(src_iter, repo_iter):
+			index = src_rorp and src_rorp.index or mir_rorp.index
+			if src_rorp and mir_rorp:
+				if not src_rorp.isreg() and src_rorp == mir_rorp:
+					log_success(src_rorp, mir_rorp)
 					continue # They must be equal, nothing else to check
-				if (src_rp.isreg() and mir_rorp.isreg() and
-					src_rp.getsize() == mir_rorp.getsize()):
+				if (src_rorp.isreg() and mir_rorp.isreg() and
+					src_rorp.getsize() == mir_rorp.getsize()):
 					mir_rorp.setfile(cls.rf_cache.get_fp(base_index + index))
 					mir_rorp.set_attached_filetype('snapshot')
 
@@ -156,6 +191,7 @@ class DataSide(backup.SourceStruct):
 		for src_rorp, mir_rorp in rorpiter.Collate2Iters(src_iter, repo_iter):
 			report = get_basic_report(src_rorp, mir_rorp)
 			if report: yield report
+			else: log_success(src_rorp, mir_rorp)
 
 	def compare_hash(cls, repo_iter):
 		"""Like above, but also compare sha1 sums of any regular files"""
@@ -174,6 +210,7 @@ class DataSide(backup.SourceStruct):
 		for src_rp, mir_rorp in rorpiter.Collate2Iters(src_iter, repo_iter):
 			report = get_basic_report(src_rp, mir_rorp, hashs_changed)
 			if report: yield report
+			else: log_success(src_rp, mir_rorp)
 
 	def compare_full(cls, src_root, repo_iter):
 		"""Given repo iter with full data attached, return report iter"""
@@ -191,6 +228,7 @@ class DataSide(backup.SourceStruct):
 			src_rp = src_root.new_index(repo_rorp.index)
 			report = get_basic_report(src_rp, repo_rorp, data_changed)
 			if report: yield report
+			else: log_success(repo_rorp)
 			
 static.MakeClass(DataSide)
 

trunk/rdiff-backup/rdiff_backup/hash.py

@@ -57,12 +57,16 @@ class Report:
 def compute_sha1(rp, compressed = 0):
 	"""Return the hex sha1 hash of given rpath"""
 	assert rp.conn is Globals.local_connection # inefficient not to do locally
-	blocksize = Globals.blocksize
-	fp = FileWrapper(rp.open("r", compressed))
-	while 1:
-		if not fp.read(blocksize): break
-	digest = fp.close().sha1_digest
+	digest = compute_sha1_fp(rp.open("r", compressed))
 	rp.set_sha1(digest)
 	return digest
 
+def compute_sha1_fp(fp, compressed = 0):
+	"""Return hex sha1 hash of given file-like object"""
+	blocksize = Globals.blocksize
+	fw = FileWrapper(fp)
+	while 1:
+		if not fw.read(blocksize): break
+	return fw.close().sha1_digest
+
 

trunk/rdiff-backup/rdiff_backup/restore.py

@@ -459,6 +459,23 @@ class RestoreFile:
 
 	def get_restore_fp(self):
 		"""Return file object of restored data"""
+		def get_fp():
+			current_fp = self.get_first_fp()
+			for inc_diff in self.relevant_incs[1:]:
+				log.Log("Applying patch %s" % (inc_diff.get_indexpath(),), 7)
+				assert inc_diff.getinctype() == 'diff'
+				delta_fp = inc_diff.open("rb", inc_diff.isinccompressed())
+				new_fp = tempfile.TemporaryFile()
+				Rdiff.write_patched_fp(current_fp, delta_fp, new_fp)
+				new_fp.seek(0)
+				current_fp = new_fp
+			return current_fp
+
+		def error_handler(exc):
+			log.Log("Error reading %s, substituting empty file." %
+					(self.mirror_rp.path,), 2)
+			return cStringIO.StringIO('')
+
 		if not self.relevant_incs[-1].isreg():
 			log.Log("""Warning: Could not restore file %s!
 
@@ -469,16 +486,7 @@ created.  This error is probably caused by data loss in the
 rdiff-backup destination directory, or a bug in rdiff-backup""" %
 					(self.mirror_rp.path, self.relevant_incs[-1].lstat()), 2)
 			return cStringIO.StringIO('')
-		current_fp = self.get_first_fp()
-		for inc_diff in self.relevant_incs[1:]:
-			log.Log("Applying patch %s" % (inc_diff.get_indexpath(),), 7)
-			assert inc_diff.getinctype() == 'diff'
-			delta_fp = inc_diff.open("rb", inc_diff.isinccompressed())
-			new_fp = tempfile.TemporaryFile()
-			Rdiff.write_patched_fp(current_fp, delta_fp, new_fp)
-			new_fp.seek(0)
-			current_fp = new_fp
-		return current_fp
+		return robust.check_common_error(error_handler, get_fp)
 
 	def get_first_fp(self):
 		"""Return first file object from relevant inc list"""

trunk/rdiff-backup/rdiff_backup/robust.py

@@ -48,7 +48,7 @@ def catch_error(exc):
 		if isinstance(exc, exception_class): return 1
 	if (isinstance(exc, EnvironmentError) and
 		# the invalid mode shows up in backups of /proc for some reason
-		(exc[0] == 'invalid mode: rb' or
+		(exc[0] in ('invalid mode: rb', 'Not a gzipped file') or
 		 errno.errorcode.has_key(exc[0]) and
 		 errno.errorcode[exc[0]] in ('EPERM', 'ENOENT', 'EACCES', 'EBUSY',
 									 'EEXIST', 'ENOTDIR',

trunk/rdiff-backup/testing/commontest.py

@@ -60,7 +60,7 @@ def rdiff_backup(source_local, dest_local, src_dir, dest_dir,
 	"""
 	if not source_local:
 		src_dir = ("'cd test1; ../%s --server'::../%s" % (RBBin, src_dir))
-	if not dest_local:
+	if dest_dir and not dest_local:
 		dest_dir = ("'cd test2/tmp; ../../%s --server'::../../%s" %
 					(RBBin, dest_dir))
 
@@ -68,7 +68,8 @@ def rdiff_backup(source_local, dest_local, src_dir, dest_dir,
 	if not (source_local and dest_local): cmdargs.append("--remote-schema %s")
 
 	if current_time: cmdargs.append("--current-time %s" % current_time)
-	cmdargs.extend([src_dir, dest_dir])
+	cmdargs.append(src_dir)
+	if dest_dir: cmdargs.append(dest_dir)
 	cmdline = " ".join(cmdargs)
 	print "Executing: ", cmdline
 	ret_val = os.system(cmdline)

trunk/rdiff-backup/testing/comparetest.py

@@ -98,5 +98,53 @@ class CompareTest(unittest.TestCase):
 		"""Test --compare-full of subdirectory, remote"""
 		self.generic_selective_test(0, "--compare-full")		
 
+	def verify(self, local):
+		"""Used for the verify tests"""
+		def change_file(rp):
+			"""Given rpath, open it, and change a byte in the middle"""
+			fp = rp.open("rb")
+			fp.seek(int(rp.getsize()/2))
+			char = fp.read(1)
+			fp.close()
+
+			fp = rp.open("wb")
+			fp.seek(int(rp.getsize()/2))
+			if char == 'a': fp.write('b')
+			else: fp.write('a')
+			fp.close()
+
+		def modify_diff():
+			"""Write to the stph_icons.h diff"""
+			l = [filename for filename in
+				 os.listdir('testfiles/output/rdiff-backup-data/increments')
+				 if filename.startswith('stph_icons.h')]
+			assert len(l) == 1, l
+			diff_rp = rpath.RPath(Globals.local_connection,
+			  'testfiles/output/rdiff-backup-data/increments/' + l[0])
+			change_file(diff_rp)
+
+		rdiff_backup(local, local, 'testfiles/output', None,
+					 extra_options = "--verify")
+		rdiff_backup(local, local, 'testfiles/output', None,
+					 extra_options = "--verify-at-time 10000")
+		modify_diff()
+		ret_val =  rdiff_backup(local, local, 'testfiles/output', None,
+					 extra_options = "--verify-at-time 10000",
+								check_return_val = 0)
+		assert ret_val, ret_val
+		change_file(rpath.RPath(Globals.local_connection,
+								'testfiles/output/stph_icons.h'))
+		ret_val =  rdiff_backup(local, local, 'testfiles/output', None,
+					 extra_options = "--verify", check_return_val = 0)
+		assert ret_val, ret_val
+
+	def testVerifyLocal(self):
+		"""Test --verify of directory, local"""
+		self.verify(1)
+
+	def testVerifyRemote(self):
+		"""Test --verify remotely"""
+		self.verify(0)
+
 if __name__ == "__main__": unittest.main()