Removed most destructive stepping operations

git-svn-id: http://svn.savannah.nongnu.org/svn/rdiff-backup@251 2b77aa54-bcbc-44c9-a7ec-4f6cf2b41109

trunk/rdiff-backup/CHANGELOG

@@ -7,6 +7,11 @@ Fixed bug with the --{include|exclude}-globbing-filelist options
 Added --list-changed-since option to list the files changed since the
 given date.
 
+Removed --mirror-only and --change-source-perms options.
+
+Removed all "resume" related functionality, like
+--checkpoint-interval.
+
 
 New in v0.11.0 (2002/10/05)
 ---------------------------

trunk/rdiff-backup/rdiff_backup/Globals.py

@@ -55,10 +55,6 @@ change_ownership = None
 # permissions).
 change_mirror_perms = (process_uid != 0)
 
-# If true, temporarily change permissions of unreadable files in
-# the source directory to make sure we can read all files.
-change_source_perms = None
-
 # If true, try to reset the atimes of the source partition.
 preserve_atime = None
 
@@ -103,24 +99,9 @@ client_conn = None
 # list.
 changed_settings = []
 
-# rdiff-backup will try to checkpoint its state every
-# checkpoint_interval seconds.  Then when resuming, at most this
-# amount of time is lost.
-checkpoint_interval = 20
-
 # The RPath of the rdiff-backup-data directory.
 rbdir = None
 
-# Indicates if a resume or a lack of resume is forced.  This
-# should be None for the default.  0 means don't resume, and 1
-# means resume.
-resume = None
-
-# If there has been an aborted backup fewer than this many seconds
-# ago, attempt to resume it where it left off instead of starting
-# a new one.
-resume_window = 7200
-
 # This string is used when recognizing and creating time strings.
 # If the time_separator is ":", then W3 datetime strings like
 # 2001-12-07T04:22:01-07:00 are produced.  It can be set to "_" to
@@ -265,7 +246,7 @@ def postset_regexp_local(name, re_string, flags):
 	if flags: globals()[name] = re.compile(re_string, flags)
 	else: globals()[name] = re.compile(re_string)
 
-def set_select(dsrpath, tuplelist, quote_mode, *filelists):
+def set_select(source, rpath, tuplelist, quote_mode, *filelists):
 	"""Initialize select object using tuplelist
 
 	Note that each list in filelists must each be passed as
@@ -275,12 +256,10 @@ def set_select(dsrpath, tuplelist, quote_mode, *filelists):
 
 	"""
 	global select_source, select_mirror
-	if dsrpath.source:
-		select_source = Select(dsrpath, quote_mode)
-		select_source.ParseArgs(tuplelist, filelists)
-	else:
-		select_mirror = Select(dsrpath, quote_mode)
-		select_mirror.ParseArgs(tuplelist, filelists)
+	sel = Select(rpath, quote_mode)
+	sel.ParseArgs(tuplelist, filelists)
+	if source: select_source = sel
+	else: select_mirror = sel
 
 
 from rpath import * # kludge to avoid circularity - not needed in this module

trunk/rdiff-backup/rdiff_backup/Main.py

@@ -25,7 +25,6 @@ from log import *
 from lazy import *
 from connection import *
 from rpath import *
-from destructive_stepping import *
 from robust import *
 from restore import *
 from highlevel import *
@@ -49,21 +48,20 @@ def parse_cmdlineoptions(arglist):
 		except IOError: Log.FatalError("Error opening file %s" % filename)
 
 	try: optlist, args = getopt.getopt(arglist, "blr:sv:V",
-		 ["backup-mode", "calculate-average", "change-source-perms",
-		  "chars-to-quote=", "checkpoint-interval=", "current-time=",
-		  "exclude=", "exclude-device-files", "exclude-filelist=",
-		  "exclude-filelist-stdin", "exclude-globbing-filelist=",
-		  "exclude-mirror=", "exclude-other-filesystems",
-		  "exclude-regexp=", "exclude-special-files", "force",
-		  "include=", "include-filelist=", "include-filelist-stdin",
+		 ["backup-mode", "calculate-average", "chars-to-quote=",
+		  "current-time=", "exclude=", "exclude-device-files",
+		  "exclude-filelist=", "exclude-filelist-stdin",
+		  "exclude-globbing-filelist=", "exclude-mirror=",
+		  "exclude-other-filesystems", "exclude-regexp=",
+		  "exclude-special-files", "force", "include=",
+		  "include-filelist=", "include-filelist-stdin",
 		  "include-globbing-filelist=", "include-regexp=",
 		  "list-changed-since=", "list-increments", "no-compression",
-		  "no-compression-regexp=", "no-hard-links", "no-resume",
-		  "null-separator", "parsable-output", "print-statistics",
-		  "quoting-char=", "remote-cmd=", "remote-schema=",
-		  "remove-older-than=", "restore-as-of=", "restrict=",
-		  "restrict-read-only=", "restrict-update-only=", "resume",
-		  "resume-window=", "server", "sleep-ratio=",
+		  "no-compression-regexp=", "no-hard-links", "null-separator",
+		  "parsable-output", "print-statistics", "quoting-char=",
+		  "remote-cmd=", "remote-schema=", "remove-older-than=",
+		  "restore-as-of=", "restrict=", "restrict-read-only=",
+		  "restrict-update-only=", "server", "sleep-ratio=",
 		  "ssh-no-compression", "terminal-verbosity=", "test-server",
 		  "verbosity=", "version", "windows-mode",
 		  "windows-time-format"])
@@ -73,13 +71,9 @@ def parse_cmdlineoptions(arglist):
 	for opt, arg in optlist:
 		if opt == "-b" or opt == "--backup-mode": action = "backup"
 		elif opt == "--calculate-average": action = "calculate-average"
-		elif opt == "--change-source-perms":
-			Globals.set('change_source_perms', 1)
 		elif opt == "--chars-to-quote":
 			Globals.set('chars_to_quote', arg)
 			Globals.set('quoting_enabled', 1)
-		elif opt == "--checkpoint-interval":
-			Globals.set_integer('checkpoint_interval', arg)
 		elif opt == "--current-time":
 			Globals.set_integer('current_time', arg)
 		elif opt == "--exclude": select_opts.append((opt, arg))
@@ -118,7 +112,6 @@ def parse_cmdlineoptions(arglist):
 		elif opt == "--no-compression-regexp":
 			Globals.set("no_compression_regexp_string", arg)
 		elif opt == "--no-hard-links": Globals.set('preserve_hardlinks', 0)
-		elif opt == '--no-resume': Globals.resume = 0
 		elif opt == "--null-separator": Globals.set("null_separator", 1)
 		elif opt == "--parsable-output": Globals.set('parsable_output', 1)
 		elif opt == "--print-statistics":
@@ -140,9 +133,6 @@ def parse_cmdlineoptions(arglist):
 		elif opt == "--restrict-update-only":
 			Globals.security_level = "update-only"
 			Globals.restrict_path = arg
-		elif opt == '--resume': Globals.resume = 1
-		elif opt == '--resume-window':
-			Globals.set_integer('resume_window', arg)
 		elif opt == "-s" or opt == "--server":
 			action = "server"
 			Globals.server = 1
@@ -263,9 +253,8 @@ def Backup(rpin, rpout):
 
 def backup_init_select(rpin, rpout):
 	"""Create Select objects on source and dest connections"""
-	rpin.conn.Globals.set_select(DSRPath(1, rpin), select_opts,
-								 None, *select_files)
-	rpout.conn.Globals.set_select(DSRPath(None, rpout), select_mirror_opts, 1)
+	rpin.conn.Globals.set_select(1, rpin, select_opts, None, *select_files)
+	rpout.conn.Globals.set_select(0, rpout, select_mirror_opts, 1)
 
 def backup_init_dirs(rpin, rpout):
 	"""Make sure rpin and rpout are valid, init data dir and logging"""
@@ -424,8 +413,8 @@ def restore_init_select(rpin, rpout):
 	the restore operation isn't.
 
 	"""
-	Globals.set_select(DSRPath(1, rpin), select_mirror_opts, None)
-	Globals.set_select(DSRPath(None, rpout), select_opts, None, *select_files)
+	Globals.set_select(1, rpin, select_mirror_opts, None)
+	Globals.set_select(0, rpout, select_opts, None, *select_files)
 
 def restore_get_root(rpin):
 	"""Return (mirror root, index) and set the data dir

trunk/rdiff-backup/rdiff_backup/SetConnections.py

@@ -197,11 +197,6 @@ def BackupInitConnections(reading_conn, writing_conn):
 	writing_conn.Globals.set("isbackup_writer", 1)
 	UpdateGlobal("backup_reader", reading_conn)
 	UpdateGlobal("backup_writer", writing_conn)
-	if (Globals.change_source_perms and
-		reading_conn.Globals.get("process_uid") == 0):
-		Log("Warning: --change_source_perms should usually not be used when\n"
-			"the reading connection is running as root, because root can\n"
-			"read all files regardless of their permissions.", 2)
 
 def CloseConnections():
 	"""Close all connections.  Run by client"""

trunk/rdiff-backup/rdiff_backup/connection.py

@@ -547,7 +547,6 @@ from connection import *
 from rpath import *
 from robust import *
 from rorpiter import *
-from destructive_stepping import *
 from selection import *
 from statistics import *
 from increment import *

trunk/rdiff-backup/rdiff_backup/highlevel.py

@@ -87,27 +87,23 @@ class HLSourceStruct:
 		"""Return diffs and finalize any dsrp changes remaining
 
 		Return a rorpiterator with files included of signatures of
-		dissimilar files.  This is the last operation run on the local
-		filestream, so finalize dsrp writes.
+		dissimilar files.
 
 		"""
 		collated = RORPIter.CollateIterators(cls.initial_dsiter2, sigiter)
-		finalizer = IterTreeReducer(DestructiveSteppingFinalizer, [])
-		def error_handler(exc, dest_sig, dsrp):
+		def error_handler(exc, dest_sig, rp):
 			Log("Error %s producing a diff of %s" %
-				(exc, dsrp and dsrp.path), 2)
+				(exc, rp and rp.path), 2)
 			return None
 			
 		def diffs():
-			for dsrp, dest_sig in collated:
+			for rp, dest_sig in collated:
 				if dest_sig:
 					if dest_sig.isplaceholder(): yield dest_sig
 					else:
 						diff = Robust.check_common_error(
-							error_handler, RORPIter.diffonce, [dest_sig, dsrp])
+							error_handler, RORPIter.diffonce, [dest_sig, rp])
 						if diff: yield diff
-				if dsrp: finalizer(dsrp.index, dsrp)
-			finalizer.Finish()
 		return diffs()
 
 MakeClass(HLSourceStruct)
@@ -117,7 +113,7 @@ class HLDestinationStruct:
 	"""Hold info used by HL on the destination side"""
 	_session_info = None # set to si if resuming
 	def split_initial_dsiter(cls):
-		"""Set initial_dsiters (iteration of all dsrps from rpath)"""
+		"""Set initial_dsiters (iteration of all rps from rpath)"""
 		result, cls.initial_dsiter2 = \
 				Iter.multiplex(Globals.select_mirror.set_iter(), 2)
 		return result
@@ -192,10 +188,10 @@ class HLDestinationStruct:
 		return RORPIter.Signatures(dissimilars)
 
 	def get_dsrp(cls, dest_rpath, index):
-		"""Return initialized dsrp based on dest_rpath with given index"""
-		dsrp = DSRPath(None, dest_rpath.conn, dest_rpath.base, index)
-		if Globals.quoting_enabled: dsrp.quote_path()
-		return dsrp
+		"""Return initialized rpath based on dest_rpath with given index"""
+		rp = RPath(dest_rpath.conn, dest_rpath.base, index)
+		if Globals.quoting_enabled: rp.quote_path()
+		return rp
 
 	def get_finalizer(cls):
 		"""Return finalizer, starting from session info if necessary"""
@@ -226,8 +222,8 @@ class HLDestinationStruct:
 	def patch_and_finalize(cls, dest_rpath, diffs):
 		"""Apply diffs and finalize"""
 		collated = RORPIter.CollateIterators(diffs, cls.initial_dsiter2)
-		finalizer = cls.get_finalizer()
-		diff_rorp, dsrp = None, None
+		#finalizer = cls.get_finalizer()
+		diff_rorp, rp = None, None
 
 		def patch(diff_rorp, dsrp):
 			if not dsrp: dsrp = cls.get_dsrp(dest_rpath, diff_rorp.index)
@@ -244,13 +240,14 @@ class HLDestinationStruct:
 			diff_rorp, dsrp = indexed_tuple
 			dsrp = Robust.check_common_error(error_handler, patch,
 											 [diff_rorp, dsrp])
-			finalizer(dsrp.index, dsrp)
-		finalizer.Finish()
+			#finalizer(dsrp.index, dsrp)
+		#finalizer.Finish()
 
 	def patch_w_datadir_writes(cls, dest_rpath, diffs, inc_rpath):
 		"""Apply diffs and finalize, with checkpointing and statistics"""
 		collated = RORPIter.CollateIterators(diffs, cls.initial_dsiter2)
-		finalizer, ITR = cls.get_finalizer(), cls.get_MirrorITR(inc_rpath)
+		#finalizer, ITR = cls.get_finalizer(), cls.get_MirrorITR(inc_rpath)
+		finalizer, ITR = None, cls.get_MirrorITR(inc_rpath)
 		MiscStats.open_dir_stats_file()
 		dsrp, finished_dsrp = None, None
 
@@ -261,10 +258,10 @@ class HLDestinationStruct:
 				if not dsrp: dsrp = cls.get_dsrp(dest_rpath, diff_rorp.index)
 				if diff_rorp and diff_rorp.isplaceholder(): diff_rorp = None
 				ITR(dsrp.index, diff_rorp, dsrp)
-				finalizer(dsrp.index, dsrp)
+				#finalizer(dsrp.index, dsrp)
 				finished_dsrp = dsrp
 			ITR.Finish()
-			finalizer.Finish()
+			#finalizer.Finish()
 		except: cls.handle_last_error(finished_dsrp, finalizer, ITR)
 
 		if Globals.preserve_hardlinks: Hardlink.final_writedata()
@@ -274,7 +271,8 @@ class HLDestinationStruct:
 	def patch_increment_and_finalize(cls, dest_rpath, diffs, inc_rpath):
 		"""Apply diffs, write increment if necessary, and finalize"""
 		collated = RORPIter.CollateIterators(diffs, cls.initial_dsiter2)
-		finalizer, ITR = cls.get_finalizer(), cls.get_ITR(inc_rpath)
+		#finalizer, ITR = cls.get_finalizer(), cls.get_ITR(inc_rpath)
+		finalizer, ITR = None, cls.get_ITR(inc_rpath)
 		MiscStats.open_dir_stats_file()
 		dsrp, finished_dsrp = None, None
 
@@ -286,10 +284,10 @@ class HLDestinationStruct:
 				if not dsrp: dsrp = cls.get_dsrp(dest_rpath, index)
 				if diff_rorp and diff_rorp.isplaceholder(): diff_rorp = None
 				ITR(index, diff_rorp, dsrp)
-				finalizer(index, dsrp)
+				#finalizer(index, dsrp)
 				finished_dsrp = dsrp
 			ITR.Finish()
-			finalizer.Finish()
+			#finalizer.Finish()
 		except: cls.handle_last_error(finished_dsrp, finalizer, ITR)
 
 		if Globals.preserve_hardlinks: Hardlink.final_writedata()
@@ -311,6 +309,5 @@ from log import *
 from rpath import *
 from robust import *
 from increment import *
-from destructive_stepping import *
 from rorpiter import *
 import Globals, Hardlink, MiscStats, metadata

trunk/rdiff-backup/rdiff_backup/increment.py

@@ -145,8 +145,6 @@ class IncrementITRB(StatsITRB):
 	4.  Directory -> Normal file:  Wait until the end, so we can
 	    process all the files in the directory.	
 
-	Remember this object needs to be pickable.
-	
 	"""
 	# Iff true, mirror file was a directory
 	mirror_isdirectory = None
@@ -299,7 +297,14 @@ class MirrorITRB(StatsITRB):
 	def start_process(self, index, diff_rorp, mirror_dsrp):
 		"""Initialize statistics and do actual writing to mirror"""
 		self.start_stats(mirror_dsrp)
-		if diff_rorp and not diff_rorp.isplaceholder():
+		if (diff_rorp and diff_rorp.isdir() or
+			not diff_rorp and mirror_dsrp.isdir()):
+			# mirror_dsrp will end up as directory, update attribs later
+			if not diff_rorp: diff_rorp = mirror_dsrp.get_rorpath()
+			if not mirror_dsrp.isdir():
+				mirror_dsrp.delete()
+				mirror_dsrp.mkdir()
+		elif diff_rorp and not diff_rorp.isplaceholder():
 			RORPIter.patchonce_action(None, mirror_dsrp, diff_rorp).execute()
 
 		self.incpref = self.inc_rpath.new_index(index)
@@ -309,6 +314,7 @@ class MirrorITRB(StatsITRB):
 		"""Update statistics when leaving"""
 		self.end_stats(self.diff_rorp, self.mirror_dsrp)
 		if self.mirror_dsrp.isdir():
+			RPathStatic.copy_attribs(self.diff_rorp, self.mirror_dsrp)
 			MiscStats.write_dir_stats_line(self, self.mirror_dsrp.index)
 
 	def can_fast_process(self, index, diff_rorp, mirror_dsrp):

trunk/rdiff-backup/rdiff_backup/restore.py

@@ -41,7 +41,6 @@ class Restore:
 		same index as mirror.
 
 		"""
-		if not isinstance(mirror, DSRPath): mirror = DSRPath(1, mirror)
 		if not isinstance(target, DSRPath): target = DSRPath(None, target)
 
 		mirror_time = Restore.get_mirror_time()
@@ -129,18 +128,16 @@ class Restore:
 		"foo/bar".
 
 		"""
-		assert isinstance(mirror, DSRPath) and isinstance(target, DSRPath)
+		assert isinstance(target, DSRPath)
 		assert mirror.index == rid.index
 
-		mirror_finalizer = IterTreeReducer(DestructiveSteppingFinalizer, ())
 		target_finalizer = IterTreeReducer(DestructiveSteppingFinalizer, ())
 		for rcd in Restore.yield_rcds(rid.index, mirror, rid,
 									  target, time, mirror_time):
 			rcd.RestoreFile()
-			if rcd.mirror: mirror_finalizer(rcd.index, rcd.mirror)
+			#if rcd.mirror: mirror_finalizer(rcd.index, rcd.mirror)
 			target_finalizer(rcd.target.index, rcd.target)
 		target_finalizer.Finish()
-		mirror_finalizer.Finish()
 
 	def yield_rcds(index, mirrorrp, rid, target, rest_time, mirror_time):
 		"""Iterate RestoreCombinedData objects starting with given args

trunk/rdiff-backup/rdiff_backup/rpath.py

@@ -288,6 +288,8 @@ class RORPath(RPathStatic):
 				(not Globals.change_ownership or self.issym())):
 				# Don't compare gid/uid for symlinks or if not change_ownership
 				pass
+			elif key == 'mtime':
+				Log("%s differs only in mtime, skipping" % (self.path,), 2)
 			elif key == 'atime' and not Globals.preserve_atime: pass
 			elif key == 'devloc' or key == 'inode' or key == 'nlink': pass
 			elif key == 'size' and not self.isreg(): pass
@@ -319,6 +321,10 @@ class RORPath(RPathStatic):
 		"""Reproduce RORPath from __getstate__ output"""
 		self.index, self.data = rorp_state
 
+	def get_rorpath(self):
+		"""Return new rorpath based on self"""
+		return RORPath(self.index, self.data.copy())
+
 	def make_placeholder(self):
 		"""Make rorp into a placeholder
 
@@ -697,19 +703,15 @@ class RPath(RORPath):
 		return self.conn.Globals.get('process_gid') == self.data['gid']
 
 	def delete(self):
-		"""Delete file at self.path
-
-		The destructive stepping allows this function to delete
-		directories even if they have files and we lack permissions.
-
-		"""
+		"""Delete file at self.path.  Recursively deletes directories."""
 		Log("Deleting %s" % self.path, 7)
 		self.setdata()
-		if not self.lstat(): return # must have been deleted in meantime
+		if not self.lstat():
+			Log("Warning: %s does not exist---deleted in meantime?"
+				% (self.path,), 2)
 		elif self.isdir():
 			itm = IterTreeReducer(RpathDeleter, [])
-			for dsrp in Select(DSRPath(None, self)).set_iter():
-				itm(dsrp.index, dsrp)
+			for rp in Select(self).set_iter(): itm(rp.index, rp)
 			itm.Finish()
 		else: self.conn.os.unlink(self.path)
 		self.setdata()
@@ -891,18 +893,17 @@ class RPathFileHook:
 import FilenameMapping
 from lazy import *
 from selection import *
-from destructive_stepping import *
 from highlevel import *
 
 class RpathDeleter(ITRBranch):
 	"""Delete a directory.  Called by RPath.delete()"""
-	def start_process(self, index, dsrp):
-		self.dsrp = dsrp
+	def start_process(self, index, rp):
+		self.rp = rp
 
 	def end_process(self):
-		if self.dsrp.isdir(): self.dsrp.rmdir()
-		else: self.dsrp.delete()
+		if self.rp.isdir(): self.rp.rmdir()
+		else: self.rp.delete()
 
-	def can_fast_process(self, index, dsrp): return not dsrp.isdir()
-	def fast_process(self, index, dsrp): dsrp.delete()
+	def can_fast_process(self, index, rp): return not rp.isdir()
+	def fast_process(self, index, rp): rp.delete()
 	

trunk/rdiff-backup/rdiff_backup/selection.py

@@ -28,7 +28,6 @@ from __future__ import generators
 import re
 from log import *
 from robust import *
-from destructive_stepping import *
 import FilenameMapping
 
 
@@ -46,7 +45,7 @@ class GlobbingError(SelectError):
 
 
 class Select:
-	"""Iterate appropriate DSRPaths in given directory
+	"""Iterate appropriate RPaths in given directory
 
 	This class acts as an iterator on account of its next() method.
 	Basically, it just goes through all the files in a directory in
@@ -66,7 +65,7 @@ class Select:
 	file in the directory gets included, so does the directory.
 
 	As mentioned above, each test takes the form of a selection
-	function.  The selection function takes a dsrp, and returns:
+	function.  The selection function takes an rpath, and returns:
 
 	None - means the test has nothing to say about the related file
 	0 - the file is excluded by the test
@@ -82,18 +81,18 @@ class Select:
 	# This re should not match normal filenames, but usually just globs
 	glob_re = re.compile("(.*[*?[]|ignorecase\\:)", re.I | re.S)
 
-	def __init__(self, dsrpath, quoted_filenames = None):
-		"""DSRPIterator initializer.  dsrp is the root directory
+	def __init__(self, rpath, quoted_filenames = None):
+		"""Select initializer.  rpath is the root directory
 
 		When files have quoted characters in them, quoted_filenames
 		should be true.  Then RPath's index will be the unquoted
 		version.
 
 		"""
-		assert isinstance(dsrpath, DSRPath)
+		assert isinstance(rpath, RPath)
 		self.selection_functions = []
-		self.dsrpath = dsrpath
-		self.prefix = self.dsrpath.path
+		self.rpath = rpath
+		self.prefix = self.rpath.path
 		self.quoting_on = Globals.quoting_enabled and quoted_filenames
 
 	def set_iter(self, starting_index = None, iterate_parents = None,
@@ -103,19 +102,19 @@ class Select:
 		Will iterate indicies greater than starting_index.  If
 		iterate_parents is true, will also include parents of
 		starting_index in iteration.  Selection function sel_func is
-		called on each dsrp and is usually self.Select.  Returns self
+		called on each rpath and is usually self.Select.  Returns self
 		just for convenience.
 
 		"""
 		if not sel_func: sel_func = self.Select
-		self.dsrpath.setdata() # this may have changed since Select init
+		self.rpath.setdata() # this may have changed since Select init
 		if starting_index is not None:
 			self.starting_index = starting_index
-			self.iter = self.iterate_starting_from(self.dsrpath,
+			self.iter = self.iterate_starting_from(self.rpath,
 						            self.iterate_starting_from, sel_func)
 		elif self.quoting_on:
-			self.iter = self.Iterate(self.dsrpath, self.Iterate, sel_func)
-		else: self.iter = self.Iterate_fast(self.dsrpath, sel_func)
+			self.iter = self.Iterate(self.rpath, self.Iterate, sel_func)
+		else: self.iter = self.Iterate_fast(self.rpath, sel_func)
 
 		# only iterate parents if we are not starting from beginning
 		self.iterate_parents = starting_index is not None and iterate_parents
@@ -123,7 +122,7 @@ class Select:
 		self.__iter__ = lambda: self
 		return self
 
-	def Iterate_fast(self, dsrpath, sel_func):
+	def Iterate_fast(self, rpath, sel_func):
 		"""Like Iterate, but don't recur, saving time
 
 		Only handles standard case (quoting off, starting from
@@ -131,116 +130,115 @@ class Select:
 
 		"""
 		def error_handler(exc, filename):
-			Log("Error initializing file %s/%s" % (dsrpath.path, filename), 2)
+			Log("Error initializing file %s/%s" % (rpath.path, filename), 2)
 			return None
 
-		def diryield(dsrpath):
-			"""Generate relevant files in directory dsrpath
+		def diryield(rpath):
+			"""Generate relevant files in directory rpath
 
-			Returns (dsrp, num) where num == 0 means dsrp should be
-			generated normally, num == 1 means the dsrp is a directory
+			Returns (rpath, num) where num == 0 means rpath should be
+			generated normally, num == 1 means the rpath is a directory
 			and should be included iff something inside is included.
 
 			"""
-			for filename in Robust.listrp(dsrpath):
-				new_dsrp = Robust.check_common_error(error_handler,
-										dsrpath.append, (filename,))
-				if new_dsrp:
-					s = sel_func(new_dsrp)
-					if s == 1: yield (new_dsrp, 0)
-					elif s == 2 and new_dsrp.isdir(): yield (new_dsrp, 1)
-
-		yield dsrpath
-		diryield_stack = [diryield(dsrpath)]
-		delayed_dsrp_stack = []
+			for filename in Robust.listrp(rpath):
+				new_rpath = Robust.check_common_error(error_handler,
+										rpath.append, (filename,))
+				if new_rpath:
+					s = sel_func(new_rpath)
+					if s == 1: yield (new_rpath, 0)
+					elif s == 2 and new_rpath.isdir(): yield (new_rpath, 1)
+
+		yield rpath
+		diryield_stack = [diryield(rpath)]
+		delayed_rp_stack = []
 
 		while diryield_stack:
-			try: dsrp, val = diryield_stack[-1].next()
+			try: rpath, val = diryield_stack[-1].next()
 			except StopIteration:
 				diryield_stack.pop()
-				if delayed_dsrp_stack: delayed_dsrp_stack.pop()
+				if delayed_rp_stack: delayed_rp_stack.pop()
 				continue
 			if val == 0:
-				if delayed_dsrp_stack:
-					for delayed_dsrp in delayed_dsrp_stack: yield delayed_dsrp
-					del delayed_dsrp_stack[:]
-				yield dsrp
-				if dsrp.isdir(): diryield_stack.append(diryield(dsrp))
+				if delayed_rp_stack:
+					for delayed_rp in delayed_rp_stack: yield delayed_rp
+					del delayed_rp_stack[:]
+				yield rpath
+				if rpath.isdir(): diryield_stack.append(diryield(rpath))
 			elif val == 1:
-				delayed_dsrp_stack.append(dsrp)
-				diryield_stack.append(diryield(dsrp))
+				delayed_rp_stack.append(rpath)
+				diryield_stack.append(diryield(rpath))
 
-	def Iterate(self, dsrpath, rec_func, sel_func):
-		"""Return iterator yielding dsrps in dsrpath
+	def Iterate(self, rpath, rec_func, sel_func):
+		"""Return iterator yielding rpaths in rpath
 
 		rec_func is usually the same as this function and is what
 		Iterate uses to find files in subdirectories.  It is used in
 		iterate_starting_from.
 
-		sel_func is the selection function to use on the dsrps.  It is
-		usually self.Select.
+		sel_func is the selection function to use on the rpaths.  It
+		is usually self.Select.
 
 		"""
-		s = sel_func(dsrpath)
+		s = sel_func(rpath)
 		if s == 0: return
 		elif s == 1: # File is included
-			yield dsrpath
-			if dsrpath.isdir():
-				for dsrp in self.iterate_in_dir(dsrpath, rec_func, sel_func):
-					yield dsrp
+			yield rpath
+			if rpath.isdir():
+				for rp in self.iterate_in_dir(rpath, rec_func, sel_func):
+					yield rp
 		elif s == 2:
-			if dsrpath.isdir(): # Directory is merely scanned
-				iid = self.iterate_in_dir(dsrpath, rec_func, sel_func)
+			if rpath.isdir(): # Directory is merely scanned
+				iid = self.iterate_in_dir(rpath, rec_func, sel_func)
 				try: first = iid.next()
-				except StopIteration: return # no files inside; skip dsrp
-				yield dsrpath
+				except StopIteration: return # no files inside; skip rp
+				yield rpath
 				yield first
-				for dsrp in iid: yield dsrp
+				for rp in iid: yield rp
 		else: assert 0, "Invalid selection result %s" % (str(s),)
 
-	def iterate_in_dir(self, dsrpath, rec_func, sel_func):
-		"""Iterate the dsrps in directory dsrpath."""
+	def iterate_in_dir(self, rpath, rec_func, sel_func):
+		"""Iterate the rpaths in directory rpath."""
 		def error_handler(exc, filename):
-			Log("Error initializing file %s/%s" % (dsrpath.path, filename), 2)
+			Log("Error initializing file %s/%s" % (rpath.path, filename), 2)
 			return None
 
 		if self.quoting_on:
-			for subdir in FilenameMapping.get_quoted_dir_children(dsrpath):
-				for dsrp in rec_func(subdir, rec_func, sel_func):
-					yield dsrp
+			for subdir in FilenameMapping.get_quoted_dir_children(rpath):
+				for rp in rec_func(subdir, rec_func, sel_func):
+					yield rp
 		else:
-			for filename in Robust.listrp(dsrpath):
-				new_dsrp = Robust.check_common_error(
-					error_handler, dsrpath.append, [filename])
-				if new_dsrp:
-					for dsrp in rec_func(new_dsrp, rec_func, sel_func):
-						yield dsrp
-
-	def iterate_starting_from(self, dsrpath, rec_func, sel_func):
+			for filename in Robust.listrp(rpath):
+				new_rp = Robust.check_common_error(
+					error_handler, rpath.append, [filename])
+				if new_rp:
+					for rp in rec_func(new_rp, rec_func, sel_func):
+						yield rp
+
+	def iterate_starting_from(self, rpath, rec_func, sel_func):
 		"""Like Iterate, but only yield indicies > self.starting_index"""
-		if dsrpath.index > self.starting_index: # past starting_index
-			for dsrp in self.Iterate(dsrpath, self.Iterate, sel_func):
-				yield dsrp
-		elif (dsrpath.index == self.starting_index[:len(dsrpath.index)]
-			  and dsrpath.isdir()):
+		if rpath.index > self.starting_index: # past starting_index
+			for rp in self.Iterate(rpath, self.Iterate, sel_func):
+				yield rp
+		elif (rpath.index == self.starting_index[:len(rpath.index)]
+			  and rpath.isdir()):
 			# May encounter starting index on this branch
-			if self.iterate_parents: yield dsrpath
-			for dsrp in self.iterate_in_dir(dsrpath,
-											self.iterate_starting_from,
-											sel_func): yield dsrp
-
-	def iterate_with_finalizer(self):
-		"""Like Iterate, but missing some options, and add finalizer"""
-		finalize = IterTreeReducer(DestructiveSteppingFinalizer, ())
-		for dsrp in self:
-			yield dsrp
-			finalize(dsrp.index, dsrp)
-		finalize.Finish()
-
-	def Select(self, dsrp):
+			if self.iterate_parents: yield rpath
+			for rp in self.iterate_in_dir(rpath, self.iterate_starting_from,
+										  sel_func): yield rp
+
+#	def iterate_with_finalizer(self):
+#		"""Like Iterate, but missing some options, and add finalizer"""
+#		finalize = IterTreeReducer(DestructiveSteppingFinalizer, ())
+#		for rp in self:
+#			yield rp
+#			finalize(rp.index, rp))
+#		finalize.Finish()
+
+	def Select(self, rp):
 		"""Run through the selection functions and return dominant val 0/1/2"""
 		for sf in self.selection_functions:
-			result = sf(dsrp)
+			result = sf(rp)
 			if result is not None: return result
 		return 1
 
@@ -353,11 +351,11 @@ probably isn't what you meant.""" %
 		tuple_list.sort()
 		i = [0] # We have to put index in list because of stupid scoping rules
 
-		def selection_function(dsrp):
+		def selection_function(rp):
 			while 1:
 				if i[0] >= len(tuple_list): return None
 				include, move_on = \
-						 self.filelist_pair_match(dsrp, tuple_list[i[0]])
+						 self.filelist_pair_match(rp, tuple_list[i[0]])
 				if move_on:
 					i[0] += 1
 					if include is None: continue # later line may match
@@ -415,12 +413,12 @@ probably isn't what you meant.""" %
 		index = tuple(filter(lambda x: x, line.split("/"))) # remove empties
 		return (index, include)
 
-	def filelist_pair_match(self, dsrp, pair):
-		"""Matches a filelist tuple against a dsrp
+	def filelist_pair_match(self, rp, pair):
+		"""Matches a filelist tuple against a rpath
 
 		Returns a pair (include, move_on).  include is None if the
 		tuple doesn't match either way, and 0/1 if the tuple excludes
-		or includes the dsrp.
+		or includes the rpath.
 
 		move_on is true if the tuple cannot match a later index, and
 		so we should move on to the next tuple in the index.
@@ -428,16 +426,16 @@ probably isn't what you meant.""" %
 		"""
 		index, include = pair
 		if include == 1:
-			if index < dsrp.index: return (None, 1)
-			if index == dsrp.index: return (1, 1)
-			elif index[:len(dsrp.index)] == dsrp.index:
+			if index < rp.index: return (None, 1)
+			if index == rp.index: return (1, 1)
+			elif index[:len(rp.index)] == rp.index:
 				return (1, None) # /foo/bar implicitly includes /foo
-			else: return (None, None) # dsrp greater, not initial sequence
+			else: return (None, None) # rp greater, not initial sequence
 		elif include == 0:
-			if dsrp.index[:len(index)] == index:
+			if rp.index[:len(index)] == index:
 				return (0, None) # /foo implicitly excludes /foo/bar
-			elif index < dsrp.index: return (None, 1)
-			else: return (None, None) # dsrp greater, not initial sequence
+			elif index < rp.index: return (None, 1)
+			else: return (None, None) # rp greater, not initial sequence
 		else: assert 0, "Include is %s, should be 0 or 1" % (include,)
 
 	def filelist_globbing_get_sfs(self, filelist_fp, inc_default, list_name):
@@ -460,9 +458,9 @@ probably isn't what you meant.""" %
 	def other_filesystems_get_sf(self, include):
 		"""Return selection function matching files on other filesystems"""
 		assert include == 0 or include == 1
-		root_devloc = self.dsrpath.getdevloc()
-		def sel_func(dsrp):
-			if dsrp.getdevloc() == root_devloc: return None
+		root_devloc = self.rpath.getdevloc()
+		def sel_func(rp):
+			if rp.getdevloc() == root_devloc: return None
 			else: return include
 		sel_func.exclude = not include
 		sel_func.name = "Match other filesystems"
@@ -476,8 +474,8 @@ probably isn't what you meant.""" %
 			Log("Error compiling regular expression %s" % regexp_string, 1)
 			raise
 		
-		def sel_func(dsrp):
-			if regexp.search(dsrp.path): return include
+		def sel_func(rp):
+			if regexp.search(rp.path): return include
 			else: return None
 
 		sel_func.exclude = not include
@@ -489,8 +487,8 @@ probably isn't what you meant.""" %
 		if self.selection_functions:
 			Log("Warning: exclude-device-files is not the first "
 				"selector.\nThis may not be what you intended", 3)
-		def sel_func(dsrp):
-			if dsrp.isdev(): return include
+		def sel_func(rp):
+			if rp.isdev(): return include
 			else: return None
 		sel_func.exclude = not include
 		sel_func.name = (include and "include" or "exclude") + " device files"
@@ -501,8 +499,8 @@ probably isn't what you meant.""" %
 		if self.selection_functions:
 			Log("Warning: exclude-special-files is not the first "
 				"selector.\nThis may not be what you intended", 3)
-		def sel_func(dsrp):
-			if dsrp.issym() or dsrp.issock() or dsrp.isfifo() or dsrp.isdev():
+		def sel_func(rp):
+			if rp.issym() or rp.issock() or rp.isfifo() or rp.isdev():
 				return include
 			else: return None
 		sel_func.exclude = not include
@@ -512,7 +510,7 @@ probably isn't what you meant.""" %
 	def glob_get_sf(self, glob_str, include):
 		"""Return selection function given by glob string"""
 		assert include == 0 or include == 1
-		if glob_str == "**": sel_func = lambda dsrp: include
+		if glob_str == "**": sel_func = lambda rp: include
 		elif not self.glob_re.match(glob_str): # normal file
 			sel_func = self.glob_get_filename_sf(glob_str, include)
 		else: sel_func = self.glob_get_normal_sf(glob_str, include)
@@ -539,14 +537,14 @@ probably isn't what you meant.""" %
 
 	def glob_get_tuple_sf(self, tuple, include):
 		"""Return selection function based on tuple"""
-		def include_sel_func(dsrp):
-			if (dsrp.index == tuple[:len(dsrp.index)] or
-				dsrp.index[:len(tuple)] == tuple):
+		def include_sel_func(rp):
+			if (rp.index == tuple[:len(rp.index)] or
+				rp.index[:len(tuple)] == tuple):
 				return 1 # /foo/bar implicitly matches /foo, vice-versa
 			else: return None
 
-		def exclude_sel_func(dsrp):
-			if dsrp.index[:len(tuple)] == tuple:
+		def exclude_sel_func(rp):
+			if rp.index[:len(tuple)] == tuple:
 				return 0 # /foo excludes /foo/bar, not vice-versa
 			else: return None
 
@@ -585,17 +583,17 @@ probably isn't what you meant.""" %
 		scan_comp_re = re_comp("^(%s)$" %
 							   "|".join(self.glob_get_prefix_res(glob_str)))
 
-		def include_sel_func(dsrp):
-			if glob_comp_re.match(dsrp.path): return 1
-			elif scan_comp_re.match(dsrp.path): return 2
+		def include_sel_func(rp):
+			if glob_comp_re.match(rp.path): return 1
+			elif scan_comp_re.match(rp.path): return 2
 			else: return None
 
-		def exclude_sel_func(dsrp):
-			if glob_comp_re.match(dsrp.path): return 0
+		def exclude_sel_func(rp):
+			if glob_comp_re.match(rp.path): return 0
 			else: return None
 
 		# Check to make sure prefix is ok
-		if not include_sel_func(self.dsrpath): raise FilePrefixError(glob_str)
+		if not include_sel_func(self.rpath): raise FilePrefixError(glob_str)
 		
 		if include: return include_sel_func
 		else: return exclude_sel_func

trunk/rdiff-backup/testing/commontest.py

@@ -173,8 +173,7 @@ def CompareRecursive(src_rp, dest_rp, compare_hardlinks = 1,
 
 	Log("Comparing %s and %s, hardlinks %s" % (src_rp.path, dest_rp.path,
 											   compare_hardlinks), 3)
-	src_select = Select(DSRPath(1, src_rp))
-	dest_select = Select(DSRPath(None, dest_rp))
+	src_select, dest_select = Select(src_rp), Select(dest_rp)
 
 	if ignore_tmp_files:
 		# Ignoring temp files can be useful when we want to check the
@@ -201,13 +200,10 @@ def CompareRecursive(src_rp, dest_rp, compare_hardlinks = 1,
 		dest_select.add_selection_func(dest_select.glob_get_tuple_sf(
 			('rdiff-backup-data',), 0))
 
-	src_select.set_iter()
-	dest_select.set_iter()
-	dsiter1, dsiter2 = src_select.iterate_with_finalizer(), \
-					   dest_select.iterate_with_finalizer()
+	dsiter1, dsiter2 = src_select.set_iter(), dest_select.set_iter()
 
 	def hardlink_equal(src_rorp, dest_rorp):
-		if src_rorp != dest_rorp: return None
+		if not src_rorp.equal_verbose(dest_rorp): return None
 		if Hardlink.rorp_eq(src_rorp, dest_rorp): return 1
 		Log("%s: %s" % (src_rorp.index, Hardlink.get_indicies(src_rorp, 1)), 3)
 		Log("%s: %s" % (dest_rorp.index,
@@ -229,7 +225,7 @@ def CompareRecursive(src_rp, dest_rp, compare_hardlinks = 1,
 				if dest_rorp.index[-1].endswith('.missing'): return 1
 		if compare_hardlinks:
 			if Hardlink.rorp_eq(src_rorp, dest_rorp): return 1
-		elif src_rorp == dest_rorp: return 1
+		elif src_rorp.equal_verbose(dest_rorp): return 1
 		Log("%s: %s" % (src_rorp.index, Hardlink.get_indicies(src_rorp, 1)), 3)
 		Log("%s: %s" % (dest_rorp.index,
 						Hardlink.get_indicies(dest_rorp, None)), 3)

trunk/rdiff-backup/testing/regressiontest.py

@@ -14,7 +14,7 @@ testfiles
 
 Globals.set('change_source_perms', 1)
 Globals.counter = 0
-Log.setverbosity(3)
+Log.setverbosity(7)
 
 class Local:
 	"""This is just a place to put increments relative to the local