Removed most destructive stepping operations

git-svn-id: http://svn.savannah.nongnu.org/svn/rdiff-backup@251 2b77aa54-bcbc-44c9-a7ec-4f6cf2b41109

trunk/rdiff-backup/CHANGELOG

@@ -7,6 +7,11 @@ Fixed bug with the --{include|exclude}-globbing-filelist options
 Added --list-changed-since option to list the files changed since the
 given date.
 
+Removed --mirror-only and --change-source-perms options.
+
+Removed all "resume" related functionality, like
+--checkpoint-interval.
+
 
 New in v0.11.0 (2002/10/05)
 ---------------------------

trunk/rdiff-backup/rdiff_backup/Globals.py

@@ -55,10 +55,6 @@ change_ownership = None
 # permissions).
 change_mirror_perms = (process_uid != 0)
 
-# If true, temporarily change permissions of unreadable files in
-# the source directory to make sure we can read all files.
-change_source_perms = None
-
 # If true, try to reset the atimes of the source partition.
 preserve_atime = None
 
@@ -103,24 +99,9 @@ client_conn = None
 # list.
 changed_settings = []
 
-# rdiff-backup will try to checkpoint its state every
-# checkpoint_interval seconds.  Then when resuming, at most this
-# amount of time is lost.
-checkpoint_interval = 20
-
 # The RPath of the rdiff-backup-data directory.
 rbdir = None
 
-# Indicates if a resume or a lack of resume is forced.  This
-# should be None for the default.  0 means don't resume, and 1
-# means resume.
-resume = None
-
-# If there has been an aborted backup fewer than this many seconds
-# ago, attempt to resume it where it left off instead of starting
-# a new one.
-resume_window = 7200
-
 # This string is used when recognizing and creating time strings.
 # If the time_separator is ":", then W3 datetime strings like
 # 2001-12-07T04:22:01-07:00 are produced.  It can be set to "_" to
@@ -265,7 +246,7 @@ def postset_regexp_local(name, re_string, flags):
 	if flags: globals()[name] = re.compile(re_string, flags)
 	else: globals()[name] = re.compile(re_string)
 
-def set_select(dsrpath, tuplelist, quote_mode, *filelists):
+def set_select(source, rpath, tuplelist, quote_mode, *filelists):
 	"""Initialize select object using tuplelist
 
 	Note that each list in filelists must each be passed as
@@ -275,12 +256,10 @@ def set_select(dsrpath, tuplelist, quote_mode, *filelists):
 
 	"""
 	global select_source, select_mirror
-	if dsrpath.source:
-		select_source = Select(dsrpath, quote_mode)
-		select_source.ParseArgs(tuplelist, filelists)
-	else:
-		select_mirror = Select(dsrpath, quote_mode)
-		select_mirror.ParseArgs(tuplelist, filelists)
+	sel = Select(rpath, quote_mode)
+	sel.ParseArgs(tuplelist, filelists)
+	if source: select_source = sel
+	else: select_mirror = sel
 
 
 from rpath import * # kludge to avoid circularity - not needed in this module

trunk/rdiff-backup/rdiff_backup/Main.py

@@ -25,7 +25,6 @@ from log import *
 from lazy import *
 from connection import *
 from rpath import *
-from destructive_stepping import *
 from robust import *
 from restore import *
 from highlevel import *
@@ -49,21 +48,20 @@ def parse_cmdlineoptions(arglist):
 		except IOError: Log.FatalError("Error opening file %s" % filename)
 
 	try: optlist, args = getopt.getopt(arglist, "blr:sv:V",
-		 ["backup-mode", "calculate-average", "change-source-perms",
-		  "chars-to-quote=", "checkpoint-interval=", "current-time=",
-		  "exclude=", "exclude-device-files", "exclude-filelist=",
-		  "exclude-filelist-stdin", "exclude-globbing-filelist=",
-		  "exclude-mirror=", "exclude-other-filesystems",
-		  "exclude-regexp=", "exclude-special-files", "force",
-		  "include=", "include-filelist=", "include-filelist-stdin",
+		 ["backup-mode", "calculate-average", "chars-to-quote=",
+		  "current-time=", "exclude=", "exclude-device-files",
+		  "exclude-filelist=", "exclude-filelist-stdin",
+		  "exclude-globbing-filelist=", "exclude-mirror=",
+		  "exclude-other-filesystems", "exclude-regexp=",
+		  "exclude-special-files", "force", "include=",
+		  "include-filelist=", "include-filelist-stdin",
 		  "include-globbing-filelist=", "include-regexp=",
 		  "list-changed-since=", "list-increments", "no-compression",
-		  "no-compression-regexp=", "no-hard-links", "no-resume",
-		  "null-separator", "parsable-output", "print-statistics",
-		  "quoting-char=", "remote-cmd=", "remote-schema=",
-		  "remove-older-than=", "restore-as-of=", "restrict=",
-		  "restrict-read-only=", "restrict-update-only=", "resume",
-		  "resume-window=", "server", "sleep-ratio=",
+		  "no-compression-regexp=", "no-hard-links", "null-separator",
+		  "parsable-output", "print-statistics", "quoting-char=",
+		  "remote-cmd=", "remote-schema=", "remove-older-than=",
+		  "restore-as-of=", "restrict=", "restrict-read-only=",
+		  "restrict-update-only=", "server", "sleep-ratio=",
 		  "ssh-no-compression", "terminal-verbosity=", "test-server",
 		  "verbosity=", "version", "windows-mode",
 		  "windows-time-format"])
@@ -73,13 +71,9 @@ def parse_cmdlineoptions(arglist):
 	for opt, arg in optlist:
 		if opt == "-b" or opt == "--backup-mode": action = "backup"
 		elif opt == "--calculate-average": action = "calculate-average"
-		elif opt == "--change-source-perms":
-			Globals.set('change_source_perms', 1)
 		elif opt == "--chars-to-quote":
 			Globals.set('chars_to_quote', arg)
 			Globals.set('quoting_enabled', 1)
-		elif opt == "--checkpoint-interval":
-			Globals.set_integer('checkpoint_interval', arg)
 		elif opt == "--current-time":
 			Globals.set_integer('current_time', arg)
 		elif opt == "--exclude": select_opts.append((opt, arg))
@@ -118,7 +112,6 @@ def parse_cmdlineoptions(arglist):
 		elif opt == "--no-compression-regexp":
 			Globals.set("no_compression_regexp_string", arg)
 		elif opt == "--no-hard-links": Globals.set('preserve_hardlinks', 0)
-		elif opt == '--no-resume': Globals.resume = 0
 		elif opt == "--null-separator": Globals.set("null_separator", 1)
 		elif opt == "--parsable-output": Globals.set('parsable_output', 1)
 		elif opt == "--print-statistics":
@@ -140,9 +133,6 @@ def parse_cmdlineoptions(arglist):
 		elif opt == "--restrict-update-only":
 			Globals.security_level = "update-only"
 			Globals.restrict_path = arg
-		elif opt == '--resume': Globals.resume = 1
-		elif opt == '--resume-window':
-			Globals.set_integer('resume_window', arg)
 		elif opt == "-s" or opt == "--server":
 			action = "server"
 			Globals.server = 1
@@ -263,9 +253,8 @@ def Backup(rpin, rpout):
 
 def backup_init_select(rpin, rpout):
 	"""Create Select objects on source and dest connections"""
-	rpin.conn.Globals.set_select(DSRPath(1, rpin), select_opts,
-								 None, *select_files)
-	rpout.conn.Globals.set_select(DSRPath(None, rpout), select_mirror_opts, 1)
+	rpin.conn.Globals.set_select(1, rpin, select_opts, None, *select_files)
+	rpout.conn.Globals.set_select(0, rpout, select_mirror_opts, 1)
 
 def backup_init_dirs(rpin, rpout):
 	"""Make sure rpin and rpout are valid, init data dir and logging"""
@@ -424,8 +413,8 @@ def restore_init_select(rpin, rpout):
 	the restore operation isn't.
 
 	"""
-	Globals.set_select(DSRPath(1, rpin), select_mirror_opts, None)
-	Globals.set_select(DSRPath(None, rpout), select_opts, None, *select_files)
+	Globals.set_select(1, rpin, select_mirror_opts, None)
+	Globals.set_select(0, rpout, select_opts, None, *select_files)
 
 def restore_get_root(rpin):
 	"""Return (mirror root, index) and set the data dir

trunk/rdiff-backup/rdiff_backup/SetConnections.py

@@ -197,11 +197,6 @@ def BackupInitConnections(reading_conn, writing_conn):
 	writing_conn.Globals.set("isbackup_writer", 1)
 	UpdateGlobal("backup_reader", reading_conn)
 	UpdateGlobal("backup_writer", writing_conn)
-	if (Globals.change_source_perms and
-		reading_conn.Globals.get("process_uid") == 0):
-		Log("Warning: --change_source_perms should usually not be used when\n"
-			"the reading connection is running as root, because root can\n"
-			"read all files regardless of their permissions.", 2)
 
 def CloseConnections():
 	"""Close all connections.  Run by client"""

trunk/rdiff-backup/rdiff_backup/connection.py

@@ -547,7 +547,6 @@ from connection import *
 from rpath import *
 from robust import *
 from rorpiter import *
-from destructive_stepping import *
 from selection import *
 from statistics import *
 from increment import *

trunk/rdiff-backup/rdiff_backup/highlevel.py

@@ -87,27 +87,23 @@ class HLSourceStruct:
 		"""Return diffs and finalize any dsrp changes remaining
 
 		Return a rorpiterator with files included of signatures of
-		dissimilar files.  This is the last operation run on the local
-		filestream, so finalize dsrp writes.
+		dissimilar files.
 
 		"""
 		collated = RORPIter.CollateIterators(cls.initial_dsiter2, sigiter)
-		finalizer = IterTreeReducer(DestructiveSteppingFinalizer, [])
-		def error_handler(exc, dest_sig, dsrp):
+		def error_handler(exc, dest_sig, rp):
 			Log("Error %s producing a diff of %s" %
-				(exc, dsrp and dsrp.path), 2)
+				(exc, rp and rp.path), 2)
 			return None
 			
 		def diffs():
-			for dsrp, dest_sig in collated:
+			for rp, dest_sig in collated:
 				if dest_sig:
 					if dest_sig.isplaceholder(): yield dest_sig
 					else:
 						diff = Robust.check_common_error(
-							error_handler, RORPIter.diffonce, [dest_sig, dsrp])
+							error_handler, RORPIter.diffonce, [dest_sig, rp])
 						if diff: yield diff
-				if dsrp: finalizer(dsrp.index, dsrp)
-			finalizer.Finish()
 		return diffs()
 
 MakeClass(HLSourceStruct)
@@ -117,7 +113,7 @@ class HLDestinationStruct:
 	"""Hold info used by HL on the destination side"""
 	_session_info = None # set to si if resuming
 	def split_initial_dsiter(cls):
-		"""Set initial_dsiters (iteration of all dsrps from rpath)"""
+		"""Set initial_dsiters (iteration of all rps from rpath)"""
 		result, cls.initial_dsiter2 = \
 				Iter.multiplex(Globals.select_mirror.set_iter(), 2)
 		return result
@@ -192,10 +188,10 @@ class HLDestinationStruct:
 		return RORPIter.Signatures(dissimilars)
 
 	def get_dsrp(cls, dest_rpath, index):
-		"""Return initialized dsrp based on dest_rpath with given index"""
-		dsrp = DSRPath(None, dest_rpath.conn, dest_rpath.base, index)
-		if Globals.quoting_enabled: dsrp.quote_path()
-		return dsrp
+		"""Return initialized rpath based on dest_rpath with given index"""
+		rp = RPath(dest_rpath.conn, dest_rpath.base, index)
+		if Globals.quoting_enabled: rp.quote_path()
+		return rp
 
 	def get_finalizer(cls):
 		"""Return finalizer, starting from session info if necessary"""
@@ -226,8 +222,8 @@ class HLDestinationStruct:
 	def patch_and_finalize(cls, dest_rpath, diffs):
 		"""Apply diffs and finalize"""
 		collated = RORPIter.CollateIterators(diffs, cls.initial_dsiter2)
-		finalizer = cls.get_finalizer()
-		diff_rorp, dsrp = None, None
+		#finalizer = cls.get_finalizer()
+		diff_rorp, rp = None, None
 
 		def patch(diff_rorp, dsrp):
 			if not dsrp: dsrp = cls.get_dsrp(dest_rpath, diff_rorp.index)
@@ -244,13 +240,14 @@ class HLDestinationStruct:
 			diff_rorp, dsrp = indexed_tuple
 			dsrp = Robust.check_common_error(error_handler, patch,
 											 [diff_rorp, dsrp])
-			finalizer(dsrp.index, dsrp)
-		finalizer.Finish()
+			#finalizer(dsrp.index, dsrp)
+		#finalizer.Finish()
 
 	def patch_w_datadir_writes(cls, dest_rpath, diffs, inc_rpath):
 		"""Apply diffs and finalize, with checkpointing and statistics"""
 		collated = RORPIter.CollateIterators(diffs, cls.initial_dsiter2)
-		finalizer, ITR = cls.get_finalizer(), cls.get_MirrorITR(inc_rpath)
+		#finalizer, ITR = cls.get_finalizer(), cls.get_MirrorITR(inc_rpath)
+		finalizer, ITR = None, cls.get_MirrorITR(inc_rpath)
 		MiscStats.open_dir_stats_file()
 		dsrp, finished_dsrp = None, None
 
@@ -261,10 +258,10 @@ class HLDestinationStruct:
 				if not dsrp: dsrp = cls.get_dsrp(dest_rpath, diff_rorp.index)
 				if diff_rorp and diff_rorp.isplaceholder(): diff_rorp = None
 				ITR(dsrp.index, diff_rorp, dsrp)
-				finalizer(dsrp.index, dsrp)
+				#finalizer(dsrp.index, dsrp)
 				finished_dsrp = dsrp
 			ITR.Finish()
-			finalizer.Finish()
+			#finalizer.Finish()
 		except: cls.handle_last_error(finished_dsrp, finalizer, ITR)
 
 		if Globals.preserve_hardlinks: Hardlink.final_writedata()
@@ -274,7 +271,8 @@ class HLDestinationStruct:
 	def patch_increment_and_finalize(cls, dest_rpath, diffs, inc_rpath):
 		"""Apply diffs, write increment if necessary, and finalize"""
 		collated = RORPIter.CollateIterators(diffs, cls.initial_dsiter2)
-		finalizer, ITR = cls.get_finalizer(), cls.get_ITR(inc_rpath)
+		#finalizer, ITR = cls.get_finalizer(), cls.get_ITR(inc_rpath)
+		finalizer, ITR = None, cls.get_ITR(inc_rpath)
 		MiscStats.open_dir_stats_file()
 		dsrp, finished_dsrp = None, None
 
@@ -286,10 +284,10 @@ class HLDestinationStruct:
 				if not dsrp: dsrp = cls.get_dsrp(dest_rpath, index)
 				if diff_rorp and diff_rorp.isplaceholder(): diff_rorp = None
 				ITR(index, diff_rorp, dsrp)
-				finalizer(index, dsrp)
+				#finalizer(index, dsrp)
 				finished_dsrp = dsrp
 			ITR.Finish()
-			finalizer.Finish()
+			#finalizer.Finish()
 		except: cls.handle_last_error(finished_dsrp, finalizer, ITR)
 
 		if Globals.preserve_hardlinks: Hardlink.final_writedata()
@@ -311,6 +309,5 @@ from log import *
 from rpath import *
 from robust import *
 from increment import *
-from destructive_stepping import *
 from rorpiter import *
 import Globals, Hardlink, MiscStats, metadata

trunk/rdiff-backup/rdiff_backup/increment.py

@@ -145,8 +145,6 @@ class IncrementITRB(StatsITRB):
 	4.  Directory -> Normal file:  Wait until the end, so we can
 	    process all the files in the directory.	
 
-	Remember this object needs to be pickable.
-	
 	"""
 	# Iff true, mirror file was a directory
 	mirror_isdirectory = None
@@ -299,7 +297,14 @@ class MirrorITRB(StatsITRB):
 	def start_process(self, index, diff_rorp, mirror_dsrp):
 		"""Initialize statistics and do actual writing to mirror"""
 		self.start_stats(mirror_dsrp)
-		if diff_rorp and not diff_rorp.isplaceholder():
+		if (diff_rorp and diff_rorp.isdir() or
+			not diff_rorp and mirror_dsrp.isdir()):
+			# mirror_dsrp will end up as directory, update attribs later
+			if not diff_rorp: diff_rorp = mirror_dsrp.get_rorpath()
+			if not mirror_dsrp.isdir():
+				mirror_dsrp.delete()
+				mirror_dsrp.mkdir()
+		elif diff_rorp and not diff_rorp.isplaceholder():
 			RORPIter.patchonce_action(None, mirror_dsrp, diff_rorp).execute()
 
 		self.incpref = self.inc_rpath.new_index(index)
@@ -309,6 +314,7 @@ class MirrorITRB(StatsITRB):
 		"""Update statistics when leaving"""
 		self.end_stats(self.diff_rorp, self.mirror_dsrp)
 		if self.mirror_dsrp.isdir():
+			RPathStatic.copy_attribs(self.diff_rorp, self.mirror_dsrp)
 			MiscStats.write_dir_stats_line(self, self.mirror_dsrp.index)
 
 	def can_fast_process(self, index, diff_rorp, mirror_dsrp):

trunk/rdiff-backup/rdiff_backup/restore.py

@@ -41,7 +41,6 @@ class Restore:
 		same index as mirror.
 
 		"""
-		if not isinstance(mirror, DSRPath): mirror = DSRPath(1, mirror)
 		if not isinstance(target, DSRPath): target = DSRPath(None, target)
 
 		mirror_time = Restore.get_mirror_time()
@@ -129,18 +128,16 @@ class Restore:
 		"foo/bar".
 
 		"""
-		assert isinstance(mirror, DSRPath) and isinstance(target, DSRPath)
+		assert isinstance(target, DSRPath)
 		assert mirror.index == rid.index
 
-		mirror_finalizer = IterTreeReducer(DestructiveSteppingFinalizer, ())
 		target_finalizer = IterTreeReducer(DestructiveSteppingFinalizer, ())
 		for rcd in Restore.yield_rcds(rid.index, mirror, rid,
 									  target, time, mirror_time):
 			rcd.RestoreFile()
-			if rcd.mirror: mirror_finalizer(rcd.index, rcd.mirror)
+			#if rcd.mirror: mirror_finalizer(rcd.index, rcd.mirror)
 			target_finalizer(rcd.target.index, rcd.target)
 		target_finalizer.Finish()
-		mirror_finalizer.Finish()
 
 	def yield_rcds(index, mirrorrp, rid, target, rest_time, mirror_time):
 		"""Iterate RestoreCombinedData objects starting with given args

trunk/rdiff-backup/rdiff_backup/rpath.py

@@ -288,6 +288,8 @@ class RORPath(RPathStatic):
 				(not Globals.change_ownership or self.issym())):
 				# Don't compare gid/uid for symlinks or if not change_ownership
 				pass
+			elif key == 'mtime':
+				Log("%s differs only in mtime, skipping" % (self.path,), 2)
 			elif key == 'atime' and not Globals.preserve_atime: pass
 			elif key == 'devloc' or key == 'inode' or key == 'nlink': pass
 			elif key == 'size' and not self.isreg(): pass
@@ -319,6 +321,10 @@ class RORPath(RPathStatic):
 		"""Reproduce RORPath from __getstate__ output"""
 		self.index, self.data = rorp_state
 
+	def get_rorpath(self):
+		"""Return new rorpath based on self"""
+		return RORPath(self.index, self.data.copy())
+
 	def make_placeholder(self):
 		"""Make rorp into a placeholder
 
@@ -697,19 +703,15 @@ class RPath(RORPath):
 		return self.conn.Globals.get('process_gid') == self.data['gid']
 
 	def delete(self):
-		"""Delete file at self.path
-
-		The destructive stepping allows this function to delete
-		directories even if they have files and we lack permissions.
-
-		"""
+		"""Delete file at self.path.  Recursively deletes directories."""
 		Log("Deleting %s" % self.path, 7)
 		self.setdata()
-		if not self.lstat(): return # must have been deleted in meantime
+		if not self.lstat():
+			Log("Warning: %s does not exist---deleted in meantime?"
+				% (self.path,), 2)
 		elif self.isdir():
 			itm = IterTreeReducer(RpathDeleter, [])
-			for dsrp in Select(DSRPath(None, self)).set_iter():
-				itm(dsrp.index, dsrp)
+			for rp in Select(self).set_iter(): itm(rp.index, rp)
 			itm.Finish()
 		else: self.conn.os.unlink(self.path)
 		self.setdata()
@@ -891,18 +893,17 @@ class RPathFileHook:
 import FilenameMapping
 from lazy import *
 from selection import *
-from destructive_stepping import *
 from highlevel import *
 
 class RpathDeleter(ITRBranch):
 	"""Delete a directory.  Called by RPath.delete()"""
-	def start_process(self, index, dsrp):
-		self.dsrp = dsrp
+	def start_process(self, index, rp):
+		self.rp = rp
 
 	def end_process(self):
-		if self.dsrp.isdir(): self.dsrp.rmdir()
-		else: self.dsrp.delete()
+		if self.rp.isdir(): self.rp.rmdir()
+		else: self.rp.delete()
 
-	def can_fast_process(self, index, dsrp): return not dsrp.isdir()
-	def fast_process(self, index, dsrp): dsrp.delete()
+	def can_fast_process(self, index, rp): return not rp.isdir()
+	def fast_process(self, index, rp): rp.delete()
 	

trunk/rdiff-backup/testing/commontest.py

@@ -173,8 +173,7 @@ def CompareRecursive(src_rp, dest_rp, compare_hardlinks = 1,
 
 	Log("Comparing %s and %s, hardlinks %s" % (src_rp.path, dest_rp.path,
 											   compare_hardlinks), 3)
-	src_select = Select(DSRPath(1, src_rp))
-	dest_select = Select(DSRPath(None, dest_rp))
+	src_select, dest_select = Select(src_rp), Select(dest_rp)
 
 	if ignore_tmp_files:
 		# Ignoring temp files can be useful when we want to check the
@@ -201,13 +200,10 @@ def CompareRecursive(src_rp, dest_rp, compare_hardlinks = 1,
 		dest_select.add_selection_func(dest_select.glob_get_tuple_sf(
 			('rdiff-backup-data',), 0))
 
-	src_select.set_iter()
-	dest_select.set_iter()
-	dsiter1, dsiter2 = src_select.iterate_with_finalizer(), \
-					   dest_select.iterate_with_finalizer()
+	dsiter1, dsiter2 = src_select.set_iter(), dest_select.set_iter()
 
 	def hardlink_equal(src_rorp, dest_rorp):
-		if src_rorp != dest_rorp: return None
+		if not src_rorp.equal_verbose(dest_rorp): return None
 		if Hardlink.rorp_eq(src_rorp, dest_rorp): return 1
 		Log("%s: %s" % (src_rorp.index, Hardlink.get_indicies(src_rorp, 1)), 3)
 		Log("%s: %s" % (dest_rorp.index,
@@ -229,7 +225,7 @@ def CompareRecursive(src_rp, dest_rp, compare_hardlinks = 1,
 				if dest_rorp.index[-1].endswith('.missing'): return 1
 		if compare_hardlinks:
 			if Hardlink.rorp_eq(src_rorp, dest_rorp): return 1
-		elif src_rorp == dest_rorp: return 1
+		elif src_rorp.equal_verbose(dest_rorp): return 1
 		Log("%s: %s" % (src_rorp.index, Hardlink.get_indicies(src_rorp, 1)), 3)
 		Log("%s: %s" % (dest_rorp.index,
 						Hardlink.get_indicies(dest_rorp, None)), 3)

trunk/rdiff-backup/testing/regressiontest.py

@@ -14,7 +14,7 @@ testfiles
 
 Globals.set('change_source_perms', 1)
 Globals.counter = 0
-Log.setverbosity(3)
+Log.setverbosity(7)
 
 class Local:
 	"""This is just a place to put increments relative to the local