Various final bug fixes for 0.7.4

git-svn-id: http://svn.savannah.nongnu.org/svn/rdiff-backup@79 2b77aa54-bcbc-44c9-a7ec-4f6cf2b41109

trunk/rdiff-backup/rdiff_backup/header.py

 #!/usr/bin/env python
 #
 # rdiff-backup -- Mirror files while keeping incremental changes
-# Version 0.7.3 released April 29, 2002
+# Version 0.7.4 released May 11, 2002
 # Copyright (C) 2001, 2002 Ben Escoto <bescoto@stanford.edu>
 #
 # This program is licensed under the GNU General Public License (GPL).

trunk/rdiff-backup/rdiff_backup/increment.py

@@ -9,6 +9,12 @@ execfile("filename_mapping.py")
 
 class Inc:
 	"""Class containing increment functions"""
+	# This is a hack.  _inc_file holds the dsrp of the latest
+	# increment file created, to be used in IncrementITR for
+	# statistics purposes.  It should be given directly to the ITR
+	# object but there didn't seem to be a good way to pass it out.
+	_inc_file = None
+
 	def Increment_action(new, mirror, incpref):
 		"""Main file incrementing function, returns RobustAction
 
@@ -95,7 +101,9 @@ class Inc:
 		while 1:
 			inctime = Resume.FindTime(rp.index, inctime)
 			incrp = get_newinc(Time.timetostring(inctime))
-			if not incrp.lstat(): return incrp
+			if not incrp.lstat(): break
+		Inc._inc_file = incrp
+		return incrp
 
 MakeStatic(Inc)
 
@@ -134,6 +142,7 @@ class IncrementITR(IterTreeReducer):
 		dsrp is the local file to be incremented
 
 		"""
+		self.init_statistics(diff_rorp, dsrp)
 		incpref = self.inc_rpath.new_index(index)
 		if Globals.quoting_enabled: incpref.quote_path()
 		if dsrp.isdir():
@@ -141,6 +150,26 @@ class IncrementITR(IterTreeReducer):
 			self.setvals(diff_rorp, dsrp, incpref)
 		else: self.init_non_dir(dsrp, diff_rorp, incpref)
 
+	def init_statistics(self, diff_rorp, dsrp):
+		"""Set initial values for various statistics
+
+		These refer to the old mirror or to new increment files.  Note
+		that changed_file_size could be bigger than total_file_size.
+		The other statistic, increment_file_size, is set later when we
+		have that information.
+
+		"""
+		if dsrp.lstat():
+			self.total_files = 1
+			self.total_file_size = dsrp.getsize()
+		else: self.total_files = self.total_file_size = 0
+		if diff_rorp:
+			self.changed_files = 1
+			if dsrp.lstat(): self.changed_file_size = dsrp.getsize()
+			else: self.changed_file_size = 0
+		else: self.changed_files = self.changed_file_size = 0
+		self.increment_file_size = 0
+
 	def override_changed(self):
 		"""Set changed flag to true
 
@@ -195,6 +224,9 @@ class IncrementITR(IterTreeReducer):
 			Robust.chain([Inc.Increment_action(diff_rorp, dsrp, incpref),
 						  RORPIter.patchonce_action(None, dsrp, diff_rorp)]
 						 ).execute()
+
+		self.increment_file_size += ((Inc._inc_file and Inc._inc_file.lstat()
+									  and Inc._inc_file.getsize()) or 0)
 		self.changed = 1
 
 	def end_process(self):
@@ -213,9 +245,32 @@ class IncrementITR(IterTreeReducer):
 			if diff_rorp:
 				RORPIter.patchonce_action(None, dsrp, diff_rorp).execute()
 
-	def branch_process(self, subinstance):
-		"""Update the has_changed flag if change in branch"""
-		if subinstance.changed: self.changed = 1	
+		self.increment_file_size += ((Inc._inc_file and Inc._inc_file.lstat()
+									  and Inc._inc_file.getsize()) or 0)
+		self.write_statistics()
 
+	def write_statistics(self):
+		"""Write the accumulated totals into file in inc directory"""
+		if not self.incpref.isdir(): return # only write for directories
+		statrp = Inc.get_inc_ext(self.incpref.append("directory_statistics"),
+								 "data")
+		tf = TempFileManager.new(statrp)
+		def init_thunk():
+			fp = tf.open("w")
+			fp.write("TotalFiles %d\n" % self.total_files)
+			fp.write("TotalFileSize %d\n" % self.total_file_size)
+			fp.write("ChangedFiles %d\n" % self.changed_files)
+			fp.write("ChangedFileSize %d\n" % self.changed_file_size)
+			fp.write("IncrementFileSize %d\n" % self.increment_file_size)
+			fp.close()
+		Robust.make_tf_robustaction(init_thunk, (tf,), (statrp,)).execute()
 
+	def branch_process(self, subinstance):
+		"""Update statistics, and the has_changed flag if change in branch"""
+		if subinstance.changed: self.changed = 1	
 
+		self.total_files += subinstance.total_files
+		self.total_file_size += subinstance.total_file_size
+		self.changed_files += subinstance.changed_files
+		self.changed_file_size += subinstance.changed_file_size
+		self.increment_file_size += subinstance.increment_file_size

trunk/rdiff-backup/rdiff_backup/restore.py

@@ -213,7 +213,7 @@ class Restore:
 			"""Add filename to the inc tuple dictionary"""
 			rp = rid.inc_rpath.append(filename)
 			if Globals.quoting_enabled: rp.quote_path()
-			if rp.isincfile():
+			if rp.isincfile() and rp.getinctype() != 'data':
 				basename = rp.getincbase_str()
 				affirm_dict_indexed(basename)
 				rid_dict[basename].inc_list.append(rp)

trunk/rdiff-backup/rdiff_backup/rpath.py

@@ -456,7 +456,6 @@ class RPath(RORPath):
 
 		if stat.S_ISREG(mode):
 			type = 'reg'
-			data['size'] = statblock[stat.ST_SIZE]
 		elif stat.S_ISDIR(mode): type = 'dir'
 		elif stat.S_ISCHR(mode):
 			type = 'dev'
@@ -471,6 +470,7 @@ class RPath(RORPath):
 		elif stat.S_ISSOCK(mode): type = 'sock'
 		else: raise RPathException("Unknown type for %s" % self.path)
 		data['type'] = type
+		data['size'] = statblock[stat.ST_SIZE]
 		data['perms'] = stat.S_IMODE(mode)
 		data['uid'] = statblock[stat.ST_UID]
 		data['gid'] = statblock[stat.ST_GID]
@@ -754,7 +754,7 @@ class RPath(RORPath):
 		if type == 'c': datatype = 'chr'
 		elif type == 'b': datatype = 'blk'
 		else: raise RPathException
-		self.data = {'type': datatype, 'devnums': (type, major, minor)}
+		self.setdata()
 
 	def getRORPath(self, include_contents = None):
 		"""Return read only version of self"""

trunk/rdiff-backup/src/header.py

 #!/usr/bin/env python
 #
 # rdiff-backup -- Mirror files while keeping incremental changes
-# Version 0.7.3 released April 29, 2002
+# Version 0.7.4 released May 11, 2002
 # Copyright (C) 2001, 2002 Ben Escoto <bescoto@stanford.edu>
 #
 # This program is licensed under the GNU General Public License (GPL).

trunk/rdiff-backup/src/increment.py

@@ -9,6 +9,12 @@ execfile("filename_mapping.py")
 
 class Inc:
 	"""Class containing increment functions"""
+	# This is a hack.  _inc_file holds the dsrp of the latest
+	# increment file created, to be used in IncrementITR for
+	# statistics purposes.  It should be given directly to the ITR
+	# object but there didn't seem to be a good way to pass it out.
+	_inc_file = None
+
 	def Increment_action(new, mirror, incpref):
 		"""Main file incrementing function, returns RobustAction
 
@@ -95,7 +101,9 @@ class Inc:
 		while 1:
 			inctime = Resume.FindTime(rp.index, inctime)
 			incrp = get_newinc(Time.timetostring(inctime))
-			if not incrp.lstat(): return incrp
+			if not incrp.lstat(): break
+		Inc._inc_file = incrp
+		return incrp
 
 MakeStatic(Inc)
 
@@ -134,6 +142,7 @@ class IncrementITR(IterTreeReducer):
 		dsrp is the local file to be incremented
 
 		"""
+		self.init_statistics(diff_rorp, dsrp)
 		incpref = self.inc_rpath.new_index(index)
 		if Globals.quoting_enabled: incpref.quote_path()
 		if dsrp.isdir():
@@ -141,6 +150,26 @@ class IncrementITR(IterTreeReducer):
 			self.setvals(diff_rorp, dsrp, incpref)
 		else: self.init_non_dir(dsrp, diff_rorp, incpref)
 
+	def init_statistics(self, diff_rorp, dsrp):
+		"""Set initial values for various statistics
+
+		These refer to the old mirror or to new increment files.  Note
+		that changed_file_size could be bigger than total_file_size.
+		The other statistic, increment_file_size, is set later when we
+		have that information.
+
+		"""
+		if dsrp.lstat():
+			self.total_files = 1
+			self.total_file_size = dsrp.getsize()
+		else: self.total_files = self.total_file_size = 0
+		if diff_rorp:
+			self.changed_files = 1
+			if dsrp.lstat(): self.changed_file_size = dsrp.getsize()
+			else: self.changed_file_size = 0
+		else: self.changed_files = self.changed_file_size = 0
+		self.increment_file_size = 0
+
 	def override_changed(self):
 		"""Set changed flag to true
 
@@ -195,6 +224,9 @@ class IncrementITR(IterTreeReducer):
 			Robust.chain([Inc.Increment_action(diff_rorp, dsrp, incpref),
 						  RORPIter.patchonce_action(None, dsrp, diff_rorp)]
 						 ).execute()
+
+		self.increment_file_size += ((Inc._inc_file and Inc._inc_file.lstat()
+									  and Inc._inc_file.getsize()) or 0)
 		self.changed = 1
 
 	def end_process(self):
@@ -213,9 +245,32 @@ class IncrementITR(IterTreeReducer):
 			if diff_rorp:
 				RORPIter.patchonce_action(None, dsrp, diff_rorp).execute()
 
-	def branch_process(self, subinstance):
-		"""Update the has_changed flag if change in branch"""
-		if subinstance.changed: self.changed = 1	
+		self.increment_file_size += ((Inc._inc_file and Inc._inc_file.lstat()
+									  and Inc._inc_file.getsize()) or 0)
+		self.write_statistics()
 
+	def write_statistics(self):
+		"""Write the accumulated totals into file in inc directory"""
+		if not self.incpref.isdir(): return # only write for directories
+		statrp = Inc.get_inc_ext(self.incpref.append("directory_statistics"),
+								 "data")
+		tf = TempFileManager.new(statrp)
+		def init_thunk():
+			fp = tf.open("w")
+			fp.write("TotalFiles %d\n" % self.total_files)
+			fp.write("TotalFileSize %d\n" % self.total_file_size)
+			fp.write("ChangedFiles %d\n" % self.changed_files)
+			fp.write("ChangedFileSize %d\n" % self.changed_file_size)
+			fp.write("IncrementFileSize %d\n" % self.increment_file_size)
+			fp.close()
+		Robust.make_tf_robustaction(init_thunk, (tf,), (statrp,)).execute()
 
+	def branch_process(self, subinstance):
+		"""Update statistics, and the has_changed flag if change in branch"""
+		if subinstance.changed: self.changed = 1	
 
+		self.total_files += subinstance.total_files
+		self.total_file_size += subinstance.total_file_size
+		self.changed_files += subinstance.changed_files
+		self.changed_file_size += subinstance.changed_file_size
+		self.increment_file_size += subinstance.increment_file_size

trunk/rdiff-backup/src/restore.py

@@ -213,7 +213,7 @@ class Restore:
 			"""Add filename to the inc tuple dictionary"""
 			rp = rid.inc_rpath.append(filename)
 			if Globals.quoting_enabled: rp.quote_path()
-			if rp.isincfile():
+			if rp.isincfile() and rp.getinctype() != 'data':
 				basename = rp.getincbase_str()
 				affirm_dict_indexed(basename)
 				rid_dict[basename].inc_list.append(rp)

trunk/rdiff-backup/src/rpath.py

@@ -456,7 +456,6 @@ class RPath(RORPath):
 
 		if stat.S_ISREG(mode):
 			type = 'reg'
-			data['size'] = statblock[stat.ST_SIZE]
 		elif stat.S_ISDIR(mode): type = 'dir'
 		elif stat.S_ISCHR(mode):
 			type = 'dev'
@@ -471,6 +470,7 @@ class RPath(RORPath):
 		elif stat.S_ISSOCK(mode): type = 'sock'
 		else: raise RPathException("Unknown type for %s" % self.path)
 		data['type'] = type
+		data['size'] = statblock[stat.ST_SIZE]
 		data['perms'] = stat.S_IMODE(mode)
 		data['uid'] = statblock[stat.ST_UID]
 		data['gid'] = statblock[stat.ST_GID]
@@ -754,7 +754,7 @@ class RPath(RORPath):
 		if type == 'c': datatype = 'chr'
 		elif type == 'b': datatype = 'blk'
 		else: raise RPathException
-		self.data = {'type': datatype, 'devnums': (type, major, minor)}
+		self.setdata()
 
 	def getRORPath(self, include_contents = None):
 		"""Return read only version of self"""

trunk/rdiff-backup/testing/commontest.py

@@ -127,7 +127,12 @@ def InternalRestore(mirror_local, dest_local, mirror_dir, dest_dir, time):
 
 	mirror_rp, dest_rp = SetConnections.InitRPs([mirror_dir, dest_dir],
 												remote_schema)
+	inc = get_increment_rp(mirror_rp, time)
+	if inc:
 		_get_main().Restore(get_increment_rp(mirror_rp, time), dest_rp)
+	else: # use alternate syntax
+		_get_main().restore_timestr = str(time)
+		_get_main().RestoreAsOf(mirror_rp, dest_rp)
 	_get_main().cleanup()
 
 def get_increment_rp(mirror_rp, time):
@@ -135,11 +140,9 @@ def get_increment_rp(mirror_rp, time):
 	data_rp = mirror_rp.append("rdiff-backup-data")
 	for filename in data_rp.listdir():
 		rp = data_rp.append(filename)
-		if (rp.isincfile() and rp.getincbase_str() == "increments" and
-			Time.stringtotime(rp.getinctime()) == time):
-			return rp
-	assert None, ("No increments.XXX.dir found in directory "
-				  "%s with that time" % data_rp.path)
+		if rp.isincfile() and rp.getincbase_str() == "increments":
+			if Time.stringtotime(rp.getinctime()) == time: return rp
+	return None # Couldn't find appropriate increment
 
 def _reset_connections(src_rp, dest_rp):
 	"""Reset some global connection information"""

trunk/rdiff-backup/testing/incrementtest.py

 import unittest, os
 
 execfile("commontest.py")
-rbexec("increment.py")
+rbexec("main.py")
 
 
 lc = Globals.local_connection
 Globals.change_source_perms = 1
-Log.setverbosity(7)
+Log.setverbosity(4)
 
 def getrp(ending):
 	return RPath(lc, "testfiles/various_file_types/" + ending)
@@ -162,4 +162,74 @@ class inctest(unittest.TestCase):
 		out2.delete()
 		out_gz.delete()
 
+class inctest2(unittest.TestCase):
+	"""Like inctest but contains more elaborate tests"""
+	def testStatistics(self):
+		"""Test the writing of statistics
+
+		The file sizes are approximate because the size of directories
+		could change with different file systems...
+
+		"""
+		Globals.compression = 1
+		Myrm("testfiles/output")
+		InternalBackup(1, 1, "testfiles/stattest1", "testfiles/output")
+		InternalBackup(1, 1, "testfiles/stattest2", "testfiles/output")
+
+		inc_base = RPath(Globals.local_connection,
+						 "testfiles/output/rdiff-backup-data/increments")
+
+		incs = Restore.get_inclist(inc_base.append("subdir").
+								   append("directory_statistics"))
+		assert len(incs) == 1
+		subdir_stats = self.parse_statistics(incs[0])
+		assert subdir_stats.total_files == 2, subdir_stats.total_files
+		assert 350000 < subdir_stats.total_file_size < 450000, \
+			   subdir_stats.total_file_size
+		assert subdir_stats.changed_files == 2, subdir_stats.changed_files
+		assert 350000 < subdir_stats.changed_file_size < 450000, \
+			   subdir_stats.changed_file_size
+		assert 10 < subdir_stats.increment_file_size < 20000, \
+			   subdir_stats.increment_file_size
+
+		incs = Restore.get_inclist(inc_base.append("directory_statistics"))
+		assert len(incs) == 1
+		root_stats = self.parse_statistics(incs[0])
+		assert root_stats.total_files == 6, root_stats.total_files
+		assert 650000 < root_stats.total_file_size < 750000, \
+			   root_stats.total_file_size
+		assert root_stats.changed_files == 4, root_stats.changed_files
+		assert 550000 < root_stats.changed_file_size < 650000, \
+			   root_stats.changed_file_size
+		assert 10 < root_stats.increment_file_size < 20000, \
+			   root_stats.increment_file_size
+
+	def parse_statistics(self, statrp):
+		"""Return StatObj from given statrp"""
+		assert statrp.isincfile() and statrp.getinctype() == "data"
+		s = StatObj()
+		fp = statrp.open("r")
+		for line in fp:
+			lsplit = line.split()
+			assert len(lsplit) == 2
+			field, num = lsplit[0], long(lsplit[1])
+			if field == "TotalFiles": s.total_files = num
+			elif field == "TotalFileSize": s.total_file_size = num
+			elif field == "ChangedFiles": s.changed_files = num
+			elif field == "ChangedFileSize": s.changed_file_size = num
+			elif field == "IncrementFileSize": s.increment_file_size = num
+			else: assert None, "Unrecognized field %s" % (field,)
+		assert not fp.close()
+		return s
+
+
+class StatObj:
+	"""Just hold various statistics"""
+	total_files = 0
+	total_file_size = 0
+	changed_files = 0
+	changed_file_size = 0
+	increment_file_size = 0
+	
+
 if __name__ == '__main__': unittest.main()

trunk/rdiff-backup/testing/restoretest.py

@@ -74,13 +74,8 @@ class RestoreTest(unittest.TestCase):
 				assert inc.isincfile()
 				inctime = Time.stringtotime(inc.getinctime())
 				rid1 = RestoreIncrementData(basename, incbase, incs)
-				rid2 = RestoreIncrementData(basename, incbase, incs)
 				rid1.sortincseq(inctime, mirror_time)
-				rid2.sortincseq(inctime + 5, mirror_time)
 				assert rid1.inc_list, rid1.inc_list
-				# Five seconds later shouldn't make a difference
-				assert rid1.inc_list == rid2.inc_list, (rid1.inc_list,
-														rid2.inc_list)
 				# oldest increment should be exactly inctime
 				ridtime = Time.stringtotime(rid1.inc_list[-1].getinctime())
 				assert ridtime == inctime, (ridtime, inctime)

trunk/rdiff-backup/testing/robusttest.py

@@ -76,7 +76,7 @@ class SaveStateTest(unittest.TestCase):
 		SaveState.record_last_file_action(last_rorp).execute()
 
 		sym_rp = RPath(Globals.local_connection,
-					   "testfiles/robust/last-file-mirrored.%s.snapshot" %
+					   "testfiles/robust/last-file-mirrored.%s.data" %
 					   Time.curtimestr)
 		assert sym_rp.issym()
 		assert sym_rp.readlink() == "increments/usr/local/bin/ls"

trunk/rdiff-backup/testing/roottest.py

@@ -13,7 +13,7 @@ Globals.counter = 0
 Log.setverbosity(4)
 
 class RootTest(unittest.TestCase):
-	dirlist1 = ["testfiles/root", "testfiles/noperms", "testfiles/increment4"]
+	dirlist1 = ["testfiles/root", "testfiles/various_file_types", "testfiles/increment4"]
 	dirlist2 = ["testfiles/increment4", "testfiles/root",
 				"testfiles/increment1"]
 	def testLocal1(self): BackupRestoreSeries(1, 1, self.dirlist1)