Final update of statistics stuff

git-svn-id: http://svn.savannah.nongnu.org/svn/rdiff-backup@109 2b77aa54-bcbc-44c9-a7ec-4f6cf2b41109

trunk/rdiff-backup/CHANGELOG

+New in v0.7.6 (2002/??/??)
+--------------------------
+
+Improved statistics support, and added --print-statistics and
+--calculate-average switches.
+
+
 New in v0.7.5 (2002/05/21)
 --------------------------
 

trunk/rdiff-backup/rdiff-backup.1

@@ -12,6 +12,9 @@ rdiff-backup \- local/remote mirror and incremental backup
 .BI "| --remove-older-than " time_interval }
 .BI [[[ user@ ] host2.foo ]:: destination_directory ]
 
+.B rdiff-backup --calculate-average
+.I statfile1 statfile2 ...
+
 .SH DESCRIPTION
 .B rdiff-backup
 is a script, written in
@@ -45,6 +48,11 @@ on other options, see the section on
 .B -b, --backup-mode
 Force backup mode even if first argument appears to be an increment file.
 .TP
+.B --calculate-average
+Enter calculate average mode.  The arguments should be a number of
+statistics files.  rdiff-backup will print the average of the listed
+statistics files and exit.
+.TP
 .B --change-source-perms
 If this option is set, rdiff-backup will try to change the mode of any
 unreadable files or unreadable/unexecutable directories in the source
@@ -199,6 +207,11 @@ or
 .B --list-increments
 switches.
 .TP
+.B --print-statistics
+If set, summary statistics will be printed after a successful backup
+If not set, this information will still be available from the
+session_statistics.<time>.data file.
+.TP
 .BI "--quoting-char " char
 Use the specified character for quoting characters specified to be
 escaped by the

trunk/rdiff-backup/rdiff_backup/header.py

 #!/usr/bin/env python
 #
 # rdiff-backup -- Mirror files while keeping incremental changes
-# Version 0.7.5 released May 21, 2002
+# Version 0.7.5.1 released May 25, 2002
 # Copyright (C) 2001, 2002 Ben Escoto <bescoto@stanford.edu>
 #
 # This program is licensed under the GNU General Public License (GPL).

trunk/rdiff-backup/rdiff_backup/highlevel.py

@@ -262,6 +262,7 @@ class HLDestinationStruct:
 			cls.check_skip_error(finalizer.Finish, dsrp)
 		except: cls.handle_last_error(dsrp, finalizer, ITR)
 		if Globals.preserve_hardlinks: Hardlink.final_writedata()
+		cls.write_statistics(ITR)
 		SaveState.checkpoint_remove()
 
 	def patch_increment_and_finalize(cls, dest_rpath, diffs, inc_rpath):
@@ -291,6 +292,7 @@ class HLDestinationStruct:
 			cls.check_skip_error(finalizer.Finish, dsrp)
 		except: cls.handle_last_error(dsrp, finalizer, ITR)
 		if Globals.preserve_hardlinks: Hardlink.final_writedata()
+		cls.write_statistics(ITR)
 		SaveState.checkpoint_remove()
 
 	def check_skip_error(cls, thunk, dsrp):
@@ -321,4 +323,19 @@ class HLDestinationStruct:
 		SaveState.touch_last_file_definitive()
 		raise
 
+	def write_statistics(cls, ITR):
+		"""Write session statistics to file, log"""
+		stat_inc = Inc.get_inc(Globals.rbdir.append("session_statistics"),
+							   Time.curtime, "data")
+		ITR.StartTime = Time.curtime
+		ITR.EndTime = time.time()
+		if Globals.preserve_hardlinks and Hardlink.final_inc:
+			# include hardlink data in size of increments
+			ITR.IncrementFileSize += Hardlink.final_inc.getsize()
+		ITR.write_stats_to_rp(stat_inc)
+		if Globals.print_statistics:
+			message = ITR.get_stats_logstring("Session statistics")
+			Log.log_to_file(message)
+			Globals.client_conn.sys.stdout.write(message)
+
 MakeClass(HLDestinationStruct)

trunk/rdiff-backup/rdiff_backup/statistics.py

@@ -54,6 +54,12 @@ class StatsObj:
 					if self.get_stat(attr) is not None]
 		return "".join(timelist + filelist)
 
+	def get_stats_logstring(self, title):
+		"""Like get_stats_string, but add header and footer"""
+		header = "-------------[ %s ]-------------" % title
+		footer = "-" * len(header)
+		return "%s\n%s%s\n" % (header, self.get_stats_string(), footer)
+
 	def init_stats_from_string(self, s):
 		"""Initialize attributes from string, return self for convenience"""
 		def error(line): raise StatsException("Bad line '%s'" % line)
@@ -64,7 +70,12 @@ class StatsObj:
 			if len(line_parts) < 2: error(line)
 			attr, value_string = line_parts[:2]
 			if not attr in self.stat_attrs: error(line)
-			try: self.set_stat(attr, long(value_string))
+			try:
+				try: val1 = long(value_string)
+				except ValueError: val1 = None
+				val2 = float(value_string)
+				if val1 == val2: self.set_stat(attr, val1) # use integer val
+				else: self.set_stat(attr, val2) # use float
 			except ValueError: error(line)
 		return self
 
@@ -111,6 +122,12 @@ class StatsObj:
 							  self.get_stat(attr)/float(len(statobj_list)))
 		return self
 
+	def get_statsobj_copy(self):
+		"""Return new StatsObj object with same stats as self"""
+		s = StatObj()
+		for attr in self.stat_attrs: s.set_stat(attr, self.get_stat(attr))
+		return s
+
 
 class StatsITR(IterTreeReducer, StatsObj):
 	"""Keep track of per directory statistics

trunk/rdiff-backup/src/globals.py

@@ -8,7 +8,7 @@ import re, os
 class Globals:
 
 	# The current version of rdiff-backup
-	version = "0.7.5"
+	version = "0.7.5.1"
 	
 	# If this is set, use this value in seconds as the current time
 	# instead of reading it from the clock.
@@ -79,6 +79,12 @@ class Globals:
 	# Connection of the backup writer
 	backup_writer = None
 
+	# True if this process is the client invoked by the user
+	isclient = None
+
+	# Connection of the client
+	client_conn = None
+
 	# This list is used by the set function below.  When a new
 	# connection is created with init_connection, its Globals class
 	# will match this one for all the variables mentioned in this
@@ -144,6 +150,9 @@ class Globals:
 	# Determines whether or not ssh will be run with the -C switch
 	ssh_compression = 1
 
+	# If true, print statistics after successful backup
+	print_statistics = None
+
 	# On the reader and writer connections, the following will be
 	# replaced by the source and mirror Select objects respectively.
 	select_source, select_mirror = None, None

trunk/rdiff-backup/src/hardlink.py

@@ -151,6 +151,7 @@ class Hardlink:
 			fp = tf.open("wb", compress)
 			cPickle.dump(dict, fp)
 			assert not fp.close()
+			tf.setdata()
 		Robust.make_tf_robustaction(init, (tf,), (rpath,)).execute()
 
 	def get_linkrp(cls, data_rpath, time, prefix):
@@ -173,14 +174,17 @@ class Hardlink:
 
 	def final_writedata(cls):
 		"""Write final checkpoint data to rbdir after successful backup"""
-		if not cls._src_index_indicies: return
+		if not cls._src_index_indicies: # no hardlinks, so writing unnecessary
+			cls.final_inc = None
+			return
 		Log("Writing hard link data", 6)
 		if Globals.compression:
-			rp = Globals.rbdir.append("hardlink_data.%s.data.gz" %
-									  Time.curtimestr)
-		else: rp = Globals.rbdir.append("hardlink_data.%s.data" %
-										Time.curtimestr)
-		cls.write_linkdict(rp, cls._src_index_indicies, Globals.compression)
+			cls.final_inc = Globals.rbdir.append("hardlink_data.%s.data.gz" %
+												 Time.curtimestr)
+		else: cls.final_inc = Globals.rbdir.append("hardlink_data.%s.data" %
+												   Time.curtimestr)
+		cls.write_linkdict(cls.final_inc,
+						   cls._src_index_indicies, Globals.compression)
 
 	def retrieve_final(cls, time):
 		"""Set source index dictionary from hardlink_data file if avail"""

trunk/rdiff-backup/src/header.py

 #!/usr/bin/env python
 #
 # rdiff-backup -- Mirror files while keeping incremental changes
-# Version 0.7.5 released May 21, 2002
+# Version 0.7.5.1 released May 25, 2002
 # Copyright (C) 2001, 2002 Ben Escoto <bescoto@stanford.edu>
 #
 # This program is licensed under the GNU General Public License (GPL).

trunk/rdiff-backup/src/highlevel.py

@@ -262,6 +262,7 @@ class HLDestinationStruct:
 			cls.check_skip_error(finalizer.Finish, dsrp)
 		except: cls.handle_last_error(dsrp, finalizer, ITR)
 		if Globals.preserve_hardlinks: Hardlink.final_writedata()
+		cls.write_statistics(ITR)
 		SaveState.checkpoint_remove()
 
 	def patch_increment_and_finalize(cls, dest_rpath, diffs, inc_rpath):
@@ -291,6 +292,7 @@ class HLDestinationStruct:
 			cls.check_skip_error(finalizer.Finish, dsrp)
 		except: cls.handle_last_error(dsrp, finalizer, ITR)
 		if Globals.preserve_hardlinks: Hardlink.final_writedata()
+		cls.write_statistics(ITR)
 		SaveState.checkpoint_remove()
 
 	def check_skip_error(cls, thunk, dsrp):
@@ -321,4 +323,19 @@ class HLDestinationStruct:
 		SaveState.touch_last_file_definitive()
 		raise
 
+	def write_statistics(cls, ITR):
+		"""Write session statistics to file, log"""
+		stat_inc = Inc.get_inc(Globals.rbdir.append("session_statistics"),
+							   Time.curtime, "data")
+		ITR.StartTime = Time.curtime
+		ITR.EndTime = time.time()
+		if Globals.preserve_hardlinks and Hardlink.final_inc:
+			# include hardlink data in size of increments
+			ITR.IncrementFileSize += Hardlink.final_inc.getsize()
+		ITR.write_stats_to_rp(stat_inc)
+		if Globals.print_statistics:
+			message = ITR.get_stats_logstring("Session statistics")
+			Log.log_to_file(message)
+			Globals.client_conn.sys.stdout.write(message)
+
 MakeClass(HLDestinationStruct)

trunk/rdiff-backup/src/main.py

@@ -24,16 +24,17 @@ class Main:
 			except IOError: Log.FatalError("Error opening file %s" % filename)
 
 		try: optlist, self.args = getopt.getopt(arglist, "blmr:sv:V",
-			 ["backup-mode", "change-source-perms",
-			  "chars-to-quote=", "checkpoint-interval=",
-			  "current-time=", "exclude=", "exclude-device-files",
-			  "exclude-filelist=", "exclude-filelist-stdin",
-			  "exclude-mirror=", "exclude-regexp=", "force",
-			  "include=", "include-filelist=",
-			  "include-filelist-stdin", "include-regexp=",
-			  "list-increments", "mirror-only", "no-compression",
-			  "no-compression-regexp=", "no-hard-links", "no-resume",
-			  "parsable-output", "quoting-char=", "remote-cmd=",
+			 ["backup-mode", "calculate-average",
+			  "change-source-perms", "chars-to-quote=",
+			  "checkpoint-interval=", "current-time=", "exclude=",
+			  "exclude-device-files", "exclude-filelist=",
+			  "exclude-filelist-stdin", "exclude-mirror=",
+			  "exclude-regexp=", "force", "include=",
+			  "include-filelist=", "include-filelist-stdin",
+			  "include-regexp=", "list-increments", "mirror-only",
+			  "no-compression", "no-compression-regexp=",
+			  "no-hard-links", "no-resume", "parsable-output",
+			  "print-statistics", "quoting-char=", "remote-cmd=",
 			  "remote-schema=", "remove-older-than=",
 			  "restore-as-of=", "resume", "resume-window=", "server",
 			  "ssh-no-compression", "terminal-verbosity=",
@@ -44,6 +45,8 @@ class Main:
 
 		for opt, arg in optlist:
 			if opt == "-b" or opt == "--backup-mode": self.action = "backup"
+			elif opt == "--calculate-average":
+				self.action = "calculate-average"
 			elif opt == "--change-source-perms":
 				Globals.set('change_source_perms', 1)
 			elif opt == "--chars-to-quote":
@@ -89,6 +92,8 @@ class Main:
 				self.restore_timestr = arg
 				self.action = "restore-as-of"
 			elif opt == "--parsable-output": Globals.set('parsable_output', 1)
+			elif opt == "--print-statistics":
+				Globals.set('print_statistics', 1)
 			elif opt == "--quoting-char":
 				Globals.set('quoting_char', arg)
 				Globals.set('quoting_enabled', 1)
@@ -160,6 +165,8 @@ class Main:
 		os.umask(077)
 		Time.setcurtime(Globals.current_time)
 		FilenameMapping.set_init_quote_vals()
+		Globals.set("isclient", 1)
+		SetConnections.UpdateGlobal("client_conn", Globals.local_connection)
 
 		# This is because I originally didn't think compiled regexps
 		# could be pickled, and so must be compiled on remote side.
@@ -177,6 +184,7 @@ class Main:
 		elif self.action == "test-server": SetConnections.TestConnections()
 		elif self.action == "list-increments": self.ListIncrements(rps[0])
 		elif self.action == "remove-older-than": self.RemoveOlderThan(rps[0])
+		elif self.action == "calculate-average": self.CalculateAverage(rps)
 		else: raise AssertionError("Unknown action " + self.action)
 
 	def cleanup(self):
@@ -444,6 +452,14 @@ Try restoring from an increment file (the filenames look like
 		else: print Manage.describe_incs_human(incs, mirror_time, mirrorrp)
 
 
+	def CalculateAverage(self, rps):
+		"""Print out the average of the given statistics files"""
+		statobjs = map(lambda rp: StatsObj().read_stats_from_rp(rp), rps)
+		average_stats = StatsObj().set_to_average(statobjs)
+		print average_stats.get_stats_logstring(
+			"Average of %d stat files" % len(rps))
+
+
 	def RemoveOlderThan(self, rootrp):
 		"""Remove all increment files older than a certain time"""
 		datadir = rootrp.append("rdiff-backup-data")

trunk/rdiff-backup/src/statistics.py

@@ -54,6 +54,12 @@ class StatsObj:
 					if self.get_stat(attr) is not None]
 		return "".join(timelist + filelist)
 
+	def get_stats_logstring(self, title):
+		"""Like get_stats_string, but add header and footer"""
+		header = "-------------[ %s ]-------------" % title
+		footer = "-" * len(header)
+		return "%s\n%s%s\n" % (header, self.get_stats_string(), footer)
+
 	def init_stats_from_string(self, s):
 		"""Initialize attributes from string, return self for convenience"""
 		def error(line): raise StatsException("Bad line '%s'" % line)
@@ -64,7 +70,12 @@ class StatsObj:
 			if len(line_parts) < 2: error(line)
 			attr, value_string = line_parts[:2]
 			if not attr in self.stat_attrs: error(line)
-			try: self.set_stat(attr, long(value_string))
+			try:
+				try: val1 = long(value_string)
+				except ValueError: val1 = None
+				val2 = float(value_string)
+				if val1 == val2: self.set_stat(attr, val1) # use integer val
+				else: self.set_stat(attr, val2) # use float
 			except ValueError: error(line)
 		return self
 
@@ -111,6 +122,12 @@ class StatsObj:
 							  self.get_stat(attr)/float(len(statobj_list)))
 		return self
 
+	def get_statsobj_copy(self):
+		"""Return new StatsObj object with same stats as self"""
+		s = StatObj()
+		for attr in self.stat_attrs: s.set_stat(attr, self.get_stat(attr))
+		return s
+
 
 class StatsITR(IterTreeReducer, StatsObj):
 	"""Keep track of per directory statistics