Added support for gzipped increments

git-svn-id: http://svn.savannah.nongnu.org/svn/rdiff-backup@23 2b77aa54-bcbc-44c9-a7ec-4f6cf2b41109

trunk/rdiff-backup/rdiff_backup/header.py

 #!/usr/bin/env python
 #
 # rdiff-backup -- Mirror files while keeping incremental changes
-# Version 0.7.0 released March 21, 2002
-# Copyright (C) 2001 Ben Escoto <bescoto@stanford.edu>
+# Version 0.7.1 released March 25, 2002
+# Copyright (C) 2001, 2002 Ben Escoto <bescoto@stanford.edu>
 #
 # This program is licensed under the GNU General Public License (GPL).
 # Distributions of rdiff-backup usually include a copy of the GPL in a
 # file called COPYING.  The GPL is also available online at
 # http://www.gnu.org/copyleft/gpl.html.
 #
-# Please send mail to me or the mailing list if you find bugs or have
-# any suggestions.
+# See http://www.stanford.edu/~bescoto/rdiff-backup for more
+# information.  Please send mail to me or the mailing list if you find
+# bugs or have any suggestions.
 
 from __future__ import nested_scopes, generators
-import os, stat, time, sys, getopt, re, cPickle, types, shutil, sha, marshal, traceback, popen2, tempfile
+import os, stat, time, sys, getopt, re, cPickle, types, shutil, sha, marshal, traceback, popen2, tempfile, gzip
 
 

trunk/rdiff-backup/rdiff_backup/highlevel.py

@@ -238,7 +238,7 @@ class HLDestinationStruct:
 
 		try:
 			while 1:
-				try: dsrp = cls.check_skip_error(error_checked)
+				try: dsrp = cls.check_skip_error(error_checked, dsrp)
 				except StopIteration: break
 				if checkpoint: SaveState.checkpoint_mirror(finalizer, dsrp)
 		except: cls.handle_last_error(dsrp, finalizer)
@@ -272,7 +272,7 @@ class HLDestinationStruct:
 
 		try:
 			while 1:
-				try: dsrp = cls.check_skip_error(error_checked)
+				try: dsrp = cls.check_skip_error(error_checked, dsrp)
 				except StopIteration: break
 				SaveState.checkpoint_inc_backup(ITR, finalizer, dsrp)
 		except: cls.handle_last_error(dsrp, finalizer, ITR)
@@ -281,7 +281,7 @@ class HLDestinationStruct:
 		if Globals.preserve_hardlinks: Hardlink.final_writedata()
 		SaveState.checkpoint_remove()
 
-	def check_skip_error(cls, thunk):
+	def check_skip_error(cls, thunk, dsrp):
 		"""Run thunk, catch certain errors skip files"""
 		try: return thunk()
 		except (IOError, OSError, SkipFileException), exp:
@@ -294,7 +294,8 @@ class HLDestinationStruct:
 							 26] # Requested by Campbell (see list) -
                                  # happens on some NT systems
 				 ))):
-				Log("Skipping file", 2)
+				Log("Skipping file because of error after %s" %
+					(dsrp and dsrp.index,), 2)
 				return None
 			else: raise
 

trunk/rdiff-backup/rdiff_backup/increment.py

@@ -43,14 +43,27 @@ class Inc:
 		
 	def makesnapshot_action(mirror, incpref):
 		"""Copy mirror to incfile, since new is quite different"""
-		snapshotrp = Inc.get_inc_ext(incpref, "snapshot")
-		return Robust.copy_with_attribs_action(mirror, snapshotrp)
+		if (mirror.isreg() and Globals.compression and
+			not Globals.no_compression_regexp.match(mirror.path)):
+			snapshotrp = Inc.get_inc_ext(incpref, "snapshot.gz")
+			return Robust.copy_with_attribs_action(mirror, snapshotrp, 1)
+		else:
+			snapshotrp = Inc.get_inc_ext(incpref, "snapshot")
+			return Robust.copy_with_attribs_action(mirror, snapshotrp, None)
 
 	def makediff_action(new, mirror, incpref):
 		"""Make incfile which is a diff new -> mirror"""
-		diff = Inc.get_inc_ext(incpref, "diff")
-		return Robust.chain([Rdiff.write_delta_action(new, mirror, diff),
-							 Robust.copy_attribs_action(mirror, diff)])
+		if (Globals.compression and
+			not Globals.no_compression_regexp.match(mirror.path)):
+			diff = Inc.get_inc_ext(incpref, "diff.gz")
+			return Robust.chain([Rdiff.write_delta_action(new, mirror,
+														  diff, 1),
+								 Robust.copy_attribs_action(mirror, diff)])
+		else: 
+			diff = Inc.get_inc_ext(incpref, "diff")
+			return Robust.chain([Rdiff.write_delta_action(new, mirror,
+														  diff, None),
+								 Robust.copy_attribs_action(mirror, diff)])
 
 	def makedir_action(mirrordir, incpref):
 		"""Make file indicating directory mirrordir has changed"""

trunk/rdiff-backup/rdiff_backup/restore.py

@@ -69,14 +69,19 @@ class Restore:
 		if inctype == "diff":
 			if not target.lstat():
 				raise RestoreError("Bad increment sequence at " + inc.path)
-			Rdiff.patch_action(target, inc).execute()
+			Rdiff.patch_action(target, inc,
+							   delta_compressed = inc.isinccompressed()
+							   ).execute()
 		elif inctype == "dir":
 			if not target.isdir():
 				if target.lstat():
 					raise RestoreError("File %s already exists" % target.path)
 				target.mkdir()
 		elif inctype == "missing": return
-		elif inctype == "snapshot": RPath.copy(inc, target)
+		elif inctype == "snapshot":
+			if inc.isinccompressed():
+				target.write_from_fileobj(inc.open("rb", compress = 1))
+			else: RPath.copy(inc, target)
 		else: raise RestoreError("Unknown inctype %s" % inctype)
 		RPath.copy_attribs(inc, target)
 

trunk/rdiff-backup/rdiff_backup/robust.py

@@ -142,13 +142,14 @@ class Robust:
 				tfl[0].rename(rpout)
 		return RobustAction(init, final, lambda e: tfl[0] and tfl[0].delete())
 
-	def copy_with_attribs_action(rorpin, rpout):
+	def copy_with_attribs_action(rorpin, rpout, compress = None):
 		"""Like copy_action but also copy attributes"""
 		tfl = [None] # Need mutable object that init and final can access
 		def init(): 
 			if not (rorpin.isdir() and rpout.isdir()): # already a dir
 				tfl[0] = TempFileManager.new(rpout)
-				if rorpin.isreg(): tfl[0].write_from_fileobj(rorpin.open("rb"))
+				if rorpin.isreg():
+					tfl[0].write_from_fileobj(rorpin.open("rb"), compress)
 				else: RPath.copy(rorpin, tfl[0])
 				if tfl[0].lstat(): # Some files, like sockets, won't be created
 					RPathStatic.copy_attribs(rorpin, tfl[0])

trunk/rdiff-backup/rdiff_backup/rpath.py

 execfile("connection.py")
-import os, stat, re, sys, shutil
+import os, stat, re, sys, shutil, gzip
 
 #######################################################################
 #
@@ -73,7 +73,7 @@ class RPathStatic:
 		try:
 			if rpout.conn is rpin.conn:
 				rpout.conn.shutil.copyfile(rpin.path, rpout.path)
-				rpout.data = {'type': rpin.data['type']}
+				rpout.setdata()
 				return
 		except AttributeError: pass
 		rpout.write_from_fileobj(rpin.open("rb"))
@@ -648,44 +648,75 @@ class RPath(RORPath):
 		"""Return similar RPath but with new index"""
 		return self.__class__(self.conn, self.base, index)
 
-	def open(self, mode):
-		"""Return open file.  Supports modes "w" and "r"."""
-		return self.conn.open(self.path, mode)
+	def open(self, mode, compress = None):
+		"""Return open file.  Supports modes "w" and "r".
+
+		If compress is true, data written/read will be gzip
+		compressed/decompressed on the fly.
+
+		"""
+		if compress: return self.conn.gzip.GzipFile(self.path, mode)
+		else: return self.conn.open(self.path, mode)
+
+	def write_from_fileobj(self, fp, compress = None):
+		"""Reads fp and writes to self.path.  Closes both when done
+
+		If compress is true, fp will be gzip compressed before being
+		written to self.
 
-	def write_from_fileobj(self, fp):
-		"""Reads fp and writes to self.path.  Closes both when done"""
+		"""
 		Log("Writing file object to " + self.path, 7)
 		assert not self.lstat(), "File %s already exists" % self.path
-		outfp = self.open("wb")
+		outfp = self.open("wb", compress = compress)
 		RPath.copyfileobj(fp, outfp)
 		if fp.close() or outfp.close():
 			raise RPathException("Error closing file")
 		self.setdata()
 
 	def isincfile(self):
-		"""Return true if path looks like an increment file"""
-		dotsplit = self.path.split(".")
-		if len(dotsplit) < 3: return None
-		timestring, ext = dotsplit[-2:]
+		"""Return true if path looks like an increment file
+
+		Also sets various inc information used by the *inc* functions.
+
+		"""
+		if self.index: dotsplit = self.index[-1].split(".")
+		else: dotsplit = self.base.split(".")
+		if dotsplit[-1] == "gz":
+			compressed = 1
+			if len(dotsplit) < 4: return None
+			timestring, ext = dotsplit[-3:-1]
+		else:
+			compressed = None
+			if len(dotsplit) < 3: return None
+			timestring, ext = dotsplit[-2:]
 		if Time.stringtotime(timestring) is None: return None
-		return (ext == "snapshot" or ext == "dir" or
-				ext == "missing" or ext == "diff")
+		if not (ext == "snapshot" or ext == "dir" or
+				ext == "missing" or ext == "diff"): return None
+		self.inc_timestr = timestring
+		self.inc_compressed = compressed
+		self.inc_type = ext
+		if compressed: self.inc_basestr = ".".join(dotsplit[:-3])
+		else: self.inc_basestr = ".".join(dotsplit[:-2])
+		return 1
+
+	def isinccompressed(self):
+		"""Return true if inc file is compressed"""
+		return self.inc_compressed
 
 	def getinctype(self):
 		"""Return type of an increment file"""
-		return self.path.split(".")[-1]
+		return self.inc_type
 
 	def getinctime(self):
 		"""Return timestring of an increment file"""
-		return self.path.split(".")[-2]
+		return self.inc_timestr
 	
 	def getincbase(self):
 		"""Return the base filename of an increment file in rp form"""
 		if self.index:
 			return self.__class__(self.conn, self.base, self.index[:-1] +
-							 ((".".join(self.index[-1].split(".")[:-2])),))
-		else: return self.__class__(self.conn,
-									".".join(self.base.split(".")[:-2]), ())
+								  (self.inc_basestr,))
+		else: return self.__class__(self.conn, self.inc_basestr)
 
 	def getincbase_str(self):
 		"""Return the base filename string of an increment file"""

trunk/rdiff-backup/src/globals.py

@@ -8,7 +8,7 @@ import re, os
 class Globals:
 
 	# The current version of rdiff-backup
-	version = "0.7.0"
+	version = "0.7.1"
 	
 	# If this is set, use this value in seconds as the current time
 	# instead of reading it from the clock.
@@ -132,6 +132,18 @@ class Globals:
 	# hardlink information regardless.
 	preserve_hardlinks = 1
 
+	# If this is false, then rdiff-backup will not compress any
+	# increments.  Default is to compress based on regexp below.
+	compression = 1
+
+	# Increments based on files whose names match this
+	# case-insensitive regular expression won't be compressed (applies
+	# to .snapshots and .diffs).  The second below is the compiled
+	# version of the first.
+	no_compression_regexp_string = ".*\\.(gz|z|bz|bz2|tgz|zip|rpm|deb|" \
+								   "mp3|ogg|avi|wmv|mpeg|mpg|rm|mov)$"
+	no_compression_regexp = None
+
 	def get(cls, name):
 		"""Return the value of something in this class"""
 		return cls.__dict__[name]
@@ -181,3 +193,15 @@ class Globals:
 		if mirror: Globals.exclude_mirror_regexps.append(compiled)
 		else: Globals.exclude_regexps.append(compiled)
 	add_regexp_local = classmethod(add_regexp_local)
+
+	def postset_regexp(cls, name, re_string, flags = None):
+		"""Compile re_string on all existing connections, set to name"""
+		for conn in Globals.connections:
+			conn.Globals.postset_regexp_local(name, re_string, flags)
+	postset_regexp = classmethod(postset_regexp)
+
+	def postset_regexp_local(cls, name, re_string, flags):
+		"""Set name to compiled re_string locally"""
+		if flags: cls.__dict__[name] = re.compile(re_string, flags)
+		else: cls.__dict__[name] = re.compile(re_string)
+	postset_regexp_local = classmethod(postset_regexp_local)

trunk/rdiff-backup/src/hardlink.py

@@ -129,7 +129,7 @@ class Hardlink:
 								cls.get_indicies(src_rorp, 1)[0])
 		dest_rpath.hardlink(dest_link_rpath.path)
 
-	def write_linkdict(cls, rpath, dict):
+	def write_linkdict(cls, rpath, dict, compress = None):
 		"""Write link data to the rbdata dir
 
 		It is stored as the a big pickled dictionary dated to match
@@ -140,7 +140,7 @@ class Hardlink:
 				rpath.conn is Globals.local_connection)
 		tf = TempFileManager.new(rpath)
 		def init():
-			fp = tf.open("wb")
+			fp = tf.open("wb", compress)
 			cPickle.dump(dict, fp)
 			assert not fp.close()
 		Robust.make_tf_robustaction(init, (tf,), (rpath,)).execute()
@@ -158,18 +158,21 @@ class Hardlink:
 		"""Return index dictionary written by write_linkdata at time"""
 		rp = cls.get_linkrp(data_rpath, time, prefix)
 		if not rp: return None
-		fp = rp.open("rb")
+		fp = rp.open("rb", rp.isinccompressed())
 		index_dict = cPickle.load(fp)
 		assert not fp.close()
 		return index_dict
 
 	def final_writedata(cls):
 		"""Write final checkpoint data to rbdir after successful backup"""
-		if cls._src_index_indicies:
-			Log("Writing hard link data", 6)
-			rp = Globals.rbdir.append("hardlink_data.%s.snapshot" %
+		if not cls._src_index_indicies: return
+		Log("Writing hard link data", 6)
+		if Globals.compression:
+			rp = Globals.rbdir.append("hardlink_data.%s.snapshot.gz" %
 									  Time.curtimestr)
-			cls.write_linkdict(rp, cls._src_index_indicies)
+		else: rp = Globals.rbdir.append("hardlink_data.%s.snapshot" %
+										Time.curtimestr)
+		cls.write_linkdict(rp, cls._src_index_indicies, Globals.compression)
 
 	def retrieve_final(cls, time):
 		"""Set source index dictionary from hardlink_data file if avail"""

trunk/rdiff-backup/src/header.py

 #!/usr/bin/env python
 #
 # rdiff-backup -- Mirror files while keeping incremental changes
-# Version 0.7.0 released March 21, 2002
-# Copyright (C) 2001 Ben Escoto <bescoto@stanford.edu>
+# Version 0.7.1 released March 25, 2002
+# Copyright (C) 2001, 2002 Ben Escoto <bescoto@stanford.edu>
 #
 # This program is licensed under the GNU General Public License (GPL).
 # Distributions of rdiff-backup usually include a copy of the GPL in a
 # file called COPYING.  The GPL is also available online at
 # http://www.gnu.org/copyleft/gpl.html.
 #
-# Please send mail to me or the mailing list if you find bugs or have
-# any suggestions.
+# See http://www.stanford.edu/~bescoto/rdiff-backup for more
+# information.  Please send mail to me or the mailing list if you find
+# bugs or have any suggestions.
 
 from __future__ import nested_scopes, generators
-import os, stat, time, sys, getopt, re, cPickle, types, shutil, sha, marshal, traceback, popen2, tempfile
+import os, stat, time, sys, getopt, re, cPickle, types, shutil, sha, marshal, traceback, popen2, tempfile, gzip
 
 

trunk/rdiff-backup/src/highlevel.py

@@ -238,7 +238,7 @@ class HLDestinationStruct:
 
 		try:
 			while 1:
-				try: dsrp = cls.check_skip_error(error_checked)
+				try: dsrp = cls.check_skip_error(error_checked, dsrp)
 				except StopIteration: break
 				if checkpoint: SaveState.checkpoint_mirror(finalizer, dsrp)
 		except: cls.handle_last_error(dsrp, finalizer)
@@ -272,7 +272,7 @@ class HLDestinationStruct:
 
 		try:
 			while 1:
-				try: dsrp = cls.check_skip_error(error_checked)
+				try: dsrp = cls.check_skip_error(error_checked, dsrp)
 				except StopIteration: break
 				SaveState.checkpoint_inc_backup(ITR, finalizer, dsrp)
 		except: cls.handle_last_error(dsrp, finalizer, ITR)
@@ -281,7 +281,7 @@ class HLDestinationStruct:
 		if Globals.preserve_hardlinks: Hardlink.final_writedata()
 		SaveState.checkpoint_remove()
 
-	def check_skip_error(cls, thunk):
+	def check_skip_error(cls, thunk, dsrp):
 		"""Run thunk, catch certain errors skip files"""
 		try: return thunk()
 		except (IOError, OSError, SkipFileException), exp:
@@ -294,7 +294,8 @@ class HLDestinationStruct:
 							 26] # Requested by Campbell (see list) -
                                  # happens on some NT systems
 				 ))):
-				Log("Skipping file", 2)
+				Log("Skipping file because of error after %s" %
+					(dsrp and dsrp.index,), 2)
 				return None
 			else: raise
 

trunk/rdiff-backup/src/increment.py

@@ -43,14 +43,27 @@ class Inc:
 		
 	def makesnapshot_action(mirror, incpref):
 		"""Copy mirror to incfile, since new is quite different"""
-		snapshotrp = Inc.get_inc_ext(incpref, "snapshot")
-		return Robust.copy_with_attribs_action(mirror, snapshotrp)
+		if (mirror.isreg() and Globals.compression and
+			not Globals.no_compression_regexp.match(mirror.path)):
+			snapshotrp = Inc.get_inc_ext(incpref, "snapshot.gz")
+			return Robust.copy_with_attribs_action(mirror, snapshotrp, 1)
+		else:
+			snapshotrp = Inc.get_inc_ext(incpref, "snapshot")
+			return Robust.copy_with_attribs_action(mirror, snapshotrp, None)
 
 	def makediff_action(new, mirror, incpref):
 		"""Make incfile which is a diff new -> mirror"""
-		diff = Inc.get_inc_ext(incpref, "diff")
-		return Robust.chain([Rdiff.write_delta_action(new, mirror, diff),
-							 Robust.copy_attribs_action(mirror, diff)])
+		if (Globals.compression and
+			not Globals.no_compression_regexp.match(mirror.path)):
+			diff = Inc.get_inc_ext(incpref, "diff.gz")
+			return Robust.chain([Rdiff.write_delta_action(new, mirror,
+														  diff, 1),
+								 Robust.copy_attribs_action(mirror, diff)])
+		else: 
+			diff = Inc.get_inc_ext(incpref, "diff")
+			return Robust.chain([Rdiff.write_delta_action(new, mirror,
+														  diff, None),
+								 Robust.copy_attribs_action(mirror, diff)])
 
 	def makedir_action(mirrordir, incpref):
 		"""Make file indicating directory mirrordir has changed"""

trunk/rdiff-backup/src/main.py

@@ -27,7 +27,8 @@ class Main:
 			  "include-from-stdin", "terminal-verbosity=",
 			  "exclude-device-files", "resume", "no-resume",
 			  "resume-window=", "windows-time-format",
-			  "checkpoint-interval=", "no-hard-links", "current-time="])
+			  "checkpoint-interval=", "no-hard-links", "current-time=",
+			  "no-compression", "no-compression-regexp="])
 		except getopt.error:
 			self.commandline_error("Error parsing commandline options")
 
@@ -45,11 +46,14 @@ class Main:
 			elif opt == "--exclude-mirror":
 				self.exclude_mirror_regstrs.append(arg)
 			elif opt == "--force": self.force = 1
-			elif opt == "--no-hard-links": Globals.set('preserve_hardlinks', 0)
 			elif opt == "--include-from-stdin": Globals.include_from_stdin = 1
 			elif opt == "-l" or opt == "--list-increments":
 				self.action = "list-increments"
 			elif opt == "-m" or opt == "--mirror-only": self.action = "mirror"
+			elif opt == "--no-compression": Globals.set("compression", None)
+			elif opt == "--no-compression-regexp":
+				Globals.set("no_compression_regexp_string", arg)
+			elif opt == "--no-hard-links": Globals.set('preserve_hardlinks', 0)
 			elif opt == '--no-resume': Globals.resume = 0
 			elif opt == "--remote-cmd": self.remote_cmd = arg
 			elif opt == "--remote-schema": self.remote_schema = arg
@@ -103,7 +107,7 @@ class Main:
 		sys.exit(1)
 
 	def misc_setup(self, rps):
-		"""Set default change ownership flag, umask, excludes"""
+		"""Set default change ownership flag, umask, regular expressions"""
 		if ((len(rps) == 2 and rps[1].conn.os.getuid() == 0) or
 			(len(rps) < 2 and os.getuid() == 0)):
 			# Allow change_ownership if destination connection is root
@@ -116,6 +120,8 @@ class Main:
 			Globals.add_regexp(regex_string, None)
 		for regex_string in self.exclude_mirror_regstrs:
 			Globals.add_regexp(regex_string, 1)
+		Globals.postset_regexp('no_compression_regexp',
+							   Globals.no_compression_regexp_string, re.I)
 
 	def take_action(self, rps):
 		"""Do whatever self.action says"""
@@ -250,7 +256,7 @@ may need to use the --exclude option.""" % (rpout.path, rpin.path), 2)
 """Warning: duplicate current_mirror files found.  Perhaps something
 went wrong during your last backup?  Using """ + mirrorrps[-1].path, 2)
 
-		timestr = self.datadir.append(mirrorrps[-1].path).getinctime()
+		timestr = mirrorrps[-1].getinctime()
 		return Time.stringtotime(timestr)
 	
 	def backup_touch_curmirror(self, rpin, rpout):

trunk/rdiff-backup/src/rdiff.py

@@ -36,37 +36,52 @@ class Rdiff:
 		return rp_new.conn.RdiffPopen(['rdiff', 'delta',
 									   rp_signature.path, rp_new.path])
 
-	def write_delta_action(basis, new, delta):
-		"""Return action writing delta which brings basis to new"""
+	def write_delta_action(basis, new, delta, compress = None):
+		"""Return action writing delta which brings basis to new
+
+		If compress is true, the output of rdiff will be gzipped
+		before written to delta.
+
+		"""
 		sig_tf = TempFileManager.new(new, None)
 		delta_tf = TempFileManager.new(delta)
 		def init():
 			Log("Writing delta %s from %s -> %s" %
 				(basis.path, new.path, delta.path), 7)
 			sig_tf.write_from_fileobj(Rdiff.get_signature(basis))
-			delta_tf.write_from_fileobj(Rdiff.get_delta(sig_tf, new))
+			delta_tf.write_from_fileobj(Rdiff.get_delta(sig_tf, new), compress)
 			sig_tf.delete()
 		return Robust.make_tf_robustaction(init, (sig_tf, delta_tf),
 										   (None, delta))
 
-	def write_delta(basis, new, delta):
+	def write_delta(basis, new, delta, compress = None):
 		"""Write rdiff delta which brings basis to new"""
-		Rdiff.write_delta_action(basis, new, delta).execute()
+		Rdiff.write_delta_action(basis, new, delta, compress).execute()
 
-	def patch_action(rp_basis, rp_delta, rp_out = None, out_tf = None):
+	def patch_action(rp_basis, rp_delta, rp_out = None,
+					 out_tf = None, delta_compressed = None):
 		"""Return RobustAction which patches rp_basis with rp_delta
 
 		If rp_out is None, put output in rp_basis.  Will use TempFile
-		out_tf it is specified.
+		out_tf it is specified.  If delta_compressed is true, the
+		delta file will be decompressed before processing with rdiff.
 
 		"""
 		if not rp_out: rp_out = rp_basis
 		else: assert rp_out.conn is rp_basis.conn
-		if not (isinstance(rp_delta, RPath) and isinstance(rp_basis, RPath)
-				and rp_basis.conn is rp_delta.conn):
-			return Rdiff.patch_fileobj_action(rp_basis, rp_delta.open('rb'),
-											  rp_out, out_tf)
-
+		if (delta_compressed or
+			not (isinstance(rp_delta, RPath) and isinstance(rp_basis, RPath)
+				 and rp_basis.conn is rp_delta.conn)):
+			if delta_compressed:
+				assert isinstance(rp_delta, RPath)
+				return Rdiff.patch_fileobj_action(rp_basis,
+												  rp_delta.open('rb', 1),
+												  rp_out, out_tf)
+			else: return Rdiff.patch_fileobj_action(rp_basis,
+													rp_delta.open('rb'),
+													rp_out, out_tf)
+
+		# Files are uncompressed on same connection, run rdiff
 		if out_tf is None: out_tf = TempFileManager.new(rp_out)
 		def init():
 			Log("Patching %s using %s to %s via %s" %
@@ -79,8 +94,8 @@ class Rdiff:
 				RdiffException("Error running %s" % cmdlist)
 		return Robust.make_tf_robustaction(init, (out_tf,), (rp_out,))
 
-	def patch_fileobj_action(rp_basis, delta_fileobj,
-							 rp_out = None, out_tf = None):
+	def patch_fileobj_action(rp_basis, delta_fileobj, rp_out = None,
+							 out_tf = None, delta_compressed = None):
 		"""Like patch_action but diff is given in fileobj form
 
 		Nest a writing of a tempfile with the actual patching to

trunk/rdiff-backup/src/restore.py

@@ -69,14 +69,19 @@ class Restore:
 		if inctype == "diff":
 			if not target.lstat():
 				raise RestoreError("Bad increment sequence at " + inc.path)
-			Rdiff.patch_action(target, inc).execute()
+			Rdiff.patch_action(target, inc,
+							   delta_compressed = inc.isinccompressed()
+							   ).execute()
 		elif inctype == "dir":
 			if not target.isdir():
 				if target.lstat():
 					raise RestoreError("File %s already exists" % target.path)
 				target.mkdir()
 		elif inctype == "missing": return
-		elif inctype == "snapshot": RPath.copy(inc, target)
+		elif inctype == "snapshot":
+			if inc.isinccompressed():
+				target.write_from_fileobj(inc.open("rb", compress = 1))
+			else: RPath.copy(inc, target)
 		else: raise RestoreError("Unknown inctype %s" % inctype)
 		RPath.copy_attribs(inc, target)
 

trunk/rdiff-backup/src/robust.py

@@ -142,13 +142,14 @@ class Robust:
 				tfl[0].rename(rpout)
 		return RobustAction(init, final, lambda e: tfl[0] and tfl[0].delete())
 
-	def copy_with_attribs_action(rorpin, rpout):
+	def copy_with_attribs_action(rorpin, rpout, compress = None):
 		"""Like copy_action but also copy attributes"""
 		tfl = [None] # Need mutable object that init and final can access
 		def init(): 
 			if not (rorpin.isdir() and rpout.isdir()): # already a dir
 				tfl[0] = TempFileManager.new(rpout)
-				if rorpin.isreg(): tfl[0].write_from_fileobj(rorpin.open("rb"))
+				if rorpin.isreg():
+					tfl[0].write_from_fileobj(rorpin.open("rb"), compress)
 				else: RPath.copy(rorpin, tfl[0])
 				if tfl[0].lstat(): # Some files, like sockets, won't be created
 					RPathStatic.copy_attribs(rorpin, tfl[0])

trunk/rdiff-backup/src/rpath.py

 execfile("connection.py")
-import os, stat, re, sys, shutil
+import os, stat, re, sys, shutil, gzip
 
 #######################################################################
 #
@@ -73,7 +73,7 @@ class RPathStatic:
 		try:
 			if rpout.conn is rpin.conn:
 				rpout.conn.shutil.copyfile(rpin.path, rpout.path)
-				rpout.data = {'type': rpin.data['type']}
+				rpout.setdata()
 				return
 		except AttributeError: pass
 		rpout.write_from_fileobj(rpin.open("rb"))
@@ -648,44 +648,75 @@ class RPath(RORPath):
 		"""Return similar RPath but with new index"""
 		return self.__class__(self.conn, self.base, index)
 
-	def open(self, mode):
-		"""Return open file.  Supports modes "w" and "r"."""
-		return self.conn.open(self.path, mode)
+	def open(self, mode, compress = None):
+		"""Return open file.  Supports modes "w" and "r".
+
+		If compress is true, data written/read will be gzip
+		compressed/decompressed on the fly.
+
+		"""
+		if compress: return self.conn.gzip.GzipFile(self.path, mode)
+		else: return self.conn.open(self.path, mode)
+
+	def write_from_fileobj(self, fp, compress = None):
+		"""Reads fp and writes to self.path.  Closes both when done
+
+		If compress is true, fp will be gzip compressed before being
+		written to self.
 
-	def write_from_fileobj(self, fp):
-		"""Reads fp and writes to self.path.  Closes both when done"""
+		"""
 		Log("Writing file object to " + self.path, 7)
 		assert not self.lstat(), "File %s already exists" % self.path
-		outfp = self.open("wb")
+		outfp = self.open("wb", compress = compress)
 		RPath.copyfileobj(fp, outfp)
 		if fp.close() or outfp.close():
 			raise RPathException("Error closing file")
 		self.setdata()
 
 	def isincfile(self):
-		"""Return true if path looks like an increment file"""
-		dotsplit = self.path.split(".")
-		if len(dotsplit) < 3: return None
-		timestring, ext = dotsplit[-2:]
+		"""Return true if path looks like an increment file
+
+		Also sets various inc information used by the *inc* functions.
+
+		"""
+		if self.index: dotsplit = self.index[-1].split(".")
+		else: dotsplit = self.base.split(".")
+		if dotsplit[-1] == "gz":
+			compressed = 1
+			if len(dotsplit) < 4: return None
+			timestring, ext = dotsplit[-3:-1]
+		else:
+			compressed = None
+			if len(dotsplit) < 3: return None
+			timestring, ext = dotsplit[-2:]
 		if Time.stringtotime(timestring) is None: return None
-		return (ext == "snapshot" or ext == "dir" or
-				ext == "missing" or ext == "diff")
+		if not (ext == "snapshot" or ext == "dir" or
+				ext == "missing" or ext == "diff"): return None
+		self.inc_timestr = timestring
+		self.inc_compressed = compressed
+		self.inc_type = ext
+		if compressed: self.inc_basestr = ".".join(dotsplit[:-3])
+		else: self.inc_basestr = ".".join(dotsplit[:-2])
+		return 1
+
+	def isinccompressed(self):
+		"""Return true if inc file is compressed"""
+		return self.inc_compressed
 
 	def getinctype(self):
 		"""Return type of an increment file"""
-		return self.path.split(".")[-1]
+		return self.inc_type
 
 	def getinctime(self):
 		"""Return timestring of an increment file"""
-		return self.path.split(".")[-2]
+		return self.inc_timestr
 	
 	def getincbase(self):
 		"""Return the base filename of an increment file in rp form"""
 		if self.index:
 			return self.__class__(self.conn, self.base, self.index[:-1] +
-							 ((".".join(self.index[-1].split(".")[:-2])),))
-		else: return self.__class__(self.conn,
-									".".join(self.base.split(".")[:-2]), ())
+								  (self.inc_basestr,))
+		else: return self.__class__(self.conn, self.inc_basestr)
 
 	def getincbase_str(self):
 		"""Return the base filename string of an increment file"""