Added Daniel Hazelbaker's resource fork code, plus detection to fs_abilities...

Added Daniel Hazelbaker's resource fork code, plus detection to fs_abilities and a new unittest file.


git-svn-id: http://svn.savannah.nongnu.org/svn/rdiff-backup@345 2b77aa54-bcbc-44c9-a7ec-4f6cf2b41109

trunk/rdiff-backup/CHANGELOG

-New in v0.13.0 (2003/07/02)
+New in v0.13.0 (2003/07/22)
 ---------------------------
 
 To prevent the buildup of confusing and error-prone options, the
@@ -17,6 +17,9 @@ Support for access control lists (ACLs) was also added.  An ACL
 capable file system and the python package pylibacl (which exports the
 posix1e module) are required.
 
+Thanks to patches by Daniel Hazelbaker, rdiff-backup now reads and
+writes Mac OS X style resource forks!
+
 Added --list-increment-sizes switch, which tells you how much space
 the various backup files take up.  (Suggested by Andrew Bressen)
 

trunk/rdiff-backup/rdiff_backup/Globals.py

@@ -71,16 +71,24 @@ read_eas = None
 
 # If true, preserve the extended attributes on the mirror directory
 # when backing up, or write them to the restore directory.  This
-# implies read_eas.
+# requires read_eas.
 write_eas = None
 
 # If true, save access control lists when backup up.
 read_acls = None
 
 # If true, write access control list information to the destination
-# when backing up or restoring.  Implies read_acls.
+# when backing up or restoring.  Requires read_acls.
 write_acls = None
 
+# If true, look for and save resource fork information when backing
+# up.
+read_resource_forks = None
+
+# If true, write resource fork information to destination when backing
+# up or restoring.  Requires read_resource_forks.
+write_resource_forks = None
+
 # This will be set as soon as the LocalConnection class loads
 local_connection = None
 

trunk/rdiff-backup/rdiff_backup/Main.py

@@ -323,29 +323,28 @@ def backup_get_mirrortime():
 
 def backup_set_fs_globals(rpin, rpout):
 	"""Use fs_abilities to set the globals that depend on filesystem"""
+	def update_bool_global(attr, bool):
+		"""If bool is not None, update Globals.attr accordingly"""
+		if Globals.get(attr) is not None:
+			SetConnections.UpdateGlobal(attr, bool)
+
 	src_fsa = fs_abilities.FSAbilities('source').init_readonly(rpin)
 	Log(str(src_fsa), 3)
-	if Globals.read_acls is None:
-		SetConnections.UpdateGlobal('read_acls', src_fsa.acls)
-	if src_fsa.eas: rpin.get_ea()
-	if Globals.read_eas is None:
-		SetConnections.UpdateGlobal('read_eas', src_fsa.eas)
-
+	update_bool_global('read_acls', src_fsa.acls)
+	update_bool_global('read_eas', src_fsa.eas)
+	update_bool_global('read_resource_forks', src_fsa.resource_forks)
+	
 	dest_fsa = fs_abilities.FSAbilities('destination').init_readwrite(
 		Globals.rbdir, override_chars_to_quote = Globals.chars_to_quote)
 	Log(str(dest_fsa), 3)
 	SetConnections.UpdateGlobal('preserve_hardlinks', dest_fsa.hardlinks)
 	SetConnections.UpdateGlobal('fsync_directories', dest_fsa.fsync_dirs)
-	if Globals.write_acls is None:
-		SetConnections.UpdateGlobal('write_acls',
-									Globals.read_acls and dest_fsa.acls)
-	if Globals.write_eas is None:
-		SetConnections.UpdateGlobal('write_eas',
-									Globals.read_eas and dest_fsa.eas)
 	SetConnections.UpdateGlobal('change_ownership', dest_fsa.ownership)
-	if Globals.change_dir_inc_perms is None:
-		SetConnections.UpdateGlobal('change_dir_inc_perms',
-									dest_fsa.dir_inc_perms)
+	update_bool_global('write_acls', Globals.read_acls and dest_fsa.acls)
+	update_bool_global('write_eas', Globals.read_eas and dest_fsa.eas)
+	update_bool_global('write_resource_forks',
+					   Globals.read_resource_forks and dest_fsa.resource_forks)
+	update_bool_global('change_dir_inc_perms', dest_fsa.dir_inc_perms)
 	SetConnections.UpdateGlobal('chars_to_quote', dest_fsa.chars_to_quote)
 	if Globals.chars_to_quote:
 		for conn in Globals.connections:
@@ -417,20 +416,22 @@ def restore_init_quoting(src_rp):
 
 def restore_set_fs_globals(target):
 	"""Use fs_abilities to set the globals that depend on filesystem"""
-	target_fsa = fs_abilities.FSAbilities().init_readwrite(target, 0)
-	if Globals.read_acls is None:
-		SetConnections.UpdateGlobal('read_acls', target_fsa.acls)
-	if Globals.write_acls is None:
-		SetConnections.UpdateGlobal('write_acls', target_fsa.acls)
-	if Globals.read_eas is None:
-		SetConnections.UpdateGlobal('read_eas', target_fsa.eas)
-	if Globals.write_eas is None:
-		SetConnections.UpdateGlobal('write_eas', target_fsa.eas)
-	if Globals.read_eas: target.get_ea()
+	def update_bool_global(attr, bool):
+		"""If bool is not None, update Globals.attr accordingly"""
+		if Globals.get(attr) is not None:
+			SetConnections.UpdateGlobal(attr, bool)
+
+	target_fsa = fs_abilities.FSAbilities('destination').init_readwrite(
+		target, 0)
+	update_bool_global('read_acls', target_fsa.acls)
+	update_bool_global('write_acls', target_fsa.acls)
+	update_bool_global('read_eas', target_fsa.eas)
+	update_bool_global('write_eas', target_fsa.eas)
 	SetConnections.UpdateGlobal('preserve_hardlinks', target_fsa.hardlinks)
 	SetConnections.UpdateGlobal('change_ownership', target_fsa.ownership)
 
-	mirror_fsa = fs_abilities.FSAbilities().init_readwrite(Globals.rbdir)
+	mirror_fsa = fs_abilities.FSAbilities('source').init_readwrite(
+		Globals.rbdir)
 	if Globals.chars_to_quote is None: # otherwise already overridden
 		if mirror_fsa.chars_to_quote:
 			SetConnections.UpdateGlobal('chars_to_quote',

trunk/rdiff-backup/rdiff_backup/fs_abilities.py

@@ -28,7 +28,7 @@ FSAbilities object describing it.
 """
 
 import errno
-import Globals, log, TempFile
+import Globals, log, TempFile, selection
 
 class FSAbilities:
 	"""Store capabilities of given file system"""
@@ -38,9 +38,10 @@ class FSAbilities:
 	eas = None # True if extended attributes supported
 	hardlinks = None # True if hard linking supported
 	fsync_dirs = None # True if directories can be fsync'd
-	read_only = None # True if capabilities were determined non-destructively
 	dir_inc_perms = None # True if regular files can have full permissions
+	resource_forks = None # True if regular_file/rsrc holds resource fork
 	name = None # Short string, not used for any technical purpose
+	read_only = None # True if capabilities were determined non-destructively
 
 	def __init__(self, name = None):
 		"""FSAbilities initializer.  name is only used in logging"""
@@ -48,31 +49,50 @@ class FSAbilities:
 
 	def __str__(self):
 		"""Return pretty printable version of self"""
-		s = ['-' * 60]
+		assert self.read_only == 0 or self.read_only == 1, self.read_only
+		s = ['-' * 65]
+
 		def addline(desc, val_text):
 			"""Add description line to s"""
 			s.append('  %s%s%s' % (desc, ' ' * (45-len(desc)), val_text))
 
-		if self.name:
-			s.append('Detected abilities for %s file system:' % (self.name,))
-		else: s.append('Detected abilities for file system')
-
-		ctq_str = (self.chars_to_quote is None and 'N/A'
-				   or repr(self.chars_to_quote))
-		addline('Characters needing quoting', ctq_str)
-
-		for desc, val in [('Ownership changing', self.ownership),
-						  ('Access control lists', self.acls),
+		def add_boolean_list(pair_list):
+			"""Add lines from list of (desc, boolean) pairs"""
+			for desc, boolean in pair_list:
+				if boolean: val_text = 'On'
+				elif boolean is None: val_text = 'N/A'
+				else:
+					assert boolean == 0
+					val_text = 'Off'
+				addline(desc, val_text)			
+
+		def get_title_line():
+			"""Add the first line, mostly for decoration"""
+			read_string = self.read_only and "read only" or "read/write"
+			if self.name:
+				return ('Detected abilities for %s (%s) file system:' %
+						(self.name, read_string))
+			else: return ('Detected abilities for %s file system' %
+						  (read_string,))
+
+		def add_ctq_line():
+			"""Get line describing chars to quote"""
+			ctq_str = (self.chars_to_quote is None and 'N/A'
+					   or repr(self.chars_to_quote))
+			addline('Characters needing quoting', ctq_str)
+
+		s.append(get_title_line())
+		if not self.read_only:
+			add_ctq_line()
+			add_boolean_list([('Ownership changing', self.ownership),
+							  ('Hard linking', self.hardlinks),
+							  ('fsync() directories', self.fsync_dirs),
+							  ('Directory inc permissions',
+							   self.dir_inc_perms)])
+		add_boolean_list([('Access control lists', self.acls),
 						  ('Extended attributes', self.eas),
-						  ('Hard linking', self.hardlinks),
-						  ('fsync() directories', self.fsync_dirs),
-						  ('Directory inc permissions', self.dir_inc_perms)]:
-			if val: val_text = 'On'
-			elif val is None: val_text = 'N/A'
-			else:
-				assert val == 0
-				val_text = 'Off'
-			addline(desc, val_text)
+						  ('Mac OS X style resource forks',
+						   self.resource_forks)])
 		s.append(s[0])
 		return '\n'.join(s)
 
@@ -90,6 +110,7 @@ class FSAbilities:
 		self.read_only = 1
 		self.set_eas(rp, 0)
 		self.set_acls(rp)
+		self.set_resource_fork_readonly(rp)
 		return self
 
 	def init_readwrite(self, rbdir, use_ctq_file = 1,
@@ -121,6 +142,7 @@ class FSAbilities:
 		self.set_eas(subdir, 1)
 		self.set_acls(subdir)
 		self.set_dir_inc_perms(subdir)
+		self.set_resource_fork_readwrite(subdir)
 		if override_chars_to_quote is None: self.set_chars_to_quote(subdir)
 		else: self.chars_to_quote = override_chars_to_quote
 		if use_ctq_file: self.compare_chars_to_quote(rbdir)
@@ -258,6 +280,49 @@ rdiff-backup-data/chars_to_quote.
 		else: self.dir_inc_perms = 0
 		test_rp.delete()
 
+	def set_resource_fork_readwrite(self, dir_rp):
+		"""Test for resource forks by writing to regular_file/rsrc"""
+		reg_rp = dir_rp.append('regfile')
+		reg_rp.touch()
+		rfork = reg_rp.append('rsrc')
+		assert not rfork.lstat()
+
+		s = 'test string---this should end up in resource fork'
+		try:
+			fp_write = rfork.open('wb')
+			fp_write.write(s)
+			assert not fp_write.close()
+
+			fp_read = rfork.open('rb')
+			s_back = fp_read.read()
+			assert not fp.read.close()
+		except (OSError, IOError), e: self.resource_forks = 0
+		else: self.resource_forks = (s_back == s)
+		reg_rp.delete()
+
+	def set_resource_fork_readonly(self, dir_rp):
+		"""Test for resource fork support by testing an regular file
+
+		Launches search for regular file in given directory.  If no
+		regular file is found, resource_fork support will be turned
+		off by default.
+
+		"""
+		for rp in selection.Select(dir_rp).set_iter():
+			if rp.isreg():
+				try:
+					rfork = rp.append('rsrc')
+					fp = rfork.open('rb')
+					fp.read()
+					assert not fp.close()
+				except (OSError, IOError), e:
+					self.resource_forks = 0
+					return
+				self.resource_forks = 1
+				return
+		self.resource_forks = 0
+
+
 def test_eas_local(rp, write):
 	"""Test ea support.  Must be called locally.  Usedy by set_eas above."""
 	assert Globals.local_connection is rp.conn

trunk/rdiff-backup/rdiff_backup/metadata.py

@@ -55,7 +55,7 @@ field names and values.
 """
 
 from __future__ import generators
-import re, gzip, os
+import re, gzip, os, binascii
 import log, Globals, rpath, Time, robust, increment, static
 
 class ParsingError(Exception):
@@ -74,6 +74,12 @@ def RORP2Record(rorpath):
 	if type == "reg":
 		str_list.append("  Size %s\n" % rorpath.getsize())
 
+		# If there is a resource fork, save it.
+		if rorpath.has_resource_fork():
+			if not rorpath.get_resource_fork(): rf = "None"
+			else: rf = binascii.hexlify(rorpath.get_resource_fork())
+			str_list.append("  ResourceFork %s\n" % (rf,))
+
 		# If file is hardlinked, add that information
 		if Globals.preserve_hardlinks:
 			numlinks = rorpath.getnumlinks()
@@ -81,6 +87,7 @@ def RORP2Record(rorpath):
 				str_list.append("  NumHardLinks %s\n" % numlinks)
 				str_list.append("  Inode %s\n" % rorpath.getinode())
 				str_list.append("  DeviceLoc %s\n" % rorpath.getdevloc())
+
 	elif type == "None": return "".join(str_list)
 	elif type == "dir" or type == "sock" or type == "fifo": pass
 	elif type == "sym":
@@ -122,6 +129,9 @@ def Record2RORP(record_string):
 			if data == "None": data_dict['type'] = None
 			else: data_dict['type'] = data
 		elif field == "Size": data_dict['size'] = long(data)
+		elif field == "ResourceFork":
+			if data == "None": data_dict['resourcefork'] = ""
+			else: data_dict['resourcefork'] = binascii.unhexlify(data)
 		elif field == "NumHardLinks": data_dict['nlink'] = int(data)
 		elif field == "Inode": data_dict['inode'] = long(data)
 		elif field == "DeviceLoc": data_dict['devloc'] = long(data)

trunk/rdiff-backup/rdiff_backup/rpath.py

@@ -157,6 +157,8 @@ def copy_attribs(rpin, rpout, acls = 1):
 	if Globals.change_ownership: apply(rpout.chown, rpin.getuidgid())
 	if Globals.change_permissions: rpout.chmod(rpin.getperms())
 	if Globals.write_acls and acls: rpout.write_acl(rpin.get_acl())
+	if Globals.write_resource_forks and rpin.isreg() and rpout.isreg():
+		rpout.write_resource_fork(rpin.get_resource_fork())
 	if not rpin.isdev(): rpout.setmtime(rpin.getmtime())
 
 def cmp_attribs(rp1, rp2):
@@ -271,6 +273,8 @@ class RORPath:
 			elif key == 'size' and not self.isreg(): pass
 			elif key == 'ea' and not Globals.read_eas: pass
 			elif key == 'acl' and not Globals.read_acls: pass
+			elif key == 'resourcefork' and not Globals.read_resource_forks:
+				pass
 			elif (key == 'inode' and
 				  (not self.isreg() or self.getnumlinks() == 1 or
 				   not Globals.compare_inode or
@@ -303,10 +307,10 @@ class RORPath:
 			elif key == 'size' and not self.isreg(): pass
 			elif key == 'perms' and not Globals.change_permissions: pass
 			elif key == 'inode': pass
-			elif (key == 'ea' and
-				  not (Globals.read_eas and Globals.write_eas)): pass
-			elif (key == 'acl' and
-				  not (Globals.read_acls and Globals.write_acls)): pass
+			elif key == 'ea' and not Globals.write_eas: pass
+			elif key == 'acl' and not Globals.write_acls: pass
+			elif key == 'resourcefork' and not Globals.write_resource_forks:
+				pass
 			elif (not other.data.has_key(key) or
 				  self.data[key] != other.data[key]): return 0
 		return 1
@@ -547,6 +551,18 @@ class RORPath:
 		"""Return extended attributes object"""
 		return self.data['ea']
 
+	def has_resource_fork(self):
+		"""True if rpath has a resourcefork parameter"""
+		return self.data.has_key('resourcefork')
+
+	def get_resource_fork(self):
+		"""Return the resource fork in binary data"""
+		return self.data['resourcefork']
+
+	def set_resource_fork(self, rfork):
+		"""Record resource fork in dictionary.  Does not write"""
+		self.data['resourcefork'] = rfork
+
 
 class RPath(RORPath):
 	"""Remote Path class - wrapper around a possibly non-local pathname
@@ -608,6 +624,8 @@ class RPath(RORPath):
 		self.data = self.conn.C.make_file_dict(self.path)
 		if Globals.read_eas and self.lstat(): self.get_ea()
 		if Globals.read_acls and self.lstat(): self.get_acl()
+		if Globals.read_resource_forks and self.isreg():
+			self.get_resource_fork()
 
 	def make_file_dict_old(self):
 		"""Create the data dictionary"""
@@ -998,6 +1016,21 @@ class RPath(RORPath):
 		ea.write_to_rp(self)
 		self.data['ea'] = ea
 
+	def get_resource_fork(self):
+		"""Return resource fork data, setting if necessary"""
+		assert self.isreg()
+		try: rfork = self.data['resourcefork']
+		except KeyError:
+			rfork = self.append('rsrc').get_data()
+			self.data['resourcefork'] = rfork
+		return rfork
+
+	def write_resource_fork(self, rfork_data):
+		"""Write new resource fork to self"""
+		fp = self.append('rsrc').open('wb')
+		fp.write(rfork_data)
+		assert not fp.close()
+
 
 class RPathFileHook:
 	"""Look like a file, but add closing hook"""

trunk/rdiff-backup/rdiff_backup/selection.py

@@ -144,7 +144,7 @@ class Select:
 				delayed_rp_stack.append(rpath)
 				diryield_stack.append(diryield(rpath))
 
-	def Iterate(self, rpath, rec_func, sel_func):
+	def Iterate(self, rp, rec_func, sel_func):
 		"""Return iterator yielding rpaths in rpath
 
 		rec_func is usually the same as this function and is what
@@ -155,21 +155,21 @@ class Select:
 		is usually self.Select.
 
 		"""
-		s = sel_func(rpath)
+		s = sel_func(rp)
 		if s == 0: return
 		elif s == 1: # File is included
-			yield rpath
-			if rpath.isdir():
-				for rp in self.iterate_in_dir(rpath, rec_func, sel_func):
-					yield rp
+			yield rp
+			if rp.isdir():
+				for rp2 in self.iterate_in_dir(rp, rec_func, sel_func):
+					yield rp2
 		elif s == 2:
-			if rpath.isdir(): # Directory is merely scanned
-				iid = self.iterate_in_dir(rpath, rec_func, sel_func)
+			if rp.isdir(): # Directory is merely scanned
+				iid = self.iterate_in_dir(rp, rec_func, sel_func)
 				try: first = iid.next()
 				except StopIteration: return # no files inside; skip rp
-				yield rpath
+				yield rp
 				yield first
-				for rp in iid: yield rp
+				for rp2 in iid: yield rp2
 		else: assert 0, "Invalid selection result %s" % (str(s),)
 
 	def listdir(self, dir_rp):

trunk/rdiff-backup/testing/fs_abilitiestest.py

@@ -18,6 +18,7 @@ class FSAbilitiesTest(unittest.TestCase):
 	ownership = (os.getuid() == 0)
 	hardlinks = fsync_dirs = 1
 	dir_inc_perms = 1
+	resource_forks = 0
 
 	# Describes MS-Windows style file system
 	#dir_to_test = "/mnt/fat"
@@ -25,7 +26,8 @@ class FSAbilitiesTest(unittest.TestCase):
 	#chars_to_quote = "^a-z0-9_ -"
 	#ownership = hardlinks = 0
 	#fsync_dirs = 1
-	#dir_inc_perms = XXX
+	#dir_inc_perms = 0
+	#resource_forks = 0
 	
 	def testReadOnly(self):
 		"""Test basic querying read only"""
@@ -35,6 +37,7 @@ class FSAbilitiesTest(unittest.TestCase):
 		assert fsa.read_only == 1, fsa.read_only
 		assert fsa.eas == self.eas, fsa.eas
 		assert fsa.acls == self.acls, fsa.acls
+		assert fsa.resource_forks == self.resource_forks, fsa.resource_forks
 
 	def testReadWrite(self):
 		"""Test basic querying read/write"""
@@ -55,6 +58,7 @@ class FSAbilitiesTest(unittest.TestCase):
 		assert fsa.hardlinks == self.hardlinks, fsa.hardlinks
 		assert fsa.fsync_dirs == self.fsync_dirs, fsa.fsync_dirs
 		assert fsa.dir_inc_perms == self.dir_inc_perms, fsa.dir_inc_perms
+		assert fsa.resource_forks == self.resource_forks, fsa.resource_forks
 
 		ctq_rp = new_dir.append("chars_to_quote")
 		assert ctq_rp.lstat()

trunk/rdiff-backup/testing/resourceforktest.py

+import unittest
+from commontest import *
+from rdiff_backup import rpath
+from rdiff_backup import metadata
+
+"""***NOTE***
+
+None of these tests should work unless your system supports resource
+forks.  So basically these tests should only be run on Mac OS X.
+
+"""
+
+Globals.read_resource_forks = Globals.write_resource_forks = 1
+
+class ResourceForkTest(unittest.TestCase):
+	"""Test dealing with Mac OS X style resource forks"""
+	tempdir = rpath.RPath(Globals.local_connection,
+						  'testfiles/resource_fork_test')
+	def make_temp(self):
+		"""Make temp directory testfiles/resource_fork_test"""
+		if self.tempdir.lstat(): self.tempdir.delete()
+		self.tempdir.mkdir()
+
+	def testBasic(self):
+		"""Test basic reading and writing of resource forks"""
+		self.make_temp()
+		rp = self.tempdir.append('test')
+		rp.touch()
+		assert rp.get_resource_fork() == '', rp.get_resource_fork()
+
+		s = 'new resource fork data'
+		rp.write_resource_fork(s)
+		assert rp.get_resource_fork() == s, rp.get_resource_fork()
+
+		rp2 = self.tempdir.append('test')
+		assert rp2.isreg()
+		assert rp2.get_resource_fork() == s, rp2.get_resource_fork()
+
+	def testRecord(self):
+		"""Test reading, writing, and comparing of records with rforks"""
+		self.make_temp()
+		rp = self.tempdir.append('test')
+		rp.touch()
+		rp.set_resource_fork('hello')
+
+		record = metadata.RORP2Record(rp)
+		#print record
+		rorp_out = metadata.Record2RORP(record)
+		assert rorp_out == rp, (rorp_out, rp)
+		assert rorp_out.get_resource_fork() == 'hello'
+
+
+if __name__ == "__main__": unittest.main()