Added ACL support, may have corrected a few EA bugs

git-svn-id: http://svn.savannah.nongnu.org/svn/rdiff-backup@341 2b77aa54-bcbc-44c9-a7ec-4f6cf2b41109

trunk/rdiff-backup/CHANGELOG

@@ -8,9 +8,14 @@ attributes, access control lists, hard links, ownership, and directory
 fsyncing.  Options such as --windows-mode, --chars-to-quote,
 --quoting-char, and --windows-restore-mode have been removed.
 
-Now rdiff-backup supports extended attributes.  To take advantage of
-this you will need the python module pyxattr and a file system that
-supports EAs.  Thanks to Greg Freemyer for discussion.
+Now rdiff-backup supports user extended attributes (EAs).  To take
+advantage of this you will need the python module pyxattr and a file
+system that supports EAs.  Thanks to Greg Freemyer for valuable
+discussion on EA and ACL support.
+
+Support for access control lists (ACLs) was also added.  An ACL
+capable file system and the python package pylibacl (which exports the
+posix1e module) are required.
 
 Added --list-increment-sizes switch, which tells you how much space
 the various backup files take up.  (Suggested by Andrew Bressen)

trunk/rdiff-backup/rdiff_backup/Globals.py

@@ -74,6 +74,13 @@ read_eas = None
 # implies read_eas.
 write_eas = None
 
+# If true, save access control lists when backup up.
+read_acls = None
+
+# If true, write access control list information to the destination
+# when backing up or restoring.  Implies read_acls.
+write_acls = None
+
 # This will be set as soon as the LocalConnection class loads
 local_connection = None
 

trunk/rdiff-backup/rdiff_backup/Main.py

@@ -320,17 +320,25 @@ def backup_get_mirrortime():
 
 def backup_set_fs_globals(rpin, rpout):
 	"""Use fs_abilities to set the globals that depend on filesystem"""
-	src_fsa = fs_abilities.FSAbilities().init_readonly(rpin)
-	SetConnections.UpdateGlobal('read_acls', src_fsa.acls)
+	src_fsa = fs_abilities.FSAbilities('source').init_readonly(rpin)
+	Log(str(src_fsa), 3)
+	if Globals.read_acls is None:
+		SetConnections.UpdateGlobal('read_acls', src_fsa.acls)
 	if src_fsa.eas: rpin.get_ea()
-	SetConnections.UpdateGlobal('read_eas', src_fsa.eas)
+	if Globals.read_eas is None:
+		SetConnections.UpdateGlobal('read_eas', src_fsa.eas)
 
-	dest_fsa = fs_abilities.FSAbilities().init_readwrite(
+	dest_fsa = fs_abilities.FSAbilities('destination').init_readwrite(
 		Globals.rbdir, override_chars_to_quote = Globals.chars_to_quote)
+	Log(str(dest_fsa), 3)
 	SetConnections.UpdateGlobal('preserve_hardlinks', dest_fsa.hardlinks)
 	SetConnections.UpdateGlobal('fsync_directories', dest_fsa.fsync_dirs)
-	SetConnections.UpdateGlobal('write_acls', dest_fsa.acls)
-	SetConnections.UpdateGlobal('write_eas', Globals.read_eas and dest_fsa.eas)
+	if Globals.write_acls is None:
+		SetConnections.UpdateGlobal('write_acls',
+									Globals.read_acls and dest_fsa.acls)
+	if Globals.write_eas is None:
+		SetConnections.UpdateGlobal('write_eas',
+									Globals.read_eas and dest_fsa.eas)
 	SetConnections.UpdateGlobal('change_ownership', dest_fsa.ownership)
 	SetConnections.UpdateGlobal('chars_to_quote', dest_fsa.chars_to_quote)
 	if Globals.chars_to_quote:
@@ -404,15 +412,19 @@ def restore_init_quoting(src_rp):
 def restore_set_fs_globals(target):
 	"""Use fs_abilities to set the globals that depend on filesystem"""
 	target_fsa = fs_abilities.FSAbilities().init_readwrite(target, 0)
-	SetConnections.UpdateGlobal('read_acls', target_fsa.acls)
-	SetConnections.UpdateGlobal('write_acls', target_fsa.acls)
-	SetConnections.UpdateGlobal('read_eas', target_fsa.eas)
-	SetConnections.UpdateGlobal('write_eas', target_fsa.eas)
+	if Globals.read_acls is None:
+		SetConnections.UpdateGlobal('read_acls', target_fsa.acls)
+	if Globals.write_acls is None:
+		SetConnections.UpdateGlobal('write_acls', target_fsa.acls)
+	if Globals.read_eas is None:
+		SetConnections.UpdateGlobal('read_eas', target_fsa.eas)
+	if Globals.write_eas is None:
+		SetConnections.UpdateGlobal('write_eas', target_fsa.eas)
 	if Globals.read_eas: target.get_ea()
 	SetConnections.UpdateGlobal('preserve_hardlinks', target_fsa.hardlinks)
 	SetConnections.UpdateGlobal('change_ownership', target_fsa.ownership)
 
-	mirror_fsa = fs_abilities.FSAbilities().init_readonly(Globals.rbdir)
+	mirror_fsa = fs_abilities.FSAbilities().init_readwrite(Globals.rbdir)
 	if Globals.chars_to_quote is None: # otherwise already overridden
 		if mirror_fsa.chars_to_quote:
 			SetConnections.UpdateGlobal('chars_to_quote',

trunk/rdiff-backup/rdiff_backup/backup.py

@@ -123,14 +123,6 @@ class DestinationStruct:
 		destination except rdiff-backup-data directory.
 
 		"""
-		def get_basic_iter():
-			"""Returns iterator of basic metadata"""
-			metadata_iter = metadata.MetadataFile.get_objects_at_time(
-				Globals.rbdir, Time.prevtime)
-			if metadata_iter: return metadata_iter
-			log.Log("Warning: Metadata file not found.\n"
-					"Metadata will be read from filesystem.", 2)
-
 		def get_iter_from_fs():
 			"""Get the combined iterator from the filesystem"""
 			sel = selection.Select(rpath)
@@ -138,10 +130,9 @@ class DestinationStruct:
 			return sel.set_iter()
 
 		if use_metadata:
-			if Globals.read_eas:
-				rorp_iter = eas_acls.ExtendedAttributesFile.\
-					  get_combined_iter_at_time(Globals.rbdir, Time.prevtime)
-			else: rorp_iter = get_basic_iter()
+			rorp_iter = eas_acls.GetCombinedMetadataIter(
+				Globals.rbdir, Time.prevtime,
+				acls = Globals.read_acls, eas = Globals.read_eas)
 			if rorp_iter: return rorp_iter
 		return get_iter_from_fs()
 
@@ -257,6 +248,7 @@ class CacheCollatedPostProcess:
 		if Globals.file_statistics: statistics.FileStats.init()
 		metadata.MetadataFile.open_file()
 		if Globals.read_eas: eas_acls.ExtendedAttributesFile.open_file()
+		if Globals.read_acls: eas_acls.AccessControlListFile.open_file()
 
 		# the following should map indicies to lists
 		# [source_rorp, dest_rorp, changed_flag, success_flag, increment]
@@ -334,6 +326,9 @@ class CacheCollatedPostProcess:
 			if Globals.read_eas and not metadata_rorp.get_ea().empty():
 				eas_acls.ExtendedAttributesFile.write_object(
 					metadata_rorp.get_ea())
+			if Globals.read_acls and not metadata_rorp.get_acl().is_basic():
+				eas_acls.AccessControlListFile.write_object(
+					metadata_rorp.get_acl())
 		if Globals.file_statistics:
 			statistics.FileStats.update(source_rorp, dest_rorp, changed, inc)
 
@@ -377,6 +372,7 @@ class CacheCollatedPostProcess:
 		while self.cache_indicies: self.shorten_cache()
 		metadata.MetadataFile.close_file()
 		if Globals.read_eas: eas_acls.ExtendedAttributesFile.close_file()
+		if Globals.read_acls: eas_acls.AccessControlListFile.close_file()
 		if Globals.print_statistics: statistics.print_active_stats()
 		if Globals.file_statistics: statistics.FileStats.close()
 		statistics.write_active_statfileobj()

trunk/rdiff-backup/rdiff_backup/connection.py

@@ -518,7 +518,8 @@ class VirtualFile:
 import Globals, Time, Rdiff, Hardlink, FilenameMapping, C, Security, \
 	   Main, rorpiter, selection, increment, statistics, manage, lazy, \
 	   iterfile, rpath, robust, restore, manage, backup, connection, \
-	   TempFile, SetConnections, librsync, log, regress, fs_abilities
+	   TempFile, SetConnections, librsync, log, regress, fs_abilities, \
+	   eas_acls
 
 Globals.local_connection = LocalConnection()
 Globals.connections.append(Globals.local_connection)

trunk/rdiff-backup/rdiff_backup/eas_acls.py

@@ -28,6 +28,8 @@ access_control_lists.<time>.snapshot.
 
 from __future__ import generators
 import base64, errno, re
+try: import posix1e
+except ImportError: pass
 import static, Globals, metadata, connection, rorpiter, log
 
 
@@ -54,6 +56,9 @@ class ExtendedAttributes:
 			if exc[0] == errno.EOPNOTSUPP: return # if not sup, consider empty
 			raise
 		for attr in attr_list:
+			if not attr.startswith('user.'):
+				# Only preserve user extended attributes
+				continue
 			try: self.attr_dict[attr] = rp.conn.xattr.getxattr(rp.path, attr)
 			except IOError, exc:
 				# File probably modified while reading, just continue
@@ -160,68 +165,231 @@ class ExtendedAttributesFile(metadata.FlatFile):
 	_extractor = EAExtractor
 	_object_to_record = staticmethod(EA2Record)
 
-	def get_combined_iter_at_time(cls, rbdir, rest_time,
-								  restrict_index = None):
-		"""Return an iter of rorps with extended attributes added"""
-		def join_eas(basic_iter, ea_iter):
-			"""Join basic_iter with ea iter"""
-			collated = rorpiter.CollateIterators(basic_iter, ea_iter)
+	def join(cls, rorp_iter, rbdir, time, restrict_index):
+		"""Add extended attribute information to existing rorp_iter"""
+		def helper(rorp_iter, ea_iter):
+			"""Add EA information in ea_iter to rorp_iter"""
+			collated = rorpiter.CollateIterators(rorp_iter, ea_iter)
 			for rorp, ea in collated:
 				assert rorp, (rorp, (ea.index, ea.attr_dict), rest_time)
 				if not ea: ea = ExtendedAttributes(rorp.index)
 				rorp.set_ea(ea)
 				yield rorp
-
-		basic_iter = metadata.MetadataFile.get_objects_at_time(
-			Globals.rbdir, rest_time, restrict_index)
-		if not basic_iter: return None
-		ea_iter = cls.get_objects_at_time(rbdir, rest_time, restrict_index)
-		if not ea_iter:
-			log.Log("Warning: Extended attributes file not found", 2)
-			ea_iter = iter([])
-		return join_eas(basic_iter, ea_iter)
+			
+		ea_iter = cls.get_objects_at_time(rbdir, time, restrict_index)
+		if ea_iter: return helper(rorp_iter, ea_iter)
+		else:
+			log.Log("Warning: Extended attributes file not found",2)
+			return rorp_iter
 
 static.MakeClass(ExtendedAttributesFile)
 
 
 class AccessControlList:
-	"""Hold a file's access control list information"""
-	def __init__(self, index, text_acl = None):
-		"""Initialize object with index and possibly text_acl"""
+	"""Hold a file's access control list information
+
+	Since ACL objects cannot be picked, store everything as text, in
+	self.acl_text and self.def_acl_text.
+
+	"""
+	def __init__(self, index, acl_text = None, def_acl_text = None):
+		"""Initialize object with index and possibly acl_text"""
 		self.index = index
-		# self.ACL is a posix1e ACL object
-		if text_acl is None: self.ACL = None
-		else: self.ACL = posix1e.ACL(text_acl)
+		if acl_text: # Check validity of ACL, reorder if necessary
+			ACL = posix1e.ACL(text=acl_text)
+			assert ACL.valid(), "Bad ACL: "+acl_text
+			self.acl_text = str(ACL)
+		else: self.acl_text = None
+
+		if def_acl_text:
+			def_ACL = posix1e.ACL(text=def_acl_text)
+			assert def_ACL.valid(), "Bad default ACL: "+def_acl_text
+			self.def_acl_text = str(def_ACL)
+		else: self.def_acl_text = None
+
+	def __str__(self):
+		"""Return human-readable string"""
+		return ("acl_text: %s\ndef_acl_text: %s" %
+				(self.acl_text, self.def_acl_text))
 
 	def __eq__(self, acl):
-		"""Compare self and other access control list"""
-		return self.index == acl.index and str(self.ACL) == str(acl.ACL)
+		"""Compare self and other access control list
+
+		Basic acl permissions are considered equal to an empty acl
+		object.
+
+		"""
+		assert isinstance(acl, self.__class__)
+		if self.index != acl.index: return 0
+		if self.is_basic(): return acl.is_basic()
+		if acl.is_basic(): return self.is_basic()
+		if self.acl_text != acl.acl_text: return 0
+		if not self.def_acl_text and not acl.def_acl_text: return 1
+		return self.def_acl_text == acl.def_acl_text
 	def __ne__(self, acl): return not self.__eq__(acl)
 	
+	def eq_verbose(self, acl):
+		"""Returns same as __eq__ but print explanation if not equal"""
+		if self.index != acl.index:
+			print "index %s not equal to index %s" % (self.index, acl.index)
+			return 0
+		if self.acl_text != acl.acl_text:
+			print "ACL texts not equal:"
+			print self.acl_text
+			print acl.acl_text
+			return 0
+		if (self.def_acl_text != acl.def_acl_text and
+			(self.def_acl_text or acl.def_acl_text)):
+			print "Unequal default acl texts:"
+			print self.def_acl_text
+			print acl.def_acl_text
+			return 0
+		return 1
+
 	def get_indexpath(self): return self.index and '/'.join(self.index) or '.'
 
+	def is_basic(self):
+		"""True if acl can be reduced to standard unix permissions
+
+		Assume that if they are only three entries, they correspond to
+		user, group, and other, and thus don't use any special ACL
+		features.
 
-def get_acl_from_rp(rp):
-	"""Return text acl from an rpath, or None if not supported"""
-	try: acl = rp.conn.posix1e.ACL(file=rp.path)
+		"""
+		if not self.acl_text and not self.def_acl_text: return 1
+		lines = self.acl_text.strip().split('\n')
+		assert len(lines) >= 3, lines
+		return len(lines) == 3 and not self.def_acl_text
+
+	def read_from_rp(self, rp):
+		"""Set self.ACL from an rpath, or None if not supported"""
+		self.acl_text, self.def_acl_text = \
+					   rp.conn.eas_acls.get_acl_text_from_rp(rp)
+
+	def write_to_rp(self, rp):
+		"""Write current access control list to RPath rp"""
+		rp.conn.eas_acls.set_rp_acl(rp, self.acl_text, self.def_acl_text)
+
+def set_rp_acl(rp, acl_text = None, def_acl_text = None):
+	"""Set given rp with ACL that acl_text defines.  rp should be local"""
+	assert rp.conn is Globals.local_connection
+	if acl_text:
+		acl = posix1e.ACL(text=acl_text)
+		assert acl.valid()
+	else: acl = posix1e.ACL()
+	acl.applyto(rp.path)
+	if rp.isdir():
+		if def_acl_text:
+			def_acl = posix1e.ACL(text=def_acl_text)
+			assert def_acl.valid()
+		else: def_acl = posix1e.ACL()
+		def_acl.applyto(rp.path, posix1e.ACL_TYPE_DEFAULT)
+
+def get_acl_text_from_rp(rp):
+	"""Returns (acl_text, def_acl_text) from an rpath.  Call locally"""
+	assert rp.conn is Globals.local_connection
+	try: acl_text = str(posix1e.ACL(file=rp.path))
 	except IOError, exc:
-		if exc[0] == errno.EOPNOTSUPP: return None
-		raise
-	return str(acl)
+		if exc[0] == errno.EOPNOTSUPP: acl_text = None
+		else: raise
+	if rp.isdir():
+		try: def_acl_text = str(posix1e.ACL(filedef=rp.path))
+		except IOError, exc:
+			if exc[0] == errno.EOPNOTSUPP: def_acl_text = None
+			else: raise
+	else: def_acl_text = None
+	return (acl_text, def_acl_text)
 
 def acl_compare_rps(rp1, rp2):
-	"""Return true if rp1 and rp2 have same acls"""
-	return get_acl_from_rp(rp1) == get_acl_from_rp(rp2)
+	"""Return true if rp1 and rp2 have same acl information"""
+	acl1 = AccessControlList(rp1.index)
+	acl1.read_from_rp(rp1)
+	acl2 = AccessControlList(rp2.index)
+	acl2.read_from_rp(rp2)
+	return acl1 == acl2
 
 
 def ACL2Record(acl):
 	"""Convert an AccessControlList object into a text record"""
-	return "# file: %s\n%s" % (acl.get_indexpath(), str(acl.ACL))
+	start = "# file: %s\n%s" % (acl.get_indexpath(), acl.acl_text)
+	if not acl.def_acl_text: return start
+	default_lines = acl.def_acl_text.strip().split('\n')
+	default_text = '\ndefault:'.join(default_lines)
+	return "%sdefault:%s\n" % (start, default_text)
 
-def Record2EA(acl):
+def Record2ACL(record):
 	"""Convert text record to an AccessControlList object"""
-	XXXX
+	lines = record.split('\n')
+	first_line = lines.pop(0)
+	if not first_line.startswith('# file: '):
+		raise metadata.ParsingError("Bad record beginning: "+ first_line)
+	filename = first_line[8:]
+	if filename == '.': index = ()
+	else: index = tuple(filename.split('/'))
+
+	normal_entries = []; default_entries = []
+	for line in lines:
+		if line.startswith('default:'): default_entries.append(line[8:])
+		else: normal_entries.append(line)
+	return AccessControlList(index, acl_text='\n'.join(normal_entries),
+							 def_acl_text='\n'.join(default_entries))
+	
+
+class ACLExtractor(EAExtractor):
+	"""Iterate AccessControlList objects from the ACL information file
+
+	Except for the record_to_object method, we can reuse everything in
+	the EAExtractor class because the file formats are so similar.
+
+	"""
+	record_to_object = staticmethod(Record2ACL)
+
+class AccessControlListFile(metadata.FlatFile):
+	"""Store/retrieve ACLs from extended attributes file"""
+	_prefix = 'access_control_lists'
+	_extractor = ACLExtractor
+	_object_to_record = staticmethod(ACL2Record)
+
+	def join(cls, rorp_iter, rbdir, time, restrict_index):
+		"""Add access control list information to existing rorp_iter"""
+		def helper(rorp_iter, acl_iter):
+			"""Add ACL information in acl_iter to rorp_iter"""
+			collated = rorpiter.CollateIterators(rorp_iter, acl_iter)
+			for rorp, acl in collated:
+				assert rorp, "Missing rorp for index %s" % (acl.index,)
+				if not acl: acl = AccessControlList(rorp.index)
+				rorp.set_acl(acl)
+				yield rorp
+
+		acl_iter = cls.get_objects_at_time(rbdir, time, restrict_index)
+		if acl_iter: return helper(rorp_iter, acl_iter)
+		else:
+			log.Log("Warning: Access Control List file not found", 2)
+			return rorp_iter
 
+static.MakeClass(AccessControlListFile)
 
 
+def GetCombinedMetadataIter(rbdir, time, restrict_index = None,
+							acls = None, eas = None):
+	"""Return iterator of rorps from metadata and related files
+
+	None will be returned if the metadata file is absent.  If acls or
+	eas is true, access control list or extended attribute information
+	will be added.
+
+	"""
+	metadata_iter = metadata.MetadataFile.get_objects_at_time(
+		rbdir, time, restrict_index)
+	if not metadata_iter:
+		log.Log("Warning, metadata file not found.\n"
+				"Metadata will be read from filesystem.", 2)
+		return None
+	if eas:
+		metadata_iter = ExtendedAttributesFile.join(
+			metadata_iter, rbdir, time, restrict_index)
+	if acls:
+		metadata_iter = AccessControlListFile.join(
+			metadata_iter, rbdir, time, restrict_index)
+	return metadata_iter
 

trunk/rdiff-backup/rdiff_backup/fs_abilities.py

@@ -39,6 +39,40 @@ class FSAbilities:
 	hardlinks = None # True if hard linking supported
 	fsync_dirs = None # True if directories can be fsync'd
 	read_only = None # True if capabilities were determined non-destructively
+	name = None # 
+
+	def __init__(self, name = None):
+		"""FSAbilities initializer.  name is only used in logging"""
+		self.name = name
+
+	def __str__(self):
+		"""Return pretty printable version of self"""
+		s = ['-' * 60]
+		def addline(desc, val_text):
+			"""Add description line to s"""
+			s.append('  %s%s%s' % (desc, ' ' * (45-len(desc)), val_text))
+
+		if self.name:
+			s.append('Detected abilities for %s file system:' % (self.name,))
+		else: s.append('Detected abilities for file system')
+
+		ctq_str = (self.chars_to_quote is None and 'N/A'
+				   or repr(self.chars_to_quote))
+		addline('Characters needing quoting', ctq_str)
+
+		for desc, val in [('Ownership changing', self.ownership),
+						  ('Access Control Lists', self.acls),
+						  ('Extended Attributes', self.eas),
+						  ('Hard linking', self.hardlinks),
+						  ('fsync() directories', self.fsync_dirs)]:
+			if val: val_text = 'On'
+			elif val is None: val_text = 'N/A'
+			else:
+				assert val == 0
+				val_text = 'Off'
+			addline(desc, val_text)
+		s.append(s[0])
+		return '\n'.join(s)
 
 	def init_readonly(self, rp):
 		"""Set variables using fs tested at RPath rp
@@ -128,7 +162,7 @@ rdiff-backup-data/chars_to_quote.
 		except (IOError, OSError), exc:
 			if exc[0] == errno.EPERM:
 				log.Log("Warning: ownership cannot be changed on filesystem "
-						"at %s" % (self.root_rp.path,), 2)
+						"at %s" % (self.root_rp.path,), 3)
 				self.ownership = 0
 			else: raise
 		else: self.ownership = 1
@@ -145,7 +179,7 @@ rdiff-backup-data/chars_to_quote.
 		except (IOError, OSError), exc:
 			if exc[0] in (errno.EOPNOTSUPP, errno.EPERM):
 				log.Log("Warning: hard linking not supported by filesystem "
-						"at %s" % (self.root_rp.path,), 2)
+						"at %s" % (self.root_rp.path,), 3)
 				self.hardlinks = 0
 			else: raise
 		else: self.hardlinks = 1
@@ -213,8 +247,8 @@ def test_eas_local(rp, write):
 	assert rp.lstat()
 	try: import xattr
 	except ImportError:
-		log.Log("Warning: Unable to import module xattr.  ACLs not "
-				"supported on filesystem at %s" % (rp.path,), 2)
+		log.Log("Unable to import module xattr.  EAs not "
+				"supported on filesystem at %s" % (rp.path,), 4)
 		return 0
 
 	try:
@@ -224,8 +258,8 @@ def test_eas_local(rp, write):
 			assert xattr.getxattr(rp.path, "user.test") == "test val"
 	except IOError, exc:
 		if exc[0] == errno.EOPNOTSUPP:
-			log.Log("Warning: Extended attributes not supported by "
-					"filesystem at %s" % (rp.path,), 2)
+			log.Log("Extended attributes not supported by "
+					"filesystem at %s" % (rp.path,), 4)
 			return 0
 		else: raise
 	else: return 1
@@ -236,16 +270,16 @@ def test_acls_local(rp):
 	assert rp.lstat()
 	try: import posix1e
 	except ImportError:
-		log.Log("Warning: Unable to import module posix1e from pylibacl "
+		log.Log("Unable to import module posix1e from pylibacl "
 				"package.\nACLs not supported on filesystem at %s" %
-				(rp.path,), 2)
+				(rp.path,), 4)
 		return 0
 
 	try: posix1e.ACL(file=rp.path)
 	except IOError, exc:
 		if exc[0] == errno.EOPNOTSUPP:
-			log.Log("Warning: ACLs appear not to be supported by "
-					"filesystem at %s" % (rp.path,), 2)
+			log.Log("ACLs appear not to be supported by "
+					"filesystem at %s" % (rp.path,), 4)
 			return 0
 		else: raise
 	else: return 1
@@ -255,8 +289,8 @@ def test_fsync_local(rp):
 	assert rp.conn is Globals.local_connection
 	try: rp.fsync()
 	except (IOError, OSError), exc:
-		log.Log("Warning: Directories on file system at %s are not "
-				"fsyncable.\nAssuming it's unnecessary." % (rp.path,), 2)
+		log.Log("Directories on file system at %s are not "
+				"fsyncable.\nAssuming it's unnecessary." % (rp.path,), 4)
 		return 0
 	else: return 1
 

trunk/rdiff-backup/rdiff_backup/increment.py

@@ -87,7 +87,9 @@ def makedir(mirrordir, incpref):
 	"""Make file indicating directory mirrordir has changed"""
 	dirsign = get_inc(incpref, "dir")
 	dirsign.touch()
-	rpath.copy_attribs(mirrordir, dirsign)
+	# Below, don't copy acls because directories can have more of them
+	# than ordinary files (they have default acls also).
+	rpath.copy_attribs(mirrordir, dirsign, acls = 0)
 	return dirsign
 
 def get_inc(rp, typestr, time = None):

trunk/rdiff-backup/rdiff_backup/restore.py

@@ -154,16 +154,13 @@ class MirrorStruct:
 
 		"""
 		if rest_time is None: rest_time = _rest_time
-		if Globals.write_eas:
-			metadata_iter = eas_acls.ExtendedAttributesFile.\
-			 get_combined_iter_at_time(
-			 Globals.rbdir, rest_time, restrict_index = cls.mirror_base.index)
-		else:
-			metadata_iter = metadata.MetadataFile.get_objects_at_time(
-			 Globals.rbdir, rest_time, restrict_index = cls.mirror_base.index)
-		if metadata_iter: rorp_iter = metadata_iter
-		elif require_metadata: log.Log.FatalError("Mirror metadata not found")
-		else:
+
+		rorp_iter = eas_acls.GetCombinedMetadataIter(
+			Globals.rbdir, rest_time, restrict_index = cls.mirror_base.index,
+			acls = Globals.write_acls, eas = Globals.write_eas)
+		if not rorp_iter:
+			if require_metadata:
+				log.Log.FatalError("Mirror metadata not found")
 			log.Log("Warning: Mirror metadata not found, "
 					"reading from directory", 2)
 			rorp_iter = cls.get_rorp_iter_from_rf(cls.root_rf)

trunk/rdiff-backup/rdiff_backup/rpath.py

@@ -143,7 +143,7 @@ def cmp(rpin, rpout):
 	elif rpin.issock(): return rpout.issock()
 	else: raise RPathException("File %s has unknown type" % rpin.path)
 
-def copy_attribs(rpin, rpout):
+def copy_attribs(rpin, rpout, acls = 1):
 	"""Change file attributes of rpout to match rpin
 
 	Only changes the chmoddable bits, uid/gid ownership, and
@@ -153,10 +153,11 @@ def copy_attribs(rpin, rpout):
 	log.Log("Copying attributes from %s to %s" % (rpin.index, rpout.path), 7)
 	check_for_files(rpin, rpout)
 	if rpin.issym(): return # symlinks have no valid attributes
+	if Globals.write_eas: rpout.write_ea(rpin.get_ea())
 	if Globals.change_ownership: apply(rpout.chown, rpin.getuidgid())
 	if Globals.change_permissions: rpout.chmod(rpin.getperms())
+	if Globals.write_acls and acls: rpout.write_acl(rpin.get_acl())
 	if not rpin.isdev(): rpout.setmtime(rpin.getmtime())
-	if Globals.write_eas: rpout.write_ea(rpin.get_ea())
 
 def cmp_attribs(rp1, rp2):
 	"""True if rp1 has the same file attributes as rp2
@@ -268,6 +269,8 @@ class RORPath:
 			elif key == 'ctime': pass
 			elif key == 'devloc' or key == 'nlink': pass
 			elif key == 'size' and not self.isreg(): pass
+			elif key == 'ea' and not Globals.read_eas: pass
+			elif key == 'acl' and not Globals.read_acls: pass
 			elif (key == 'inode' and
 				  (not self.isreg() or self.getnumlinks() == 1 or
 				   not Globals.compare_inode or
@@ -300,13 +303,17 @@ class RORPath:
 			elif key == 'size' and not self.isreg(): pass
 			elif key == 'perms' and not Globals.change_permissions: pass
 			elif key == 'inode': pass
+			elif (key == 'ea' and
+				  not (Globals.read_eas and Globals.write_eas)): pass
+			elif (key == 'acl' and
+				  not (Globals.read_acls and Globals.write_acls)): pass
 			elif (not other.data.has_key(key) or
 				  self.data[key] != other.data[key]): return 0
 		return 1
 
 	def equal_verbose(self, other, check_index = 1,
 					  compare_inodes = 0, compare_ownership = 0,
-					  compare_eas = 0):
+					  compare_acls = 0, compare_eas = 0):
 		"""Like __eq__, but log more information.  Useful when testing"""
 		if check_index and self.index != other.index:
 			log.Log("Index %s != index %s" % (self.index, other.index), 2)
@@ -325,6 +332,7 @@ class RORPath:
 			elif key == 'inode' and (not self.isreg() or not compare_inodes):
 				pass
 			elif key == 'ea' and not compare_eas: pass
+			elif key == 'acl' and not compare_acls: pass
 			elif (not other.data.has_key(key) or
 				  self.data[key] != other.data[key]):
 				if not other.data.has_key(key):
@@ -523,6 +531,14 @@ class RORPath:
 															   self.index)
 			self.file_already_open = None
 
+	def set_acl(self, acl):
+		"""Record access control list in dictionary.  Does not write"""
+		self.data['acl'] = acl
+
+	def get_acl(self):
+		"""Return access control list object from dictionary"""
+		return self.data['acl']
+
 	def set_ea(self, ea):
 		"""Record extended attributes in dictionary.  Does not write"""
 		self.data['ea'] = ea
@@ -591,6 +607,7 @@ class RPath(RORPath):
 		"""Set data dictionary using C extension"""
 		self.data = self.conn.C.make_file_dict(self.path)
 		if Globals.read_eas and self.lstat(): self.get_ea()
+		if Globals.read_acls and self.lstat(): self.get_acl()
 
 	def make_file_dict_old(self):
 		"""Create the data dictionary"""
@@ -949,6 +966,20 @@ class RPath(RORPath):
 		assert not fp.close()
 		return s
 
+	def get_acl(self):
+		"""Return access control list object, setting if necessary"""
+		try: acl = self.data['acl']
+		except KeyError:
+			acl = eas_acls.AccessControlList(self.index)
+			if not self.issym(): acl.read_from_rp(self)
+			self.data['acl'] = acl
+		return acl
+
+	def write_acl(self, acl):
+		"""Change access control list of rp"""
+		acl.write_to_rp(self)
+		self.data['acl'] = acl
+
 	def get_ea(self):
 		"""Return extended attributes object, setting if necessary"""
 		try: ea = self.data['ea']

trunk/rdiff-backup/testing/commontest.py

@@ -168,7 +168,7 @@ def _reset_connections(src_rp, dest_rp):
 def CompareRecursive(src_rp, dest_rp, compare_hardlinks = 1,
 					 equality_func = None, exclude_rbdir = 1,
 					 ignore_tmp_files = None, compare_ownership = 0,
-					 compare_eas = 0):
+					 compare_eas = 0, compare_acls = 0):
 	"""Compare src_rp and dest_rp, which can be directories
 
 	This only compares file attributes, not the actual data.  This
@@ -180,8 +180,9 @@ def CompareRecursive(src_rp, dest_rp, compare_hardlinks = 1,
 	src_rp.setdata()
 	dest_rp.setdata()
 
-	Log("Comparing %s and %s, hardlinks %s, eas %s" %
-		(src_rp.path, dest_rp.path, compare_hardlinks, compare_eas), 3)
+	Log("Comparing %s and %s, hardlinks %s, eas %s, acls %s" %
+		(src_rp.path, dest_rp.path, compare_hardlinks,
+		 compare_eas, compare_acls), 3)
 	src_select = selection.Select(src_rp)
 	dest_select = selection.Select(dest_rp)
 
@@ -226,6 +227,10 @@ def CompareRecursive(src_rp, dest_rp, compare_hardlinks = 1,
 			Log("Different EAs in files %s and %s" %
 				(src_rorp.get_indexpath(), dest_rorp.get_indexpath()), 3)
 			return None
+		if compare_acls and not eas_acls.acl_compare_rps(src_rorp, dest_rorp):
+			Log("Different ACLs in files %s and %s" %
+				(src_rorp.get_indexpath(), dest_rorp.get_indexpath()), 3)
+			return None
 		return 1
 
 	def rbdir_equal(src_rorp, dest_rorp):
@@ -245,6 +250,10 @@ def CompareRecursive(src_rp, dest_rp, compare_hardlinks = 1,
 			Log("Different EAs in files %s and %s" %
 				(src_rorp.get_indexpath(), dest_rorp.get_indexpath()))
 			return None
+		if compare_acls and not eas_acls.acl_compare_rps(src_rorp, dest_rorp):
+			Log("Different ACLs in files %s and %s" %
+				(src_rorp.get_indexpath(), dest_rorp.get_indexpath()), 3)
+			return None
 		if compare_hardlinks:
 			if Hardlink.rorp_eq(src_rorp, dest_rorp): return 1
 		elif src_rorp.equal_verbose(dest_rorp,
@@ -285,7 +294,8 @@ def BackupRestoreSeries(source_local, dest_local, list_of_dirnames,
 						dest_dirname = "testfiles/output",
 						restore_dirname = "testfiles/rest_out",
 						compare_backups = 1,
-						compare_eas = 0):
+						compare_eas = 0,
+						compare_acls = 0):
 	"""Test backing up/restoring of a series of directories
 
 	The dirnames correspond to a single directory at different times.
@@ -297,6 +307,8 @@ def BackupRestoreSeries(source_local, dest_local, list_of_dirnames,
 	Globals.set('preserve_hardlinks', compare_hardlinks)
 	Globals.set('write_eas', compare_eas)
 	Globals.set('read_eas', compare_eas)
+	Globals.set('write_acls', compare_acls)
+	Globals.set('read_acls', compare_acls)
 	time = 10000
 	dest_rp = rpath.RPath(Globals.local_connection, dest_dirname)
 	restore_rp = rpath.RPath(Globals.local_connection, restore_dirname)
@@ -312,7 +324,8 @@ def BackupRestoreSeries(source_local, dest_local, list_of_dirnames,
 		_reset_connections(src_rp, dest_rp)
 		if compare_backups:
 			assert CompareRecursive(src_rp, dest_rp, compare_hardlinks,
-									compare_eas = compare_eas)
+									compare_eas = compare_eas,
+									compare_acls = compare_acls)
 
 	time = 10000
 	for dirname in list_of_dirnames[:-1]:
@@ -321,7 +334,9 @@ def BackupRestoreSeries(source_local, dest_local, list_of_dirnames,
 		InternalRestore(dest_local, source_local, dest_dirname,
 						restore_dirname, time)
 		src_rp = rpath.RPath(Globals.local_connection, dirname)
-		assert CompareRecursive(src_rp, restore_rp, compare_eas = compare_eas)
+		assert CompareRecursive(src_rp, restore_rp,
+								compare_eas = compare_eas,
+								compare_acls = compare_acls)
 
 		# Restore should default back to newest time older than it
 		# with a backup then.

trunk/rdiff-backup/testing/eas_aclstest.py

@@ -62,7 +62,12 @@ class EATest(unittest.TestCase):
 			assert 0, "We shouldn't have gotten this far"
 
 	def make_backup_dirs(self):
-		"""Create testfiles/ea_test[12] directories"""
+		"""Create testfiles/ea_test[12] directories
+
+		Goal is to set range of extended attributes, to give good test
+		to extended attribute code.
+
+		"""
 		if self.ea_testdir1.lstat(): self.ea_testdir1.delete()
 		if self.ea_testdir2.lstat(): self.ea_testdir2.delete()
 		self.ea_testdir1.mkdir()
@@ -132,4 +137,208 @@ class EATest(unittest.TestCase):
 				   'testfiles/empty', 'testfiles/ea_test1']
 		BackupRestoreSeries(None, None, dirlist, compare_eas = 1)
 
+
+
+class ACLTest(unittest.TestCase):
+	"""Test access control lists"""
+	sample_acl = AccessControlList((),"""user::rwx
+user:root:rwx
+group::r-x
+group:root:r-x
+mask::r-x
+other::---""")
+	dir_acl = AccessControlList((), """user::rwx
+user:root:rwx
+group::r-x
+group:root:r-x
+mask::r-x
+other::---""",
+								"""user::rwx
+user:root:---
+group::r-x
+mask::r-x
+other::---""")
+	acl1 = AccessControlList(('1',), """user::r--
+user:ben:---
+group::---
+group:root:---
+mask::---
+other::---""")
+	acl2 = AccessControlList(('2',), """user::rwx
+group::r-x
+group:ben:rwx
+mask::---
+other::---""")
+	acl3 = AccessControlList(('3',), """user::rwx
+user:root:---
+group::r-x
+mask::---
+other::---""")
+	empty_acl = AccessControlList((), "user::rwx\ngroup::---\nother::---")
+	acl_testdir1 = rpath.RPath(Globals.local_connection, 'testfiles/acl_test1')
+	acl_testdir2 = rpath.RPath(Globals.local_connection, 'testfiles/acl_test2')
+	def make_temp(self):
+		"""Make temp directory testfile/output"""
+		if tempdir.lstat(): tempdir.delete()
+		tempdir.mkdir()
+
+	def testBasic(self):
+		"""Test basic writing and reading of ACLs"""
+		self.make_temp()
+		new_acl = AccessControlList(())
+		tempdir.chmod(0700)
+		new_acl.read_from_rp(tempdir)
+		assert new_acl.is_basic(), new_acl.acl_text
+		assert not new_acl == self.sample_acl
+		assert new_acl != self.sample_acl
+		assert new_acl == self.empty_acl, \
+			   (new_acl.acl_text, self.empty_acl.acl_text)
+
+		self.sample_acl.write_to_rp(tempdir)
+		new_acl.read_from_rp(tempdir)
+		assert new_acl.acl_text == self.sample_acl.acl_text, \
+			   (new_acl.acl_text, self.sample_acl.acl_text)
+		assert new_acl == self.sample_acl
+		
+	def testBasicDir(self):
+		"""Test reading and writing of ACL w/ defaults to directory"""
+		self.make_temp()
+		new_acl = AccessControlList(())
+		new_acl.read_from_rp(tempdir)
+		assert new_acl.is_basic()
+		assert new_acl != self.dir_acl
+
+		self.dir_acl.write_to_rp(tempdir)
+		new_acl.read_from_rp(tempdir)
+		assert not new_acl.is_basic()
+		if not new_acl == self.dir_acl:
+			assert new_acl.eq_verbose(self.dir_acl)
+			assert 0, "Shouldn't be here---eq != eq_verbose?"
+
+	def testRecord(self):
+		"""Test writing a record and reading it back"""
+		record = ACL2Record(self.sample_acl)
+		new_acl = Record2ACL(record)
+		assert new_acl == self.sample_acl
+
+		record2 = ACL2Record(self.dir_acl)
+		new_acl2 = Record2ACL(record2)
+		if not new_acl2 == self.dir_acl:
+			assert new_acl2.eq_verbose(self.dir_acl)
+			assert 0
+
+	def make_backup_dirs(self):
+		"""Create testfiles/acl_test[12] directories"""
+		if self.acl_testdir1.lstat(): self.acl_testdir1.delete()
+		if self.acl_testdir2.lstat(): self.acl_testdir2.delete()
+		self.acl_testdir1.mkdir()
+		rp1_1 = self.acl_testdir1.append('1')
+		rp1_2 = self.acl_testdir1.append('2')
+		rp1_3 = self.acl_testdir1.append('3')
+		map(rpath.RPath.touch, [rp1_1, rp1_2, rp1_3])
+		self.dir_acl.write_to_rp(self.acl_testdir1)
+		self.acl1.write_to_rp(rp1_1)
+		self.acl2.write_to_rp(rp1_2)
+		self.acl3.write_to_rp(rp1_3)
+
+		self.acl_testdir2.mkdir()
+		rp2_1, rp2_2, rp2_3 = map(self.acl_testdir2.append, ('1', '2', '3'))
+		map(rpath.RPath.touch, (rp2_1, rp2_2, rp2_3))
+		self.sample_acl.write_to_rp(self.acl_testdir2)
+		self.acl3.write_to_rp(rp2_1)
+		self.acl1.write_to_rp(rp2_2)
+		self.acl2.write_to_rp(rp2_3)
+		
+	def testIterate(self):
+		"""Test writing several records and then reading them back"""
+		self.make_backup_dirs()
+		rp1 = self.acl_testdir1.append('1')
+		rp2 = self.acl_testdir1.append('2')
+		rp3 = self.acl_testdir1.append('3')
+
+		# Now write records corresponding to above rps into file
+		Globals.rbdir = tempdir
+		Time.setcurtime(10000)
+		AccessControlListFile.open_file()
+		for rp in [self.acl_testdir1, rp1, rp2, rp3]:
+			acl = AccessControlList(rp.index)
+			acl.read_from_rp(rp)
+			AccessControlListFile.write_object(acl)
+		AccessControlListFile.close_file()
+
+		# Read back records and compare
+		acl_iter = AccessControlListFile.get_objects_at_time(tempdir, 10000)
+		assert acl_iter, "No acl file found"
+		dir_acl_reread = acl_iter.next()
+		assert dir_acl_reread == self.dir_acl
+		acl1_reread = acl_iter.next()
+		assert acl1_reread == self.acl1
+		acl2_reread = acl_iter.next()
+		assert acl2_reread == self.acl2
+		acl3_reread = acl_iter.next()
+		assert acl3_reread == self.acl3
+		try: extra = acl_iter.next()
+		except StopIteration: pass
+		else: assert 0, "Got unexpected object: " + repr(extra)
+
+	def testSeriesLocal(self):
+		"""Test backing up and restoring directories with ACLs locally"""
+		self.make_backup_dirs()
+		dirlist = ['testfiles/acl_test1', 'testfiles/empty',
+				   'testfiles/acl_test2', 'testfiles/acl_test1']
+		BackupRestoreSeries(1, 1, dirlist, compare_acls = 1)
+
+	def testSeriesRemote(self):
+		"""Test backing up, restoring directories with EA remotely"""
+		self.make_backup_dirs()
+		dirlist = ['testfiles/acl_test1', 'testfiles/acl_test2',
+				   'testfiles/empty', 'testfiles/acl_test1']
+		BackupRestoreSeries(None, None, dirlist, compare_acls = 1)
+
+
+class CombinedTest(unittest.TestCase):
+	"""Test backing up and restoring directories with both EAs and ACLs"""
+	combo_testdir1 = rpath.RPath(Globals.local_connection,
+								 'testfiles/ea_acl_test1')
+	combo_testdir2 = rpath.RPath(Globals.local_connection,
+								 'testfiles/ea_acl_test2')
+	def make_backup_dirs(self):
+		"""Create testfiles/ea_acl_test[12] directories"""
+		if self.combo_testdir1.lstat(): self.combo_testdir1.delete()
+		if self.combo_testdir2.lstat(): self.combo_testdir2.delete()
+		self.combo_testdir1.mkdir()
+		rp1_1, rp1_2, rp1_3 = map(self.combo_testdir1.append, ('1', '2', '3'))
+		map(rpath.RPath.touch, [rp1_1, rp1_2, rp1_3])
+		ACLTest.dir_acl.write_to_rp(self.combo_testdir1)
+		EATest.sample_ea.write_to_rp(self.combo_testdir1)
+		ACLTest.acl1.write_to_rp(rp1_1)
+		EATest.ea2.write_to_rp(rp1_2)
+		ACLTest.acl3.write_to_rp(rp1_3)
+		EATest.ea3.write_to_rp(rp1_3)
+
+		self.combo_testdir2.mkdir()
+		rp2_1, rp2_2, rp2_3 = map(self.combo_testdir2.append, ('1', '2', '3'))
+		map(rpath.RPath.touch, [rp2_1, rp2_2, rp2_3])
+		ACLTest.sample_acl.write_to_rp(self.combo_testdir2)
+		EATest.ea1.write_to_rp(rp2_1)
+		EATest.ea3.write_to_rp(rp2_2)
+		ACLTest.acl2.write_to_rp(rp2_2)
+
+	def testSeriesLocal(self):
+		"""Test backing up and restoring EAs/ACLs locally"""
+		self.make_backup_dirs()
+		dirlist = ['testfiles/ea_acl_test1', 'testfiles/ea_acl_test2',
+				   'testfiles/empty', 'testfiles/ea_acl_test1']
+		BackupRestoreSeries(1, 1, dirlist,
+							compare_eas = 1, compare_acls = 1)
+
+	def testSeriesRemote(self):
+		"""Test backing up and restoring EAs/ACLs locally"""
+		self.make_backup_dirs()
+		dirlist = ['testfiles/ea_acl_test1', 'testfiles/empty',
+				   'testfiles/ea_acl_test2', 'testfiles/ea_acl_test1']
+		BackupRestoreSeries(None, None, dirlist,
+							compare_eas = 1, compare_acls = 1)
+
+
 if __name__ == "__main__": unittest.main()

trunk/rdiff-backup/testing/fs_abilitiestest.py

@@ -28,7 +28,8 @@ class FSAbilitiesTest(unittest.TestCase):
 	def testReadOnly(self):
 		"""Test basic querying read only"""
 		base_dir = rpath.RPath(Globals.local_connection, self.dir_to_test)
-		fsa = fs_abilities.FSAbilities().init_readonly(base_dir)
+		fsa = fs_abilities.FSAbilities('read-only').init_readonly(base_dir)
+		print fsa
 		assert fsa.read_only == 1, fsa.read_only
 		assert fsa.eas == self.eas, fsa.eas
 		assert fsa.acls == self.acls, fsa.acls
@@ -41,8 +42,9 @@ class FSAbilitiesTest(unittest.TestCase):
 		new_dir.setdata()
 		new_dir.mkdir()
 		t = time.time()
-		fsa = fs_abilities.FSAbilities().init_readwrite(new_dir)
+		fsa = fs_abilities.FSAbilities('read/write').init_readwrite(new_dir)
 		print "Time elapsed = ", time.time() - t
+		print fsa
 		assert fsa.read_only == 0, fsa.read_only
 		assert fsa.eas == self.eas, fsa.eas
 		assert fsa.acls == self.acls, fsa.acls