Refactored fs_abilities, also don't quote if case-insensitive->case-insensitive

git-svn-id: http://svn.savannah.nongnu.org/svn/rdiff-backup@648 2b77aa54-bcbc-44c9-a7ec-4f6cf2b41109

trunk/rdiff-backup/CHANGELOG

 New in v1.1.0 (????/??/??)
 --------------------------
 
+Refactored fs_abilities for more flexibility.  In particular, avoid
+quoting if both source and destination file systems are
+case-insensitive.
+
 When possible, fsync using a writable file descriptor.  This may help
 with cygwin.  (Requested/tested by Dave Kempe.)
 

trunk/rdiff-backup/rdiff_backup/Main.py

@@ -301,7 +301,7 @@ def Backup(rpin, rpout):
 	SetConnections.BackupInitConnections(rpin.conn, rpout.conn)
 	backup_check_dirs(rpin, rpout)
 	backup_set_rbdir(rpin, rpout)
-	backup_set_fs_globals(rpin, rpout)
+	rpout.conn.fs_abilities.backup_set_globals(rpin)
 	if Globals.chars_to_quote: rpout = backup_quoted_rpaths(rpout)
 	init_user_group_mapping(rpout.conn)
 	backup_final_init(rpout)
@@ -353,7 +353,6 @@ def backup_check_dirs(rpin, rpout):
 def backup_set_rbdir(rpin, rpout):
 	"""Initialize data dir and logging"""
 	global incdir
-	SetConnections.UpdateGlobal('rbdir', Globals.rbdir)
 	incdir = Globals.rbdir.append_path("increments")
 
 	assert rpout.lstat(), (rpout.path, rpout.lstat())
@@ -370,6 +369,7 @@ want to update or overwrite it, run rdiff-backup with the --force
 option.""" % rpout.path)
 
 	if not Globals.rbdir.lstat(): Globals.rbdir.mkdir()
+	SetConnections.UpdateGlobal('rbdir', Globals.rbdir)
 
 def backup_warn_if_infinite_regress(rpin, rpout):
 	"""Warn user if destination area contained in source area"""
@@ -407,53 +407,6 @@ def backup_final_init(rpout):
 	inc_base = Globals.rbdir.append_path("increments")
 	if not inc_base.lstat(): inc_base.mkdir()
 
-def backup_set_fs_globals(rpin, rpout):
-	"""Use fs_abilities to set the globals that depend on filesystem"""
-	def update_triple(src_support, dest_support, attr_triple):
-		"""Update global settings for feature based on fsa results"""
-		active_attr, write_attr, conn_attr = attr_triple
-		if Globals.get(active_attr) == 0: return # don't override 0
-		for attr in attr_triple: SetConnections.UpdateGlobal(attr, None)
-		if not src_support: return # if source doesn't support, nothing
-		SetConnections.UpdateGlobal(active_attr, 1)
-		rpin.conn.Globals.set_local(conn_attr, 1)
-		if dest_support:
-			SetConnections.UpdateGlobal(write_attr, 1)
-			rpout.conn.Globals.set_local(conn_attr, 1)
-
-	src_fsa = rpin.conn.fs_abilities.get_fsabilities_readonly('source', rpin)
-	Log(str(src_fsa), 4)
-	dest_fsa = rpout.conn.fs_abilities.get_fsabilities_readwrite(
-		'destination', Globals.rbdir, 1, Globals.chars_to_quote)
-	Log(str(dest_fsa), 4)
-
-	update_triple(src_fsa.eas, dest_fsa.eas,
-				  ('eas_active', 'eas_write', 'eas_conn'))
-	update_triple(src_fsa.acls, dest_fsa.acls,
-				  ('acls_active', 'acls_write', 'acls_conn'))
-	update_triple(src_fsa.resource_forks, dest_fsa.resource_forks,
-				  ('resource_forks_active', 'resource_forks_write',
-				   'resource_forks_conn'))
-
-	update_triple(src_fsa.carbonfile, dest_fsa.carbonfile,
-				  ('carbonfile_active', 'carbonfile_write', 'carbonfile_conn'))
-	if src_fsa.carbonfile and not Globals.carbonfile_active:
-		Log("Source may have carbonfile support, but support defaults to "
-			"off.\n  Use --carbonfile to enable.", 5)
-
-	if Globals.never_drop_acls and not Globals.acls_active:
-		Log.FatalError("--never-drop-acls specified, but ACL support\n"
-					   "disabled on destination filesystem")
-
-	if Globals.preserve_hardlinks != 0:
-		SetConnections.UpdateGlobal('preserve_hardlinks', dest_fsa.hardlinks)
-	SetConnections.UpdateGlobal('fsync_directories', dest_fsa.fsync_dirs)
-	SetConnections.UpdateGlobal('change_ownership', dest_fsa.ownership)
-	SetConnections.UpdateGlobal('chars_to_quote', dest_fsa.chars_to_quote)
-	if not dest_fsa.high_perms:
-		SetConnections.UpdateGlobal('permission_mask', 0777)
-	if Globals.chars_to_quote: FilenameMapping.set_init_quote_vals()
-	
 def backup_touch_curmirror_local(rpin, rpout):
 	"""Make a file like current_mirror.time.data to record time
 
@@ -496,7 +449,7 @@ def Restore(src_rp, dest_rp, restore_as_of = None):
 	"""
 	if not restore_root_set: assert restore_set_root(src_rp)
 	restore_check_paths(src_rp, dest_rp, restore_as_of)
-	restore_set_fs_globals(dest_rp)
+	dest_rp.conn.fs_abilities.restore_set_globals(dest_rp)
 	init_user_group_mapping(dest_rp.conn)
 	src_rp = restore_init_quoting(src_rp)
 	restore_check_backup_dir(restore_root, src_rp, restore_as_of)
@@ -521,51 +474,6 @@ def restore_init_quoting(src_rp):
 		'rbdir', FilenameMapping.get_quotedrpath(Globals.rbdir))
 	return FilenameMapping.get_quotedrpath(src_rp)
 
-def restore_set_fs_globals(target):
-	"""Use fs_abilities to set the globals that depend on filesystem"""
-	def update_triple(src_support, dest_support, attr_triple):
-		"""Update global settings for feature based on fsa results"""
-		active_attr, write_attr, conn_attr = attr_triple
-		if Globals.get(active_attr) == 0: return # don't override 0
-		for attr in attr_triple: SetConnections.UpdateGlobal(attr, None)
-		if not dest_support: return # if dest doesn't support, do nothing
-		SetConnections.UpdateGlobal(active_attr, 1)
-		target.conn.Globals.set_local(conn_attr, 1)
-		target.conn.Globals.set_local(write_attr, 1)
-		if src_support: Globals.rbdir.conn.Globals.set_local(conn_attr, 1)
-
-	target_fsa = target.conn.fs_abilities.get_fsabilities_readwrite(
-		'destination', target, 0, Globals.chars_to_quote)
-	Log(str(target_fsa), 4)
-	mirror_fsa = Globals.rbdir.conn.fs_abilities.get_fsabilities_restoresource(
-		Globals.rbdir)
-	Log(str(mirror_fsa), 4)
-
-	update_triple(mirror_fsa.eas, target_fsa.eas,
-				  ('eas_active', 'eas_write', 'eas_conn'))
-	update_triple(mirror_fsa.acls, target_fsa.acls,
-				  ('acls_active', 'acls_write', 'acls_conn'))
-	update_triple(mirror_fsa.resource_forks, target_fsa.resource_forks,
-				  ('resource_forks_active', 'resource_forks_write',
-				   'resource_forks_conn'))
-	update_triple(mirror_fsa.carbonfile, target_fsa.carbonfile,
-				  ('carbonfile_active', 'carbonfile_write', 'carbonfile_conn'))
-	if Globals.never_drop_acls and not Globals.acls_active:
-		Log.FatalError("--never-drop-acls specified, but ACL support\n"
-					   "disabled on destination filesystem")
-
-	if Globals.preserve_hardlinks != 0:
-		SetConnections.UpdateGlobal('preserve_hardlinks', target_fsa.hardlinks)
-	SetConnections.UpdateGlobal('change_ownership', target_fsa.ownership)
-	if not target_fsa.high_perms:
-		SetConnections.UpdateGlobal('permission_mask', 0777)
-
-	if Globals.chars_to_quote is None: # otherwise already overridden
-		if mirror_fsa.chars_to_quote:
-			SetConnections.UpdateGlobal('chars_to_quote',
-										mirror_fsa.chars_to_quote)
-		else: SetConnections.UpdateGlobal('chars_to_quote', "")
-
 def restore_set_select(mirror_rp, target):
 	"""Set the selection iterator on both side from command line args
 
@@ -662,6 +570,8 @@ def restore_set_root(rpin):
 
 	restore_root = parent_dir
 	Log("Using mirror root directory %s" % restore_root.path, 6)
+	if restore_root.conn is Globals.local_connection:
+		Security.reset_restrict_path(restore_root)
 	SetConnections.UpdateGlobal('rbdir',
 								restore_root.append_path("rdiff-backup-data"))
 	if not Globals.rbdir.isdir():
@@ -702,49 +612,10 @@ def require_root_set(rp):
 	if not restore_set_root(rp):
 		Log.FatalError(("Bad directory %s.\n" % (rp.path,)) +
 		  "It doesn't appear to be an rdiff-backup destination dir")
-	single_set_fs_globals(Globals.rbdir)
+	Globals.rbdir.conn.fs_abilities.single_set_globals(Globals.rbdir)
 	if Globals.chars_to_quote: return restore_init_quoting(rp)
 	else: return rp
 	
-def single_set_fs_globals(rbdir):
-	"""Use fs_abilities to set globals that depend on filesystem.
-
-	This is appropriate for listing increments, or any other operation
-	that depends only on the one file system.
-
-	"""
-	def update_triple(fsa_support, attr_triple):
-		"""Update global settings based on fsa result"""
-		active_attr, write_attr, conn_attr = attr_triple
-		if Globals.get(active_attr) == 0: return # don't override 0
-		for attr in attr_triple: SetConnections.UpdateGlobal(attr, None)
-		if not fsa_support: return
-		SetConnections.UpdateGlobal(active_attr, 1)
-		SetConnections.UpdateGlobal(write_attr, 1)
-		rbdir.conn.Globals.set_local(conn_attr, 1)
-
-	fsa = rbdir.conn.fs_abilities.get_fsabilities_readwrite('archive',
-								   rbdir, 1, Globals.chars_to_quote)
-	Log(str(fsa), 4)
-
-	update_triple(fsa.eas, ('eas_active', 'eas_write', 'eas_conn'))
-	update_triple(fsa.acls, ('acls_active', 'acls_write', 'acls_conn'))
-	update_triple(fsa.resource_forks,
-				  ('resource_forks_active', 'resource_forks_write',
-				   'resource_forks_conn'))
-	update_triple(fsa.carbonfile,
-				  ('carbonfile_active', 'carbonfile_write', 'carbonfile_conn'))
-
-	if Globals.preserve_hardlinks != 0:
-		SetConnections.UpdateGlobal('preserve_hardlinks', fsa.hardlinks)
-	SetConnections.UpdateGlobal('fsync_directories', fsa.fsync_dirs)
-	SetConnections.UpdateGlobal('change_ownership', fsa.ownership)
-	if not fsa.high_perms: SetConnections.UpdateGlobal('permission_mask', 0777)
-	SetConnections.UpdateGlobal('chars_to_quote', fsa.chars_to_quote)
-	if Globals.chars_to_quote:
-		for conn in Globals.connections:
-			conn.FilenameMapping.set_init_quote_vals()
-
 
 def ListIncrementSizes(rp):
 	"""Print out a summary of the increments """

trunk/rdiff-backup/rdiff_backup/Security.py

@@ -47,13 +47,17 @@ file_requests = {'os.listdir':0, 'C.make_file_dict':0, 'os.chmod':0,
 				 'os.utime':0, 'os.lchown':0, 'os.link':1, 'os.symlink':1,
 				 'os.mkdir':0, 'os.makedirs':0}
 				 
-
 def initialize(action, cmdpairs):
 	"""Initialize allowable request list and chroot"""
 	global allowed_requests
 	set_security_level(action, cmdpairs)
 	set_allowed_requests(Globals.security_level)
 
+def reset_restrict_path(rp):
+	"""Reset restrict path to be within rpath"""
+	assert rp.conn is Globals.local_connection
+	Globals.restrict_path = rp.normalize().path
+
 def set_security_level(action, cmdpairs):
 	"""If running client, set security level and restrict_path
 
@@ -137,8 +141,7 @@ def set_allowed_requests(sec_level):
 				  "Hardlink.initialize_dictionaries", "user_group.uid2uname",
 				  "user_group.gid2gname"])
 	if sec_level == "read-only" or sec_level == "all":
-		l.extend(["fs_abilities.get_fsabilities_readonly",
-				  "fs_abilities.get_fsabilities_restoresource",
+		l.extend(["fs_abilities.get_readonly_fsa",
 				  "restore.MirrorStruct.set_mirror_and_rest_times",
 				  "restore.MirrorStruct.set_mirror_select",
 				  "restore.MirrorStruct.initialize_rf_cache",
@@ -161,14 +164,16 @@ def set_allowed_requests(sec_level):
 				  "Globals.ITRB.increment_stat",
 				  "statistics.record_error",
 				  "log.ErrorLog.write_if_open",
-				  "fs_abilities.get_fsabilities_readwrite"])
+				  "fs_abilities.backup_set_globals"])
 	if sec_level == "all":
 		l.extend(["os.mkdir", "os.chown", "os.lchown", "os.rename",
-				  "os.unlink", "os.remove", "os.chmod",
+				  "os.unlink", "os.remove", "os.chmod", "os.makedirs",
 				  "backup.DestinationStruct.patch",
 				  "restore.TargetStruct.get_initial_iter",
 				  "restore.TargetStruct.patch",
 				  "restore.TargetStruct.set_target_select",
+				  "fs_abilities.restore_set_globals",
+				  "fs_abilities.single_set_globals",
 				  "regress.Regress", "manage.delete_earlier_than_local"])
 	if Globals.server:
 		l.extend(["SetConnections.init_connection_remote",
@@ -200,8 +205,7 @@ def vet_request(request, arglist):
 	if security_level == "override": return
 	if request.function_string in allowed_requests: return
 	if request.function_string in ("Globals.set", "Globals.set_local"):
-		if Globals.server and arglist[0] not in disallowed_server_globals:
-			return
+		if arglist[0] not in disallowed_server_globals: return
 	raise_violation(request, arglist)
 
 def vet_rpath(rpath):

trunk/rdiff-backup/rdiff_backup/eas_acls.py

@@ -585,6 +585,11 @@ def rpath_acl_get(rp):
 	return acl
 rpath.acl_get = rpath_acl_get
 
+def rpath_get_blank_acl(index):
+	"""Get a blank AccessControlLists object (override rpath function)"""
+	return AccessControlLists(index)
+rpath.get_blank_acl = rpath_get_blank_acl
+
 def rpath_ea_get(rp):
 	"""Get extended attributes of given rpath
 
@@ -595,3 +600,9 @@ def rpath_ea_get(rp):
 	if not rp.issym(): ea.read_from_rp(rp)
 	return ea
 rpath.ea_get = rpath_ea_get
+
+def rpath_get_blank_ea(index):
+	"""Get a blank ExtendedAttributes object (override rpath function)"""
+	return ExtendedAttributes(index)
+rpath.get_blank_ea = rpath_get_blank_ea
+

trunk/rdiff-backup/rdiff_backup/fs_abilities.py

@@ -28,11 +28,12 @@ FSAbilities object describing it.
 """
 
 import errno, os
-import Globals, log, TempFile, selection, robust
+import Globals, log, TempFile, selection, robust, SetConnections, \
+	   static, FilenameMapping
 
 class FSAbilities:
 	"""Store capabilities of given file system"""
-	chars_to_quote = None # Hold characters not allowable in file names
+	extended_filenames = None # True if filenames can handle ":" etc.
 	case_sensitive = None # True if "foobar" and "FoObAr" are different files
 	ownership = None # True if chown works on this filesystem
 	acls = None # True if access control lists supported
@@ -40,7 +41,7 @@ class FSAbilities:
 	hardlinks = None # True if hard linking supported
 	fsync_dirs = None # True if directories can be fsync'd
 	dir_inc_perms = None # True if regular files can have full permissions
-	resource_forks = None # True if regular_file/..namedfork/rsrc holds resource fork
+	resource_forks = None # True if system supports resource forks
 	carbonfile = None # True if Mac Carbon file data is supported. 
 	name = None # Short string, not used for any technical purpose
 	read_only = None # True if capabilities were determined non-destructively
@@ -78,21 +79,15 @@ class FSAbilities:
 			else: return ('Detected abilities for %s file system' %
 						  (read_string,))
 
-		def add_ctq_line():
-			"""Get line describing chars to quote"""
-			ctq_str = (self.chars_to_quote is None and 'N/A'
-					   or repr(self.chars_to_quote))
-			addline('Characters needing quoting', ctq_str)
-
 		s.append(get_title_line())
 		if not self.read_only:
-			add_ctq_line()
 			add_boolean_list([('Ownership changing', self.ownership),
 							  ('Hard linking', self.hardlinks),
 							  ('fsync() directories', self.fsync_dirs),
 							  ('Directory inc permissions',
 							   self.dir_inc_perms),
-							  ('High-bit permissions', self.high_perms)])
+							  ('High-bit permissions', self.high_perms),
+							  ('Extended filenames', self.extended_filenames)])
 		add_boolean_list([('Access control lists', self.acls),
 						  ('Extended attributes', self.eas),
 						  ('Case sensitivity', self.case_sensitive),
@@ -122,20 +117,13 @@ class FSAbilities:
 		self.set_case_sensitive_readonly(rp)
 		return self
 
-	def init_readwrite(self, rbdir, use_ctq_file = 1,
-					   override_chars_to_quote = None):
+	def init_readwrite(self, rbdir):
 		"""Set variables using fs tested at rp_base.  Run locally.
 
 		This method creates a temp directory in rp_base and writes to
 		it in order to test various features.  Use on a file system
 		that will be written to.
 
-		This sets self.chars_to_quote, self.ownership, self.acls,
-		self.eas, self.hardlinks, and self.fsync_dirs.
-
-		If user_ctq_file is true, try reading the "chars_to_quote"
-		file in directory.
-
 		"""
 		assert rbdir.conn is Globals.local_connection
 		if not rbdir.isdir():
@@ -143,9 +131,11 @@ class FSAbilities:
 			rbdir.mkdir()
 		self.root_rp = rbdir
 		self.read_only = 0
-
 		subdir = TempFile.new_in_dir(rbdir)
 		subdir.mkdir()
+
+		self.set_extended_filenames(subdir)
+		self.set_case_sensitive_readwrite(subdir)
 		self.set_ownership(subdir)
 		self.set_hardlinks(subdir)
 		self.set_fsync_dirs(subdir)
@@ -155,39 +145,10 @@ class FSAbilities:
 		self.set_resource_fork_readwrite(subdir)
 		self.set_carbonfile()
 		self.set_high_perms_readwrite(subdir)
-		if override_chars_to_quote is None: self.set_chars_to_quote(subdir)
-		else: self.chars_to_quote = override_chars_to_quote
-		if use_ctq_file: self.compare_chars_to_quote(rbdir)
+
 		subdir.delete()
 		return self
 
-	def compare_chars_to_quote(self, rbdir):
-		"""Read chars_to_quote file, compare with current settings"""
-		assert self.chars_to_quote is not None
-		ctq_rp = rbdir.append("chars_to_quote")
-		def write_new_chars():
-			"""Replace old chars_to_quote file with new value"""
-			if ctq_rp.lstat(): ctq_rp.delete()
-			fp = ctq_rp.open("wb")
-			fp.write(self.chars_to_quote)
-			assert not fp.close()
-
-		if not ctq_rp.lstat(): write_new_chars()
-		else:
-			old_chars = ctq_rp.get_data()
-			if old_chars != self.chars_to_quote:
-				if self.chars_to_quote == "":
-					log.Log("Warning: File system no longer needs quoting, "
-							"but will retain for backwards compatibility.", 2)
-					self.chars_to_quote = old_chars
-				else: log.Log.FatalError("""New quoting requirements
-
-This may be caused when you copy an rdiff-backup directory from a
-normal file system on to a windows one that cannot support the same
-characters.  If you want to risk it, remove the file
-rdiff-backup-data/chars_to_quote.
-""")
-
 	def set_ownership(self, testdir):
 		"""Set self.ownership to true iff testdir's ownership can be changed"""
 		tmp_rp = testdir.append("foo")
@@ -196,7 +157,7 @@ rdiff-backup-data/chars_to_quote.
 		try:
 			tmp_rp.chown(uid+1, gid+1) # just choose random uid/gid
 			tmp_rp.chown(0, 0)
-		except (IOError, OSError), exc: self.ownership = 0
+		except (IOError, OSError): self.ownership = 0
 		else: self.ownership = 1
 		tmp_rp.delete()
 
@@ -209,7 +170,7 @@ rdiff-backup-data/chars_to_quote.
 			hl_dest.hardlink(hl_source.path)
 			if hl_source.getinode() != hl_dest.getinode():
 				raise IOError(errno.EOPNOTSUPP, "Hard links don't compare")
-		except (IOError, OSError), exc:
+		except (IOError, OSError):
 			log.Log("Warning: hard linking not supported by filesystem "
 					"at %s" % (self.root_rp.path,), 3)
 			self.hardlinks = 0
@@ -219,58 +180,34 @@ rdiff-backup-data/chars_to_quote.
 		"""Set self.fsync_dirs if directories can be fsync'd"""
 		assert testdir.conn is Globals.local_connection
 		try: testdir.fsync()
-		except (IOError, OSError), exc:
+		except (IOError, OSError):
 			log.Log("Directories on file system at %s are not fsyncable.\n"
 					"Assuming it's unnecessary." % (testdir.path,), 4)
 			self.fsync_dirs = 0
 		else: self.fsync_dirs = 1
 
-	def set_chars_to_quote(self, subdir):
-		"""Set self.chars_to_quote by trying to write various paths"""
-		def is_case_sensitive():
-			"""Return true if file system is case sensitive"""
-			upper_a = subdir.append("A")
-			upper_a.touch()
-			lower_a = subdir.append("a")
-			if lower_a.lstat():
-				lower_a.delete()
-				upper_a.setdata()
-				assert not upper_a.lstat()
-				self.case_sensitive = 0
-			else:
-				upper_a.delete()
-				self.case_sensitive = 1
-			return self.case_sensitive
+	def set_extended_filenames(self, subdir):
+		"""Set self.extended_filenames by trying to write a path"""
+		assert not self.read_only
+
+		# Make sure ordinary filenames ok
+		ordinary_filename = '5-_ a.'
+		ord_rp = subdir.append(ordinary_filename)
+		ord_rp.touch()
+		assert ord_rp.lstat()
+		ord_rp.delete()
 
-		def supports_unusual_chars():
-			"""Test handling of several chars sometimes not supported"""
-			for filename in [':', '\\', chr(175)]:
+		extended_filename = ':\\' + chr(175)
 		try:
-					rp = subdir.append(filename)
-					rp.touch()
+			ext_rp = subdir.append(extended_filename)
+			ext_rp.touch()
 		except (IOError, OSError):
-					assert not rp.lstat()
-					return 0
-				else:
-					assert rp.lstat()
-					rp.delete()
-			return 1
-
-		def sanity_check():
-			"""Make sure basic filenames writable"""
-			for filename in ['5-_ a.']:
-				rp = subdir.append(filename)
-				rp.touch()
-				assert rp.lstat()
-				rp.delete()
-
-		sanity_check()
-		if is_case_sensitive():
-			if supports_unusual_chars(): self.chars_to_quote = ""
-			else: self.chars_to_quote = "^A-Za-z0-9_ -."
+			assert not ext_rp.lstat()
+			self.extended_filenames = 0
 		else:
-			if supports_unusual_chars(): self.chars_to_quote = "A-Z;"
-			else: self.chars_to_quote = "^a-z0-9_ -."
+			assert ext_rp.lstat()
+			ext_rp.delete()
+			self.extended_filenames = 1
 
 	def set_acls(self, rp):
 		"""Set self.acls based on rp.  Does not write.  Needs to be local"""
@@ -285,11 +222,26 @@ rdiff-backup-data/chars_to_quote.
 			return
 
 		try: posix1e.ACL(file=rp.path)
-		except IOError, exc:
+		except IOError:
 			log.Log("ACLs not supported by filesystem at %s" % (rp.path,), 4)
 			self.acls = 0
 		else: self.acls = 1
 
+	def set_case_sensitive_readwrite(self, subdir):
+		"""Determine if directory at rp is case sensitive by writing"""
+		assert not self.read_only
+		upper_a = subdir.append("A")
+		upper_a.touch()
+		lower_a = subdir.append("a")
+		if lower_a.lstat():
+			lower_a.delete()
+			upper_a.setdata()
+			assert not upper_a.lstat()
+			self.case_sensitive = 0
+		else:
+			upper_a.delete()
+			self.case_sensitive = 1
+
 	def set_case_sensitive_readonly(self, rp):
 		"""Determine if directory at rp is case sensitive without writing"""
 		def find_letter(subdir):
@@ -349,7 +301,7 @@ rdiff-backup-data/chars_to_quote.
 			if write:
 				xattr.setxattr(rp.path, "user.test", "test val")
 				assert xattr.getxattr(rp.path, "user.test") == "test val"
-		except IOError, exc:
+		except IOError:
 			log.Log("Extended attributes not supported by "
 					"filesystem at %s" % (rp.path,), 4)
 			self.eas = 0
@@ -402,7 +354,7 @@ rdiff-backup-data/chars_to_quote.
 			fp_read = open(os.path.join(reg_rp.path, '..namedfork', 'rsrc'), 'rb')
 			s_back = fp_read.read()
 			assert not fp_read.close()
-		except (OSError, IOError), e: self.resource_forks = 0
+		except (OSError, IOError): self.resource_forks = 0
 		else: self.resource_forks = (s_back == s)
 		reg_rp.delete()
 
@@ -421,7 +373,7 @@ rdiff-backup-data/chars_to_quote.
 					fp = rfork.open('rb')
 					fp.read()
 					assert not fp.close()
-				except (OSError, IOError), e:
+				except (OSError, IOError):
 					self.resource_forks = 0
 					return
 				self.resource_forks = 1
@@ -435,27 +387,268 @@ rdiff-backup-data/chars_to_quote.
 		try:
 			tmp_rp.chmod(07000)
 			tmp_rp.chmod(07777)
-		except (OSError, IOError), e: self.high_perms = 0
+		except (OSError, IOError): self.high_perms = 0
 		else: self.high_perms = 1
 		tmp_rp.delete()
 
-def get_fsabilities_readonly(desc_string, rp):
-	"""Return an FSAbilities object with given description_string
+def get_readonly_fsa(desc_string, rp):
+	"""Return an fsa with given description_string
 
-	Will be initialized read_only with given RPath rp.
+	Will be initialized read_only with given RPath rp.  We separate
+	this out into a separate function so the request can be vetted by
+	the security module.
 
 	"""
 	return FSAbilities(desc_string).init_readonly(rp)
 
-def get_fsabilities_readwrite(desc_string, rb, use_ctq_file = 1, ctq = None):
-	"""Like above but initialize read/write and pass other arguments"""
-	return FSAbilities(desc_string).init_readwrite(
-		rb, use_ctq_file = use_ctq_file, override_chars_to_quote = ctq)
-
-def get_fsabilities_restoresource(rp):
-	"""Used when restoring, get abilities of source directory"""
-	fsa = FSAbilities('source').init_readonly(rp)
-	ctq_rp = rp.append("chars_to_quote")
-	if ctq_rp.lstat(): fsa.chars_to_quote = ctq_rp.get_data()
-	else: fsa.chars_to_quote = ""
-	return fsa
+
+class SetGlobals:
+	"""Various functions for setting Globals vars given FSAbilities above
+
+	Container for BackupSetGlobals and RestoreSetGlobals (don't use directly)
+
+	"""
+	def __init__(self, in_conn, out_conn, src_fsa, dest_fsa):
+		"""Just store some variables for use below"""
+		self.in_conn, self.out_conn = in_conn, out_conn
+		self.src_fsa, self.dest_fsa = src_fsa, dest_fsa
+
+	def set_eas(self):
+		self.update_triple(self.src_fsa.eas, self.dest_fsa.eas,
+						   ('eas_active', 'eas_write', 'eas_conn'))
+
+	def set_acls(self):
+		self.update_triple(self.src_fsa.acls, self.dest_fsa.acls,
+						   ('acls_active', 'acls_write', 'acls_conn'))
+		if Globals.never_drop_acls and not Globals.acls_active:
+			Log.FatalError("--never-drop-acls specified, but ACL support\n"
+						   "missing from destination filesystem")
+
+	def set_resource_forks(self):
+		self.update_triple(self.src_fsa.resource_forks,
+						   self.dest_fsa.resource_forks,
+						   ('resource_forks_active', 'resource_forks_write',
+							'resource_forks_conn'))
+
+	def set_carbonfile(self):
+		self.update_triple(self.src_fsa.carbonfile, self.dest_fsa.carbonfile,
+			  ('carbonfile_active', 'carbonfile_write', 'carbonfile_conn'))
+		if self.src_fsa.carbonfile and not Globals.carbonfile_active:
+			Log("Source may have carbonfile support, but support defaults to "
+				"off.\n  Use --carbonfile to enable.", 5)
+
+	def set_hardlinks(self):
+		if Globals.preserve_hardlinks != 0:
+			SetConnections.UpdateGlobal('preserve_hardlinks',
+										self.dest_fsa.hardlinks)
+
+	def set_fsync_directories(self):
+		SetConnections.UpdateGlobal('fsync_directories',
+									self.dest_fsa.fsync_dirs)
+
+	def set_change_ownership(self):
+		SetConnections.UpdateGlobal('change_ownership',
+									self.dest_fsa.ownership)
+
+	def set_high_perms(self):
+		if not self.dest_fsa.high_perms:
+			SetConnections.UpdateGlobal('permission_mask', 0777)
+
+
+class BackupSetGlobals(SetGlobals):
+	"""Functions for setting fsa related globals for backup session"""
+	def update_triple(self, src_support, dest_support, attr_triple):
+		"""Many of the settings have a common form we can handle here"""
+		active_attr, write_attr, conn_attr = attr_triple
+		if Globals.get(active_attr) == 0: return # don't override 0
+		for attr in attr_triple: SetConnections.UpdateGlobal(attr, None)
+		if not src_support: return # if source doesn't support, nothing
+		SetConnections.UpdateGlobal(active_attr, 1)
+		self.in_conn.Globals.set_local(conn_attr, 1)
+		if dest_support:
+			SetConnections.UpdateGlobal(write_attr, 1)
+			self.out_conn.Globals.set_local(conn_attr, 1)
+
+	def set_chars_to_quote(self, rbdir):
+		"""Set chars_to_quote setting for backup session
+
+		Unlike the other options, the chars_to_quote setting also
+		depends on the current settings in the rdiff-backup-data
+		directory, not just the current fs features.
+
+		"""
+		ctq = self.compare_ctq_file(rbdir, self.get_ctq_from_fsas())
+
+		SetConnections.UpdateGlobal('chars_to_quote', ctq)
+		if Globals.chars_to_quote: FilenameMapping.set_init_quote_vals()
+
+	def get_ctq_from_fsas(self):
+		"""Determine chars_to_quote just from filesystems, no ctq file"""
+		if not self.src_fsa.case_sensitive and self.dest_fsa.case_sensitive:
+			if self.dest_fsa.extended_filenames:
+				return "A-Z;" # Quote upper case and quoting char
+			else: return "^a-z0-9_ -." # quote everything but basic chars
+
+		if self.dest_fsa.extended_filenames:
+			return "" # Don't quote anything
+		else: return "^A-Za-z0-9_ -."
+
+	def compare_ctq_file(self, rbdir, suggested_ctq):
+		"""Compare ctq file with suggested result, return actual ctq"""
+		ctq_rp = rbdir.append("chars_to_quote")
+		if not ctq_rp.lstat():
+			if Globals.chars_to_quote is None: actual_ctq = suggested_ctq
+			else: actual_ctq = Globals.chars_to_quote
+			ctq_rp.write_string(actual_ctq)
+			return actual_ctq
+
+		if Globals.chars_to_quote is None: actual_ctq = ctq_rp.get_data()
+		else: actual_ctq = Globals.chars_to_quote # Globals override
+
+		if actual_ctq == suggested_ctq: return actual_ctq
+		if suggested_ctq == "":
+			log.Log("Warning: File system no longer needs quoting, "
+					"but we will retain for backwards compatibility.", 2)
+			return actual_ctq
+		log.Log.FatalError("""New quoting requirements!
+
+The quoting this session appears to need do not match those in
+
+%s
+
+This may be caused when you copy an rdiff-backup repository from a
+normal file system onto a windows one that cannot support the same
+characters, or if you backup a case-sensitive file system onto a
+case-insensitive one that previously only had case-insensitive ones
+backed up onto it.
+
+If you want to risk it, remove the file
+rdiff-backup-data/chars_to_quote.""" % (ctq_rp.path,))
+
+
+class RestoreSetGlobals(SetGlobals):
+	"""Functions for setting fsa-related globals for restore session"""
+	def update_triple(self, src_support, dest_support, attr_triple):
+		"""Update global settings for feature based on fsa results
+
+		This is slightly different from BackupSetGlobals.update_triple
+		because (using the mirror_metadata file) rpaths from the
+		source may have more information than the file system
+		supports.
+
+		"""
+		active_attr, write_attr, conn_attr = attr_triple
+		if Globals.get(active_attr) == 0: return # don't override 0
+		for attr in attr_triple: SetConnections.UpdateGlobal(attr, None)
+		if not dest_support: return # if dest doesn't support, do nothing
+		SetConnections.UpdateGlobal(active_attr, 1)
+		self.out_conn.Globals.set_local(conn_attr, 1)
+		self.out_conn.Globals.set_local(write_attr, 1)
+		if src_support: self.in_conn.Globals.set_local(conn_attr, 1)
+
+	def set_chars_to_quote(self, rbdir):
+		"""Set chars_to_quote from rdiff-backup-data dir"""
+		if Globals.chars_to_quote is not None: return # already overridden
+		
+		ctq_rp = rbdir.append("chars_to_quote")
+		if ctq_rp.lstat():
+			SetConnections.UpdateGlobal("chars_to_quote", ctq_rp.get_data())
+		else:
+			log.Log("Warning: chars_to_quote file not found,\n"
+					"assuming no quoting in backup repository.", 2)
+			SetConnections.UpdateGlobal("chars_to_quote", "")
+
+
+class SingleSetGlobals(RestoreSetGlobals):
+	"""For setting globals when dealing only with one filesystem"""
+	def __init__(self, conn, fsa):
+		self.conn = conn
+		self.dest_fsa = fsa
+
+	def update_triple(self, fsa_support, attr_triple):
+		"""Update global vars from single fsa test"""
+		active_attr, write_attr, conn_attr = attr_triple
+		if Globals.get(active_attr) == 0: return # don't override 0
+		for attr in attr_triple: SetConnections.UpdateGlobal(attr, None)
+		if not fsa_support: return
+		SetConnections.UpdateGlobal(active_attr, 1)
+		SetConnections.UpdateGlobal(write_attr, 1)
+		self.conn.Globals.set_local(conn_attr, 1)
+
+	def set_eas(self):
+		self.update_triple(self.dest_fsa.eas,
+						   ('eas_active', 'eas_write', 'eas_conn'))
+	def set_acls(self):
+		self.update_triple(self.dest_fsa.acls,
+						  ('acls_active', 'acls_write', 'acls_conn'))
+	def set_resource_forks(self):
+		self.update_triple(self.dest_fsa.resource_forks,
+						   ('resource_forks_active',
+							'resource_forks_write', 'resource_forks_conn'))
+	def set_carbonfile(self):
+		self.update_triple(self.dest_fsa.carbonfile,
+			 ('carbonfile_active', 'carbonfile_write', 'carbonfile_conn'))
+
+
+def backup_set_globals(rpin):
+	"""Given rps for source filesystem and repository, set fsa globals
+
+	This should be run on the destination connection, because we may
+	need to write a new chars_to_quote file.
+
+	"""
+	assert Globals.rbdir.conn is Globals.local_connection
+	src_fsa = rpin.conn.fs_abilities.get_readonly_fsa('source', rpin)
+	log.Log(str(src_fsa), 4)
+	dest_fsa = FSAbilities('destination').init_readwrite(Globals.rbdir)
+	log.Log(str(dest_fsa), 4)
+
+	bsg = BackupSetGlobals(rpin.conn, Globals.rbdir.conn, src_fsa, dest_fsa)
+	bsg.set_eas()
+	bsg.set_acls()
+	bsg.set_resource_forks()
+	bsg.set_carbonfile()
+	bsg.set_hardlinks()
+	bsg.set_fsync_directories()
+	bsg.set_change_ownership()
+	bsg.set_high_perms()
+	bsg.set_chars_to_quote(Globals.rbdir)
+
+def restore_set_globals(rpout):
+	"""Set fsa related globals for restore session, given in/out rps"""
+	assert rpout.conn is Globals.local_connection
+	src_fsa = Globals.rbdir.conn.fs_abilities.get_readonly_fsa(
+		                  'rdiff-backup repository', Globals.rbdir)
+	log.Log(str(src_fsa), 4)
+	dest_fsa = FSAbilities('restore target').init_readwrite(rpout)
+	log.Log(str(dest_fsa), 4)
+
+	rsg = RestoreSetGlobals(Globals.rbdir.conn, rpout.conn, src_fsa, dest_fsa)
+	rsg.set_eas()
+	rsg.set_acls()
+	rsg.set_resource_forks()
+	rsg.set_carbonfile()
+	rsg.set_hardlinks()
+	# No need to fsync anything when restoring
+	rsg.set_change_ownership()
+	rsg.set_high_perms()
+	rsg.set_chars_to_quote(Globals.rbdir)
+
+def single_set_globals(rp, read_only = None):
+	"""Set fsa related globals for operation on single filesystem"""
+	if read_only:
+		fsa = rp.conn.fs_abilities.get_readonly_fsa(rp.path, rp)
+	else: fsa = FSAbilities(rp.path).init_readwrite(rp)
+	log.Log(str(fsa), 4)
+
+	ssg = SingleSetGlobals(rp.conn, fsa)
+	ssg.set_eas()
+	ssg.set_acls()
+	ssg.set_resource_forks()
+	ssg.set_carbonfile()
+	if not read_only:
+		ssg.set_hardlinks()
+		ssg.set_change_ownership()
+		ssg.set_high_perms()
+	ssg.set_chars_to_quote(Globals.rbdir)
+

trunk/rdiff-backup/rdiff_backup/rorpiter.py

@@ -164,10 +164,6 @@ def FillInIter(rpiter, rootrp):
 	(2,5).  This is used when we need to process directories before or
 	after processing a file in that directory.
 
-	If start_index is given, start with start_index instead of ().
-	The indicies of rest of the rorps should also start with
-	start_index.
-
 	"""
 	# Handle first element as special case
 	first_rp = rpiter.next() # StopIteration gets passed upwards

trunk/rdiff-backup/rdiff_backup/rpath.py

@@ -155,7 +155,8 @@ def copy_attribs(rpin, rpout):
 	if Globals.change_ownership:
 		rpout.chown(*rpout.conn.user_group.map_rpath(rpin))
 	if rpin.issym(): return # symlinks don't have times or perms
-	if Globals.resource_forks_write and rpin.isreg():
+	if (Globals.resource_forks_write and rpin.isreg() and
+		rpin.has_resource_fork()):
 		rpout.write_resource_fork(rpin.get_resource_fork())
 	if Globals.carbonfile_write and rpin.isreg():
 		rpout.write_carbonfile(rpin.get_carbonfile())
@@ -176,7 +177,8 @@ def copy_attribs_inc(rpin, rpout):
 	check_for_files(rpin, rpout)
 	if Globals.change_ownership: apply(rpout.chown, rpin.getuidgid())
 	if rpin.issym(): return # symlinks don't have times or perms
-	if Globals.resource_forks_write and rpin.isreg() and rpout.isreg():
+	if (Globals.resource_forks_write and rpin.isreg() and
+		rpin.has_resource_fork() and rpout.isreg()):
 		rpout.write_resource_fork(rpin.get_resource_fork())
 	if Globals.carbonfile_write and rpin.isreg() and rpout.isreg():
 		rpout.write_carbonfile(rpin.get_carbonfile())
@@ -604,7 +606,10 @@ class RORPath:
 
 	def get_acl(self):
 		"""Return access control list object from dictionary"""
-		return self.data['acl']
+		try: return self.data['acl']
+		except KeyError:
+			acl = self.data['acl'] = get_blank_acl(self.index)
+			return acl
 
 	def set_ea(self, ea):
 		"""Record extended attributes in dictionary.  Does not write"""
@@ -612,7 +617,10 @@ class RORPath:
 
 	def get_ea(self):
 		"""Return extended attributes object"""
-		return self.data['ea']
+		try: return self.data['ea']
+		except KeyError:
+			ea = self.data['ea'] = get_blank_ea(self.index)
+			return ea
 
 	def has_carbonfile(self):
 		"""True if rpath has a carbonfile parameter"""
@@ -1243,7 +1251,9 @@ def setdata_local(rpath):
 	if Globals.carbonfile_conn and rpath.isreg(): rpath.get_carbonfile()
 
 
-# These two are overwritten by the eas_acls.py module.  We can't
+# These functions are overwritten by the eas_acls.py module.  We can't
 # import that module directly because of circular dependency problems.
 def acl_get(rp): assert 0
+def get_blank_acl(index): assert 0
 def ea_get(rp): assert 0
+def get_blank_ea(index): assert 0