Some older systems don't support unicode, so check for support before using it.

git-svn-id: http://svn.savannah.nongnu.org/svn/rdiff-backup@1054 2b77aa54-bcbc-44c9-a7ec-4f6cf2b41109

trunk/rdiff-backup/rdiff_backup/FilenameMapping.py

@@ -159,7 +159,7 @@ class QuotedRPath(rpath.RPath):
 
 		"""
 		path = self.path
-		if type(path) != unicode:
+		if type(path) != unicode and Globals.use_unicode_paths:
 			path = unicode(path, 'utf-8')
 		return map(unquote, self.conn.os.listdir(path))
 

trunk/rdiff-backup/rdiff_backup/Globals.py

@@ -159,6 +159,10 @@ rbdir = None
 chars_to_quote = None
 quoting_char = ';'
 
+
+# Some systems don't support unicode in filenames
+use_unicode_paths = None
+
 # If true, the timestamps use the following format: "2008-09-01T04-49-04-07-00"
 # (instead of "2008-09-01T04:49:04-07:00"). This creates timestamps which
 # don't need to be escaped on Windows.

trunk/rdiff-backup/rdiff_backup/eas_acls.py

@@ -38,6 +38,11 @@ import static, Globals, eas_acls, connection, metadata, rorpiter, log, C, \
 # triggers a warning.
 dropped_acl_names = {}
 
+def encode(str_):
+	if type(str_) == unicode:
+		return str_.encode('utf-8')
+	return str_
+
 class ExtendedAttributes:
 	"""Hold a file's extended attribute information"""
 	def __init__(self, index, attr_dict = None):
@@ -57,7 +62,7 @@ class ExtendedAttributes:
 	def read_from_rp(self, rp):
 		"""Set the extended attributes from an rpath"""
 		try:
-			attr_list = rp.conn.xattr.listxattr(rp.path.encode('utf-8'),
+			attr_list = rp.conn.xattr.listxattr(encode(rp.path),
 												rp.issym())
 		except IOError, exc:
 			if exc[0] in (errno.EOPNOTSUPP, errno.EPERM, errno.ETXTBSY):
@@ -75,7 +80,7 @@ class ExtendedAttributes:
 				continue
 			try:
 				self.attr_dict[attr] = \
-					rp.conn.xattr.getxattr(rp.path.encode('utf-8'),
+					rp.conn.xattr.getxattr(encode(rp.path),
 											attr, rp.issym())
 			except IOError, exc:
 				# File probably modified while reading, just continue
@@ -88,10 +93,10 @@ class ExtendedAttributes:
 	def clear_rp(self, rp):
 		"""Delete all the extended attributes in rpath"""
 		try:
-			for name in rp.conn.xattr.listxattr(rp.path.encode('utf-8'),
+			for name in rp.conn.xattr.listxattr(encode(rp.path),
 												rp.issym()):
 				try:
-					rp.conn.xattr.removexattr(rp.path.encode('utf-8'),
+					rp.conn.xattr.removexattr(encode(rp.path),
 											name, rp.issym())
 				except IOError, exc:
 					# SELinux attributes cannot be removed, and we don't want
@@ -115,7 +120,7 @@ class ExtendedAttributes:
 		self.clear_rp(rp)
 		for (name, value) in self.attr_dict.iteritems():
 			try:
-				rp.conn.xattr.setxattr(rp.path.encode('utf-8'), name,
+				rp.conn.xattr.setxattr(encode(rp.path), name,
 										value, 0, rp.issym())
 			except IOError, exc:
 				# Mac and Linux attributes have different namespaces, so
@@ -154,13 +159,13 @@ def ea_compare_rps(rp1, rp2):
 
 def EA2Record(ea):
 	"""Convert ExtendedAttributes object to text record"""
-	str_list = ['# file: %s' % C.acl_quote(ea.get_indexpath().encode('utf-8'))]
+	str_list = ['# file: %s' % C.acl_quote(encode(ea.get_indexpath()))]
 	for (name, val) in ea.attr_dict.iteritems():
 		if not val: str_list.append(name)
 		else:
 			encoded_val = base64.encodestring(val).replace('\n', '')
 			try:
-				str_list.append('%s=0s%s' % (C.acl_quote(name.encode('utf-8')),
+				str_list.append('%s=0s%s' % (C.acl_quote(encode(name)),
 											encoded_val))
 			except UnicodeEncodeError:
 				log.Log("Warning: unable to store Unicode extended attribute %s"
@@ -175,7 +180,7 @@ def Record2EA(record):
 		raise metadata.ParsingError("Bad record beginning: " + first[:8])
 	filename = first[8:]
 	if filename == '.': index = ()
-	else: index = tuple(unicode(C.acl_unquote(filename.encode('utf-8')),
+	else: index = tuple(unicode(C.acl_unquote(encode(filename)),
 											'utf-8').split('/'))
 	ea = ExtendedAttributes(index)
 
@@ -201,7 +206,7 @@ class EAExtractor(metadata.FlatExtractor):
 	def filename_to_index(self, filename):
 		"""Convert possibly quoted filename to index tuple"""
 		if filename == '.': return ()
-		else: return tuple(C.acl_unquote(filename.encode('utf-8')).split('/'))
+		else: return tuple(C.acl_unquote(encode(filename)).split('/'))
 
 class ExtendedAttributesFile(metadata.FlatFile):
 	"""Store/retrieve EAs from extended_attributes file"""
@@ -386,7 +391,7 @@ def set_rp_acl(rp, entry_list = None, default_entry_list = None,
 	else: acl = posix1e.ACL()
 
 	try:
-		acl.applyto(rp.path.encode('utf-8'))
+		acl.applyto(encode(rp.path))
 	except IOError, exc:
 		if exc[0] == errno.EOPNOTSUPP:
 			log.Log("Warning: unable to set ACL on %s: %s" % 
@@ -398,12 +403,12 @@ def set_rp_acl(rp, entry_list = None, default_entry_list = None,
 		if default_entry_list:
 			def_acl = list_to_acl(default_entry_list, map_names)
 		else: def_acl = posix1e.ACL()
-		def_acl.applyto(rp.path.encode('utf-8'), posix1e.ACL_TYPE_DEFAULT)
+		def_acl.applyto(encode(rp.path), posix1e.ACL_TYPE_DEFAULT)
 
 def get_acl_lists_from_rp(rp):
 	"""Returns (acl_list, def_acl_list) from an rpath.  Call locally"""
 	assert rp.conn is Globals.local_connection
-	try: acl = posix1e.ACL(file=rp.path.encode('utf-8'))
+	try: acl = posix1e.ACL(file=encode(rp.path))
 	except IOError, exc:
 		if exc[0] == errno.EOPNOTSUPP:
 			acl = None
@@ -413,7 +418,7 @@ def get_acl_lists_from_rp(rp):
 			acl = None
 		else: raise
 	if rp.isdir():
-		try: def_acl = posix1e.ACL(filedef=rp.path.encode('utf-8'))
+		try: def_acl = posix1e.ACL(filedef=encode(rp.path))
 		except IOError, exc:
 			if exc[0] == errno.EOPNOTSUPP:
 				def_acl = None
@@ -541,7 +546,7 @@ def acl_compare_rps(rp1, rp2):
 def ACL2Record(acl):
 	"""Convert an AccessControlLists object into a text record"""
 	return '# file: %s\n%s\n' % \
-		(C.acl_quote(acl.get_indexpath().encode('utf-8')), str(acl))
+		(C.acl_quote(encode(acl.get_indexpath())), str(acl))
 
 def Record2ACL(record):
 	"""Convert text record to an AccessControlLists object"""
@@ -551,7 +556,7 @@ def Record2ACL(record):
 		raise metadata.ParsingError("Bad record beginning: "+ first_line)
 	filename = first_line[8:]
 	if filename == '.': index = ()
-	else: index = tuple(unicode(C.acl_unquote(filename.encode('utf-8')),
+	else: index = tuple(unicode(C.acl_unquote(encode(filename)),
 						'utf-8').split('/'))
 	return AccessControlLists(index, record[newline_pos:])
 

trunk/rdiff-backup/rdiff_backup/fs_abilities.py

@@ -34,6 +34,7 @@ import Globals, log, TempFile, selection, robust, SetConnections, \
 class FSAbilities:
 	"""Store capabilities of given file system"""
 	extended_filenames = None # True if filenames can have non-ASCII chars
+	unicode_filenames = None # True if we can use unicode in paths
 	win_reserved_filenames = None # True if filenames can't have ",*,: etc.
 	case_sensitive = None # True if "foobar" and "FoObAr" are different files
 	ownership = None # True if chown works on this filesystem
@@ -97,7 +98,8 @@ class FSAbilities:
 							  ('Symlink permissions', self.symlink_perms),
 							  ('Extended filenames', self.extended_filenames),
 							  ('Windows reserved filenames',
-							   self.win_reserved_filenames)])
+							   self.win_reserved_filenames),
+							  ('Unicode filenames', self.unicode_filenames)])
 		add_boolean_list([('Access control lists', self.acls),
 						  ('Extended attributes', self.eas),
 						  ('Windows access control lists', self.win_acls),
@@ -153,6 +155,7 @@ class FSAbilities:
 
 		self.set_extended_filenames(subdir)
 		self.set_win_reserved_filenames(subdir)
+		self.set_unicode_filenames(subdir)
 		self.set_case_sensitive_readwrite(subdir)
 		self.set_ownership(subdir)
 		self.set_hardlinks(subdir)
@@ -268,6 +271,24 @@ class FSAbilities:
 			else:
 				self.win_reserved_filenames = 0
 
+	def set_unicode_filenames(self, subdir):
+		"""Set self.unicode_filenames by trying to write a path"""
+		assert not self.read_only
+
+		# Try a unicode filename
+		unicode_filename = u'\u3046\u3069\u3093\u5c4b.txt'
+		unicode_rp = None
+		try:
+			unicode_rp = subdir.append(unicode_filename)
+			unicode_rp.touch()
+		except UnicodeEncodeError:
+			if unicode_rp: assert not unicode_rp.lstat()
+			self.unicode_filenames = 0
+		else:
+			assert unicode_rp.lstat()
+			unicode_rp.delete()
+			self.unicode_filenames = 1
+
 	def set_acls(self, rp):
 		"""Set self.acls based on rp.  Does not write.  Needs to be local"""
 		assert Globals.local_connection is rp.conn
@@ -317,6 +338,7 @@ class FSAbilities:
 			where the list is of the directory containing rp.
 
 			"""
+			subdir.path = unicode(subdir.path)
 			l = robust.listrp(subdir)
 			for filename in l:
 				if filename != filename.swapcase():
@@ -679,6 +701,10 @@ class SetGlobals:
 		self.update_triple(self.src_fsa.win_acls, self.dest_fsa.win_acls,
 			  ('win_acls_active', 'win_acls_write', 'win_acls_conn'))
 
+	def set_unicode_filenames(self):
+		SetConnections.UpdateGlobal('use_unicode_paths',
+									self.dest_fsa.unicode_filenames)
+
 	def set_resource_forks(self):
 		self.update_triple(self.src_fsa.resource_forks,
 						   self.dest_fsa.resource_forks,
@@ -963,6 +989,7 @@ def backup_set_globals(rpin, force):
 	bsg.set_eas()
 	bsg.set_acls()
 	bsg.set_win_acls()
+	bsg.set_unicode_filenames()
 	bsg.set_resource_forks()
 	bsg.set_carbonfile()
 	bsg.set_hardlinks()
@@ -991,6 +1018,7 @@ def restore_set_globals(rpout):
 	rsg.set_eas()
 	rsg.set_acls()
 	rsg.set_win_acls()
+	rsg.set_unicode_filenames()
 	rsg.set_resource_forks()
 	rsg.set_carbonfile()
 	rsg.set_hardlinks()

trunk/rdiff-backup/rdiff_backup/rpath.py

@@ -309,23 +309,23 @@ def make_file_dict_python(filename):
 	data = {}
 	mode = statblock[stat.ST_MODE]
 
-	if stat.S_ISREG(mode): type = 'reg'
-	elif stat.S_ISDIR(mode): type = 'dir'
+	if stat.S_ISREG(mode): type_ = 'reg'
+	elif stat.S_ISDIR(mode): type_ = 'dir'
 	elif stat.S_ISCHR(mode):
-		type = 'dev'
+		type_ = 'dev'
 		s = statblock.st_rdev
 		data['devnums'] = ('c',) + (s >> 8, s & 0xff)
 	elif stat.S_ISBLK(mode):
-		type = 'dev'
+		type_ = 'dev'
 		s = statblock.st_rdev
 		data['devnums'] = ('b',) + (s >> 8, s & 0xff)
-	elif stat.S_ISFIFO(mode): type = 'fifo'
+	elif stat.S_ISFIFO(mode): type_ = 'fifo'
 	elif stat.S_ISLNK(mode):
-		type = 'sym'
+		type_ = 'sym'
 		data['linkname'] = os.readlink(filename)
-	elif stat.S_ISSOCK(mode): type = 'sock'
+	elif stat.S_ISSOCK(mode): type_ = 'sock'
 	else: raise C.UnknownFileError(filename)
-	data['type'] = type
+	data['type'] = type_
 	data['size'] = statblock[stat.ST_SIZE]
 	data['perms'] = stat.S_IMODE(mode)
 	data['uid'] = statblock[stat.ST_UID]
@@ -335,12 +335,16 @@ def make_file_dict_python(filename):
 	data['nlink'] = statblock[stat.ST_NLINK]
 
 	if os.name == 'nt':
-		attribs = win32file.GetFileAttributesW(filename)
+		global type
+		if type(filename) == unicode:
+			attribs = win32file.GetFileAttributesW(filename)
+		else:
+			attribs = win32file.GetFileAttributes(filename)
 		if attribs & winnt.FILE_ATTRIBUTE_REPARSE_POINT:
 			data['type'] = 'sym'
 			data['linkname'] = None
 
-	if not (type == 'sym' or type == 'dev'):
+	if not (type_ == 'sym' or type_ == 'dev'):
 		# mtimes on symlinks and dev files don't work consistently
 		data['mtime'] = long(statblock[stat.ST_MTIME])
 		data['atime'] = long(statblock[stat.ST_ATIME])
@@ -1000,7 +1004,7 @@ class RPath(RORPath):
 		path = self.path
 		# Use pass in unicode to os.listdir, so that the returned
 		# entries are in unicode.
-		if type(path) != unicode:
+		if type(path) != unicode and Globals.use_unicode_paths:
 			path = unicode(path, 'utf-8')
 		return self.conn.os.listdir(path)