Many changes - added extended attribute support and file system

ability detection


git-svn-id: http://svn.savannah.nongnu.org/svn/rdiff-backup@334 2b77aa54-bcbc-44c9-a7ec-4f6cf2b41109

trunk/rdiff-backup/TODO

+Use ctime to check whether files have been changed
 
+Include some option to summarize space taken up
 
 ---------[ Medium term ]---------------------------------------
 

trunk/rdiff-backup/rdiff-backup.1

@@ -56,23 +56,14 @@ ability to restore previous versions of that file.
 .SH OPTIONS
 .TP
 .B -b, --backup-mode
-Force backup mode even if first argument appears to be an increment file.
+Force backup mode even if first argument appears to be an increment or
+mirror file.
 .TP
 .B --calculate-average
 Enter calculate average mode.  The arguments should be a number of
 statistics files.  rdiff-backup will print the average of the listed
 statistics files and exit.
 .TP
-.BI "--chars-to-quote " chars
-If this option is set, any characters in
-.I chars
-present in filenames on the source side will be quoted on the
-destination side, so that they do not appear in filenames on the
-remote side.  See
-.B --quoting-char
-and
-.BR --windows-mode .
-.TP
 .B --check-destination-dir
 If an rdiff-backup session fails, running rdiff-backup with this
 option on the destination dir will undo the failed directory.  This
@@ -132,8 +123,7 @@ See the
 section for more information.
 .TP
 .B --exclude-special-files
-Exclude all device files, fifos, sockets, and symlinks.  This option
-is implied by --windows-mode.
+Exclude all device files, fifos, sockets, and symlinks.
 .TP
 .B --force
 Authorize the updating or overwriting of a destination path.
@@ -202,7 +192,7 @@ In this mode rdiff-backup is similar to rsync (but usually
 slower).
 .TP
 .B --no-compare-inode
-This relative esoteric option prevents rdiff-backup from flagging a
+This relatively esoteric option prevents rdiff-backup from flagging a
 file as changed when its inode changes.  This option may be useful if
 you are backing up two different directories to the same rdiff-backup
 destination directory.  The downside is that hard link information may
@@ -255,13 +245,6 @@ session statistics file.  See the
 .B STATISTICS
 section for more information.
 .TP
-.BI "--quoting-char " char
-Use the specified character for quoting characters specified to be
-escaped by the
-.B --chars-to-quote
-option.  The default is the semicolon ";".  See also
-.BR --windows-mode .
-.TP
 .BI "-r, --restore-as-of " restore_time
 Restore the specified directory as it was as of
 .IR restore_time .
@@ -273,10 +256,6 @@ and see the
 .B RESTORING
 section for more information on restoring.
 .TP
-.BI "--remote-cmd " command
-This command has been depreciated as of version 0.4.1.  Use
---remote-schema instead.
-.TP
 .BI "--remote-schema " schema
 Specify an alternate method of connecting to a remote computer.  This
 is necessary to get rdiff-backup not to use ssh for remote backups, or
@@ -354,20 +333,6 @@ is noisiest).  This determines how much is written to the log file.
 .TP
 .B "-V, --version"
 Print the current version and exit
-.TP
-.B --windows-mode
-This option quotes characters not allowable on windows, and does not
-try to preserve ownership, hardlinks, or permissions on the
-destination side.  It is appropriate when backing up a normal unix
-file system to a windows one such as VFS, or a file system with
-similar limitations.  Because metadata is stored in a separate regular
-file, this option does not prevent all data from being restored.
-.TP
-.B --windows-restore
-This option turns on windows quoting, but does not disable
-permissions, hard linking, or ownership.  Use this when restoring from
-an rdiff-backup directory on a windows file system to a unix file
-system.
 
 .SH EXAMPLES
 Simplest case---backup directory foo to directory bar, with increments

trunk/rdiff-backup/rdiff_backup/FilenameMapping.py

-# Copyright 2002 Ben Escoto
+# Copyright 2002, 2003 Ben Escoto
 #
 # This file is part of rdiff-backup.
 #
@@ -29,7 +29,7 @@ handle that error.)
 
 """
 
-import re
+import re, types
 import Globals, log, rpath
 
 max_filename_length = 255
@@ -66,12 +66,14 @@ def set_init_quote_vals_local():
 def init_quoting_regexps():
 	"""Compile quoting regular expressions"""
 	global chars_to_quote_regexp, unquoting_regexp
+	assert chars_to_quote and type(chars_to_quote) is types.StringType, \
+		   "Chars to quote: '%s'" % (chars_to_quote,)
 	try:
 		chars_to_quote_regexp = \
 				 re.compile("[%s]|%s" % (chars_to_quote, quoting_char), re.S)
 		unquoting_regexp = re.compile("%s[0-9]{3}" % quoting_char, re.S)
 	except re.error:
-		log.Log.FatalError("Error '%s' when processing char quote list %s" %
+		log.Log.FatalError("Error '%s' when processing char quote list '%s'" %
 						   (re.error, chars_to_quote))
 
 def quote(path):
@@ -131,8 +133,16 @@ class QuotedRPath(rpath.RPath):
 
 	def isincfile(self):
 		"""Return true if path indicates increment, sets various variables"""
-		result = rpath.RPath.isincfile(self)
-		if result: self.inc_basestr = unquote(self.inc_basestr)
+		if not self.index: # consider the last component as quoted
+			dirname, basename = self.dirsplit()
+			temp_rp = rpath.RPath(self.conn, dirname, (unquote(basename),))
+			result = temp_rp.isincfile()
+			if result:
+				self.inc_basestr = unquote(temp_rp.inc_basestr)
+				self.inc_timestr = unquote(temp_rp.inc_timestr)
+		else:
+			result = rpath.RPath.isincfile(self)
+			if result: self.inc_basestr = unquote(self.inc_basestr)
 		return result
 		
 def get_quotedrpath(rp, separate_basename = 0):

trunk/rdiff-backup/rdiff_backup/Globals.py

@@ -66,6 +66,14 @@ change_mirror_perms = (process_uid != 0)
 # If true, try to reset the atimes of the source partition.
 preserve_atime = None
 
+# If true, save the extended attributes when backing up.
+read_eas = None
+
+# If true, preserve the extended attributes on the mirror directory
+# when backing up, or write them to the restore directory.  This
+# implies read_eas.
+write_eas = None
+
 # This will be set as soon as the LocalConnection class loads
 local_connection = None
 
@@ -112,10 +120,12 @@ rbdir = None
 
 # quoting_enabled is true if we should quote certain characters in
 # filenames on the source side (see FilenameMapping for more
-# info).  chars_to_quote is a string whose characters should be
-# quoted, and quoting_char is the character to quote with.
-quoting_enabled = None
-chars_to_quote = "A-Z:"
+# info).
+
+# chars_to_quote is a string whose characters should be quoted.  It
+# should be true if certain characters in filenames on the source side
+# should be escaped (see FilenameMapping for more info).
+chars_to_quote = None
 quoting_char = ';'
 
 # If true, emit output intended to be easily readable by a

trunk/rdiff-backup/rdiff_backup/Security.py

@@ -76,8 +76,9 @@ def set_security_level(action, cmdpairs):
 			rdir = tempfile.gettempdir()
 		elif islocal(cp1):
 			sec_level = "read-only"
-			rdir = Main.restore_get_root(rpath.RPath(Globals.local_connection,
-													 getpath(cp1)))[0].path
+			Main.restore_get_root(rpath.RPath(Globals.local_connection,
+											  getpath(cp1)))
+			rdir = Main.restore_root.path
 		else:
 			assert islocal(cp2)
 			sec_level = "all"

trunk/rdiff-backup/rdiff_backup/SetConnections.py

-# Copyright 2002 Ben Escoto
+# Copyright 2002, 2003 Ben Escoto
 #
 # This file is part of rdiff-backup.
 #
@@ -27,7 +27,7 @@ the related connections.
 
 import os
 from log import Log
-import Globals, FilenameMapping, connection, rpath
+import Globals, connection, rpath
 
 # This is the schema that determines how rdiff-backup will open a
 # pipe to the remote system.  If the file is given as A::B, %s will
@@ -178,7 +178,6 @@ def init_connection_settings(conn):
 	conn.log.Log.setterm_verbosity(Log.term_verbosity)
 	for setting_name in Globals.changed_settings:
 		conn.Globals.set(setting_name, Globals.get(setting_name))
-	FilenameMapping.set_init_quote_vals()
 
 def init_connection_remote(conn_number):
 	"""Run on server side to tell self that have given conn_number"""
@@ -203,8 +202,6 @@ def BackupInitConnections(reading_conn, writing_conn):
 	writing_conn.Globals.set("isbackup_writer", 1)
 	UpdateGlobal("backup_reader", reading_conn)
 	UpdateGlobal("backup_writer", writing_conn)
-	if writing_conn.os.getuid() == 0 and Globals.change_ownership != 0:
-		UpdateGlobal('change_ownership', 1)
 
 def CloseConnections():
 	"""Close all connections.  Run by client"""

trunk/rdiff-backup/rdiff_backup/backup.py

-# Copyright 2002 Ben Escoto
+# Copyright 2002, 2003 Ben Escoto
 #
 # This file is part of rdiff-backup.
 #
@@ -22,7 +22,8 @@
 from __future__ import generators
 import errno
 import Globals, metadata, rorpiter, TempFile, Hardlink, robust, increment, \
-	   rpath, static, log, selection, Time, Rdiff, statistics, iterfile
+	   rpath, static, log, selection, Time, Rdiff, statistics, iterfile, \
+	   eas_acls
 
 def Mirror(src_rpath, dest_rpath):
 	"""Turn dest_rpath into a copy of src_rpath"""
@@ -122,16 +123,27 @@ class DestinationStruct:
 		destination except rdiff-backup-data directory.
 
 		"""
-		if use_metadata:
-			metadata_iter = metadata.GetMetadata_at_time(Globals.rbdir,
-														 Time.prevtime)
+		def get_basic_iter():
+			"""Returns iterator of basic metadata"""
+			metadata_iter = metadata.MetadataFile.get_objects_at_time(
+				Globals.rbdir, Time.prevtime)
 			if metadata_iter: return metadata_iter
 			log.Log("Warning: Metadata file not found.\n"
 					"Metadata will be read from filesystem.", 2)
 
-		sel = selection.Select(rpath)
-		sel.parse_rbdir_exclude()
-		return sel.set_iter()
+		def get_iter_from_fs():
+			"""Get the combined iterator from the filesystem"""
+			sel = selection.Select(rpath)
+			sel.parse_rbdir_exclude()
+			return sel.set_iter()
+
+		if use_metadata:
+			if Globals.read_eas:
+				rorp_iter = eas_acls.ExtendedAttributesFile.\
+					  get_combined_iter_at_time(Globals.rbdir, Time.prevtime)
+			else: rorp_iter = get_basic_iter()
+			if rorp_iter: return rorp_iter
+		return get_iter_from_fs()
 
 	def set_rorp_cache(cls, baserp, source_iter, for_increment):
 		"""Initialize cls.CCPP, the destination rorp cache
@@ -243,7 +255,8 @@ class CacheCollatedPostProcess:
 		self.cache_size = cache_size
 		self.statfileobj = statistics.init_statfileobj()
 		if Globals.file_statistics: statistics.FileStats.init()
-		metadata.OpenMetadata()
+		metadata.MetadataFile.open_file()
+		if Globals.read_eas: eas_acls.ExtendedAttributesFile.open_file()
 
 		# the following should map indicies to lists
 		# [source_rorp, dest_rorp, changed_flag, success_flag, increment]
@@ -317,7 +330,10 @@ class CacheCollatedPostProcess:
 			metadata_rorp = source_rorp
 		else: metadata_rorp = None
 		if metadata_rorp and metadata_rorp.lstat():
-			metadata.WriteMetadata(metadata_rorp)
+			metadata.MetadataFile.write_object(metadata_rorp)
+			if Globals.read_eas and not metadata_rorp.get_ea().empty():
+				eas_acls.ExtendedAttributesFile.write_object(
+					metadata_rorp.get_ea())
 		if Globals.file_statistics:
 			statistics.FileStats.update(source_rorp, dest_rorp, changed, inc)
 
@@ -359,7 +375,8 @@ class CacheCollatedPostProcess:
 	def close(self):
 		"""Process the remaining elements in the cache"""
 		while self.cache_indicies: self.shorten_cache()
-		metadata.CloseMetadata()
+		metadata.MetadataFile.close_file()
+		if Globals.read_eas: eas_acls.ExtendedAttributesFile.close_file()
 		if Globals.print_statistics: statistics.print_active_stats()
 		if Globals.file_statistics: statistics.FileStats.close()
 		statistics.write_active_statfileobj()

trunk/rdiff-backup/rdiff_backup/connection.py

@@ -22,6 +22,11 @@
 from __future__ import generators
 import types, os, tempfile, cPickle, shutil, traceback, pickle, \
 	   socket, sys, gzip
+# The following EA and ACL modules may be used if available
+try: import xattr
+except ImportError: pass
+try: import posix1e
+except ImportError: pass
 
 
 class ConnectionError(Exception): pass
@@ -513,7 +518,7 @@ class VirtualFile:
 import Globals, Time, Rdiff, Hardlink, FilenameMapping, C, Security, \
 	   Main, rorpiter, selection, increment, statistics, manage, lazy, \
 	   iterfile, rpath, robust, restore, manage, backup, connection, \
-	   TempFile, SetConnections, librsync, log, regress
+	   TempFile, SetConnections, librsync, log, regress, fs_abilities
 
 Globals.local_connection = LocalConnection()
 Globals.connections.append(Globals.local_connection)

trunk/rdiff-backup/rdiff_backup/eas_acls.py

+# Copyright 2003 Ben Escoto
+#
+# This file is part of rdiff-backup.
+#
+# rdiff-backup is free software; you can redistribute it and/or modify
+# under the terms of the GNU General Public License as published by the
+# Free Software Foundation; either version 2 of the License, or (at your
+# option) any later version.
+#
+# rdiff-backup is distributed in the hope that it will be useful, but
+# WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with rdiff-backup; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
+# USA
+
+"""Store and retrieve extended attributes and access control lists
+
+Not all file systems will have EAs and ACLs, but if they do, store
+this information in separate files in the rdiff-backup-data directory,
+called extended_attributes.<time>.snapshot and
+access_control_lists.<time>.snapshot.
+
+"""
+
+from __future__ import generators
+import base64, errno, re
+import static, Globals, metadata, connection, rorpiter, log
+
+
+class ExtendedAttributes:
+	"""Hold a file's extended attribute information"""
+	def __init__(self, index, attr_dict = None):
+		"""Initialize EA object with no attributes"""
+		self.index = index
+		if attr_dict is None: self.attr_dict = {}
+		else: self.attr_dict = attr_dict
+
+	def __eq__(self, ea):
+		"""Equal if all attributes and index are equal"""
+		assert isinstance(ea, ExtendedAttributes)
+		return ea.index == self.index and ea.attr_dict == self.attr_dict
+
+	def __ne__(self, ea): return not self.__eq__(ea)
+
+	def get_indexpath(self): return self.index and '/'.join(self.index) or '.'
+
+	def read_from_rp(self, rp):
+		"""Set the extended attributes from an rpath"""
+		try: attr_list = rp.conn.xattr.listxattr(rp.path)
+		except IOError, exc:
+			if exc[0] == errno.EOPNOTSUPP: return # if not sup, consider empty
+			raise
+		for attr in attr_list:
+			try: self.attr_dict[attr] = rp.conn.xattr.getxattr(rp.path, attr)
+			except IOError, exc:
+				# File probably modified while reading, just continue
+				if exc[0] == errno.ENODATA: continue
+				elif exc[0] == errno.ENOENT: break
+				else: raise
+
+	def clear_rp(self, rp):
+		"""Delete all the extended attributes in rpath"""
+		for name in rp.conn.xattr.listxattr(rp.path):
+			rp.conn.xattr.removexattr(rp.path, name)
+
+	def write_to_rp(self, rp):
+		"""Write extended attributes to rpath rp"""
+		self.clear_rp(rp)
+		for (name, value) in self.attr_dict.iteritems():
+			rp.conn.xattr.setxattr(rp.path, name, value)
+
+	def get(self, name):
+		"""Return attribute attached to given name"""
+		return self.attr_dict[name]
+
+	def set(self, name, value = ""):
+		"""Set given name to given value.  Does not write to disk"""
+		self.attr_dict[name] = value
+
+	def delete(self, name):
+		"""Delete value associated with given name"""
+		del self.attr_dict[name]
+
+	def empty(self):
+		"""Return true if no extended attributes are set"""
+		return not self.attr_dict
+
+def compare_rps(rp1, rp2):
+	"""Return true if rp1 and rp2 have same extended attributes"""
+	ea1 = ExtendedAttributes(rp1.index)
+	ea1.read_from_rp(rp1)
+	ea2 = ExtendedAttributes(rp2.index)
+	ea2.read_from_rp(rp2)
+	return ea1 == ea2
+
+
+def EA2Record(ea):
+	"""Convert ExtendedAttributes object to text record"""
+	str_list = ['# file: %s' % ea.get_indexpath()]
+	for (name, val) in ea.attr_dict.iteritems():
+		if not val: str_list.append(name)
+		else:
+			encoded_val = base64.encodestring(val).replace('\n', '')
+			str_list.append('%s=0s%s' % (name, encoded_val))
+	return '\n'.join(str_list)+'\n'
+
+def Record2EA(record):
+	"""Convert text record to ExtendedAttributes object"""
+	lines = record.split('\n')
+	first = lines.pop(0)
+	if not first[:8] == "# file: ":
+		raise metadata.ParsingError("Bad record beginning: " + first[:8])
+	filename = first[8:]
+	if filename == '.': index = ()
+	else: index = tuple(filename.split('/'))
+	ea = ExtendedAttributes(index)
+
+	for line in lines:
+		line = line.strip()
+		if not line: continue
+		assert line[0] != '#', line
+		eq_pos = line.find('=')
+		if eq_pos == -1: ea.set(line)
+		else:
+			name = line[:eq_pos]
+			assert line[eq_pos+1:eq_pos+3] == '0s', \
+				   "Currently only base64 encoding supported"
+			encoded_val = line[eq_pos+3:]
+			ea.set(name, base64.decodestring(encoded_val))
+	return ea
+
+def quote_path(path):
+	"""Quote a path for use EA/ACL records.
+
+	Right now no quoting!!!  Change this to reflect the updated
+	quoting style of getfattr/setfattr when they are changed.
+
+	"""
+	return path
+
+
+class EAExtractor(metadata.FlatExtractor):
+	"""Iterate ExtendedAttributes objects from the EA information file"""
+	record_boundary_regexp = re.compile("\\n# file:")
+	record_to_object = staticmethod(Record2EA)
+	def get_index_re(self, index):
+		"""Find start of EA record with given index"""
+		if not index: indexpath = '.'
+		else: indexpath = '/'.join(index)
+		# Right now there is no quoting, due to a bug in
+		# getfacl/setfacl.  Replace later when bug fixed.
+		return re.compile('(^|\\n)(# file: %s\\n)' % indexpath)
+
+class ExtendedAttributesFile(metadata.FlatFile):
+	"""Store/retrieve EAs from extended_attributes file"""
+	_prefix = "extended_attributes"
+	_extractor = EAExtractor
+	_object_to_record = staticmethod(EA2Record)
+
+	def get_combined_iter_at_time(cls, rbdir, rest_time,
+								  restrict_index = None):
+		"""Return an iter of rorps with extended attributes added"""
+		def join_eas(basic_iter, ea_iter):
+			"""Join basic_iter with ea iter"""
+			collated = rorpiter.CollateIterators(basic_iter, ea_iter)
+			for rorp, ea in collated:
+				assert rorp, (rorp, (ea.index, ea.attr_dict), rest_time)
+				if not ea: ea = ExtendedAttributes(rorp.index)
+				rorp.set_ea(ea)
+				yield rorp
+
+		basic_iter = metadata.MetadataFile.get_objects_at_time(
+			Globals.rbdir, rest_time, restrict_index)
+		if not basic_iter: return None
+		ea_iter = cls.get_objects_at_time(rbdir, rest_time, restrict_index)
+		if not ea_iter:
+			log.Log("Warning: Extended attributes file not found", 2)
+			ea_iter = iter([])
+		return join_eas(basic_iter, ea_iter)
+
+static.MakeClass(ExtendedAttributesFile)

trunk/rdiff-backup/rdiff_backup/fs_abilities.py

-# Copyright 2002 Ben Escoto
+# Copyright 2003 Ben Escoto
 #
 # This file is part of rdiff-backup.
 #
@@ -50,12 +50,14 @@ class FSAbilities:
 		Only self.acls and self.eas are set.
 
 		"""
+		self.root_rp = rp
 		self.read_only = 1
 		self.set_eas(rp, 0)
 		self.set_acls(rp)
 		return self
 
-	def init_readwrite(self, rbdir, use_ctq_file = 1):
+	def init_readwrite(self, rbdir, use_ctq_file = 1,
+					   override_chars_to_quote = None):
 		"""Set variables using fs tested at rp_base
 
 		This method creates a temp directory in rp_base and writes to
@@ -69,17 +71,21 @@ class FSAbilities:
 		file in directory.
 
 		"""
-		assert rbdir.isdir()
+		if not rbdir.isdir():
+			assert not rbdir.lstat(), (rbdir.path, rbdir.lstat())
+			rbdir.mkdir()
+		self.root_rp = rbdir
 		self.read_only = 0
 
-		subdir = TempFile.new_in_dir(rbdir)
+		subdir = rbdir.conn.TempFile.new_in_dir(rbdir)
 		subdir.mkdir()
 		self.set_ownership(subdir)
 		self.set_hardlinks(subdir)
 		self.set_fsync_dirs(subdir)
 		self.set_eas(subdir, 1)
 		self.set_acls(subdir)
-		self.set_chars_to_quote(subdir)
+		if override_chars_to_quote is None: self.set_chars_to_quote(subdir)
+		else: self.chars_to_quote = override_chars_to_quote
 		if use_ctq_file: self.compare_chars_to_quote(rbdir)
 		subdir.delete()
 		return self
@@ -95,19 +101,14 @@ class FSAbilities:
 			fp.write(self.chars_to_quote)
 			assert not fp.close()
 
-		def get_old_chars():
-			fp = ctq_rp.open("rb")
-			old_chars = fp.read()
-			assert not fp.close()
-			return old_chars
-
 		if not ctq_rp.lstat(): write_new_chars()
 		else:
-			old_chars = get_old_chars()
+			old_chars = ctq_rp.get_data()
 			if old_chars != self.chars_to_quote:
 				if self.chars_to_quote == "":
 					log.Log("Warning: File system no longer needs quoting, "
 							"but will retain for backwards compatibility.", 2)
+					self.chars_to_quote = old_chars
 				else: log.FatalError("""New quoting requirements
 
 This may be caused when you copy an rdiff-backup directory from a
@@ -127,7 +128,7 @@ rdiff-backup-data/chars_to_quote.
 		except (IOError, OSError), exc:
 			if exc[0] == errno.EPERM:
 				log.Log("Warning: ownership cannot be changed on filesystem "
-						"at device %s" % (testdir.getdevloc(),), 2)
+						"at %s" % (self.root_rp.path,), 2)
 				self.ownership = 0
 			else: raise
 		else: self.ownership = 1
@@ -143,21 +144,15 @@ rdiff-backup-data/chars_to_quote.
 			assert hl_source.getinode() == hl_dest.getinode()
 		except (IOError, OSError), exc:
 			if exc[0] in (errno.EOPNOTSUPP, errno.EPERM):
-				log.Log("Warning: hard linking not supported by filesystem %s"
-						% (testdir.getdevloc(),), 2)
+				log.Log("Warning: hard linking not supported by filesystem "
+						"at %s" % (self.root_rp.path,), 2)
 				self.hardlinks = 0
 			else: raise
 		else: self.hardlinks = 1
 
 	def set_fsync_dirs(self, testdir):
 		"""Set self.fsync_dirs if directories can be fsync'd"""
-		try: testdir.fsync()
-		except (IOError, OSError), exc:
-			log.Log("Warning: Directories on file system at %s are not "
-					"fsyncable.\nAssuming it's unnecessary." %
-					(testdir.getdevloc(),), 2)
-			self.fsync_dirs = 0
-		else: self.fsync_dirs = 1
+		self.fsync_dirs = testdir.conn.fs_abilities.test_fsync_local(testdir)
 
 	def set_chars_to_quote(self, subdir):
 		"""Set self.chars_to_quote by trying to write various paths"""
@@ -189,7 +184,7 @@ rdiff-backup-data/chars_to_quote.
 
 		def sanity_check():
 			"""Make sure basic filenames writable"""
-			for filename in ['5-_ a']:
+			for filename in ['5-_ a.']:
 				rp = subdir.append(filename)
 				rp.touch()
 				assert rp.lstat()
@@ -198,57 +193,70 @@ rdiff-backup-data/chars_to_quote.
 		sanity_check()
 		if is_case_sensitive():
 			if supports_unusual_chars(): self.chars_to_quote = ""
-			else: self.chars_to_quote = "^A-Za-z0-9_ -"
+			else: self.chars_to_quote = "^A-Za-z0-9_ -."
 		else:
 			if supports_unusual_chars(): self.chars_to_quote = "A-Z;"
-			else: self.chars_to_quote = "^a-z0-9_ -"
+			else: self.chars_to_quote = "^a-z0-9_ -."
 
 	def set_acls(self, rp):
 		"""Set self.acls based on rp.  Does not write.  Needs to be local"""
-		assert Globals.local_connection is rp.conn
-		assert rp.lstat()
-		try: import posix1e
-		except ImportError:
-			log.Log("Warning: Unable to import module posix1e from pylibacl "
-					"package.\nACLs not supported on device %s" %
-					(rp.getdevloc(),), 2)
-			self.acls = 0
-			return
-
-		try: posix1e.ACL(file=rp.path)
-		except IOError, exc:
-			if exc[0] == errno.EOPNOTSUPP:
-				log.Log("Warning: ACLs appear not to be supported by "
-						"filesystem on device %s" % (rp.getdevloc(),), 2)
-				self.acls = 0
-			else: raise
-		else: self.acls = 1
+		self.acls = rp.conn.fs_abilities.test_acls_local(rp)
 		
 	def set_eas(self, rp, write):
-		"""Set extended attributes from rp.  Run locally.
-
-		Tests writing if write is true.
-
-		"""
-		assert Globals.local_connection is rp.conn
-		assert rp.lstat()
-		try: import xattr
-		except ImportError:
-			log.Log("Warning: Unable to import module xattr.  ACLs not "
-					"supported on device %s" % (rp.getdevloc(),), 2)
-			self.eas = 0
-			return
-
-		try:
-			xattr.listxattr(rp.path)
-			if write:
-				xattr.setxattr(rp.path, "user.test", "test val")
-				assert xattr.getxattr(rp.path, "user.test") == "test val"
-		except IOError, exc:
-			if exc[0] == errno.EOPNOTSUPP:
-				log.Log("Warning: Extended attributes not supported by "
-						"filesystem on device %s" % (rp.getdevloc(),), 2)
-				self.eas = 0
-			else: raise
-		else: self.eas = 1
+		"""Set extended attributes from rp. Tests writing if write is true."""
+		self.eas = rp.conn.fs_abilities.test_eas_local(rp, write)
+
+
+def test_eas_local(rp, write):
+	"""Test ea support.  Must be called locally.  Usedy by set_eas above."""
+	assert Globals.local_connection is rp.conn
+	assert rp.lstat()
+	try: import xattr
+	except ImportError:
+		log.Log("Warning: Unable to import module xattr.  ACLs not "
+				"supported on filesystem at %s" % (rp.path,), 2)
+		return 0
+
+	try:
+		xattr.listxattr(rp.path)
+		if write:
+			xattr.setxattr(rp.path, "user.test", "test val")
+			assert xattr.getxattr(rp.path, "user.test") == "test val"
+	except IOError, exc:
+		if exc[0] == errno.EOPNOTSUPP:
+			log.Log("Warning: Extended attributes not supported by "
+					"filesystem at %s" % (rp.path,), 2)
+			return 0
+		else: raise
+	else: return 1
+
+def test_acls_local(rp):
+	"""Test acl support.  Call locally.  Does not write."""
+	assert Globals.local_connection is rp.conn
+	assert rp.lstat()
+	try: import posix1e
+	except ImportError:
+		log.Log("Warning: Unable to import module posix1e from pylibacl "
+				"package.\nACLs not supported on filesystem at %s" %
+				(rp.path,), 2)
+		return 0
+
+	try: posix1e.ACL(file=rp.path)
+	except IOError, exc:
+		if exc[0] == errno.EOPNOTSUPP:
+			log.Log("Warning: ACLs appear not to be supported by "
+					"filesystem at %s" % (rp.path,), 2)
+			return 0
+		else: raise
+	else: return 1
+
+def test_fsync_local(rp):
+	"""Test fsyncing directories locally"""
+	assert rp.conn is Globals.local_connection
+	try: rp.fsync()
+	except (IOError, OSError), exc:
+		log.Log("Warning: Directories on file system at %s are not "
+				"fsyncable.\nAssuming it's unnecessary." % (rp.path,), 2)
+		return 0
+	else: return 1
 

trunk/rdiff-backup/rdiff_backup/regress.py

@@ -124,7 +124,8 @@ def iterate_raw_rfs(mirror_rp, inc_rp):
 
 def yield_metadata():
 	"""Iterate rorps from metadata file, if any are available"""
-	metadata_iter = metadata.GetMetadata_at_time(Globals.rbdir, regress_time)
+	metadata_iter = metadata.MetadataFile.get_objects_at_time(Globals.rbdir,
+															  regress_time)
 	if metadata_iter: return metadata_iter
 	log.Log.FatalError("No metadata for time %s found, cannot regress"
 					   % Time.timetopretty(regress_time))

trunk/rdiff-backup/rdiff_backup/restore.py

-# Copyright 2002 Ben Escoto
+# Copyright 2002, 2003 Ben Escoto
 #
 # This file is part of rdiff-backup.
 #
@@ -22,7 +22,7 @@
 from __future__ import generators
 import tempfile, os, cStringIO
 import Globals, Time, Rdiff, Hardlink, rorpiter, selection, rpath, \
-	   log, static, robust, metadata, statistics, TempFile
+	   log, static, robust, metadata, statistics, TempFile, eas_acls
 
 
 # This should be set to selection.Select objects over the source and
@@ -154,8 +154,13 @@ class MirrorStruct:
 
 		"""
 		if rest_time is None: rest_time = _rest_time
-		metadata_iter = metadata.GetMetadata_at_time(Globals.rbdir,
-				 rest_time, restrict_index = cls.mirror_base.index)
+		if Globals.write_eas:
+			metadata_iter = eas_acls.ExtendedAttributesFile.\
+			 get_combined_iter_at_time(
+			 Globals.rbdir, rest_time, restrict_index = cls.mirror_base.index)
+		else:
+			metadata_iter = metadata.MetadataFile.get_objects_at_time(
+			 Globals.rbdir, rest_time, restrict_index = cls.mirror_base.index)
 		if metadata_iter: rorp_iter = metadata_iter
 		elif require_metadata: log.Log.FatalError("Mirror metadata not found")
 		else:

trunk/rdiff-backup/rdiff_backup/rpath.py

-# Copyright 2002 Ben Escoto
+# Copyright 2002, 2003 Ben Escoto
 #
 # This file is part of rdiff-backup.
 #
@@ -156,6 +156,7 @@ def copy_attribs(rpin, rpout):
 	if Globals.change_ownership: apply(rpout.chown, rpin.getuidgid())
 	if Globals.change_permissions: rpout.chmod(rpin.getperms())
 	if not rpin.isdev(): rpout.setmtime(rpin.getmtime())
+	if Globals.write_eas: rpout.write_ea(rpin.get_ea())
 
 def cmp_attribs(rp1, rp2):
 	"""True if rp1 has the same file attributes as rp2
@@ -301,7 +302,8 @@ class RORPath:
 		return 1
 
 	def equal_verbose(self, other, check_index = 1,
-					  compare_inodes = 0, compare_ownership = 0):
+					  compare_inodes = 0, compare_ownership = 0,
+					  compare_eas = 0):
 		"""Like __eq__, but log more information.  Useful when testing"""
 		if check_index and self.index != other.index:
 			log.Log("Index %s != index %s" % (self.index, other.index), 2)
@@ -318,6 +320,7 @@ class RORPath:
 			elif key == 'size' and not self.isreg(): pass
 			elif key == 'inode' and (not self.isreg() or not compare_inodes):
 				pass
+			elif key == 'ea' and not compare_eas: pass
 			elif (not other.data.has_key(key) or
 				  self.data[key] != other.data[key]):
 				if not other.data.has_key(key):
@@ -512,6 +515,14 @@ class RORPath:
 															   self.index)
 			self.file_already_open = None
 
+	def set_ea(self, ea):
+		"""Record extended attributes in dictionary.  Does not write"""
+		self.data['ea'] = ea
+
+	def get_ea(self):
+		"""Return extended attributes object"""
+		return self.data['ea']
+
 
 class RPath(RORPath):
 	"""Remote Path class - wrapper around a possibly non-local pathname
@@ -546,7 +557,7 @@ class RPath(RORPath):
 			else: self.path = "/".join((base,) + index)
 		self.file = None
 		if data or base is None: self.data = data
-		else: self.data = self.conn.C.make_file_dict(self.path)
+		else: self.setdata()
 
 	def __str__(self):
 		return "Path: %s\nIndex: %s\nData: %s" % (self.path, self.index,
@@ -571,6 +582,7 @@ class RPath(RORPath):
 	def setdata(self):
 		"""Set data dictionary using C extension"""
 		self.data = self.conn.C.make_file_dict(self.path)
+		if Globals.read_eas and self.lstat(): self.get_ea()
 
 	def make_file_dict_old(self):
 		"""Create the data dictionary"""
@@ -727,7 +739,7 @@ class RPath(RORPath):
 		log.Log("Deleting %s" % self.path, 7)
 		if self.isdir():
 			try: self.rmdir()
-			except os.error: shutil.rmtree(self.path)
+			except os.error: self.conn.shutil.rmtree(self.path)
 		else: self.conn.os.unlink(self.path)
 		self.setdata()
 
@@ -929,6 +941,24 @@ class RPath(RORPath):
 		assert not fp.close()
 		return s
 
+	def get_ea(self):
+		"""Return extended attributes object, setting if necessary"""
+		try: ea = self.data['ea']
+		except KeyError:
+			ea = eas_acls.ExtendedAttributes(self.index)
+			if not self.issym():
+				# Don't read from symlinks because they will be
+				# followed.  Update this when llistxattr,
+				# etc. available
+				ea.read_from_rp(self)
+			self.data['ea'] = ea
+		return ea
+
+	def write_ea(self, ea):
+		"""Change extended attributes of rp"""
+		ea.write_to_rp(self)
+		self.data['ea'] = ea
+
 
 class RPathFileHook:
 	"""Look like a file, but add closing hook"""
@@ -945,3 +975,4 @@ class RPathFileHook:
 		self.closing_thunk()
 		return result
 
+import eas_acls # Put at end to avoid regress

trunk/rdiff-backup/testing/commontest.py

@@ -3,7 +3,7 @@ import os, sys
 from rdiff_backup.log import Log
 from rdiff_backup.rpath import RPath
 from rdiff_backup import Globals, Hardlink, SetConnections, Main, \
-	 selection, lazy, Time, rpath
+	 selection, lazy, Time, rpath, eas_acls
 
 RBBin = "../rdiff-backup"
 SourceDir = "../rdiff_backup"
@@ -143,7 +143,7 @@ def InternalRestore(mirror_local, dest_local, mirror_dir, dest_dir, time):
 	if inc: Main.Restore(get_increment_rp(mirror_rp, time), dest_rp)
 	else: # use alternate syntax
 		Main.restore_timestr = str(time)
-		Main.RestoreAsOf(mirror_rp, dest_rp)
+		Main.Restore(mirror_rp, dest_rp, restore_as_of = 1)
 	Main.cleanup()
 
 def get_increment_rp(mirror_rp, time):
@@ -166,7 +166,8 @@ def _reset_connections(src_rp, dest_rp):
 
 def CompareRecursive(src_rp, dest_rp, compare_hardlinks = 1,
 					 equality_func = None, exclude_rbdir = 1,
-					 ignore_tmp_files = None, compare_ownership = 0):
+					 ignore_tmp_files = None, compare_ownership = 0,
+					 compare_eas = 0):
 	"""Compare src_rp and dest_rp, which can be directories
 
 	This only compares file attributes, not the actual data.  This
@@ -178,8 +179,8 @@ def CompareRecursive(src_rp, dest_rp, compare_hardlinks = 1,
 	src_rp.setdata()
 	dest_rp.setdata()
 
-	Log("Comparing %s and %s, hardlinks %s" % (src_rp.path, dest_rp.path,
-											   compare_hardlinks), 3)
+	Log("Comparing %s and %s, hardlinks %s, eas %s" %
+		(src_rp.path, dest_rp.path, compare_hardlinks, compare_eas), 3)
 	src_select = selection.Select(src_rp)
 	dest_select = selection.Select(dest_rp)
 
@@ -214,11 +215,17 @@ def CompareRecursive(src_rp, dest_rp, compare_hardlinks = 1,
 		if not src_rorp.equal_verbose(dest_rorp,
 									  compare_ownership = compare_ownership):
 			return None
-		if Hardlink.rorp_eq(src_rorp, dest_rorp): return 1
-		Log("%s: %s" % (src_rorp.index, Hardlink.get_indicies(src_rorp, 1)), 3)
-		Log("%s: %s" % (dest_rorp.index,
-						Hardlink.get_indicies(dest_rorp, None)), 3)
-		return None
+		if not Hardlink.rorp_eq(src_rorp, dest_rorp):
+			Log("%s: %s" % (src_rorp.index,
+							Hardlink.get_indicies(src_rorp, 1)), 3)
+			Log("%s: %s" % (dest_rorp.index,
+							Hardlink.get_indicies(dest_rorp, None)), 3)
+			return None
+		if compare_eas and not eas_acls.compare_rps(src_rorp, dest_rorp):
+			Log("Different EAs in files %s and %s" %
+				(src_rorp.get_indexpath(), dest_rorp.get_indexpath()), 3)
+			return None
+		return 1
 
 	def rbdir_equal(src_rorp, dest_rorp):
 		"""Like hardlink_equal, but make allowances for data directories"""
@@ -233,6 +240,10 @@ def CompareRecursive(src_rp, dest_rp, compare_hardlinks = 1,
 				if dest_rorp.index[-1].endswith('gz'): return 1
 				# Don't compare .missing increments because they don't matter
 				if dest_rorp.index[-1].endswith('.missing'): return 1
+		if compare_eas and not eas_acls.compare_rps(src_rorp, dest_rorp):
+			Log("Different EAs in files %s and %s" %
+				(src_rorp.get_indexpath(), dest_rorp.get_indexpath()))
+			return None
 		if compare_hardlinks:
 			if Hardlink.rorp_eq(src_rorp, dest_rorp): return 1
 		elif src_rorp.equal_verbose(dest_rorp,
@@ -272,7 +283,8 @@ def BackupRestoreSeries(source_local, dest_local, list_of_dirnames,
 						compare_hardlinks = 1,
 						dest_dirname = "testfiles/output",
 						restore_dirname = "testfiles/rest_out",
-						compare_backups = 1):
+						compare_backups = 1,
+						compare_eas = 0):
 	"""Test backing up/restoring of a series of directories
 
 	The dirnames correspond to a single directory at different times.
@@ -282,6 +294,8 @@ def BackupRestoreSeries(source_local, dest_local, list_of_dirnames,
 
 	"""
 	Globals.set('preserve_hardlinks', compare_hardlinks)
+	Globals.set('write_eas', compare_eas)
+	Globals.set('read_eas', compare_eas)
 	time = 10000
 	dest_rp = rpath.RPath(Globals.local_connection, dest_dirname)
 	restore_rp = rpath.RPath(Globals.local_connection, restore_dirname)
@@ -296,7 +310,8 @@ def BackupRestoreSeries(source_local, dest_local, list_of_dirnames,
 		time += 10000
 		_reset_connections(src_rp, dest_rp)
 		if compare_backups:
-			assert CompareRecursive(src_rp, dest_rp, compare_hardlinks)
+			assert CompareRecursive(src_rp, dest_rp, compare_hardlinks,
+									compare_eas = compare_eas)
 
 	time = 10000
 	for dirname in list_of_dirnames[:-1]:
@@ -305,7 +320,7 @@ def BackupRestoreSeries(source_local, dest_local, list_of_dirnames,
 		InternalRestore(dest_local, source_local, dest_dirname,
 						restore_dirname, time)
 		src_rp = rpath.RPath(Globals.local_connection, dirname)
-		assert CompareRecursive(src_rp, restore_rp)
+		assert CompareRecursive(src_rp, restore_rp, compare_eas = compare_eas)
 
 		# Restore should default back to newest time older than it
 		# with a backup then.

trunk/rdiff-backup/testing/eas_aclstest.py

+import unittest, os, time
+from commontest import *
+from rdiff_backup.eas_acls import *
+from rdiff_backup import Globals, rpath, Time
+
+tempdir = rpath.RPath(Globals.local_connection, "testfiles/output")
+
+class EATest(unittest.TestCase):
+	"""Test extended attributes"""
+	sample_ea = ExtendedAttributes(
+		(), {'user.empty':'', 'user.not_empty':'foobar', 'user.third':'hello',
+			 'user.binary':chr(0)+chr(1)+chr(2)+chr(140)+'/="',
+			 'user.multiline':"""This is a fairly long extended attribute.
+			 Encoding it will require several lines of
+			 base64.""" + chr(177)*300})
+	empty_ea = ExtendedAttributes(())
+	ea1 = ExtendedAttributes(('1',), sample_ea.attr_dict.copy())
+	ea1.delete('user.not_empty')
+	ea2 = ExtendedAttributes(('2',), sample_ea.attr_dict.copy())
+	ea2.set('user.third', 'Another random attribute')
+	ea3 = ExtendedAttributes(('3',))
+	ea4 = ExtendedAttributes(('4',), {'user.deleted': 'File to be deleted'})
+	ea_testdir1 = rpath.RPath(Globals.local_connection, "testfiles/ea_test1")
+	ea_testdir2 = rpath.RPath(Globals.local_connection, "testfiles/ea_test2")
+
+	def make_temp(self):
+		"""Make temp directory testfiles/output"""
+		if tempdir.lstat(): tempdir.delete()
+		tempdir.mkdir()
+
+	def testBasic(self):
+		"""Test basic writing and reading of extended attributes"""
+		self.make_temp()
+		new_ea = ExtendedAttributes(())
+		new_ea.read_from_rp(tempdir)
+		assert not new_ea.attr_dict
+		assert not new_ea == self.sample_ea
+		assert new_ea != self.sample_ea
+		assert new_ea == self.empty_ea
+
+		self.sample_ea.write_to_rp(tempdir)
+		new_ea.read_from_rp(tempdir)
+		assert new_ea.attr_dict == self.sample_ea.attr_dict, \
+			   (new_ea.attr_dict, self.sample_ea.attr_dict)
+		assert new_ea == self.sample_ea
+
+	def testRecord(self):
+		"""Test writing a record and reading it back"""
+		record = EA2Record(self.sample_ea)
+		new_ea = Record2EA(record)
+		if not new_ea == self.sample_ea:
+			new_list = new_ea.attr_dict.keys()
+			sample_list = self.sample_ea.attr_dict.keys()
+			new_list.sort()
+			sample_list.sort()
+			assert new_list == sample_list, (new_list, sample_list)
+			for name in new_list:
+				assert self.sample_ea.get(name) == new_ea.get(name), \
+					   (self.sample_ea.get(name), new_ea.get(name))
+			assert self.sample_ea.index == new_ea.index, \
+				   (self.sample_ea.index, new_ea.index)
+			assert 0, "We shouldn't have gotten this far"
+
+	def make_backup_dirs(self):
+		"""Create testfiles/ea_test[12] directories"""
+		if self.ea_testdir1.lstat(): self.ea_testdir1.delete()
+		if self.ea_testdir2.lstat(): self.ea_testdir2.delete()
+		self.ea_testdir1.mkdir()
+		rp1_1 = self.ea_testdir1.append('1')
+		rp1_2 = self.ea_testdir1.append('2')
+		rp1_3 = self.ea_testdir1.append('3')
+		rp1_4 = self.ea_testdir1.append('4')
+		map(rpath.RPath.touch, [rp1_1, rp1_2, rp1_3, rp1_4])
+		self.sample_ea.write_to_rp(self.ea_testdir1)
+		self.ea1.write_to_rp(rp1_1)
+		self.ea2.write_to_rp(rp1_2)
+		self.ea4.write_to_rp(rp1_4)
+
+		self.ea_testdir2.mkdir()
+		rp2_1 = self.ea_testdir2.append('1')
+		rp2_2 = self.ea_testdir2.append('2')
+		rp2_3 = self.ea_testdir2.append('3')
+		map(rpath.RPath.touch, [rp2_1, rp2_2, rp2_3])
+		self.ea3.write_to_rp(self.ea_testdir2)
+		self.sample_ea.write_to_rp(rp2_1)
+		self.ea1.write_to_rp(rp2_2)
+		self.ea2.write_to_rp(rp2_3)
+
+	def testIterate(self):
+		"""Test writing several records and then reading them back"""
+		self.make_backup_dirs()
+		rp1 = self.ea_testdir1.append('1')
+		rp2 = self.ea_testdir1.append('2')
+		rp3 = self.ea_testdir1.append('3')
+
+		# Now write records corresponding to above rps into file
+		Globals.rbdir = tempdir
+		Time.setcurtime(10000)
+		ExtendedAttributesFile.open_file()
+		for rp in [self.ea_testdir1, rp1, rp2, rp3]:
+			ea = ExtendedAttributes(rp.index)
+			ea.read_from_rp(rp)
+			ExtendedAttributesFile.write_object(ea)
+		ExtendedAttributesFile.close_file()
+
+		# Read back records and compare
+		ea_iter = ExtendedAttributesFile.get_objects_at_time(tempdir, 10000)
+		assert ea_iter, "No extended_attributes.<time> file found"
+		sample_ea_reread = ea_iter.next()
+		assert sample_ea_reread == self.sample_ea
+		ea1_reread = ea_iter.next()
+		assert ea1_reread == self.ea1
+		ea2_reread = ea_iter.next()
+		assert ea2_reread == self.ea2
+		ea3_reread = ea_iter.next()
+		assert ea3_reread == self.ea3
+		try: ea_iter.next()
+		except StopIteration: pass
+		else: assert 0, "Expected end to iterator"
+
+	def testSeriesLocal(self):
+		"""Test backing up and restoring directories with EAs locally"""
+		self.make_backup_dirs()
+		dirlist = ['testfiles/ea_test1', 'testfiles/empty',
+				   'testfiles/ea_test2', 'testfiles/ea_test1']
+		BackupRestoreSeries(1, 1, dirlist, compare_eas = 1)
+
+	def testSeriesRemote(self):
+		"""Test backing up, restoring directories with EA remotely"""
+		self.make_backup_dirs()
+		dirlist = ['testfiles/ea_test1', 'testfiles/ea_test2',
+				   'testfiles/empty', 'testfiles/ea_test1']
+		BackupRestoreSeries(None, None, dirlist, compare_eas = 1)
+
+if __name__ == "__main__": unittest.main()

trunk/rdiff-backup/testing/finaltest.py

@@ -35,7 +35,7 @@ class Local:
 	vft2_in = get_local_rp('vft2_out')
 
 	timbar_in = get_local_rp('increment1/timbar.pyc')
-	timbar_out = get_local_rp('../timbar.pyc') # in cur directory
+	timbar_out = get_local_rp('timbar.pyc') # in cur directory
 
 	wininc2 = get_local_rp('win-increment2')
 	wininc3 = get_local_rp('win-increment3')
@@ -105,7 +105,7 @@ class PathSetter(unittest.TestCase):
 		"""Remove any temp directories created by previous tests"""
 		assert not os.system(MiscDir + '/myrm testfiles/output* '
 							 'testfiles/restoretarget* testfiles/vft_out '
-							 'timbar.pyc testfiles/vft2_out')
+							 'testfiles/timbar.pyc testfiles/vft2_out')
 
 	def runtest(self):
 		self.delete_tmpdirs()
@@ -155,7 +155,7 @@ class PathSetter(unittest.TestCase):
 
 		timbar_paths = self.getinc_paths("timbar.pyc.",
 						 "testfiles/output/rdiff-backup-data/increments")
-		self.exec_rb(None, timbar_paths[0])
+		self.exec_rb(None, timbar_paths[0], 'testfiles/timbar.pyc')
 		self.refresh(Local.timbar_in, Local.timbar_out)
 		assert Local.timbar_in.equal_loose(Local.timbar_out)
 
@@ -242,7 +242,7 @@ class Final(PathSetter):
 		self.exec_rb(None, '../../../../../../proc', 'testfiles/procoutput')
 
 	def testWindowsMode(self):
-		"""Test backup with the --windows-mode option
+		"""Test backup with quoting enabled
 
 		We need to delete from the increment? directories long file
 		names, because quoting adds too many extra letters.
@@ -260,30 +260,35 @@ class Final(PathSetter):
 			delete_long(Local.wininc3)
 
 		old_schema = self.rb_schema
-		self.rb_schema = old_schema + " --windows-mode "
+		self.rb_schema = old_schema+" --override-chars-to-quote '^a-z0-9_ -.' "
 		self.set_connections(None, None, None, None)
 
 		self.delete_tmpdirs()
-		
 		# Back up increment2, this contains a file with colons
 		self.exec_rb(20000, 'testfiles/win-increment2', 'testfiles/output')
+		self.rb_schema = old_schema # Quoting setting should now be saved
 		time.sleep(1)
 
 		# Back up increment3
 		self.exec_rb(30000, 'testfiles/win-increment3', 'testfiles/output')
 
-		# Start restore
-		self.rb_schema = old_schema + ' --windows-restore '
-		Globals.time_separator = "_"
+		# Start restore of increment 2
+		Globals.chars_to_quote = '^a-z0-9_ -.'
 		inc_paths = self.getinc_paths("increments.",
 									  "testfiles/output/rdiff-backup-data", 1)
-		Globals.time_separator = ":"
+		Globals.chars_to_quote = None
 		assert len(inc_paths) == 1, inc_paths
-		# Restore increment2
 		self.exec_rb(None, inc_paths[0], 'testfiles/restoretarget2')
 		assert CompareRecursive(Local.wininc2, Local.rpout2,
 								compare_hardlinks = 0)
 
+		# Restore increment 3 again, using different syntax
+		self.rb_schema = old_schema + '-r 30000 '
+		self.exec_rb(None, 'testfiles/output', 'testfiles/restoretarget3')
+		assert CompareRecursive(Local.wininc3, Local.rpout3,
+								compare_hardlinks = 0)
+		self.rb_schema = old_schema
+
 		# Now check to make sure no ":" in output directory
 		popen_fp = os.popen("find testfiles/output -name '*:*' | wc")
 		wc_output = popen_fp.read()

trunk/rdiff-backup/testing/metadatatest.py

@@ -40,13 +40,16 @@ class MetadataTest(unittest.TestCase):
 
 	def testIterator(self):
 		"""Test writing RORPs to file and iterating them back"""
+		def write_rorp_iter_to_file(rorp_iter, file):
+			for rorp in rorp_iter: file.write(RORP2Record(rorp))
+
 		l = self.get_rpaths()
 		fp = cStringIO.StringIO()
 		write_rorp_iter_to_file(iter(l), fp)
 		fp.seek(0)
 		cstring = fp.read()
 		fp.seek(0)
-		outlist = list(rorp_extractor(fp).iterate())
+		outlist = list(RorpExtractor(fp).iterate())
 		assert len(l) == len(outlist), (len(l), len(outlist))
 		for i in range(len(l)):
 			if not l[i].equal_verbose(outlist[i]):
@@ -65,18 +68,19 @@ class MetadataTest(unittest.TestCase):
 		rpath_iter = selection.Select(rootrp).set_iter()
 
 		start_time = time.time()
-		OpenMetadata(temprp)
-		for rp in rpath_iter: WriteMetadata(rp)
-		CloseMetadata()
+		MetadataFile.open_file(temprp)
+		for rp in rpath_iter: MetadataFile.write_object(rp)
+		MetadataFile.close_file()
 		print "Writing metadata took %s seconds" % (time.time() - start_time)
 		return temprp
 
 	def testSpeed(self):
 		"""Test testIterator on 10000 files"""
 		temprp = self.write_metadata_to_temp()
+		MetadataFile._rp = temprp
 		
 		start_time = time.time(); i = 0
-		for rorp in GetMetadata(temprp): i += 1
+		for rorp in MetadataFile.get_objects(): i += 1
 		print "Reading %s metadata entries took %s seconds." % \
 			  (i, time.time() - start_time)
 
@@ -98,11 +102,35 @@ class MetadataTest(unittest.TestCase):
 
 		"""
 		temprp = self.write_metadata_to_temp()
+		MetadataFile._rp = temprp
 		start_time = time.time(); i = 0
-		for rorp in GetMetadata(temprp, ("subdir3", "subdir10")): i += 1
+		for rorp in MetadataFile.get_objects(("subdir3", "subdir10")): i += 1
 		print "Reading %s metadata entries took %s seconds." % \
 			  (i, time.time() - start_time)
 		assert i == 51
 
+	def test_write(self):
+		"""Test writing to metadata file, then reading back contents"""
+		global tempdir
+		temprp = tempdir.append("write_test.gz")
+		if temprp.lstat(): temprp.delete()
+
+		self.make_temp()
+		rootrp = rpath.RPath(Globals.local_connection,
+							 "testfiles/various_file_types")
+		dirlisting = rootrp.listdir()
+		dirlisting.sort()
+		rps = map(rootrp.append, dirlisting)
+
+		assert not temprp.lstat()
+		MetadataFile.open_file(temprp)
+		for rp in rps: MetadataFile.write_object(rp)
+		MetadataFile.close_file()
+		assert temprp.lstat()
+
+		reread_rps = list(MetadataFile.get_objects())
+		assert len(reread_rps) == len(rps), (len(reread_rps), len(rps))
+		for i in range(len(reread_rps)):
+			assert reread_rps[i] == rps[i], i
 
 if __name__ == "__main__": unittest.main()

trunk/rdiff-backup/testing/regressiontest.py

@@ -12,7 +12,7 @@ testfiles
 
 Globals.set('change_source_perms', 1)
 Globals.counter = 0
-log.Log.setverbosity(3)
+log.Log.setverbosity(7)
 
 def get_local_rp(extension):
 	return rpath.RPath(Globals.local_connection, "testfiles/" + extension)
@@ -172,16 +172,12 @@ class IncrementTest1(unittest.TestCase):
 		hl2.hardlink(hl1.path)
 		
 		Myrm(Local.rpout.path)
-		old_settings = (Globals.quoting_enabled, Globals.chars_to_quote,
-						Globals.quoting_char)
-		Globals.quoting_enabled = 1
+		old_chars = Globals.chars_to_quote
 		Globals.chars_to_quote = 'A-Z'
-		Globals.quoting_char = ';'
 		InternalBackup(1, 1, hldir.path, Local.rpout.path, current_time = 1)
 		InternalBackup(1, 1, "testfiles/empty", Local.rpout.path,
 					   current_time = 10000)
-		(Globals.quoting_enabled, Globals.chars_to_quote,
-		 Globals.quoting_char) = old_settings
+		Globals.chars_to_quote = old_chars
 
 	def test_long_socket(self):
 		"""Test backing up a directory with long sockets in them
@@ -389,8 +385,10 @@ class MirrorTest(PathSetter):
 		Main.force = 1
 		assert not rpout.append("rdiff-backup-data").lstat()
 		Main.misc_setup([rpin, rpout])
+		Main.backup_check_dirs(rpin, rpout)
+		Main.backup_set_fs_globals(rpin, rpout)
+		Main.backup_set_rbdir(rpin, rpout)
 		Main.backup_set_select(rpin)
-		Main.backup_init_dirs(rpin, rpout)
 		backup.Mirror(rpin, rpout)
 		log.ErrorLog.close()
 		log.Log.close_logfile()

trunk/rdiff-backup/testing/restoretest.py

@@ -143,22 +143,6 @@ class RestoreTest(unittest.TestCase):
 						"testfiles/output", 5000)
 		assert CompareRecursive(inc1_rp, target_rp, compare_hardlinks = 0)
 
-#	def testRestoreCorrupt(self):
-#		"""Test restoring a partially corrupt archive
-#
-#		The problem here is that a directory is missing from what is
-#		to be restored, but because the previous backup was aborted in
-#		the middle, some of the files in that directory weren't marked
-#		as .missing.
-#
-#		"""
-#		Myrm("testfiles/output")
-#		InternalRestore(1, 1, "testfiles/restoretest4", "testfiles/output",
-#						10000)
-#		assert os.lstat("testfiles/output")
-#		self.assertRaises(OSError, os.lstat, "testfiles/output/tmp")
-#		self.assertRaises(OSError, os.lstat, "testfiles/output/rdiff-backup")
-
 	def testRestoreNoincs(self):
 		"""Test restoring a directory with no increments, just mirror"""
 		Myrm("testfiles/output")