Some refactoring of metadata code

git-svn-id: http://svn.savannah.nongnu.org/svn/rdiff-backup@668 2b77aa54-bcbc-44c9-a7ec-4f6cf2b41109

trunk/rdiff-backup/rdiff_backup/backup.py

@@ -23,7 +23,7 @@ from __future__ import generators
 import errno
 import Globals, metadata, rorpiter, TempFile, Hardlink, robust, increment, \
 	   rpath, static, log, selection, Time, Rdiff, statistics, iterfile, \
-	   eas_acls, hash
+	   hash
 
 def Mirror(src_rpath, dest_rpath):
 	"""Turn dest_rpath into a copy of src_rpath"""
@@ -135,10 +135,9 @@ class DestinationStruct:
 			sel.parse_rbdir_exclude()
 			return sel.set_iter()
 
+		metadata.SetManager()
 		if use_metadata:
-			rorp_iter = eas_acls.GetCombinedMetadataIter(
-				Globals.rbdir, Time.prevtime,
-				acls = Globals.acls_active, eas = Globals.eas_active)
+			rorp_iter = metadata.ManagerObj.GetAtTime(Time.prevtime)
 			if rorp_iter: return rorp_iter
 		return get_iter_from_fs()
 
@@ -273,9 +272,7 @@ class CacheCollatedPostProcess:
 
 		self.statfileobj = statistics.init_statfileobj()
 		if Globals.file_statistics: statistics.FileStats.init()
-		metadata.MetadataFile.open_file()
-		if Globals.eas_active: eas_acls.ExtendedAttributesFile.open_file()
-		if Globals.acls_active: eas_acls.AccessControlListFile.open_file()
+		self.metawriter = metadata.ManagerObj.GetWriter()
 
 		# the following should map indicies to lists
 		# [source_rorp, dest_rorp, changed_flag, success_flag, increment]
@@ -381,13 +378,7 @@ class CacheCollatedPostProcess:
 			self.statfileobj.add_changed(source_rorp, dest_rorp)
 
 		if metadata_rorp and metadata_rorp.lstat():
-			metadata.MetadataFile.write_object(metadata_rorp)
-			if Globals.eas_active and not metadata_rorp.get_ea().empty():
-				eas_acls.ExtendedAttributesFile.write_object(
-					metadata_rorp.get_ea())
-			if Globals.acls_active and not metadata_rorp.get_acl().is_basic():
-				eas_acls.AccessControlListFile.write_object(
-					metadata_rorp.get_acl())
+			self.metawriter.write_object(metadata_rorp)
 		if Globals.file_statistics:
 			statistics.FileStats.update(source_rorp, dest_rorp, changed, inc)
 
@@ -451,9 +442,8 @@ class CacheCollatedPostProcess:
 		while self.dir_perms_list:
 			dir_rp, perms = self.dir_perms_list.pop()
 			dir_rp.chmod(perms)
-		metadata.MetadataFile.close_file()
-		if Globals.eas_active: eas_acls.ExtendedAttributesFile.close_file()
-		if Globals.acls_active: eas_acls.AccessControlListFile.close_file()
+		self.metawriter.close()
+
 		if Globals.print_statistics: statistics.print_active_stats()
 		if Globals.file_statistics: statistics.FileStats.close()
 		statistics.write_active_statfileobj()

trunk/rdiff-backup/rdiff_backup/eas_acls.py

@@ -30,7 +30,7 @@ from __future__ import generators
 import base64, errno, re
 try: import posix1e
 except ImportError: pass
-import static, Globals, metadata, connection, rorpiter, log, C, \
+import static, Globals, eas_acls, connection, metadata, rorpiter, log, C, \
 	   rpath, user_group
 
 # When an ACL gets dropped, put name in dropped_acl_names.  This is
@@ -170,24 +170,13 @@ class ExtendedAttributesFile(metadata.FlatFile):
 	_extractor = EAExtractor
 	_object_to_record = staticmethod(EA2Record)
 
-	def join(cls, rorp_iter, rbdir, time, restrict_index):
-		"""Add extended attribute information to existing rorp_iter"""
-		def helper(rorp_iter, ea_iter):
-			"""Add EA information in ea_iter to rorp_iter"""
-			collated = rorpiter.CollateIterators(rorp_iter, ea_iter)
-			for rorp, ea in collated:
-				assert rorp, (rorp, (ea.index, ea.attr_dict), time)
-				if not ea: ea = ExtendedAttributes(rorp.index)
-				rorp.set_ea(ea)
-				yield rorp
-			
-		ea_iter = cls.get_objects_at_time(rbdir, time, restrict_index)
-		if not ea_iter:
-			log.Log("Warning: Extended attributes file not found", 2)
-			ea_iter = iter([])
-		return helper(rorp_iter, ea_iter)
-
-static.MakeClass(ExtendedAttributesFile)
+def join_ea_iter(rorp_iter, ea_iter):
+	"""Update a rorp iter by adding the information from ea_iter"""
+	for rorp, ea in rorpiter.CollateIterators(rorp_iter, ea_iter):
+		assert rorp, "Missing rorp for index %s" % (ea.index,)
+		if not ea: ea = ExtendedAttributes(rorp.index)
+		rorp.set_ea(ea)
+		yield rorp
 
 
 class AccessControlLists:
@@ -521,49 +510,14 @@ class AccessControlListFile(metadata.FlatFile):
 	_extractor = ACLExtractor
 	_object_to_record = staticmethod(ACL2Record)
 
-	def join(cls, rorp_iter, rbdir, time, restrict_index):
-		"""Add access control list information to existing rorp_iter"""
-		def helper(rorp_iter, acl_iter):
-			"""Add ACL information in acl_iter to rorp_iter"""
-			collated = rorpiter.CollateIterators(rorp_iter, acl_iter)
-			for rorp, acl in collated:
-				assert rorp, "Missing rorp for index %s" % (acl.index,)
-				if not acl: acl = AccessControlLists(rorp.index)
-				rorp.set_acl(acl)
-				yield rorp
-
-		acl_iter = cls.get_objects_at_time(rbdir, time, restrict_index)
-		if not acl_iter:
-			log.Log("Warning: Access Control List file not found", 2)
-			acl_iter = iter([])
-		return helper(rorp_iter, acl_iter)
-
-static.MakeClass(AccessControlListFile)
-
-
-def GetCombinedMetadataIter(rbdir, time, restrict_index = None,
-							acls = None, eas = None):
-	"""Return iterator of rorps from metadata and related files
-
-	None will be returned if the metadata file is absent.  If acls or
-	eas is true, access control list or extended attribute information
-	will be added.
-
-	"""
-	metadata_iter = metadata.MetadataFile.get_objects_at_time(
-		rbdir, time, restrict_index)
-	if not metadata_iter:
-		log.Log("Warning, metadata file not found.\n"
-				"Metadata will be read from filesystem.", 2)
-		return None
-	if eas:
-		metadata_iter = ExtendedAttributesFile.join(
-			metadata_iter, rbdir, time, restrict_index)
-	if acls:
-		metadata_iter = AccessControlListFile.join(
-			metadata_iter, rbdir, time, restrict_index)
-	return metadata_iter
-
+def join_acl_iter(rorp_iter, acl_iter):
+	"""Update a rorp iter by adding the information from acl_iter"""
+	for rorp, acl in rorpiter.CollateIterators(rorp_iter, acl_iter):
+		assert rorp, "Missing rorp for index %s" % (acl.index,)
+		if not acl: acl = AccessControlLists(rorp.index)
+		rorp.set_acl(acl)
+		yield rorp
+	
 
 def rpath_acl_get(rp):
 	"""Get acls of given rpath rp.

trunk/rdiff-backup/rdiff_backup/regress.py

@@ -143,8 +143,8 @@ def iterate_raw_rfs(mirror_rp, inc_rp):
 
 def yield_metadata():
 	"""Iterate rorps from metadata file, if any are available"""
-	metadata_iter = metadata.MetadataFile.get_objects_at_time(Globals.rbdir,
-															  regress_time)
+	metadata.SetManager()
+	metadata_iter = metadata.ManagerObj.GetAtTime(regress_time)
 	if metadata_iter: return metadata_iter
 	log.Log.FatalError("No metadata for time %s found, cannot regress"
 					   % Time.timetopretty(regress_time))

trunk/rdiff-backup/rdiff_backup/restore.py

@@ -22,7 +22,7 @@
 from __future__ import generators
 import tempfile, os, cStringIO
 import Globals, Time, Rdiff, Hardlink, rorpiter, selection, rpath, \
-	   log, static, robust, metadata, statistics, TempFile, eas_acls, hash
+	   log, static, robust, metadata, statistics, TempFile, hash
 
 
 class RestoreError(Exception): pass
@@ -177,9 +177,9 @@ class MirrorStruct:
 		"""
 		if rest_time is None: rest_time = cls._rest_time
 
-		rorp_iter = eas_acls.GetCombinedMetadataIter(
-			Globals.rbdir, rest_time, restrict_index = cls.mirror_base.index,
-			acls = Globals.acls_active, eas = Globals.eas_active)
+		if not metadata.ManagerObj: metadata.SetManager()
+		rorp_iter = metadata.ManagerObj.GetAtTime(rest_time,
+												  cls.mirror_base.index)
 		if not rorp_iter:
 			if require_metadata:
 				log.Log.FatalError("Mirror metadata not found")

trunk/rdiff-backup/testing/metadatatest.py

@@ -60,7 +60,7 @@ class MetadataTest(unittest.TestCase):
 	def write_metadata_to_temp(self):
 		"""If necessary, write metadata of bigdir to file metadata.gz"""
 		global tempdir
-		temprp = tempdir.append("metadata.gz")
+		temprp = tempdir.append("mirror_metadata.2005-11-03T14:51:06-06:00.snapshot.gz")
 		if temprp.lstat(): return temprp
 
 		self.make_temp()
@@ -68,19 +68,19 @@ class MetadataTest(unittest.TestCase):
 		rpath_iter = selection.Select(rootrp).set_iter()
 
 		start_time = time.time()
-		MetadataFile.open_file(temprp)
-		for rp in rpath_iter: MetadataFile.write_object(rp)
-		MetadataFile.close_file()
+		mf = MetadataFile(temprp, 'w')
+		for rp in rpath_iter: mf.write_object(rp)
+		mf.close()
 		print "Writing metadata took %s seconds" % (time.time() - start_time)
 		return temprp
 
 	def testSpeed(self):
 		"""Test testIterator on 10000 files"""
 		temprp = self.write_metadata_to_temp()
-		MetadataFile._rp = temprp
+		mf = MetadataFile(temprp, 'r')
 		
 		start_time = time.time(); i = 0
-		for rorp in MetadataFile.get_objects(): i += 1
+		for rorp in mf.get_objects(): i += 1
 		print "Reading %s metadata entries took %s seconds." % \
 			  (i, time.time() - start_time)
 
@@ -102,9 +102,9 @@ class MetadataTest(unittest.TestCase):
 
 		"""
 		temprp = self.write_metadata_to_temp()
-		MetadataFile._rp = temprp
+		mf = MetadataFile(temprp, 'r')
 		start_time = time.time(); i = 0
-		for rorp in MetadataFile.get_objects(("subdir3", "subdir10")): i += 1
+		for rorp in mf.get_objects(("subdir3", "subdir10")): i += 1
 		print "Reading %s metadata entries took %s seconds." % \
 			  (i, time.time() - start_time)
 		assert i == 51
@@ -112,7 +112,7 @@ class MetadataTest(unittest.TestCase):
 	def test_write(self):
 		"""Test writing to metadata file, then reading back contents"""
 		global tempdir
-		temprp = tempdir.append("write_test.gz")
+		temprp = tempdir.append("mirror_metadata.2005-11-03T12:51:06-06:00.snapshot.gz")
 		if temprp.lstat(): temprp.delete()
 
 		self.make_temp()
@@ -123,14 +123,42 @@ class MetadataTest(unittest.TestCase):
 		rps = map(rootrp.append, dirlisting)
 
 		assert not temprp.lstat()
-		MetadataFile.open_file(temprp)
-		for rp in rps: MetadataFile.write_object(rp)
-		MetadataFile.close_file()
+		write_mf = MetadataFile(temprp, 'w')
+		for rp in rps: write_mf.write_object(rp)
+		write_mf.close()
 		assert temprp.lstat()
 
-		reread_rps = list(MetadataFile.get_objects())
+		reread_rps = list(MetadataFile(temprp, 'r').get_objects())
 		assert len(reread_rps) == len(rps), (len(reread_rps), len(rps))
 		for i in range(len(reread_rps)):
 			assert reread_rps[i] == rps[i], i
 
+	def test_patch(self):
+		"""Test combining 3 iters of metadata rorps"""
+		self.make_temp()
+		os.system('cp -a testfiles/various_file_types/* ' + tempdir.path)
+
+		rp1 = tempdir.append('regular_file')
+		rp2 = tempdir.append('subdir')
+		rp3 = rp2.append('subdir_file')
+		rp4 = tempdir.append('test')
+
+		rp1new = tempdir.append('regular_file')
+		rp1new.chmod(0)
+		zero = rpath.RORPath(('test',))
+
+		current = [rp1, rp2, rp3]
+		diff1 = [rp1, rp4]
+		diff2 = [rp1new, rp2, zero]
+
+		output = patch(iter(current), iter(diff1), iter(diff2))
+		out1 = output.next()
+		assert out1 is rp1new, out1
+		out2 = output.next()
+		assert out2 is rp2, out2
+		out3 = output.next()
+		assert out3 is rp3, out3
+		self.assertRaises(StopIteration, output.next)
+
+
 if __name__ == "__main__": unittest.main()