Changed testfiles, testing code to deal with Select instead of DS.Iterate

git-svn-id: http://svn.savannah.nongnu.org/svn/rdiff-backup@44 2b77aa54-bcbc-44c9-a7ec-4f6cf2b41109

trunk/rdiff-backup/testing/commontest.py

@@ -73,8 +73,7 @@ def InternalBackup(source_local, dest_local, src_dir, dest_dir,
 				   % (SourceDir, dest_dir)
 
 	rpin, rpout = SetConnections.InitRPs([src_dir, dest_dir], remote_schema)
-	Globals.postset_regexp('no_compression_regexp',
-						   Globals.no_compression_regexp_string, re.I)
+	_get_main().misc_setup([rpin, rpout])
 	_get_main().Backup(rpin, rpout)
 	_get_main().cleanup()
 
@@ -91,11 +90,11 @@ def InternalMirror(source_local, dest_local, src_dir, dest_dir,
 				   % (SourceDir, dest_dir)
 
 	rpin, rpout = SetConnections.InitRPs([src_dir, dest_dir], remote_schema)
+	_get_main().misc_setup([rpin, rpout])
 	if not rpout.lstat(): rpout.mkdir()
 	if checkpointing: # rdiff-backup-data must exist to checkpoint
 		data_dir = rpout.append("rdiff-backup-data")
 		if not data_dir.lstat(): data_dir.mkdir()
-		Globals.add_regexp(data_dir.path, 1)
 		SetConnections.UpdateGlobal('rbdir', data_dir)
 	HighLevel.Mirror(rpin, rpout, checkpointing)
 	_get_main().cleanup()
@@ -140,10 +139,7 @@ def _reset_connections(src_rp, dest_rp):
 	#Globals.connections = [Globals.local_connection]
 	#Globals.connection_dict = {0: Globals.local_connection}
 	SetConnections.UpdateGlobal('rbdir', None)
-	SetConnections.UpdateGlobal('exclude_regexps', [])
-	SetConnections.UpdateGlobal('exclude_mirror_regexps', [])
-	Globals.add_regexp(dest_rp.append("rdiff-backup-data").path, 1)
-	Globals.add_regexp(src_rp.append("rdiff-backup-data").path, None)
+	_get_main().misc_setup([src_rp, dest_rp])
 
 def _get_main():
 	"""Set Globals.Main if it doesn't exist, and return"""
@@ -166,8 +162,13 @@ def CompareRecursive(src_rp, dest_rp, compare_hardlinks = 1):
 
 	Log("Comparing %s and %s, hardlinks %s" % (src_rp.path, dest_rp.path,
 											   compare_hardlinks), 3)
-	dsiter1, dsiter2 = map(DestructiveStepping.Iterate_with_Finalizer,
-						   [src_rp, dest_rp], [1, None])
+	src_select, dest_select = Select(src_rp, 1), Select(dest_rp, None)
+	src_select.parse_rbdir_exclude()
+	dest_select.parse_rbdir_exclude()
+	src_select.set_iter()
+	dest_select.set_iter()
+	dsiter1, dsiter2 = src_select.iterate_with_finalizer(), \
+					   dest_select.iterate_with_finalizer()
 
 	def hardlink_equal(src_rorp, dest_rorp):
 		if src_rorp != dest_rorp: return None

trunk/rdiff-backup/testing/destructive_steppingtest.py

 from __future__ import generators
 import unittest
 execfile("commontest.py")
-rbexec("destructive_stepping.py")
+rbexec("selection.py")
 
 
 
@@ -15,13 +15,15 @@ class DSTest(unittest.TestCase):
 	def testDSIter(self):
 		"""Testing destructive stepping iterator from baserp"""
 		for i in range(2):
-			ds_iter = DestructiveStepping.Iterate_with_Finalizer(
-				self.noperms, 1)
+			sel = Select(self.noperms, 1)
+			sel.set_iter()
+			ds_iter = sel.iterate_with_finalizer()
 			noperms = ds_iter.next()
 			assert noperms.isdir() and noperms.getperms() == 0
 
 			bar = ds_iter.next()
-			assert bar.isreg() and bar.getperms() == 0
+			assert bar.isreg() and bar.getperms() == 0, \
+				   "%s %s" % (bar.isreg(), bar.getperms())
 			barbuf = bar.open("rb").read()
 			assert len(barbuf) > 0
 			
@@ -36,37 +38,4 @@ class DSTest(unittest.TestCase):
 
 			self.assertRaises(StopIteration, ds_iter.next)
 
-	def testIterate_from(self):
-		"""Tests basic iteration by Iterate_from"""
-		iter = DestructiveStepping.Iterate_from(self.iteration_dir, 1)
-		l = []
-		for rp in iter: l.append(rp.index)
-		assert l == [(),
-					 ('1',),
-					 ('2',),
-					 ('3',), ('3','2'), ('3','3'),
-					 ('4',),
-					 ('5',), ('5','1'), ('5','2'), ('5','2','1'),
-					 ('6',), ('6','3'),
-                             ('6','3','1'), ('6','3','2'), ('6','4'),
-					 ('7',)], l
-
-	def testIterate_from_index(self):
-		"""Test iteration from a given index"""
-		iter = DestructiveStepping.Iterate_from(self.iteration_dir, 1, ('3',))
-		l = []
-		for rp in iter: l.append(rp.index)
-		assert l == [('3','2'), ('3','3'),
-					 ('4',),
-					 ('5',), ('5','1'), ('5','2'), ('5','2','1'),
-					 ('6',), ('6','3'),
-					         ('6','3','1'), ('6','3','2'), ('6','4'),
-					 ('7',)], l
-		iter = DestructiveStepping.Iterate_from(self.iteration_dir, 1,
-												('6','3'))
-		l = []
-		for rp in iter: l.append(rp.index)
-		assert l == [('6','3','1'), ('6','3','2'), ('6', '4'),
-					 ('7',)], l
-
 if __name__ == "__main__": unittest.main()

trunk/rdiff-backup/testing/hardlinktest.py

@@ -3,7 +3,7 @@ execfile("commontest.py")
 rbexec("main.py")
 
 
-Log.setverbosity(6)
+Log.setverbosity(7)
 
 class HardlinkTest(unittest.TestCase):
 	"""Test cases for Hard links"""
@@ -73,7 +73,7 @@ class HardlinkTest(unittest.TestCase):
 		"""See if the partial inode dictionary is correct"""
 		Globals.preserve_hardlinks = 1
 		reset_hardlink_dicts()
-		for dsrp in DestructiveStepping.Iterate_from(self.hardlink_dir3, 1):
+		for dsrp in Select(self.hardlink_dir3, 1).set_iter():
 			Hardlink.add_rorp(dsrp, 1)
 		
 		assert len(Hardlink._src_inode_indicies.keys()) == 3, \
@@ -90,7 +90,7 @@ class HardlinkTest(unittest.TestCase):
 		"""Same as testBuildingDict but test destination building"""
 		Globals.preserve_hardlinks = 1
 		reset_hardlink_dicts()
-		for dsrp in DestructiveStepping.Iterate_from(self.hardlink_dir3, None):
+		for dsrp in Select(self.hardlink_dir3, None).set_iter():
 			Hardlink.add_rorp(dsrp, None)
 		
 		assert len(Hardlink._dest_inode_indicies.keys()) == 3, \
@@ -106,7 +106,7 @@ class HardlinkTest(unittest.TestCase):
 	def testCompletedDict(self):
 		"""See if the hardlink dictionaries are built correctly"""
 		reset_hardlink_dicts()
-		for dsrp in DestructiveStepping.Iterate_from(self.hardlink_dir1, 1):
+		for dsrp in Select(self.hardlink_dir1, 1).set_iter():
 			Hardlink.add_rorp(dsrp, 1)
 		assert Hardlink._src_inode_indicies == {}, \
 			   Hardlink._src_inode_indicies
@@ -119,7 +119,7 @@ class HardlinkTest(unittest.TestCase):
 		assert Hardlink._src_index_indicies == dict
 
 		reset_hardlink_dicts()
-		for dsrp in DestructiveStepping.Iterate_from(self.hardlink_dir2, 1):
+		for dsrp in Select(self.hardlink_dir2, 1).set_iter():
 			Hardlink.add_rorp(dsrp, 1)
 		assert Hardlink._src_inode_indicies == {}, \
 			   Hardlink._src_inode_indicies

trunk/rdiff-backup/testing/rdifftest.py

 import unittest, random
 
 execfile("commontest.py")
-rbexec("destructive_stepping.py")
+rbexec("selection.py")
 
 
 Log.setverbosity(6)

trunk/rdiff-backup/testing/regressiontest.py

@@ -66,12 +66,6 @@ class PathSetter(unittest.TestCase):
 		self.rbdir.mkdir()
 		SetConnections.UpdateGlobal('rbdir', self.rbdir)
 
-		# Better safe than sorry - cover all possibilities
-		Globals.add_regexp("testfiles/output/rdiff-backup-data", 1)
-		Globals.add_regexp("./testfiles/output/rdiff-backup-data", 1)
-		Globals.add_regexp("../testfiles/output/rdiff-backup-data", 1)
-		Globals.add_regexp("../../testfiles/output/rdiff-backup-data", 1)
-		
 	def setPathnames(self, src_path, src_return, dest_path, dest_return):
 		"""Start servers which will run in src_path and dest_path respectively
 
@@ -97,6 +91,7 @@ class PathSetter(unittest.TestCase):
 		os.system(MiscDir+"/myrm testfiles/output* testfiles/restoretarget* "
 				  "testfiles/noperms_output testfiles/root_output "
 				  "testfiles/unreadable_out")
+
 		self.inc1rp = self.get_src_rp("testfiles/increment1")
 		self.inc2rp = self.get_src_rp('testfiles/increment2')
 		self.inc3rp = self.get_src_rp('testfiles/increment3')
@@ -296,7 +291,7 @@ class MirrorTest(PathSetter):
 		self.setPathnames(None, None, None, None)
 		Time.setcurtime()
 		SaveState.init_filenames(None)
-		HighLevel.Mirror(self.noperms, self.noperms_out, None)
+		self.Mirror(self.noperms, self.noperms_out, None)
 		# Can't compare because we don't have the permissions to do it right
 		#assert CompareRecursive(Local.noperms, Local.noperms_out)
 
@@ -305,7 +300,7 @@ class MirrorTest(PathSetter):
 		self.setPathnames('test1', '../', 'test2/tmp', '../../')
 		Time.setcurtime()
 		SaveState.init_filenames(None)
-		HighLevel.Mirror(self.noperms, self.noperms_out, checkpoint=None)
+		self.Mirror(self.noperms, self.noperms_out, checkpoint=None)
 		#assert CompareRecursive(Local.noperms, Local.noperms_out)
 
 	def testPermSkipLocal(self):
@@ -314,9 +309,9 @@ class MirrorTest(PathSetter):
 		Globals.change_source_perms = None
 		Time.setcurtime()
 		SaveState.init_filenames(None)
-		HighLevel.Mirror(self.one_unreadable, self.one_unreadable_out, checkpoint=None)
+		self.Mirror(self.one_unreadable, self.one_unreadable_out, checkpoint=None)
 		Globals.change_source_perms = 1
-		HighLevel.Mirror(self.one_unreadable, self.one_unreadable_out)
+		self.Mirror(self.one_unreadable, self.one_unreadable_out)
 		# Could add test, but for now just make sure it doesn't exit
 
 	def testPermSkipRemote(self):
@@ -325,9 +320,9 @@ class MirrorTest(PathSetter):
 		Globals.change_source_perms = None
 		Time.setcurtime()
 		SaveState.init_filenames(None)
-		HighLevel.Mirror(self.one_unreadable, self.one_unreadable_out)
+		self.Mirror(self.one_unreadable, self.one_unreadable_out)
 		Globals.change_source_perms = 1
-		HighLevel.Mirror(self.one_unreadable, self.one_unreadable_out)
+		self.Mirror(self.one_unreadable, self.one_unreadable_out)
 		# Could add test, but for now just make sure it doesn't exit
 
 	def refresh(self, *rps):
@@ -365,11 +360,11 @@ class MirrorTest(PathSetter):
 		Globals.change_ownership = Globals.change_source_perms = None
 		self.refresh(self.rootfiles2, self.rootfiles_out2,
 				Local.rootfiles2, Local.rootfiles_out2) # add uid/gid info
-		HighLevel.Mirror(self.rootfiles2, self.rootfiles_out2)
+		self.Mirror(self.rootfiles2, self.rootfiles_out2)
 		assert CompareRecursive(Local.rootfiles2, Local.rootfiles_out2)
 		self.refresh(self.rootfiles2, self.rootfiles_out2,
 				Local.rootfiles2, Local.rootfiles_out2) # remove that info
-		HighLevel.Mirror(self.rootfiles21, self.rootfiles_out2)
+		self.Mirror(self.rootfiles21, self.rootfiles_out2)
 		assert CompareRecursive(Local.rootfiles21, Local.rootfiles_out2)
 		self.refresh(self.rootfiles21, self.rootfiles_out2,
 				Local.rootfiles21, Local.rootfiles_out2) # remove that info
@@ -396,12 +391,12 @@ class MirrorTest(PathSetter):
 		Time.setcurtime()
 		SaveState.init_filenames(None)
 		assert self.rbdir.lstat()
-		HighLevel.Mirror(self.inc1rp, self.rpout)
+		self.Mirror(self.inc1rp, self.rpout)
 		assert CompareRecursive(Local.inc1rp, Local.rpout)
 
 		self.deleteoutput()
 
-		HighLevel.Mirror(self.inc2rp, self.rpout)
+		self.Mirror(self.inc2rp, self.rpout)
 		assert CompareRecursive(Local.inc2rp, Local.rpout)
 
 	def run_partial_test(self):
@@ -410,11 +405,16 @@ class MirrorTest(PathSetter):
 
 		Time.setcurtime()
 		SaveState.init_filenames(None)		
-		HighLevel.Mirror(self.inc1rp, self.rpout)
+		self.Mirror(self.inc1rp, self.rpout)
 		#RPath.copy_attribs(self.inc1rp, self.rpout)
 		assert CompareRecursive(Local.inc1rp, Local.rpout)
 
-		HighLevel.Mirror(self.inc2rp, self.rpout)
+		self.Mirror(self.inc2rp, self.rpout)
 		assert CompareRecursive(Local.inc2rp, Local.rpout)
 
+	def Mirror(self, rpin, rpout, checkpoint = 1):
+		"""Like HighLevel.Mirror, but run misc_setup first"""
+		_get_main().misc_setup([rpin, rpout])
+		HighLevel.Mirror(rpin, rpout, checkpoint)
+
 if __name__ == "__main__": unittest.main()

trunk/rdiff-backup/testing/selectiontest.py

@@ -10,7 +10,7 @@ class MatchingTest(unittest.TestCase):
 
 	def setUp(self):
 		self.root = DSRPath(Globals.local_connection, "testfiles/select")
-		self.Select = Select(self.root)
+		self.Select = Select(self.root, 1)
 
 	def testRegexp(self):
 		"""Test regular expression selection func"""
@@ -21,13 +21,13 @@ class MatchingTest(unittest.TestCase):
 
 		sf2 = self.Select.regexp_get_sf("hello", 0)
 		assert sf2(self.makedsrp("hello")) == 0
-		assert sf2(self.makedsrp("hello_there")) == None
+		assert sf2(self.makedsrp("foohello_there")) == 0
+		assert sf2(self.makedsrp("foo")) == None
 
 	def testTupleInclude(self):
 		"""Test include selection function made from a regular filename"""
-		sf1 = self.Select.glob_get_sf("foo", 1) # should warn
-		assert sf1(13) == None
-		assert sf1("foo") == None
+		self.assertRaises(FilePrefixError,
+						  self.Select.glob_get_filename_sf, "foo", 1)
 
 		sf2 = self.Select.glob_get_sf("testfiles/select/usr/local/bin/", 1)
 		assert sf2(self.makeext("usr")) == 1
@@ -39,9 +39,8 @@ class MatchingTest(unittest.TestCase):
 
 	def testTupleExclude(self):
 		"""Test exclude selection function made from a regular filename"""
-		sf1 = self.Select.glob_get_sf("foo", 0) # should warn
-		assert sf1(13) == None
-		assert sf1("foo") == None
+		self.assertRaises(FilePrefixError,
+						  self.Select.glob_get_filename_sf, "foo", 0)
 
 		sf2 = self.Select.glob_get_sf("testfiles/select/usr/local/bin/", 0)
 		assert sf2(self.makeext("usr")) == None
@@ -58,10 +57,10 @@ class MatchingTest(unittest.TestCase):
 		assert sf1(self.makeext("")) == 1
 
 		sf2 = self.Select.glob_get_sf("**.py", 1)
-		assert sf1(self.makeext("foo")) == 2
-		assert sf1(self.makeext("usr/local/bin")) == 2
-		assert sf1(self.makeext("what/ever.py")) == 1
-		assert sf1(self.makeext("what/ever.py/foo")) == 1
+		assert sf2(self.makeext("foo")) == 2
+		assert sf2(self.makeext("usr/local/bin")) == 2
+		assert sf2(self.makeext("what/ever.py")) == 1
+		assert sf2(self.makeext("what/ever.py/foo")) == 1
 
 	def testGlobStarExclude(self):
 		"""Test a few glob excludes, including **"""
@@ -69,10 +68,10 @@ class MatchingTest(unittest.TestCase):
 		assert sf1(self.makeext("/usr/local/bin")) == 0
 
 		sf2 = self.Select.glob_get_sf("**.py", 0)
-		assert sf1(self.makeext("foo")) == None
-		assert sf1(self.makeext("usr/local/bin")) == None
-		assert sf1(self.makeext("what/ever.py")) == 0
-		assert sf1(self.makeext("what/ever.py/foo")) == 0
+		assert sf2(self.makeext("foo")) == None, sf2(self.makeext("foo"))
+		assert sf2(self.makeext("usr/local/bin")) == None
+		assert sf2(self.makeext("what/ever.py")) == 0
+		assert sf2(self.makeext("what/ever.py/foo")) == 0
 
 	def testFilelistInclude(self):
 		"""Test included filelist"""
@@ -143,19 +142,73 @@ testfiles/select/1/1
 		assert sf(self.makeext("2")) == None
 		assert sf(self.makeext("3")) == 0
 
+	def testGlobRE(self):
+		"""testGlobRE - test translation of shell pattern to regular exp"""
+		assert self.Select.glob_to_re("hello") == "hello"
+		assert self.Select.glob_to_re(".e?ll**o") == "\\.e[^/]ll.*o"
+		r = self.Select.glob_to_re("[abc]el[^de][!fg]h")
+		assert r == "[abc]el[^de][^fg]h", r
+		r = self.Select.glob_to_re("/usr/*/bin/")
+		assert r == "\\/usr\\/[^/]*\\/bin\\/", r
+		assert self.Select.glob_to_re("[a.b/c]") == "[a.b/c]"
+		r = self.Select.glob_to_re("[a*b-c]e[!]]")
+		assert r == "[a*b-c]e[^]]", r
+
+	def testGlobSFException(self):
+		"""testGlobSFException - see if globbing errors returned"""
+		self.assertRaises(GlobbingError, self.Select.glob_get_normal_sf,
+						  "testfiles/select/hello//there", 1)
+		self.assertRaises(FilePrefixError,
+						  self.Select.glob_get_sf, "testfiles/whatever", 1)
+		self.assertRaises(FilePrefixError,
+						  self.Select.glob_get_sf, "testfiles/?hello", 0)
+		assert self.Select.glob_get_normal_sf("**", 1)
+
+	def testIgnoreCase(self):
+		"""testIgnoreCase - try a few expressions with ignorecase:"""
+		sf = self.Select.glob_get_sf("ignorecase:testfiles/SeLect/foo/bar", 1)
+		assert sf(self.makeext("FOO/BAR")) == 1
+		assert sf(self.makeext("foo/bar")) == 1
+		assert sf(self.makeext("fOo/BaR")) == 1
+		self.assertRaises(FilePrefixError, self.Select.glob_get_sf,
+						  "ignorecase:tesfiles/sect/foo/bar", 1)
+						  
+	def testRoot(self):
+		"""testRoot - / may be a counterexample to several of these.."""
+		root = DSRPath(Globals.local_connection, "/")
+		select = Select(root, 1)
+
+		assert select.glob_get_sf("/", 1)(root) == 1
+		assert select.glob_get_sf("/foo", 1)(root) == 1
+		assert select.glob_get_sf("/foo/bar", 1)(root) == 1
+		assert select.glob_get_sf("/", 0)(root) == 0
+		assert select.glob_get_sf("/foo", 0)(root) == None
+
+		assert select.glob_get_sf("**.py", 1)(root) == 2
+		assert select.glob_get_sf("**", 1)(root) == 1
+		assert select.glob_get_sf("ignorecase:/", 1)(root) == 1
+		assert select.glob_get_sf("**.py", 0)(root) == None
+		assert select.glob_get_sf("**", 0)(root) == 0
+		assert select.glob_get_sf("/foo/*", 0)(root) == None
+
+		select.filelist_get_sf(StringIO.StringIO("/"), 1, "test")(root) == 1
+		select.filelist_get_sf(StringIO.StringIO("/foo/bar"), 1,
+							   "test")(root) == 1
+		select.filelist_get_sf(StringIO.StringIO("/"), 0, "test")(root) == 0
+		select.filelist_get_sf(StringIO.StringIO("/foo/bar"), 0,
+							   "test")(root) == None
+
 
 class ParseArgsTest(unittest.TestCase):
 	"""Test argument parsing"""
 	def ParseTest(self, tuplelist, indicies):
 		"""No error if running select on tuple goes over indicies"""
 		self.root = DSRPath(Globals.local_connection, "testfiles/select")
-		self.Select = Select(self.root)
+		self.Select = Select(self.root, 1)
 		self.Select.ParseArgs(tuplelist)
 		self.Select.set_iter()
-		print self.Select.next # just make sure it exists
-		for i in self.Select: print i
 		assert Iter.equal(Iter.map(lambda dsrp: dsrp.index, self.Select),
-						  iter(indicies))
+						  iter(indicies), verbose = 1)
 
 	def testParse(self):
 		"""Test just one include, all exclude"""
@@ -173,5 +226,47 @@ class ParseArgsTest(unittest.TestCase):
 					   [(), ('1',), ('1', '1'), ('1', '1', '2'),
 						('1', '1', '3')])
 
-		
+	def testGlob(self):
+		"""Test globbing expression"""
+		self.ParseTest([("--exclude", "**[3-5]"),
+						("--include", "testfiles/select/1"),
+						("--exclude", "**")],
+					   [(), ('1',), ('1', '1'),
+						('1', '1', '1'), ('1', '1', '2'),
+						('1', '2'), ('1', '2', '1'), ('1', '2', '2')])
+		self.ParseTest([("--include", "testfiles/select**/2"),
+						("--exclude", "**")],
+					   [(), ('1',), ('1', '1'),
+						('1', '1', '2'),
+						('1', '2'),
+						('1', '2', '1'), ('1', '2', '2'), ('1', '2', '3'),
+						('1', '3'),
+						('1', '3', '2'),
+						('2',), ('2', '1'),
+						('2', '1', '1'), ('2', '1', '2'), ('2', '1', '3'),
+						('2', '2'),
+						('2', '2', '1'), ('2', '2', '2'), ('2', '2', '3'),
+						('2', '3'),
+						('2', '3', '1'), ('2', '3', '2'), ('2', '3', '3'),
+						('3',), ('3', '1'),
+						('3', '1', '2'),
+						('3', '2'),
+						('3', '2', '1'), ('3', '2', '2'), ('3', '2', '3'),
+						('3', '3'),
+						('3', '3', '2')])
+
+	def testParseStartingFrom(self):
+		"""Test parse, this time starting from inside"""
+		self.root = DSRPath(Globals.local_connection, "testfiles/select")
+		self.Select = Select(self.root, 1)
+		self.Select.ParseArgs([("--include", "testfiles/select/1/1"),
+							   ("--exclude", "**")])
+		self.Select.set_iter(('1', '1'))
+		assert Iter.equal(Iter.map(lambda dsrp: dsrp.index, self.Select),
+						  iter([("1", '1', '1'),
+								('1', '1', '2'),
+								('1', '1', '3')]),
+						  verbose = 1)
+
+
 if __name__ == "__main__": unittest.main()