Commit 10309d51 authored by ben's avatar ben

Many changes, see changelog for 0.7.4


git-svn-id: http://svn.savannah.nongnu.org/svn/rdiff-backup@73 2b77aa54-bcbc-44c9-a7ec-4f6cf2b41109
parent cea2c3bc
......@@ -97,11 +97,15 @@ def InternalMirror(source_local, dest_local, src_dir, dest_dir,
_get_main().misc_setup([rpin, rpout])
_get_main().backup_init_select(rpin, rpout)
if not rpout.lstat(): rpout.mkdir()
if checkpointing: # rdiff-backup-data must exist to checkpoint
if checkpointing: # use rdiff-backup-data dir to checkpoint
data_dir = rpout.append("rdiff-backup-data")
if not data_dir.lstat(): data_dir.mkdir()
SetConnections.UpdateGlobal('rbdir', data_dir)
HighLevel.Mirror(rpin, rpout, checkpointing)
else: # just use root directory to hold checkpoints
SetConnections.UpdateGlobal('rbdir', rpout)
SetConnections.BackupInitConnections(rpin.conn, rpout.conn)
SaveState.init_filenames(None)
HighLevel.Mirror(rpin, rpout, checkpointing, None, write_finaldata = None)
_get_main().cleanup()
def InternalRestore(mirror_local, dest_local, mirror_dir, dest_dir, time):
......@@ -167,7 +171,8 @@ def CompareRecursive(src_rp, dest_rp, compare_hardlinks = 1,
Log("Comparing %s and %s, hardlinks %s" % (src_rp.path, dest_rp.path,
compare_hardlinks), 3)
src_select, dest_select = Select(src_rp, 1), Select(dest_rp, None)
src_select = Select(DSRPath(1, src_rp))
dest_select = Select(DSRPath(None, dest_rp))
src_select.parse_rbdir_exclude()
dest_select.parse_rbdir_exclude()
src_select.set_iter()
......@@ -200,11 +205,13 @@ def reset_hardlink_dicts():
Hardlink._src_index_indicies = {}
Hardlink._dest_inode_indicies = {}
Hardlink._dest_index_indicies = {}
Hardlink._restore_index_path = {}
def BackupRestoreSeries(source_local, dest_local, list_of_dirnames,
compare_hardlinks = 1,
dest_dirname = "testfiles/output",
restore_dirname = "testfiles/rest_out"):
restore_dirname = "testfiles/rest_out",
compare_backups = 1):
"""Test backing up/restoring of a series of directories
The dirnames correspond to a single directory at different times.
......@@ -227,7 +234,8 @@ def BackupRestoreSeries(source_local, dest_local, list_of_dirnames,
InternalBackup(source_local, dest_local, dirname, dest_dirname, time)
time += 10000
_reset_connections(src_rp, dest_rp)
assert CompareRecursive(src_rp, dest_rp, compare_hardlinks)
if compare_backups:
assert CompareRecursive(src_rp, dest_rp, compare_hardlinks)
time = 10000
for dirname in list_of_dirnames[:-1]:
......@@ -237,6 +245,11 @@ def BackupRestoreSeries(source_local, dest_local, list_of_dirnames,
restore_dirname, time)
src_rp = RPath(Globals.local_connection, dirname)
assert CompareRecursive(src_rp, restore_rp)
# Restore should default back to newest time older than it
# with a backup then.
if time == 20000: time = 21000
time += 10000
def MirrorTest(source_local, dest_local, list_of_dirnames,
......
......@@ -3,7 +3,7 @@ import unittest
execfile("commontest.py")
rbexec("selection.py")
Log.setverbosity(4)
class DSTest(unittest.TestCase):
def setUp(self):
......@@ -15,11 +15,11 @@ class DSTest(unittest.TestCase):
def testDSIter(self):
"""Testing destructive stepping iterator from baserp"""
for i in range(2):
sel = Select(self.noperms, 1)
sel.set_iter()
sel = Select(DSRPath(1, self.noperms)).set_iter()
ds_iter = sel.iterate_with_finalizer()
noperms = ds_iter.next()
assert noperms.isdir() and noperms.getperms() == 0
assert noperms.isdir() and noperms.getperms() == 0, \
(noperms.isdir(), noperms.getperms())
bar = ds_iter.next()
assert bar.isreg() and bar.getperms() == 0, \
......
......@@ -32,6 +32,7 @@ class Local:
vft_in = get_local_rp('vft_out')
vft_out = get_local_rp('increment2/various_file_types')
vft2_in = get_local_rp('vft2_out')
timbar_in = get_local_rp('increment1/timbar.pyc')
timbar_out = get_local_rp('../timbar.pyc') # in cur directory
......@@ -53,9 +54,10 @@ class PathSetter(unittest.TestCase):
if dest_pre: self.dest_prefix = "%s::%s" % (dest_pre, dest_back)
else: self.dest_prefix = './'
def exec_rb(self, *args):
def exec_rb(self, time, *args):
"""Run rdiff-backup on given arguments"""
arglist = []
if time: arglist.append("--current-time %s" % str(time))
arglist.append(self.src_prefix + args[0])
if len(args) > 1:
arglist.append(self.dest_prefix + args[1])
......@@ -65,32 +67,45 @@ class PathSetter(unittest.TestCase):
print "executing " + cmdstr
assert not os.system(cmdstr)
def exec_rb_restore(self, time, *args):
"""Restore using rdiff-backup's new syntax and given time"""
arglist = []
arglist.append("--restore-as-of %s" % str(time))
arglist.append(self.src_prefix + args[0])
if len(args) > 1:
arglist.append(self.dest_prefix + args[1])
assert len(args) == 2
cmdstr = self.rb_schema + " ".join(arglist)
print "Restoring via cmdline: " + cmdstr
assert not os.system(cmdstr)
def delete_tmpdirs(self):
"""Remove any temp directories created by previous tests"""
assert not os.system(MiscDir + '/myrm testfiles/output* '
'testfiles/restoretarget* testfiles/vft_out '
'timbar.pyc')
'timbar.pyc testfiles/vft2_out')
def runtest(self):
self.delete_tmpdirs()
# Backing up increment1
self.exec_rb('testfiles/increment1', 'testfiles/output')
self.exec_rb(10000, 'testfiles/increment1', 'testfiles/output')
assert CompareRecursive(Local.inc1rp, Local.rpout)
time.sleep(1)
# Backing up increment2
self.exec_rb('testfiles/increment2', 'testfiles/output')
self.exec_rb(20000, 'testfiles/increment2', 'testfiles/output')
assert CompareRecursive(Local.inc2rp, Local.rpout)
time.sleep(1)
# Backing up increment3
self.exec_rb('testfiles/increment3', 'testfiles/output')
self.exec_rb(30000, 'testfiles/increment3', 'testfiles/output')
assert CompareRecursive(Local.inc3rp, Local.rpout)
time.sleep(1)
# Backing up increment4
self.exec_rb('testfiles/increment4', 'testfiles/output')
self.exec_rb(40000, 'testfiles/increment4', 'testfiles/output')
assert CompareRecursive(Local.inc4rp, Local.rpout)
# Getting restore rps
......@@ -99,30 +114,35 @@ class PathSetter(unittest.TestCase):
assert len(inc_paths) == 3
# Restoring increment1
self.exec_rb(inc_paths[0], 'testfiles/restoretarget1')
self.exec_rb(None, inc_paths[0], 'testfiles/restoretarget1')
assert CompareRecursive(Local.inc1rp, Local.rpout1)
# Restoring increment2
self.exec_rb(inc_paths[1], 'testfiles/restoretarget2')
self.exec_rb(None, inc_paths[1], 'testfiles/restoretarget2')
assert CompareRecursive(Local.inc2rp, Local.rpout2)
# Restoring increment3
self.exec_rb(inc_paths[2], 'testfiles/restoretarget3')
self.exec_rb(None, inc_paths[2], 'testfiles/restoretarget3')
assert CompareRecursive(Local.inc3rp, Local.rpout3)
# Test restoration of a few random files
vft_paths = self.getinc_paths("various_file_types.",
"testfiles/output/rdiff-backup-data/increments")
self.exec_rb(vft_paths[1], 'testfiles/vft_out')
self.exec_rb(None, vft_paths[1], 'testfiles/vft_out')
self.refresh(Local.vft_in, Local.vft_out)
assert CompareRecursive(Local.vft_in, Local.vft_out)
timbar_paths = self.getinc_paths("timbar.pyc.",
"testfiles/output/rdiff-backup-data/increments")
self.exec_rb(timbar_paths[0])
self.exec_rb(None, timbar_paths[0])
self.refresh(Local.timbar_in, Local.timbar_out)
assert RPath.cmp_with_attribs(Local.timbar_in, Local.timbar_out)
self.exec_rb_restore(25000, 'testfiles/output/various_file_types',
'testfiles/vft2_out')
self.refresh(Local.vft2_in, Local.vft_out)
assert CompareRecursive(Local.vft2_in, Local.vft_out)
# Make sure too many increment files not created
assert len(self.getinc_paths("nochange.",
"testfiles/output/rdiff-backup-data/increments")) == 0
......@@ -222,13 +242,15 @@ class FinalCorrupt(PathSetter):
self.delete_tmpdirs()
assert not os.system("cp -a testfiles/corruptbackup testfiles/output")
self.set_connections(None, None, None, None)
self.exec_rb('testfiles/corruptbackup_source', 'testfiles/output')
self.exec_rb(None, 'testfiles/corruptbackup_source',
'testfiles/output')
def testBackupOverlayRemote(self):
"""Like above but destination is remote"""
self.delete_tmpdirs()
assert not os.system("cp -a testfiles/corruptbackup testfiles/output")
self.set_connections(None, None, "test1/", '../')
self.exec_rb('testfiles/corruptbackup_source', 'testfiles/output')
self.exec_rb(None, 'testfiles/corruptbackup_source',
'testfiles/output')
if __name__ == "__main__": unittest.main()
......@@ -73,7 +73,7 @@ class HardlinkTest(unittest.TestCase):
"""See if the partial inode dictionary is correct"""
Globals.preserve_hardlinks = 1
reset_hardlink_dicts()
for dsrp in Select(self.hardlink_dir3, 1).set_iter():
for dsrp in Select(DSRPath(1, self.hardlink_dir3)).set_iter():
Hardlink.add_rorp(dsrp, 1)
assert len(Hardlink._src_inode_indicies.keys()) == 3, \
......@@ -90,7 +90,7 @@ class HardlinkTest(unittest.TestCase):
"""Same as testBuildingDict but test destination building"""
Globals.preserve_hardlinks = 1
reset_hardlink_dicts()
for dsrp in Select(self.hardlink_dir3, None).set_iter():
for dsrp in Select(DSRPath(None, self.hardlink_dir3)).set_iter():
Hardlink.add_rorp(dsrp, None)
assert len(Hardlink._dest_inode_indicies.keys()) == 3, \
......@@ -106,7 +106,7 @@ class HardlinkTest(unittest.TestCase):
def testCompletedDict(self):
"""See if the hardlink dictionaries are built correctly"""
reset_hardlink_dicts()
for dsrp in Select(self.hardlink_dir1, 1).set_iter():
for dsrp in Select(DSRPath(1, self.hardlink_dir1)).set_iter():
Hardlink.add_rorp(dsrp, 1)
assert Hardlink._src_inode_indicies == {}, \
Hardlink._src_inode_indicies
......@@ -119,7 +119,7 @@ class HardlinkTest(unittest.TestCase):
assert Hardlink._src_index_indicies == dict
reset_hardlink_dicts()
for dsrp in Select(self.hardlink_dir2, 1).set_iter():
for dsrp in Select(DSRPath(1, self.hardlink_dir2)).set_iter():
Hardlink.add_rorp(dsrp, 1)
assert Hardlink._src_inode_indicies == {}, \
Hardlink._src_inode_indicies
......
from __future__ import generators
import unittest
import unittest, pickle
execfile("commontest.py")
rbexec("lazy.py")
......@@ -219,108 +219,90 @@ class MultiplexTest(Iterators):
assert Iter.equal(i2, self.one_to_100())
class index:
"""This is just used below to test the iter tree reducer"""
def __init__(self, index):
self.index = index
class ITRadder(IterTreeReducer):
def start_process(self, index):
self.total = 0
def end_process(self):
if self.base_index:
summand = self.base_index[-1]
#print "Adding ", summand
self.total += summand
class TreeReducerTest(unittest.TestCase):
def setUp(self):
self.i1 = iter(map(index, [(), (1,), (2,), (3,)]))
self.i2 = iter(map(index, [(0,), (0,1), (0,1,0), (0,1,1),
(0,2), (0,2,1), (0,3)]))
def branch_process(self, subinstance):
#print "Adding subinstance ", subinstance.total
self.total += subinstance.total
self.i1a = iter(map(index, [(), (1,)]))
self.i1b = iter(map(index, [(2,), (3,)]))
self.i2a = iter(map(index, [(0,), (0,1), (0,1,0)]))
self.i2b = iter(map(index, [(0,1,1), (0,2)]))
self.i2c = iter(map(index, [(0,2,1), (0,3)]))
class ITRadder2(IterTreeReducer):
def start_process(self, index):
self.total = 0
# The four following are used to make an ITR later
def number_of_index(self, index_obj):
if not index_obj.index: return 0
else: return index_obj.index[-1]
def end_process(self):
#print "Adding ", self.base_index
self.total += reduce(lambda x,y: x+y, self.base_index, 0)
def sum_index(self, index_obj):
return reduce(lambda x,y: x+y, index_obj.index, 0)
def branch_process(self, subinstance):
#print "Adding branch ", subinstance.total
self.total += subinstance.total
def add2(self, x, y):
#print "adding %d and %d" % (x,y)
return x+y
class TreeReducerTest(unittest.TestCase):
def setUp(self):
self.i1 = [(), (1,), (2,), (3,)]
self.i2 = [(0,), (0,1), (0,1,0), (0,1,1), (0,2), (0,2,1), (0,3)]
def add3(self, x,y,z):
#print "ignoring %s, adding %d and %d" % (x,y,z)
return y+z
self.i1a = [(), (1,)]
self.i1b = [(2,), (3,)]
self.i2a = [(0,), (0,1), (0,1,0)]
self.i2b = [(0,1,1), (0,2)]
self.i2c = [(0,2,1), (0,3)]
def testTreeReducer(self):
"""testing IterTreeReducer"""
itm = IterTreeReducer(self.number_of_index, self.add2, 0, self.add3)
for elem in self.i1:
val = itm(elem)
itm = ITRadder()
for index in self.i1:
val = itm(index)
assert val, elem.index
itm.calculate_final_val()
assert itm.getresult() == 6, itm.getresult()
itm.Finish()
assert itm.total == 6, itm.total
itm2 = IterTreeReducer(self.sum_index, self.add2, 0, self.add3)
for elem in self.i2:
val = itm2(elem)
if elem.index == (): assert not val
itm2 = ITRadder2()
for index in self.i2:
val = itm2(index)
if index == (): assert not val
else: assert val
assert itm2.getresult() == 12, itm2.getresult()
itm2.Finish()
assert itm2.total == 12, itm2.total
def testTreeReducerState(self):
"""Test saving and recreation of an IterTreeReducer"""
itm1a = IterTreeReducer(self.number_of_index, self.add2, 0, self.add3)
for elem in self.i1a:
val = itm1a(elem)
assert val, elem.index
itm1b = IterTreeReducer(self.number_of_index, self.add2, 0, self.add3,
itm1a.getstate())
for elem in self.i1b:
val = itm1b(elem)
assert val, elem.index
itm1b.calculate_final_val()
assert itm1b.getresult() == 6, itm1b.getresult()
itm2a = IterTreeReducer(self.sum_index, self.add2, 0, self.add3)
for elem in self.i2a:
val = itm2a(elem)
if elem.index == (): assert not val
itm1a = ITRadder()
for index in self.i1a:
val = itm1a(index)
assert val, index
itm1b = pickle.loads(pickle.dumps(itm1a))
for index in self.i1b:
val = itm1b(index)
assert val, index
itm1b.Finish()
assert itm1b.total == 6, itm1b.total
itm2a = ITRadder2()
for index in self.i2a:
val = itm2a(index)
if index == (): assert not val
else: assert val
itm2b = IterTreeReducer(self.sum_index, self.add2, 0, self.add3,
itm2a.getstate())
for elem in self.i2b:
val = itm2b(elem)
if elem.index == (): assert not val
itm2b = pickle.loads(pickle.dumps(itm2a))
for index in self.i2b:
val = itm2b(index)
if index == (): assert not val
else: assert val
itm2c = IterTreeReducer(self.sum_index, self.add2, 0, self.add3,
itm2b.getstate())
for elem in self.i2c:
val = itm2c(elem)
if elem.index == (): assert not val
itm2c = pickle.loads(pickle.dumps(itm2b))
for index in self.i2c:
val = itm2c(index)
if index == (): assert not val
else: assert val
assert itm2c.getresult() == 12, itm2c.getresult()
def testTreeReducer2(self):
"""Another test of the tree reducer"""
assert Iter.len(self.i1) == 4
hit_021_02 = [None, None]
def helper(indexobj, elem_init, branch_result):
if indexobj.index == (0,2):
assert hit_021_02[0]
hit_021_02[1] = 1
elif indexobj.index == (0,2,1):
assert not hit_021_02[1]
hit_021_02[0] = 1
return None
itm = IterTreeReducer(lambda x: None, lambda x,y: None, None, helper)
for elem in self.i2: itm(elem)
itm.getresult()
assert hit_021_02 == [1,1]
itm2c.Finish()
assert itm2c.total == 12, itm2c.total
if __name__ == "__main__": unittest.main()
#!/bin/sh
# This script will create the testing/restoretest3 directory as it
# needs to be for one of the tests in restoretest.py to work.
rm -rf testfiles/restoretest3
rdiff-backup --current-time 10000 testfiles/increment1 testfiles/restoretest3
rdiff-backup --current-time 20000 testfiles/increment2 testfiles/restoretest3
rdiff-backup --current-time 30000 testfiles/increment3 testfiles/restoretest3
rdiff-backup --current-time 40000 testfiles/increment4 testfiles/restoretest3
......@@ -11,8 +11,15 @@ class RestoreTest(unittest.TestCase):
"""Test Restore class"""
prefix = "testfiles/restoretest/"
def maketesttuples(self, basename):
"""Make testing tuples from available files starting with prefix"""
"""Make testing tuples from available files starting with prefix
tuples is a sorted (oldest to newest) list of pairs (rp1, rp2)
where rp1 is an increment file and rp2 is the same but without
the final extension. incs is a list of all increment files.
"""
dirlist = os.listdir(self.prefix)
dirlist.sort()
baselist = filter(lambda f: f.startswith(basename), dirlist)
rps = map(lambda f: RPath(lc, self.prefix+f), baselist)
incs = filter(lambda rp: rp.isincfile(), rps)
......@@ -31,8 +38,12 @@ class RestoreTest(unittest.TestCase):
print "Processing file " + pair[0].path
if rptarget.lstat(): rptarget.delete()
rest_time = Time.stringtotime(pair[0].getinctime())
sorted_incs = Restore.sortincseq(rest_time, incs)
Restore.RestoreFile(rest_time, rpbase, (), sorted_incs, rptarget)
rid = RestoreIncrementData((), rpbase, incs)
rid.sortincseq(rest_time, 10000000000) # pick some really late time
rcd = RestoreCombinedData(rid, rpbase, rptarget)
rcd.RestoreFile()
#sorted_incs = Restore.sortincseq(rest_time, incs)
#Restore.RestoreFile(rest_time, rpbase, (), sorted_incs, rptarget)
rptarget.setdata()
if not rptarget.lstat(): assert not pair[1].lstat()
elif not pair[1].lstat(): assert not rptarget.lstat()
......@@ -53,17 +64,27 @@ class RestoreTest(unittest.TestCase):
"""
for basename in ['ocaml', 'mf']:
tuples, incs = self.maketesttuples(basename)
completed_dict = {}
for i in range(len(tuples)):
pair = tuples[i]
rest_time = Time.stringtotime(pair[0].getinctime())
sorted_incs = Restore.sortincseq(rest_time, incs)
key = sorted_incs[-1].path
assert not completed_dict.has_key(key)
completed_dict[key] = 1
for inc in incs: assert completed_dict[inc.path] == 1
tuples, unused = self.maketesttuples(basename)
incs = [tuple[0] for tuple in tuples]
# Now we need a time newer than any inc
mirror_time = Time.stringtotime(incs[-1].getinctime()) + 10000
for inc, incbase in tuples:
assert inc.isincfile()
inctime = Time.stringtotime(inc.getinctime())
rid1 = RestoreIncrementData(basename, incbase, incs)
rid2 = RestoreIncrementData(basename, incbase, incs)
rid1.sortincseq(inctime, mirror_time)
rid2.sortincseq(inctime + 5, mirror_time)
assert rid1.inc_list, rid1.inc_list
# Five seconds later shouldn't make a difference
assert rid1.inc_list == rid2.inc_list, (rid1.inc_list,
rid2.inc_list)
# oldest increment should be exactly inctime
ridtime = Time.stringtotime(rid1.inc_list[-1].getinctime())
assert ridtime == inctime, (ridtime, inctime)
def testRestorefiles(self):
"""Testing restoration of files one at a time"""
......
......@@ -5,12 +5,12 @@ rbexec("highlevel.py")
class MatchingTest(unittest.TestCase):
"""Test matching of file names against various selection functions"""
def makedsrp(self, path): return DSRPath(Globals.local_connection, path)
def makedsrp(self, path): return DSRPath(1, Globals.local_connection, path)
def makeext(self, path): return self.root.new_index(tuple(path.split("/")))
def setUp(self):
self.root = DSRPath(Globals.local_connection, "testfiles/select")
self.Select = Select(self.root, 1)
self.root = DSRPath(1, Globals.local_connection, "testfiles/select")
self.Select = Select(self.root)
def testRegexp(self):
"""Test regular expression selection func"""
......@@ -177,8 +177,8 @@ testfiles/select/1/1
def testRoot(self):
"""testRoot - / may be a counterexample to several of these.."""
root = DSRPath(Globals.local_connection, "/")
select = Select(root, 1)
root = DSRPath(1, Globals.local_connection, "/")
select = Select(root)
assert select.glob_get_sf("/", 1)(root) == 1
assert select.glob_get_sf("/foo", 1)(root) == 1
......@@ -205,8 +205,8 @@ class ParseArgsTest(unittest.TestCase):
"""Test argument parsing"""
def ParseTest(self, tuplelist, indicies):
"""No error if running select on tuple goes over indicies"""
self.root = DSRPath(Globals.local_connection, "testfiles/select")
self.Select = Select(self.root, 1)
self.root = DSRPath(1, Globals.local_connection, "testfiles/select")
self.Select = Select(self.root)
self.Select.ParseArgs(tuplelist)
self.Select.set_iter()
assert Iter.equal(Iter.map(lambda dsrp: dsrp.index, self.Select),
......@@ -259,8 +259,8 @@ class ParseArgsTest(unittest.TestCase):
def testParseStartingFrom(self):
"""Test parse, this time starting from inside"""
self.root = DSRPath(Globals.local_connection, "testfiles/select")
self.Select = Select(self.root, 1)
self.root = DSRPath(1, Globals.local_connection, "testfiles/select")
self.Select = Select(self.root)
self.Select.ParseArgs([("--include", "testfiles/select/1/1"),
("--exclude", "**")])
self.Select.set_iter(('1', '1'))
......
......@@ -90,5 +90,14 @@ class TimeTest(unittest.TestCase):
assert g2t('2001-05-12') == t
assert g2t('2001/05/12') == t
assert g2t('5/12/2001') == t
assert g2t('123456') == 123456
def testGenericStringErrors(self):
"""Test genstrtotime on some bad strings"""
g2t = Time.genstrtotime
self.assertRaises(TimeException, g2t, "hello")
self.assertRaises(TimeException, g2t, "")
self.assertRaises(TimeException, g2t, "3q")
if __name__ == '__main__': unittest.main()
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment