Commit 58073772 authored by bescoto's avatar bescoto

Some refactoring of metadata code


git-svn-id: http://svn.savannah.nongnu.org/svn/rdiff-backup@668 2b77aa54-bcbc-44c9-a7ec-4f6cf2b41109
parent 25183ca0
......@@ -23,7 +23,7 @@ from __future__ import generators
import errno
import Globals, metadata, rorpiter, TempFile, Hardlink, robust, increment, \
rpath, static, log, selection, Time, Rdiff, statistics, iterfile, \
eas_acls, hash
hash
def Mirror(src_rpath, dest_rpath):
"""Turn dest_rpath into a copy of src_rpath"""
......@@ -135,10 +135,9 @@ class DestinationStruct:
sel.parse_rbdir_exclude()
return sel.set_iter()
metadata.SetManager()
if use_metadata:
rorp_iter = eas_acls.GetCombinedMetadataIter(
Globals.rbdir, Time.prevtime,
acls = Globals.acls_active, eas = Globals.eas_active)
rorp_iter = metadata.ManagerObj.GetAtTime(Time.prevtime)
if rorp_iter: return rorp_iter
return get_iter_from_fs()
......@@ -273,9 +272,7 @@ class CacheCollatedPostProcess:
self.statfileobj = statistics.init_statfileobj()
if Globals.file_statistics: statistics.FileStats.init()
metadata.MetadataFile.open_file()
if Globals.eas_active: eas_acls.ExtendedAttributesFile.open_file()
if Globals.acls_active: eas_acls.AccessControlListFile.open_file()
self.metawriter = metadata.ManagerObj.GetWriter()
# the following should map indicies to lists
# [source_rorp, dest_rorp, changed_flag, success_flag, increment]
......@@ -381,13 +378,7 @@ class CacheCollatedPostProcess:
self.statfileobj.add_changed(source_rorp, dest_rorp)
if metadata_rorp and metadata_rorp.lstat():
metadata.MetadataFile.write_object(metadata_rorp)
if Globals.eas_active and not metadata_rorp.get_ea().empty():
eas_acls.ExtendedAttributesFile.write_object(
metadata_rorp.get_ea())
if Globals.acls_active and not metadata_rorp.get_acl().is_basic():
eas_acls.AccessControlListFile.write_object(
metadata_rorp.get_acl())
self.metawriter.write_object(metadata_rorp)
if Globals.file_statistics:
statistics.FileStats.update(source_rorp, dest_rorp, changed, inc)
......@@ -451,9 +442,8 @@ class CacheCollatedPostProcess:
while self.dir_perms_list:
dir_rp, perms = self.dir_perms_list.pop()
dir_rp.chmod(perms)
metadata.MetadataFile.close_file()
if Globals.eas_active: eas_acls.ExtendedAttributesFile.close_file()
if Globals.acls_active: eas_acls.AccessControlListFile.close_file()
self.metawriter.close()
if Globals.print_statistics: statistics.print_active_stats()
if Globals.file_statistics: statistics.FileStats.close()
statistics.write_active_statfileobj()
......
......@@ -30,7 +30,7 @@ from __future__ import generators
import base64, errno, re
try: import posix1e
except ImportError: pass
import static, Globals, metadata, connection, rorpiter, log, C, \
import static, Globals, eas_acls, connection, metadata, rorpiter, log, C, \
rpath, user_group
# When an ACL gets dropped, put name in dropped_acl_names. This is
......@@ -170,25 +170,14 @@ class ExtendedAttributesFile(metadata.FlatFile):
_extractor = EAExtractor
_object_to_record = staticmethod(EA2Record)
def join(cls, rorp_iter, rbdir, time, restrict_index):
"""Add extended attribute information to existing rorp_iter"""
def helper(rorp_iter, ea_iter):
"""Add EA information in ea_iter to rorp_iter"""
collated = rorpiter.CollateIterators(rorp_iter, ea_iter)
for rorp, ea in collated:
assert rorp, (rorp, (ea.index, ea.attr_dict), time)
def join_ea_iter(rorp_iter, ea_iter):
"""Update a rorp iter by adding the information from ea_iter"""
for rorp, ea in rorpiter.CollateIterators(rorp_iter, ea_iter):
assert rorp, "Missing rorp for index %s" % (ea.index,)
if not ea: ea = ExtendedAttributes(rorp.index)
rorp.set_ea(ea)
yield rorp
ea_iter = cls.get_objects_at_time(rbdir, time, restrict_index)
if not ea_iter:
log.Log("Warning: Extended attributes file not found", 2)
ea_iter = iter([])
return helper(rorp_iter, ea_iter)
static.MakeClass(ExtendedAttributesFile)
class AccessControlLists:
"""Hold a file's access control list information
......@@ -521,49 +510,14 @@ class AccessControlListFile(metadata.FlatFile):
_extractor = ACLExtractor
_object_to_record = staticmethod(ACL2Record)
def join(cls, rorp_iter, rbdir, time, restrict_index):
"""Add access control list information to existing rorp_iter"""
def helper(rorp_iter, acl_iter):
"""Add ACL information in acl_iter to rorp_iter"""
collated = rorpiter.CollateIterators(rorp_iter, acl_iter)
for rorp, acl in collated:
def join_acl_iter(rorp_iter, acl_iter):
"""Update a rorp iter by adding the information from acl_iter"""
for rorp, acl in rorpiter.CollateIterators(rorp_iter, acl_iter):
assert rorp, "Missing rorp for index %s" % (acl.index,)
if not acl: acl = AccessControlLists(rorp.index)
rorp.set_acl(acl)
yield rorp
acl_iter = cls.get_objects_at_time(rbdir, time, restrict_index)
if not acl_iter:
log.Log("Warning: Access Control List file not found", 2)
acl_iter = iter([])
return helper(rorp_iter, acl_iter)
static.MakeClass(AccessControlListFile)
def GetCombinedMetadataIter(rbdir, time, restrict_index = None,
acls = None, eas = None):
"""Return iterator of rorps from metadata and related files
None will be returned if the metadata file is absent. If acls or
eas is true, access control list or extended attribute information
will be added.
"""
metadata_iter = metadata.MetadataFile.get_objects_at_time(
rbdir, time, restrict_index)
if not metadata_iter:
log.Log("Warning, metadata file not found.\n"
"Metadata will be read from filesystem.", 2)
return None
if eas:
metadata_iter = ExtendedAttributesFile.join(
metadata_iter, rbdir, time, restrict_index)
if acls:
metadata_iter = AccessControlListFile.join(
metadata_iter, rbdir, time, restrict_index)
return metadata_iter
def rpath_acl_get(rp):
"""Get acls of given rpath rp.
......
This diff is collapsed.
......@@ -143,8 +143,8 @@ def iterate_raw_rfs(mirror_rp, inc_rp):
def yield_metadata():
"""Iterate rorps from metadata file, if any are available"""
metadata_iter = metadata.MetadataFile.get_objects_at_time(Globals.rbdir,
regress_time)
metadata.SetManager()
metadata_iter = metadata.ManagerObj.GetAtTime(regress_time)
if metadata_iter: return metadata_iter
log.Log.FatalError("No metadata for time %s found, cannot regress"
% Time.timetopretty(regress_time))
......
......@@ -22,7 +22,7 @@
from __future__ import generators
import tempfile, os, cStringIO
import Globals, Time, Rdiff, Hardlink, rorpiter, selection, rpath, \
log, static, robust, metadata, statistics, TempFile, eas_acls, hash
log, static, robust, metadata, statistics, TempFile, hash
class RestoreError(Exception): pass
......@@ -177,9 +177,9 @@ class MirrorStruct:
"""
if rest_time is None: rest_time = cls._rest_time
rorp_iter = eas_acls.GetCombinedMetadataIter(
Globals.rbdir, rest_time, restrict_index = cls.mirror_base.index,
acls = Globals.acls_active, eas = Globals.eas_active)
if not metadata.ManagerObj: metadata.SetManager()
rorp_iter = metadata.ManagerObj.GetAtTime(rest_time,
cls.mirror_base.index)
if not rorp_iter:
if require_metadata:
log.Log.FatalError("Mirror metadata not found")
......
......@@ -60,7 +60,7 @@ class MetadataTest(unittest.TestCase):
def write_metadata_to_temp(self):
"""If necessary, write metadata of bigdir to file metadata.gz"""
global tempdir
temprp = tempdir.append("metadata.gz")
temprp = tempdir.append("mirror_metadata.2005-11-03T14:51:06-06:00.snapshot.gz")
if temprp.lstat(): return temprp
self.make_temp()
......@@ -68,19 +68,19 @@ class MetadataTest(unittest.TestCase):
rpath_iter = selection.Select(rootrp).set_iter()
start_time = time.time()
MetadataFile.open_file(temprp)
for rp in rpath_iter: MetadataFile.write_object(rp)
MetadataFile.close_file()
mf = MetadataFile(temprp, 'w')
for rp in rpath_iter: mf.write_object(rp)
mf.close()
print "Writing metadata took %s seconds" % (time.time() - start_time)
return temprp
def testSpeed(self):
"""Test testIterator on 10000 files"""
temprp = self.write_metadata_to_temp()
MetadataFile._rp = temprp
mf = MetadataFile(temprp, 'r')
start_time = time.time(); i = 0
for rorp in MetadataFile.get_objects(): i += 1
for rorp in mf.get_objects(): i += 1
print "Reading %s metadata entries took %s seconds." % \
(i, time.time() - start_time)
......@@ -102,9 +102,9 @@ class MetadataTest(unittest.TestCase):
"""
temprp = self.write_metadata_to_temp()
MetadataFile._rp = temprp
mf = MetadataFile(temprp, 'r')
start_time = time.time(); i = 0
for rorp in MetadataFile.get_objects(("subdir3", "subdir10")): i += 1
for rorp in mf.get_objects(("subdir3", "subdir10")): i += 1
print "Reading %s metadata entries took %s seconds." % \
(i, time.time() - start_time)
assert i == 51
......@@ -112,7 +112,7 @@ class MetadataTest(unittest.TestCase):
def test_write(self):
"""Test writing to metadata file, then reading back contents"""
global tempdir
temprp = tempdir.append("write_test.gz")
temprp = tempdir.append("mirror_metadata.2005-11-03T12:51:06-06:00.snapshot.gz")
if temprp.lstat(): temprp.delete()
self.make_temp()
......@@ -123,14 +123,42 @@ class MetadataTest(unittest.TestCase):
rps = map(rootrp.append, dirlisting)
assert not temprp.lstat()
MetadataFile.open_file(temprp)
for rp in rps: MetadataFile.write_object(rp)
MetadataFile.close_file()
write_mf = MetadataFile(temprp, 'w')
for rp in rps: write_mf.write_object(rp)
write_mf.close()
assert temprp.lstat()
reread_rps = list(MetadataFile.get_objects())
reread_rps = list(MetadataFile(temprp, 'r').get_objects())
assert len(reread_rps) == len(rps), (len(reread_rps), len(rps))
for i in range(len(reread_rps)):
assert reread_rps[i] == rps[i], i
def test_patch(self):
"""Test combining 3 iters of metadata rorps"""
self.make_temp()
os.system('cp -a testfiles/various_file_types/* ' + tempdir.path)
rp1 = tempdir.append('regular_file')
rp2 = tempdir.append('subdir')
rp3 = rp2.append('subdir_file')
rp4 = tempdir.append('test')
rp1new = tempdir.append('regular_file')
rp1new.chmod(0)
zero = rpath.RORPath(('test',))
current = [rp1, rp2, rp3]
diff1 = [rp1, rp4]
diff2 = [rp1new, rp2, zero]
output = patch(iter(current), iter(diff1), iter(diff2))
out1 = output.next()
assert out1 is rp1new, out1
out2 = output.next()
assert out2 is rp2, out2
out3 = output.next()
assert out3 is rp3, out3
self.assertRaises(StopIteration, output.next)
if __name__ == "__main__": unittest.main()
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment