Skip to content

R
rdiff-backup
Commit cea2c3bc authored by ben's avatar ben
Browse files
Options

Lots of changes, see changelog for 0.7.4. 


git-svn-id: http://svn.savannah.nongnu.org/svn/rdiff-backup@72 2b77aa54-bcbc-44c9-a7ec-4f6cf2b41109
parent 8a3e9334
Expand all Hide whitespace changes
Inline Side-by-side
Showing with 879 additions and 643 deletions
trunk/rdiff-backup/rdiff_backup/connection.py
View file @ cea2c3bc
...@@ -92,6 +92,7 @@ class LowLevelPipeConnection(Connection): ...@@ -92,6 +92,7 @@ class LowLevelPipeConnection(Connection):
b - string b - string
q - quit signal q - quit signal
t - TempFile t - TempFile
d - DSRPath
R - RPath R - RPath
r - RORPath only r - RORPath only
c - PipeConnection object c - PipeConnection object
...@@ -118,6 +119,7 @@ class LowLevelPipeConnection(Connection): ...@@ -118,6 +119,7 @@ class LowLevelPipeConnection(Connection):
if type(obj) is types.StringType: self._putbuf(obj, req_num) if type(obj) is types.StringType: self._putbuf(obj, req_num)
elif isinstance(obj, Connection): self._putconn(obj, req_num) elif isinstance(obj, Connection): self._putconn(obj, req_num)
elif isinstance(obj, TempFile): self._puttempfile(obj, req_num) elif isinstance(obj, TempFile): self._puttempfile(obj, req_num)
elif isinstance(obj, DSRPath): self._putdsrpath(obj, req_num)
elif isinstance(obj, RPath): self._putrpath(obj, req_num) elif isinstance(obj, RPath): self._putrpath(obj, req_num)
elif isinstance(obj, RORPath): self._putrorpath(obj, req_num) elif isinstance(obj, RORPath): self._putrorpath(obj, req_num)
elif ((hasattr(obj, "read") or hasattr(obj, "write")) elif ((hasattr(obj, "read") or hasattr(obj, "write"))
...@@ -148,6 +150,11 @@ class LowLevelPipeConnection(Connection): ...@@ -148,6 +150,11 @@ class LowLevelPipeConnection(Connection):
tempfile.index, tempfile.data) tempfile.index, tempfile.data)
self._write("t", cPickle.dumps(tf_repr, 1), req_num) self._write("t", cPickle.dumps(tf_repr, 1), req_num)
def _putdsrpath(self, dsrpath, req_num):
"""Put DSRPath into pipe. See _putrpath"""
dsrpath_repr = (dsrpath.conn.conn_number, dsrpath.getstatedict())
self._write("d", cPickle.dumps(dsrpath_repr, 1), req_num)
def _putrpath(self, rpath, req_num): def _putrpath(self, rpath, req_num):
"""Put an rpath into the pipe """Put an rpath into the pipe
...@@ -219,23 +226,22 @@ class LowLevelPipeConnection(Connection): ...@@ -219,23 +226,22 @@ class LowLevelPipeConnection(Connection):
ord(header_string[1]), ord(header_string[1]),
self._s2l(header_string[2:])) self._s2l(header_string[2:]))
except IndexError: raise ConnectionError() except IndexError: raise ConnectionError()
if format_string == "o": result = cPickle.loads(self._read(length)) if format_string == "q": raise ConnectionQuit("Received quit signal")
elif format_string == "b": result = self._read(length)
elif format_string == "f": data = self._read(length)
result = VirtualFile(self, int(self._read(length))) if format_string == "o": result = cPickle.loads(data)
elif format_string == "b": result = data
elif format_string == "f": result = VirtualFile(self, int(data))
elif format_string == "i": elif format_string == "i":
result = RORPIter.FromFile(BufferedRead( result = RORPIter.FromFile(BufferedRead(VirtualFile(self,
VirtualFile(self, int(self._read(length))))) int(data))))
elif format_string == "t": elif format_string == "t": result = self._gettempfile(data)
result = self._gettempfile(self._read(length)) elif format_string == "r": result = self._getrorpath(data)
elif format_string == "r": elif format_string == "R": result = self._getrpath(data)
result = self._getrorpath(self._read(length)) elif format_string == "d": result = self._getdsrpath(data)
elif format_string == "R": result = self._getrpath(self._read(length))
elif format_string == "c":
result = Globals.connection_dict[int(self._read(length))]
else: else:
assert format_string == "q", header_string assert format_string == "c", header_string
raise ConnectionQuit("Received quit signal") result = Globals.connection_dict[int(data)]
Log.conn("received", result, req_num) Log.conn("received", result, req_num)
return (req_num, result) return (req_num, result)
...@@ -255,6 +261,15 @@ class LowLevelPipeConnection(Connection): ...@@ -255,6 +261,15 @@ class LowLevelPipeConnection(Connection):
conn_number, base, index, data = cPickle.loads(raw_rpath_buf) conn_number, base, index, data = cPickle.loads(raw_rpath_buf)
return RPath(Globals.connection_dict[conn_number], base, index, data) return RPath(Globals.connection_dict[conn_number], base, index, data)
def _getdsrpath(self, raw_dsrpath_buf):
"""Return DSRPath object indicated by buf"""
conn_number, state_dict = cPickle.loads(raw_dsrpath_buf)
empty_dsrp = DSRPath("bypass", Globals.local_connection, None)
empty_dsrp.__setstate__(state_dict)
empty_dsrp.conn = Globals.connection_dict[conn_number]
empty_dsrp.file = None
return empty_dsrp
def _close(self): def _close(self):
"""Close the pipes associated with the connection""" """Close the pipes associated with the connection"""
self.outpipe.close() self.outpipe.close()
......
trunk/rdiff-backup/rdiff_backup/destructive_stepping.py
View file @ cea2c3bc
from __future__ import generators from __future__ import generators
import types
execfile("rorpiter.py") execfile("rorpiter.py")
####################################################################### #######################################################################
...@@ -40,13 +41,17 @@ class DSRPath(RPath): ...@@ -40,13 +41,17 @@ class DSRPath(RPath):
otherwise use the same arguments as the RPath initializer. otherwise use the same arguments as the RPath initializer.
""" """
if len(args) == 2 and isinstance(args[0], RPath): if len(args) == 1 and isinstance(args[0], RPath):
rp = args[0] rp = args[0]
RPath.__init__(self, rp.conn, rp.base, rp.index) RPath.__init__(self, rp.conn, rp.base, rp.index)
else: RPath.__init__(self, *args) else: RPath.__init__(self, *args)
self.set_delays(source) if source != "bypass":
self.set_init_perms(source) # "bypass" val is used when unpackaging over connection
assert source is None or source is 1
self.source = source
self.set_delays(source)
self.set_init_perms(source)
def set_delays(self, source): def set_delays(self, source):
"""Delay writing permissions and times where appropriate""" """Delay writing permissions and times where appropriate"""
...@@ -59,13 +64,14 @@ class DSRPath(RPath): ...@@ -59,13 +64,14 @@ class DSRPath(RPath):
# Now get atime right away if possible # Now get atime right away if possible
if self.data.has_key('atime'): self.newatime = self.data['atime'] if self.data.has_key('atime'): self.newatime = self.data['atime']
else: self.newatime = None else: self.newatime = None
else: self.delay_atime = None
if source: if source:
self.delay_mtime = None # we'll never change mtime of source file self.delay_mtime = None # we'll never change mtime of source file
else: else:
self.delay_mtime = 1 self.delay_mtime = 1
# Save mtime now for a dir, because it might inadvertantly change # Save mtime now for a dir, because it might inadvertantly change
if self.isdir(): self.newmtime = self.getmtime() if self.isdir(): self.newmtime = self.data['mtime']
else: self.newmtime = None else: self.newmtime = None
def set_init_perms(self, source): def set_init_perms(self, source):
...@@ -75,26 +81,30 @@ class DSRPath(RPath): ...@@ -75,26 +81,30 @@ class DSRPath(RPath):
self.chmod_bypass(0400) self.chmod_bypass(0400)
else: self.warn("No read permissions") else: self.warn("No read permissions")
elif self.isdir(): elif self.isdir():
if source and (not self.readable() or self.executable()): if source and (not self.readable() or not self.executable()):
if Globals.change_source_perms and self.isowner(): if Globals.change_source_perms and self.isowner():
self.chmod_bypass(0500) self.chmod_bypass(0500)
else: warn("No read or exec permission") else: self.warn("No read or exec permission")
elif not source and not self.hasfullperms(): elif not source and not self.hasfullperms():
self.chmod_bypass(0700) self.chmod_bypass(0700)
def warn(self, err): def warn(self, err):
Log("Received error '%s' when dealing with file %s, skipping..." Log("Received error '%s' when dealing with file %s, skipping..."
% (err, self.path), 1) % (err, self.path), 1)
raise DSRPermError(self.path) raise DSRPPermError(self.path)
def __getstate__(self): def __getstate__(self):
"""Return picklable state. See RPath __getstate__.""" """Return picklable state. See RPath __getstate__."""
assert self.conn is Globals.local_connection # Can't pickle a conn assert self.conn is Globals.local_connection # Can't pickle a conn
return self.getstatedict()
def getstatedict(self):
"""Return dictionary containing the attributes we can save"""
pickle_dict = {} pickle_dict = {}
for attrib in ['index', 'data', 'delay_perms', 'newperms', for attrib in ['index', 'data', 'delay_perms', 'newperms',
'delay_atime', 'newatime', 'delay_atime', 'newatime',
'delay_mtime', 'newmtime', 'delay_mtime', 'newmtime',
'path', 'base']: 'path', 'base', 'source']:
if self.__dict__.has_key(attrib): if self.__dict__.has_key(attrib):
pickle_dict[attrib] = self.__dict__[attrib] pickle_dict[attrib] = self.__dict__[attrib]
return pickle_dict return pickle_dict
...@@ -110,10 +120,17 @@ class DSRPath(RPath): ...@@ -110,10 +120,17 @@ class DSRPath(RPath):
if self.delay_perms: self.newperms = self.data['perms'] = permissions if self.delay_perms: self.newperms = self.data['perms'] = permissions
else: RPath.chmod(self, permissions) else: RPath.chmod(self, permissions)
def getperms(self):
"""Return dsrp's intended permissions"""
if self.delay_perms and self.newperms is not None:
return self.newperms
else: return self.data['perms']
def chmod_bypass(self, permissions): def chmod_bypass(self, permissions):
"""Change permissions without updating the data dictionary""" """Change permissions without updating the data dictionary"""
self.delay_perms = 1 self.delay_perms = 1
if self.newperms is None: self.newperms = self.getperms() if self.newperms is None: self.newperms = self.getperms()
Log("DSRP: Perm bypass %s to %o" % (self.path, permissions), 8)
self.conn.os.chmod(self.path, permissions) self.conn.os.chmod(self.path, permissions)
def settime(self, accesstime, modtime): def settime(self, accesstime, modtime):
...@@ -129,11 +146,25 @@ class DSRPath(RPath): ...@@ -129,11 +146,25 @@ class DSRPath(RPath):
if self.delay_mtime: self.newmtime = self.data['mtime'] = modtime if self.delay_mtime: self.newmtime = self.data['mtime'] = modtime
else: RPath.setmtime(self, modtime) else: RPath.setmtime(self, modtime)
def getmtime(self):
"""Return dsrp's intended modification time"""
if self.delay_mtime and self.newmtime is not None:
return self.newmtime
else: return self.data['mtime']
def getatime(self):
"""Return dsrp's intended access time"""
if self.delay_atime and self.newatime is not None:
return self.newatime
else: return self.data['atime']
def write_changes(self): def write_changes(self):
"""Write saved up permission/time changes""" """Write saved up permission/time changes"""
if not self.lstat(): return # File has been deleted in meantime if not self.lstat(): return # File has been deleted in meantime
if self.delay_perms and self.newperms is not None: if self.delay_perms and self.newperms is not None:
Log("Finalizing permissions of dsrp %s to %s" %
(self.path, self.newperms), 8)
RPath.chmod(self, self.newperms) RPath.chmod(self, self.newperms)
do_atime = self.delay_atime and self.newatime is not None do_atime = self.delay_atime and self.newatime is not None
...@@ -145,6 +176,19 @@ class DSRPath(RPath): ...@@ -145,6 +176,19 @@ class DSRPath(RPath):
elif not do_atime and do_mtime: elif not do_atime and do_mtime:
RPath.setmtime(self, self.newmtime) RPath.setmtime(self, self.newmtime)
def newpath(self, newpath, index = ()):
"""Return similar DSRPath but with new path"""
return self.__class__(self.source, self.conn, newpath, index)
def append(self, ext):
"""Return similar DSRPath with new extension"""
return self.__class__(self.source, self.conn, self.base,
self.index + (ext,))
def new_index(self, index):
"""Return similar DSRPath with new index"""
return self.__class__(self.source, self.conn, self.base, index)
class DestructiveSteppingFinalizer(IterTreeReducer): class DestructiveSteppingFinalizer(IterTreeReducer):
"""Finalizer that can work on an iterator of dsrpaths """Finalizer that can work on an iterator of dsrpaths
...@@ -155,11 +199,12 @@ class DestructiveSteppingFinalizer(IterTreeReducer): ...@@ -155,11 +199,12 @@ class DestructiveSteppingFinalizer(IterTreeReducer):
coming back to it. coming back to it.
""" """
dsrpath = None
def start_process(self, index, dsrpath): def start_process(self, index, dsrpath):
self.dsrpath = dsrpath self.dsrpath = dsrpath
def end_process(self): def end_process(self):
self.dsrpath.write_changes() if self.dsrpath: self.dsrpath.write_changes()
trunk/rdiff-backup/rdiff_backup/highlevel.py
View file @ cea2c3bc
...@@ -24,12 +24,14 @@ class HighLevel: ...@@ -24,12 +24,14 @@ class HighLevel:
accompanying diagram. accompanying diagram.
""" """
def Mirror(src_rpath, dest_rpath, checkpoint = 1, session_info = None): def Mirror(src_rpath, dest_rpath, checkpoint = 1,
session_info = None, write_finaldata = 1):
"""Turn dest_rpath into a copy of src_rpath """Turn dest_rpath into a copy of src_rpath
Checkpoint true means to checkpoint periodically, otherwise Checkpoint true means to checkpoint periodically, otherwise
not. If session_info is given, try to resume Mirroring from not. If session_info is given, try to resume Mirroring from
that point. that point. If write_finaldata is true, save extra data files
like hardlink_data. If it is false, make a complete mirror.
""" """
SourceS = src_rpath.conn.HLSourceStruct SourceS = src_rpath.conn.HLSourceStruct
...@@ -40,7 +42,8 @@ class HighLevel: ...@@ -40,7 +42,8 @@ class HighLevel:
src_init_dsiter = SourceS.split_initial_dsiter() src_init_dsiter = SourceS.split_initial_dsiter()
dest_sigiter = DestS.get_sigs(dest_rpath, src_init_dsiter) dest_sigiter = DestS.get_sigs(dest_rpath, src_init_dsiter)
diffiter = SourceS.get_diffs_and_finalize(dest_sigiter) diffiter = SourceS.get_diffs_and_finalize(dest_sigiter)
DestS.patch_and_finalize(dest_rpath, diffiter, checkpoint) DestS.patch_and_finalize(dest_rpath, diffiter,
checkpoint, write_finaldata)
dest_rpath.setdata() dest_rpath.setdata()
...@@ -61,24 +64,6 @@ class HighLevel: ...@@ -61,24 +64,6 @@ class HighLevel:
dest_rpath.setdata() dest_rpath.setdata()
inc_rpath.setdata() inc_rpath.setdata()
def Restore(rest_time, mirror_base, rel_index, baseinc_tup, target_base):
"""Like Restore.RestoreRecursive but check arguments"""
if (Globals.preserve_hardlinks != 0 and
Hardlink.retrieve_final(rest_time)):
Log("Hard link information found, attempting to preserve "
"hard links.", 4)
SetConnections.UpdateGlobal('preserve_hardlinks', 1)
else: SetConnections.UpdateGlobal('preserve_hardlinks', None)
if not isinstance(target_base, DSRPath):
target_base = DSRPath(target_base.conn, target_base.base,
target_base.index, target_base.data)
if not isinstance(mirror_base, DSRPath):
mirror_base = DSRPath(mirror_base.conn, mirror_base.base,
mirror_base.index, mirror_base.data)
Restore.RestoreRecursive(rest_time, mirror_base, rel_index,
baseinc_tup, target_base)
MakeStatic(HighLevel) MakeStatic(HighLevel)
...@@ -164,7 +149,7 @@ class HLDestinationStruct: ...@@ -164,7 +149,7 @@ class HLDestinationStruct:
def compare(src_rorp, dest_dsrp): def compare(src_rorp, dest_dsrp):
"""Return dest_dsrp if they are different, None if the same""" """Return dest_dsrp if they are different, None if the same"""
if not dest_dsrp: if not dest_dsrp:
dest_dsrp = DSRPath(baserp.conn, baserp.base, src_rorp.index) dest_dsrp = cls.get_dsrp(baserp, src_rorp.index)
if dest_dsrp.lstat(): if dest_dsrp.lstat():
Log("Warning: Found unexpected destination file %s, " Log("Warning: Found unexpected destination file %s, "
"not processing it." % dest_dsrp.path, 2) "not processing it." % dest_dsrp.path, 2)
...@@ -203,8 +188,9 @@ class HLDestinationStruct: ...@@ -203,8 +188,9 @@ class HLDestinationStruct:
def get_dsrp(cls, dest_rpath, index): def get_dsrp(cls, dest_rpath, index):
"""Return initialized dsrp based on dest_rpath with given index""" """Return initialized dsrp based on dest_rpath with given index"""
return DSRPath(source = None, dest_rpath.conn, dsrp = DSRPath(None, dest_rpath.conn, dest_rpath.base, index)
dest_rpath.base, index) if Globals.quoting_enabled: dsrp.quote_path()
return dsrp
def get_finalizer(cls): def get_finalizer(cls):
"""Return finalizer, starting from session info if necessary""" """Return finalizer, starting from session info if necessary"""
...@@ -216,9 +202,13 @@ class HLDestinationStruct: ...@@ -216,9 +202,13 @@ class HLDestinationStruct:
"""Return ITR, starting from state if necessary""" """Return ITR, starting from state if necessary"""
if cls._session_info and cls._session_info.ITR: if cls._session_info and cls._session_info.ITR:
return cls._session_info.ITR return cls._session_info.ITR
else: return IncrementITR(inc_rpath) else:
iitr = IncrementITR(inc_rpath)
iitr.override_changed()
return iitr
def patch_and_finalize(cls, dest_rpath, diffs, checkpoint = 1): def patch_and_finalize(cls, dest_rpath, diffs,
checkpoint = 1, write_finaldata = 1):
"""Apply diffs and finalize""" """Apply diffs and finalize"""
collated = RORPIter.CollateIterators(diffs, cls.initial_dsiter2) collated = RORPIter.CollateIterators(diffs, cls.initial_dsiter2)
finalizer = cls.get_finalizer() finalizer = cls.get_finalizer()
...@@ -242,7 +232,7 @@ class HLDestinationStruct: ...@@ -242,7 +232,7 @@ class HLDestinationStruct:
if checkpoint: SaveState.checkpoint_mirror(finalizer, dsrp) if checkpoint: SaveState.checkpoint_mirror(finalizer, dsrp)
except: cls.handle_last_error(dsrp, finalizer) except: cls.handle_last_error(dsrp, finalizer)
finalizer.Finish() finalizer.Finish()
if Globals.preserve_hardlinks and Globals.rbdir: if Globals.preserve_hardlinks and write_finaldata:
Hardlink.final_writedata() Hardlink.final_writedata()
if checkpoint: SaveState.checkpoint_remove() if checkpoint: SaveState.checkpoint_remove()
...@@ -300,8 +290,7 @@ class HLDestinationStruct: ...@@ -300,8 +290,7 @@ class HLDestinationStruct:
Log.exception(1) Log.exception(1)
if ITR: SaveState.checkpoint_inc_backup(ITR, finalizer, dsrp, 1) if ITR: SaveState.checkpoint_inc_backup(ITR, finalizer, dsrp, 1)
else: SaveState.checkpoint_mirror(finalizer, dsrp, 1) else: SaveState.checkpoint_mirror(finalizer, dsrp, 1)
if Globals.preserve_hardlinks: if Globals.preserve_hardlinks: Hardlink.final_checkpoint(Globals.rbdir)
Hardlink.final_checkpoint(Globals.rbdir)
SaveState.touch_last_file_definitive() SaveState.touch_last_file_definitive()
raise raise
......
trunk/rdiff-backup/rdiff_backup/increment.py
View file @ cea2c3bc
execfile("selection.py") execfile("filename_mapping.py")
####################################################################### #######################################################################
# #
...@@ -85,10 +85,12 @@ class Inc: ...@@ -85,10 +85,12 @@ class Inc:
"""Get new increment rp with given time suffix""" """Get new increment rp with given time suffix"""
addtostr = lambda s: "%s.%s.%s" % (s, timestr, typestr) addtostr = lambda s: "%s.%s.%s" % (s, timestr, typestr)
if rp.index: if rp.index:
return rp.__class__(rp.conn, rp.base, rp.index[:-1] + incrp = rp.__class__(rp.conn, rp.base, rp.index[:-1] +
(addtostr(rp.index[-1]),)) (addtostr(rp.index[-1]),))
else: return rp.__class__(rp.conn, addtostr(rp.base), rp.index) else: incrp = rp.__class__(rp.conn, addtostr(rp.base), rp.index)
if Globals.quoting_enabled: incrp.quote_path()
return incrp
inctime = 0 inctime = 0
while 1: while 1:
inctime = Resume.FindTime(rp.index, inctime) inctime = Resume.FindTime(rp.index, inctime)
...@@ -123,7 +125,7 @@ class IncrementITR(IterTreeReducer): ...@@ -123,7 +125,7 @@ class IncrementITR(IterTreeReducer):
def __init__(self, inc_rpath): def __init__(self, inc_rpath):
"""Set inc_rpath, an rpath of the base of the tree""" """Set inc_rpath, an rpath of the base of the tree"""
self.inc_rpath = inc_rpath self.inc_rpath = inc_rpath
IterTreeReducer.__init__(inc_rpath) IterTreeReducer.__init__(self, inc_rpath)
def start_process(self, index, diff_rorp, dsrp): def start_process(self, index, diff_rorp, dsrp):
"""Initial processing of file """Initial processing of file
...@@ -133,11 +135,21 @@ class IncrementITR(IterTreeReducer): ...@@ -133,11 +135,21 @@ class IncrementITR(IterTreeReducer):
""" """
incpref = self.inc_rpath.new_index(index) incpref = self.inc_rpath.new_index(index)
if Globals.quoting_enabled: incpref.quote_path()
if dsrp.isdir(): if dsrp.isdir():
self.init_dir(dsrp, diff_rorp, incpref) self.init_dir(dsrp, diff_rorp, incpref)
self.setvals(diff_rorp, dsrp, incpref) self.setvals(diff_rorp, dsrp, incpref)
else: self.init_non_dir(dsrp, diff_rorp, incpref) else: self.init_non_dir(dsrp, diff_rorp, incpref)
def override_changed(self):
"""Set changed flag to true
This is used only at the top level of a backup, to make sure
that a marker is created recording every backup session.
"""
self.changed = 1
def setvals(self, diff_rorp, dsrp, incpref): def setvals(self, diff_rorp, dsrp, incpref):
"""Record given values in state dict since in directory """Record given values in state dict since in directory
...@@ -162,7 +174,7 @@ class IncrementITR(IterTreeReducer): ...@@ -162,7 +174,7 @@ class IncrementITR(IterTreeReducer):
""" """
if not (incpref.lstat() and incpref.isdir()): incpref.mkdir() if not (incpref.lstat() and incpref.isdir()): incpref.mkdir()
if diff_rorp and diff_rorp.isreg() and diff_rorp.file: if diff_rorp and diff_rorp.isreg() and diff_rorp.file:
tf = TempFileManager(dsrp) tf = TempFileManager.new(dsrp)
RPathStatic.copy_with_attribs(diff_rorp, tf) RPathStatic.copy_with_attribs(diff_rorp, tf)
tf.set_attached_filetype(diff_rorp.get_attached_filetype()) tf.set_attached_filetype(diff_rorp.get_attached_filetype())
self.directory_replacement = tf self.directory_replacement = tf
...@@ -170,7 +182,7 @@ class IncrementITR(IterTreeReducer): ...@@ -170,7 +182,7 @@ class IncrementITR(IterTreeReducer):
def init_non_dir(self, dsrp, diff_rorp, incpref): def init_non_dir(self, dsrp, diff_rorp, incpref):
"""Process a non directory file (initial pass)""" """Process a non directory file (initial pass)"""
if not diff_rorp: return # no diff, so no change necessary if not diff_rorp: return # no diff, so no change necessary
if diff_rorp.isreg and (dsrp.isreg() or diff_rorp.isflaglinked()): if diff_rorp.isreg() and (dsrp.isreg() or diff_rorp.isflaglinked()):
tf = TempFileManager.new(dsrp) tf = TempFileManager.new(dsrp)
def init_thunk(): def init_thunk():
if diff_rorp.isflaglinked(): if diff_rorp.isflaglinked():
...@@ -180,8 +192,8 @@ class IncrementITR(IterTreeReducer): ...@@ -180,8 +192,8 @@ class IncrementITR(IterTreeReducer):
Inc.Increment_action(tf, dsrp, incpref).execute() Inc.Increment_action(tf, dsrp, incpref).execute()
Robust.make_tf_robustaction(init_thunk, (tf,), (dsrp,)).execute() Robust.make_tf_robustaction(init_thunk, (tf,), (dsrp,)).execute()
else: else:
Robust.chain([Inc.Increment_action(diff_rorp, dsrp, incref), Robust.chain([Inc.Increment_action(diff_rorp, dsrp, incpref),
RORPIter.patchonce_action(none, dsrp, diff_rorp)] RORPIter.patchonce_action(None, dsrp, diff_rorp)]
).execute() ).execute()
self.changed = 1 self.changed = 1
...@@ -207,89 +219,3 @@ class IncrementITR(IterTreeReducer): ...@@ -207,89 +219,3 @@ class IncrementITR(IterTreeReducer):
def make_patch_increment_ITR(inc_rpath, initial_state = None):
"""Return IterTreeReducer that patches and increments"""
def base_init(indexed_tuple):
"""Patch if appropriate, return (a,b) tuple
a is true if found directory and thus didn't take action
if a is false, b is true if some changes were made
if a is true, b is the rp of a temporary file used to hold
the diff_rorp's data (for dir -> normal file change), and
false if none was necessary.
"""
diff_rorp, dsrp = indexed_tuple
incpref = inc_rpath.new_index(indexed_tuple.index)
if dsrp.isdir(): return init_dir(dsrp, diff_rorp, incpref)
else: return init_non_dir(dsrp, diff_rorp, incpref)
def init_dir(dsrp, diff_rorp, incpref):
"""Initial processing of a directory
Make the corresponding directory right away, but wait
until the end to write the replacement. However, if the
diff_rorp contains data, we must write it locally before
continuing, or else that data will be lost in the stream.
"""
if not (incpref.lstat() and incpref.isdir()): incpref.mkdir()
if diff_rorp and diff_rorp.isreg() and diff_rorp.file:
tf = TempFileManager.new(dsrp)
RPathStatic.copy_with_attribs(diff_rorp, tf)
tf.set_attached_filetype(diff_rorp.get_attached_filetype())
return (1, tf)
else: return (1, None)
def init_non_dir(dsrp, diff_rorp, incpref):
"""Initial processing of non-directory
If a reverse diff is called for it is generated by apply
the forwards diff first on a temporary file.
"""
if diff_rorp:
if diff_rorp.isreg() and (dsrp.isreg() or
diff_rorp.isflaglinked()):
tf = TempFileManager.new(dsrp)
def init_thunk():
if diff_rorp.isflaglinked():
Hardlink.link_rp(diff_rorp, tf, dsrp)
else: Rdiff.patch_with_attribs_action(dsrp, diff_rorp,
tf).execute()
Inc.Increment_action(tf, dsrp, incpref).execute()
Robust.make_tf_robustaction(init_thunk, (tf,),
(dsrp,)).execute()
else:
Robust.chain([Inc.Increment_action(diff_rorp, dsrp,
incpref),
RORPIter.patchonce_action(
None, dsrp, diff_rorp)]).execute()
return (None, 1)
return (None, None)
def base_final(base_tuple, base_init_tuple, changed):
"""Patch directory if not done, return true iff made change"""
if base_init_tuple[0]: # was directory
diff_rorp, dsrp = base_tuple
if changed or diff_rorp:
if base_init_tuple[1]: diff_rorp = base_init_tuple[1]
Inc.Increment(diff_rorp, dsrp,
inc_rpath.new_index(base_tuple.index))
if diff_rorp:
RORPIter.patchonce_action(None, dsrp,
diff_rorp).execute()
if isinstance(diff_rorp, TempFile): diff_rorp.delete()
return 1
return None
else: # changed iff base_init_tuple says it was
return base_init_tuple[1]
return IterTreeReducer(base_init, lambda x,y: x or y, None,
base_final, initial_state)
trunk/rdiff-backup/rdiff_backup/lazy.py
View file @ cea2c3bc
...@@ -260,7 +260,7 @@ class IterTreeReducer: ...@@ -260,7 +260,7 @@ class IterTreeReducer:
""" """
index = args[0] index = args[0]
assert type(index) is types.TupleType assert type(index) is types.TupleType, type(index)
if self.index is None: if self.index is None:
self.start_process(*args) self.start_process(*args)
......
trunk/rdiff-backup/rdiff_backup/manage.py
View file @ cea2c3bc
...@@ -12,37 +12,53 @@ class Manage: ...@@ -12,37 +12,53 @@ class Manage:
"""Return Increments objects given the rdiff-backup data directory""" """Return Increments objects given the rdiff-backup data directory"""
return map(IncObj, Manage.find_incrps_with_base(datadir, "increments")) return map(IncObj, Manage.find_incrps_with_base(datadir, "increments"))
def find_incrps_with_base(dir_rp, basename): def get_file_type(rp):
"""Return list of incfiles with given basename in dir_rp""" """Returns one of "regular", "directory", "missing", or "special"."""
rps = map(dir_rp.append, dir_rp.listdir()) if not rp.lstat(): return "missing"
incrps = filter(RPath.isincfile, rps) elif rp.isdir(): return "directory"
result = filter(lambda rp: rp.getincbase_str() == basename, incrps) elif rp.isreg(): return "regular"
Log("find_incrps_with_base: found %d incs" % len(result), 6) else: return "special"
return result
def describe_root_incs(datadir): def get_inc_type(inc):
"""Return file type increment represents"""
assert inc.isincfile()
type = inc.getinctype()
if type == "dir": return "directory"
elif type == "diff": return "regular"
elif type == "missing": return "missing"
elif type == "snapshot": return Manage.get_file_type(inc)
else: assert None, "Unknown type %s" % (type,)
def describe_incs_parsable(incs, mirror_time, mirrorrp):
"""Return a string parsable by computer describing the increments
Each line is a time in seconds of the increment, and then the
type of the file. It will be sorted oldest to newest. For example:
10000 regular
20000 directory
30000 special
40000 missing
50000 regular <- last will be the current mirror
"""
incpairs = [(Time.stringtotime(inc.getinctime()), inc) for inc in incs]
incpairs.sort()
result = ["%s %s" % (time, Manage.get_inc_type(inc))
for time, inc in incpairs]
result.append("%s %s" % (mirror_time, Manage.get_file_type(mirrorrp)))
return "\n".join(result)
def describe_incs_human(incs, mirror_time, mirrorrp):
"""Return a string describing all the the root increments""" """Return a string describing all the the root increments"""
result = [] incpairs = [(Time.stringtotime(inc.getinctime()), inc) for inc in incs]
currentrps = Manage.find_incrps_with_base(datadir, "current_mirror") incpairs.sort()
if not currentrps:
Log("Warning: no current mirror marker found", 1) result = ["Found %d increments:" % len(incpairs)]
elif len(currentrps) > 1: for time, inc in incpairs:
Log("Warning: multiple mirror markers found", 1) result.append(" %s %s" %
for rp in currentrps: (inc.dirsplit()[1], Time.timetopretty(time)))
result.append("Found mirror marker %s" % rp.path) result.append("Current mirror: %s" % Time.timetopretty(mirror_time))
result.append("Indicating latest mirror taken at %s" %
Time.stringtopretty(rp.getinctime()))
result.append("---------------------------------------------"
"-------------")
# Sort so they are in reverse order by time
time_w_incobjs = map(lambda io: (-io.time, io),
Manage.get_incobjs(datadir))
time_w_incobjs.sort()
incobjs = map(lambda x: x[1], time_w_incobjs)
result.append("Found %d increments:" % len(incobjs))
result.append("\n------------------------------------------\n".join(
map(IncObj.full_description, incobjs)))
return "\n".join(result) return "\n".join(result)
def delete_earlier_than(baserp, time): def delete_earlier_than(baserp, time):
...@@ -53,6 +69,11 @@ class Manage: ...@@ -53,6 +69,11 @@ class Manage:
rdiff-backup-data directory should be the root of the tree. rdiff-backup-data directory should be the root of the tree.
""" """
baserp.conn.Manage.delete_earlier_than_local(baserp, time)
def delete_earlier_than_local(baserp, time):
"""Like delete_earlier_than, but run on local connection for speed"""
assert baserp.conn is Globals.local_connection
def yield_files(rp): def yield_files(rp):
yield rp yield rp
if rp.isdir(): if rp.isdir():
......
trunk/rdiff-backup/rdiff_backup/restore.py
View file @ cea2c3bc
This diff is collapsed.
trunk/rdiff-backup/rdiff_backup/robust.py
View file @ cea2c3bc
...@@ -252,7 +252,7 @@ class TempFile(RPath): ...@@ -252,7 +252,7 @@ class TempFile(RPath):
if self.isdir() and not rp_dest.isdir(): if self.isdir() and not rp_dest.isdir():
# Cannot move a directory directly over another file # Cannot move a directory directly over another file
rp_dest.delete() rp_dest.delete()
if (isinstance(rp_dest, DSRPath) and rp_dest.perms_delayed if (isinstance(rp_dest, DSRPath) and rp_dest.delay_perms
and not self.hasfullperms()): and not self.hasfullperms()):
# If we are moving to a delayed perm directory, delay # If we are moving to a delayed perm directory, delay
# permission change on destination. # permission change on destination.
...@@ -531,7 +531,7 @@ class Resume: ...@@ -531,7 +531,7 @@ class Resume:
Log("Last backup dated %s was aborted, but we aren't " Log("Last backup dated %s was aborted, but we aren't "
"resuming it." % Time.timetopretty(si.time), 2) "resuming it." % Time.timetopretty(si.time), 2)
return None return None
assert 0 assert None
MakeClass(Resume) MakeClass(Resume)
......
trunk/rdiff-backup/rdiff_backup/rpath.py
View file @ cea2c3bc
...@@ -168,6 +168,17 @@ class RPathStatic: ...@@ -168,6 +168,17 @@ class RPathStatic:
rp_dest.data = rp_source.data rp_dest.data = rp_source.data
rp_source.data = {'type': None} rp_source.data = {'type': None}
# If we are moving to a DSRPath, assume that the current times
# are the intended ones. We need to save them now in case
# they are changed later.
if isinstance(rp_dest, DSRPath):
if rp_dest.delay_mtime:
if 'mtime' in rp_dest.data:
rp_dest.setmtime(rp_dest.data['mtime'])
if rp_dest.delay_atime:
if 'atime' in rp_dest.data:
rp_dest.setatime(rp_dest.data['atime'])
def tupled_lstat(filename): def tupled_lstat(filename):
"""Like os.lstat, but return only a tuple, or None if os.error """Like os.lstat, but return only a tuple, or None if os.error
...@@ -413,7 +424,7 @@ class RPath(RORPath): ...@@ -413,7 +424,7 @@ class RPath(RORPath):
self.base = base self.base = base
self.path = apply(os.path.join, (base,) + self.index) self.path = apply(os.path.join, (base,) + self.index)
self.file = None self.file = None
if data: self.data = data if data or base is None: self.data = data
else: self.setdata() else: self.setdata()
def __str__(self): def __str__(self):
...@@ -493,6 +504,12 @@ class RPath(RORPath): ...@@ -493,6 +504,12 @@ class RPath(RORPath):
s = self.conn.reval("lambda path: os.lstat(path).st_rdev", self.path) s = self.conn.reval("lambda path: os.lstat(path).st_rdev", self.path)
return (s >> 8, s & 0xff) return (s >> 8, s & 0xff)
def quote_path(self):
"""Set path from quoted version of index"""
quoted_list = [FilenameMapping.quote(path) for path in self.index]
self.path = apply(os.path.join, [self.base] + quoted_list)
self.setdata()
def chmod(self, permissions): def chmod(self, permissions):
"""Wrapper around os.chmod""" """Wrapper around os.chmod"""
self.conn.os.chmod(self.path, permissions) self.conn.os.chmod(self.path, permissions)
...@@ -594,7 +611,8 @@ class RPath(RORPath): ...@@ -594,7 +611,8 @@ class RPath(RORPath):
if not self.lstat(): return # must have been deleted in meantime if not self.lstat(): return # must have been deleted in meantime
elif self.isdir(): elif self.isdir():
itm = RpathDeleter() itm = RpathDeleter()
for dsrp in Select(self, None).set_iter(): itm(dsrp.index, dsrp) for dsrp in Select(DSRPath(None, self)).set_iter():
itm(dsrp.index, dsrp)
itm.Finish() itm.Finish()
else: self.conn.os.unlink(self.path) else: self.conn.os.unlink(self.path)
self.setdata() self.setdata()
...@@ -616,7 +634,7 @@ class RPath(RORPath): ...@@ -616,7 +634,7 @@ class RPath(RORPath):
self.path.split("/"))) self.path.split("/")))
if self.path[0] == "/": newpath = "/" + newpath if self.path[0] == "/": newpath = "/" + newpath
elif not newpath: newpath = "." elif not newpath: newpath = "."
return self.__class__(self.conn, newpath, ()) return self.newpath(newpath)
def dirsplit(self): def dirsplit(self):
"""Returns a tuple of strings (dirname, basename) """Returns a tuple of strings (dirname, basename)
...@@ -635,10 +653,20 @@ class RPath(RORPath): ...@@ -635,10 +653,20 @@ class RPath(RORPath):
comps = normed.path.split("/") comps = normed.path.split("/")
return "/".join(comps[:-1]), comps[-1] return "/".join(comps[:-1]), comps[-1]
def newpath(self, newpath, index = ()):
"""Return new RPath with the same connection but different path"""
return self.__class__(self.conn, newpath, index)
def append(self, ext): def append(self, ext):
"""Return new RPath with same connection by adjoing ext""" """Return new RPath with same connection by adjoing ext"""
return self.__class__(self.conn, self.base, self.index + (ext,)) return self.__class__(self.conn, self.base, self.index + (ext,))
def append_path(self, ext, new_index = ()):
"""Like append, but add ext to path instead of to index"""
assert not self.index # doesn't make sense if index isn't ()
return self.__class__(self.conn, os.path.join(self.base, ext),
new_index)
def new_index(self, index): def new_index(self, index):
"""Return similar RPath but with new index""" """Return similar RPath but with new index"""
return self.__class__(self.conn, self.base, index) return self.__class__(self.conn, self.base, index)
......
trunk/rdiff-backup/rdiff_backup/selection.py
View file @ cea2c3bc
...@@ -60,21 +60,20 @@ class Select: ...@@ -60,21 +60,20 @@ class Select:
# This re should not match normal filenames, but usually just globs # This re should not match normal filenames, but usually just globs
glob_re = re.compile("(.*[*?[]|ignorecase\\:)", re.I | re.S) glob_re = re.compile("(.*[*?[]|ignorecase\\:)", re.I | re.S)
def __init__(self, rpath, source): def __init__(self, dsrpath, quoted_filenames = None):
"""DSRPIterator initializer. """DSRPIterator initializer. dsrp is the root directory
rpath is the root dir. Source is true if rpath is the root of When files have quoted characters in them, quoted_filenames
the source directory, and false for the mirror directory should be true. Then RPath's index will be the unquoted
version.
""" """
assert isinstance(rpath, RPath) assert isinstance(dsrpath, DSRPath)
self.selection_functions = [] self.selection_functions = []
self.source = source self.dsrpath = dsrpath
if isinstance(rpath, DSRPath): self.dsrpath = rpath
else: self.dsrpath = DSRPath(rpath.conn, rpath.base,
rpath.index, rpath.data)
self.prefix = self.dsrpath.path self.prefix = self.dsrpath.path
self.quoting_on = Globals.quoting_enabled and quoted_filenames
def set_iter(self, starting_index = None, sel_func = None): def set_iter(self, starting_index = None, sel_func = None):
"""Initialize more variables, get ready to iterate """Initialize more variables, get ready to iterate
...@@ -106,7 +105,7 @@ class Select: ...@@ -106,7 +105,7 @@ class Select:
""" """
s = sel_func(dsrpath) s = sel_func(dsrpath)
if s === 0: return if s == 0: return
elif s == 1: # File is included elif s == 1: # File is included
yield dsrpath yield dsrpath
if dsrpath.isdir(): if dsrpath.isdir():
...@@ -122,11 +121,15 @@ class Select: ...@@ -122,11 +121,15 @@ class Select:
def iterate_in_dir(self, dsrpath, rec_func, sel_func): def iterate_in_dir(self, dsrpath, rec_func, sel_func):
"""Iterate the dsrps in directory dsrpath.""" """Iterate the dsrps in directory dsrpath."""
dir_listing = dsrpath.listdir() if self.quoting_on:
dir_listing.sort() for subdir in FilenameMapping.get_quoted_dir_children(dsrpath):
for filename in dir_listing: for dsrp in rec_func(subdir, rec_func, sel_func): yield dsrp
for dsrp in rec_func(dsrpath.append(filename), rec_func, sel_func): else:
yield dsrp dir_listing = dsrpath.listdir()
dir_listing.sort()
for filename in dir_listing:
for dsrp in rec_func(dsrpath.append(filename),
rec_func, sel_func): yield dsrp
def iterate_starting_from(self, dsrpath, rec_func, sel_func): def iterate_starting_from(self, dsrpath, rec_func, sel_func):
"""Like Iterate, but only yield indicies > self.starting_index""" """Like Iterate, but only yield indicies > self.starting_index"""
......
trunk/rdiff-backup/src/Make
View file @ cea2c3bc
...@@ -23,9 +23,9 @@ def mystrip(filename): ...@@ -23,9 +23,9 @@ def mystrip(filename):
files = ["globals.py", "static.py", "lazy.py", "log.py", "ttime.py", files = ["globals.py", "static.py", "lazy.py", "log.py", "ttime.py",
"iterfile.py", "rdiff.py", "connection.py", "rpath.py", "iterfile.py", "rdiff.py", "connection.py", "rpath.py",
"hardlink.py", "robust.py", "rorpiter.py", "hardlink.py", "robust.py", "rorpiter.py",
"destructive_stepping.py", "selection.py", "increment.py", "destructive_stepping.py", "selection.py",
"restore.py", "manage.py", "highlevel.py", "filename_mapping.py", "increment.py", "restore.py",
"setconnections.py", "main.py"] "manage.py", "highlevel.py", "setconnections.py", "main.py"]
os.system("cp header.py rdiff-backup") os.system("cp header.py rdiff-backup")
......
trunk/rdiff-backup/src/connection.py
View file @ cea2c3bc
...@@ -92,6 +92,7 @@ class LowLevelPipeConnection(Connection): ...@@ -92,6 +92,7 @@ class LowLevelPipeConnection(Connection):
b - string b - string
q - quit signal q - quit signal
t - TempFile t - TempFile
d - DSRPath
R - RPath R - RPath
r - RORPath only r - RORPath only
c - PipeConnection object c - PipeConnection object
...@@ -118,6 +119,7 @@ class LowLevelPipeConnection(Connection): ...@@ -118,6 +119,7 @@ class LowLevelPipeConnection(Connection):
if type(obj) is types.StringType: self._putbuf(obj, req_num) if type(obj) is types.StringType: self._putbuf(obj, req_num)
elif isinstance(obj, Connection): self._putconn(obj, req_num) elif isinstance(obj, Connection): self._putconn(obj, req_num)
elif isinstance(obj, TempFile): self._puttempfile(obj, req_num) elif isinstance(obj, TempFile): self._puttempfile(obj, req_num)
elif isinstance(obj, DSRPath): self._putdsrpath(obj, req_num)
elif isinstance(obj, RPath): self._putrpath(obj, req_num) elif isinstance(obj, RPath): self._putrpath(obj, req_num)
elif isinstance(obj, RORPath): self._putrorpath(obj, req_num) elif isinstance(obj, RORPath): self._putrorpath(obj, req_num)
elif ((hasattr(obj, "read") or hasattr(obj, "write")) elif ((hasattr(obj, "read") or hasattr(obj, "write"))
...@@ -148,6 +150,11 @@ class LowLevelPipeConnection(Connection): ...@@ -148,6 +150,11 @@ class LowLevelPipeConnection(Connection):
tempfile.index, tempfile.data) tempfile.index, tempfile.data)
self._write("t", cPickle.dumps(tf_repr, 1), req_num) self._write("t", cPickle.dumps(tf_repr, 1), req_num)
def _putdsrpath(self, dsrpath, req_num):
"""Put DSRPath into pipe. See _putrpath"""
dsrpath_repr = (dsrpath.conn.conn_number, dsrpath.getstatedict())
self._write("d", cPickle.dumps(dsrpath_repr, 1), req_num)
def _putrpath(self, rpath, req_num): def _putrpath(self, rpath, req_num):
"""Put an rpath into the pipe """Put an rpath into the pipe
...@@ -219,23 +226,22 @@ class LowLevelPipeConnection(Connection): ...@@ -219,23 +226,22 @@ class LowLevelPipeConnection(Connection):
ord(header_string[1]), ord(header_string[1]),
self._s2l(header_string[2:])) self._s2l(header_string[2:]))
except IndexError: raise ConnectionError() except IndexError: raise ConnectionError()
if format_string == "o": result = cPickle.loads(self._read(length)) if format_string == "q": raise ConnectionQuit("Received quit signal")
elif format_string == "b": result = self._read(length)
elif format_string == "f": data = self._read(length)
result = VirtualFile(self, int(self._read(length))) if format_string == "o": result = cPickle.loads(data)
elif format_string == "b": result = data
elif format_string == "f": result = VirtualFile(self, int(data))
elif format_string == "i": elif format_string == "i":
result = RORPIter.FromFile(BufferedRead( result = RORPIter.FromFile(BufferedRead(VirtualFile(self,
VirtualFile(self, int(self._read(length))))) int(data))))
elif format_string == "t": elif format_string == "t": result = self._gettempfile(data)
result = self._gettempfile(self._read(length)) elif format_string == "r": result = self._getrorpath(data)
elif format_string == "r": elif format_string == "R": result = self._getrpath(data)
result = self._getrorpath(self._read(length)) elif format_string == "d": result = self._getdsrpath(data)
elif format_string == "R": result = self._getrpath(self._read(length))
elif format_string == "c":
result = Globals.connection_dict[int(self._read(length))]
else: else:
assert format_string == "q", header_string assert format_string == "c", header_string
raise ConnectionQuit("Received quit signal") result = Globals.connection_dict[int(data)]
Log.conn("received", result, req_num) Log.conn("received", result, req_num)
return (req_num, result) return (req_num, result)
...@@ -255,6 +261,15 @@ class LowLevelPipeConnection(Connection): ...@@ -255,6 +261,15 @@ class LowLevelPipeConnection(Connection):
conn_number, base, index, data = cPickle.loads(raw_rpath_buf) conn_number, base, index, data = cPickle.loads(raw_rpath_buf)
return RPath(Globals.connection_dict[conn_number], base, index, data) return RPath(Globals.connection_dict[conn_number], base, index, data)
def _getdsrpath(self, raw_dsrpath_buf):
"""Return DSRPath object indicated by buf"""
conn_number, state_dict = cPickle.loads(raw_dsrpath_buf)
empty_dsrp = DSRPath("bypass", Globals.local_connection, None)
empty_dsrp.__setstate__(state_dict)
empty_dsrp.conn = Globals.connection_dict[conn_number]
empty_dsrp.file = None
return empty_dsrp
def _close(self): def _close(self):
"""Close the pipes associated with the connection""" """Close the pipes associated with the connection"""
self.outpipe.close() self.outpipe.close()
......
trunk/rdiff-backup/src/destructive_stepping.py
View file @ cea2c3bc
from __future__ import generators from __future__ import generators
import types
execfile("rorpiter.py") execfile("rorpiter.py")
####################################################################### #######################################################################
...@@ -40,13 +41,17 @@ class DSRPath(RPath): ...@@ -40,13 +41,17 @@ class DSRPath(RPath):
otherwise use the same arguments as the RPath initializer. otherwise use the same arguments as the RPath initializer.
""" """
if len(args) == 2 and isinstance(args[0], RPath): if len(args) == 1 and isinstance(args[0], RPath):
rp = args[0] rp = args[0]
RPath.__init__(self, rp.conn, rp.base, rp.index) RPath.__init__(self, rp.conn, rp.base, rp.index)
else: RPath.__init__(self, *args) else: RPath.__init__(self, *args)
self.set_delays(source) if source != "bypass":
self.set_init_perms(source) # "bypass" val is used when unpackaging over connection
assert source is None or source is 1
self.source = source
self.set_delays(source)
self.set_init_perms(source)
def set_delays(self, source): def set_delays(self, source):
"""Delay writing permissions and times where appropriate""" """Delay writing permissions and times where appropriate"""
...@@ -59,13 +64,14 @@ class DSRPath(RPath): ...@@ -59,13 +64,14 @@ class DSRPath(RPath):
# Now get atime right away if possible # Now get atime right away if possible
if self.data.has_key('atime'): self.newatime = self.data['atime'] if self.data.has_key('atime'): self.newatime = self.data['atime']
else: self.newatime = None else: self.newatime = None
else: self.delay_atime = None
if source: if source:
self.delay_mtime = None # we'll never change mtime of source file self.delay_mtime = None # we'll never change mtime of source file
else: else:
self.delay_mtime = 1 self.delay_mtime = 1
# Save mtime now for a dir, because it might inadvertantly change # Save mtime now for a dir, because it might inadvertantly change
if self.isdir(): self.newmtime = self.getmtime() if self.isdir(): self.newmtime = self.data['mtime']
else: self.newmtime = None else: self.newmtime = None
def set_init_perms(self, source): def set_init_perms(self, source):
...@@ -75,26 +81,30 @@ class DSRPath(RPath): ...@@ -75,26 +81,30 @@ class DSRPath(RPath):
self.chmod_bypass(0400) self.chmod_bypass(0400)
else: self.warn("No read permissions") else: self.warn("No read permissions")
elif self.isdir(): elif self.isdir():
if source and (not self.readable() or self.executable()): if source and (not self.readable() or not self.executable()):
if Globals.change_source_perms and self.isowner(): if Globals.change_source_perms and self.isowner():
self.chmod_bypass(0500) self.chmod_bypass(0500)
else: warn("No read or exec permission") else: self.warn("No read or exec permission")
elif not source and not self.hasfullperms(): elif not source and not self.hasfullperms():
self.chmod_bypass(0700) self.chmod_bypass(0700)
def warn(self, err): def warn(self, err):
Log("Received error '%s' when dealing with file %s, skipping..." Log("Received error '%s' when dealing with file %s, skipping..."
% (err, self.path), 1) % (err, self.path), 1)
raise DSRPermError(self.path) raise DSRPPermError(self.path)
def __getstate__(self): def __getstate__(self):
"""Return picklable state. See RPath __getstate__.""" """Return picklable state. See RPath __getstate__."""
assert self.conn is Globals.local_connection # Can't pickle a conn assert self.conn is Globals.local_connection # Can't pickle a conn
return self.getstatedict()
def getstatedict(self):
"""Return dictionary containing the attributes we can save"""
pickle_dict = {} pickle_dict = {}
for attrib in ['index', 'data', 'delay_perms', 'newperms', for attrib in ['index', 'data', 'delay_perms', 'newperms',
'delay_atime', 'newatime', 'delay_atime', 'newatime',
'delay_mtime', 'newmtime', 'delay_mtime', 'newmtime',
'path', 'base']: 'path', 'base', 'source']:
if self.__dict__.has_key(attrib): if self.__dict__.has_key(attrib):
pickle_dict[attrib] = self.__dict__[attrib] pickle_dict[attrib] = self.__dict__[attrib]
return pickle_dict return pickle_dict
...@@ -110,10 +120,17 @@ class DSRPath(RPath): ...@@ -110,10 +120,17 @@ class DSRPath(RPath):
if self.delay_perms: self.newperms = self.data['perms'] = permissions if self.delay_perms: self.newperms = self.data['perms'] = permissions
else: RPath.chmod(self, permissions) else: RPath.chmod(self, permissions)
def getperms(self):
"""Return dsrp's intended permissions"""
if self.delay_perms and self.newperms is not None:
return self.newperms
else: return self.data['perms']
def chmod_bypass(self, permissions): def chmod_bypass(self, permissions):
"""Change permissions without updating the data dictionary""" """Change permissions without updating the data dictionary"""
self.delay_perms = 1 self.delay_perms = 1
if self.newperms is None: self.newperms = self.getperms() if self.newperms is None: self.newperms = self.getperms()
Log("DSRP: Perm bypass %s to %o" % (self.path, permissions), 8)
self.conn.os.chmod(self.path, permissions) self.conn.os.chmod(self.path, permissions)
def settime(self, accesstime, modtime): def settime(self, accesstime, modtime):
...@@ -129,11 +146,25 @@ class DSRPath(RPath): ...@@ -129,11 +146,25 @@ class DSRPath(RPath):
if self.delay_mtime: self.newmtime = self.data['mtime'] = modtime if self.delay_mtime: self.newmtime = self.data['mtime'] = modtime
else: RPath.setmtime(self, modtime) else: RPath.setmtime(self, modtime)
def getmtime(self):
"""Return dsrp's intended modification time"""
if self.delay_mtime and self.newmtime is not None:
return self.newmtime
else: return self.data['mtime']
def getatime(self):
"""Return dsrp's intended access time"""
if self.delay_atime and self.newatime is not None:
return self.newatime
else: return self.data['atime']
def write_changes(self): def write_changes(self):
"""Write saved up permission/time changes""" """Write saved up permission/time changes"""
if not self.lstat(): return # File has been deleted in meantime if not self.lstat(): return # File has been deleted in meantime
if self.delay_perms and self.newperms is not None: if self.delay_perms and self.newperms is not None:
Log("Finalizing permissions of dsrp %s to %s" %
(self.path, self.newperms), 8)
RPath.chmod(self, self.newperms) RPath.chmod(self, self.newperms)
do_atime = self.delay_atime and self.newatime is not None do_atime = self.delay_atime and self.newatime is not None
...@@ -145,6 +176,19 @@ class DSRPath(RPath): ...@@ -145,6 +176,19 @@ class DSRPath(RPath):
elif not do_atime and do_mtime: elif not do_atime and do_mtime:
RPath.setmtime(self, self.newmtime) RPath.setmtime(self, self.newmtime)
def newpath(self, newpath, index = ()):
"""Return similar DSRPath but with new path"""
return self.__class__(self.source, self.conn, newpath, index)
def append(self, ext):
"""Return similar DSRPath with new extension"""
return self.__class__(self.source, self.conn, self.base,
self.index + (ext,))
def new_index(self, index):
"""Return similar DSRPath with new index"""
return self.__class__(self.source, self.conn, self.base, index)
class DestructiveSteppingFinalizer(IterTreeReducer): class DestructiveSteppingFinalizer(IterTreeReducer):
"""Finalizer that can work on an iterator of dsrpaths """Finalizer that can work on an iterator of dsrpaths
...@@ -155,11 +199,12 @@ class DestructiveSteppingFinalizer(IterTreeReducer): ...@@ -155,11 +199,12 @@ class DestructiveSteppingFinalizer(IterTreeReducer):
coming back to it. coming back to it.
""" """
dsrpath = None
def start_process(self, index, dsrpath): def start_process(self, index, dsrpath):
self.dsrpath = dsrpath self.dsrpath = dsrpath
def end_process(self): def end_process(self):
self.dsrpath.write_changes() if self.dsrpath: self.dsrpath.write_changes()
trunk/rdiff-backup/src/globals.py
View file @ cea2c3bc
...@@ -8,7 +8,7 @@ import re, os ...@@ -8,7 +8,7 @@ import re, os
class Globals: class Globals:
# The current version of rdiff-backup # The current version of rdiff-backup
version = "0.7.3" version = "0.7.4"
# If this is set, use this value in seconds as the current time # If this is set, use this value in seconds as the current time
# instead of reading it from the clock. # instead of reading it from the clock.
...@@ -108,6 +108,18 @@ class Globals: ...@@ -108,6 +108,18 @@ class Globals:
# under MS windows NT. # under MS windows NT.
time_separator = ":" time_separator = ":"
# quoting_enabled is true if we should quote certain characters in
# filenames on the source side (see FilenameMapping for more
# info). chars_to_quote is a string whose characters should be
# quoted, and quoting_char is the character to quote with.
quoting_enabled = None
chars_to_quote = ""
quoting_char = ';'
# If true, emit output intended to be easily readable by a
# computer. False means output is intended for humans.
parsable_output = None
# If true, then hardlinks will be preserved to mirror and recorded # If true, then hardlinks will be preserved to mirror and recorded
# in the increments directory. There is also a difference here # in the increments directory. There is also a difference here
# between None and 0. When restoring, None or 1 means to preserve # between None and 0. When restoring, None or 1 means to preserve
...@@ -180,12 +192,12 @@ class Globals: ...@@ -180,12 +192,12 @@ class Globals:
else: cls.__dict__[name] = re.compile(re_string) else: cls.__dict__[name] = re.compile(re_string)
postset_regexp_local = classmethod(postset_regexp_local) postset_regexp_local = classmethod(postset_regexp_local)
def set_select(cls, source, dsrpath, tuplelist): def set_select(cls, dsrpath, tuplelist, quote_mode = None):
"""Initialize select object using tuplelist""" """Initialize select object using tuplelist"""
if source: if dsrpath.source:
cls.select_source = Select(dsrpath, 1) cls.select_source = Select(dsrpath, quote_mode)
cls.select_source.ParseArgs(tuplelist) cls.select_source.ParseArgs(tuplelist)
else: else:
cls.select_mirror = Select(dsrpath, None) cls.select_mirror = Select(dsrpath, quote_mode)
cls.select_mirror.ParseArgs(tuplelist) cls.select_mirror.ParseArgs(tuplelist)
set_select = classmethod(set_select) set_select = classmethod(set_select)
trunk/rdiff-backup/src/hardlink.py
View file @ cea2c3bc
...@@ -28,6 +28,11 @@ class Hardlink: ...@@ -28,6 +28,11 @@ class Hardlink:
_src_index_indicies = {} _src_index_indicies = {}
_dest_index_indicies = {} _dest_index_indicies = {}
# When a linked file is restored, its path is added to this dict,
# so it can be found when later paths being restored are linked to
# it.
_restore_index_path = {}
def get_inode_key(cls, rorp): def get_inode_key(cls, rorp):
"""Return rorp's key for _inode_ dictionaries""" """Return rorp's key for _inode_ dictionaries"""
return (rorp.getinode(), rorp.getdevloc()) return (rorp.getinode(), rorp.getdevloc())
...@@ -101,26 +106,29 @@ class Hardlink: ...@@ -101,26 +106,29 @@ class Hardlink:
"""True if rorp's index is already linked to something on src side""" """True if rorp's index is already linked to something on src side"""
return len(cls.get_indicies(rorp, 1)) >= 2 return len(cls.get_indicies(rorp, 1)) >= 2
def restore_link(cls, mirror_rel_index, rpath): def restore_link(cls, index, rpath):
"""Restores a linked file by linking it """Restores a linked file by linking it
When restoring, all the hardlink data is already present, and When restoring, all the hardlink data is already present, and
we can only link to something already written. Returns true we can only link to something already written. In either
if succeeded in creating rpath, false if must restore rpath case, add to the _restore_index_path dict, so we know later
normally. that the file is available for hard
linking.
Returns true if succeeded in creating rpath, false if must
restore rpath normally.
""" """
full_index = mirror_rel_index + rpath.index if index not in cls._src_index_indicies: return None
if not cls._src_index_indicies.has_key(full_index): return None for linked_index in cls._src_index_indicies[index]:
truncated_list = [] if linked_index in cls._restore_index_path:
for index in cls._src_index_indicies[full_index]: srcpath = cls._restore_index_path[linked_index]
if index[:len(mirror_rel_index)] == mirror_rel_index: Log("Restoring %s by hard linking to %s" %
truncated_list.append(index[len(mirror_rel_index):]) (rpath.path, srcpath), 6)
rpath.hardlink(srcpath)
if not truncated_list or truncated_list[0] >= rpath.index: return None return 1
srclink = RPath(rpath.conn, rpath.base, truncated_list[0]) cls._restore_index_path[index] = rpath.path
rpath.hardlink(srclink.path) return None
return 1
def link_rp(cls, src_rorp, dest_rpath, dest_root = None): def link_rp(cls, src_rorp, dest_rpath, dest_root = None):
"""Make dest_rpath into a link analogous to that of src_rorp""" """Make dest_rpath into a link analogous to that of src_rorp"""
......
trunk/rdiff-backup/src/highlevel.py
View file @ cea2c3bc
...@@ -24,12 +24,14 @@ class HighLevel: ...@@ -24,12 +24,14 @@ class HighLevel:
accompanying diagram. accompanying diagram.
""" """
def Mirror(src_rpath, dest_rpath, checkpoint = 1, session_info = None): def Mirror(src_rpath, dest_rpath, checkpoint = 1,
session_info = None, write_finaldata = 1):
"""Turn dest_rpath into a copy of src_rpath """Turn dest_rpath into a copy of src_rpath
Checkpoint true means to checkpoint periodically, otherwise Checkpoint true means to checkpoint periodically, otherwise
not. If session_info is given, try to resume Mirroring from not. If session_info is given, try to resume Mirroring from
that point. that point. If write_finaldata is true, save extra data files
like hardlink_data. If it is false, make a complete mirror.
""" """
SourceS = src_rpath.conn.HLSourceStruct SourceS = src_rpath.conn.HLSourceStruct
...@@ -40,7 +42,8 @@ class HighLevel: ...@@ -40,7 +42,8 @@ class HighLevel:
src_init_dsiter = SourceS.split_initial_dsiter() src_init_dsiter = SourceS.split_initial_dsiter()
dest_sigiter = DestS.get_sigs(dest_rpath, src_init_dsiter) dest_sigiter = DestS.get_sigs(dest_rpath, src_init_dsiter)
diffiter = SourceS.get_diffs_and_finalize(dest_sigiter) diffiter = SourceS.get_diffs_and_finalize(dest_sigiter)
DestS.patch_and_finalize(dest_rpath, diffiter, checkpoint) DestS.patch_and_finalize(dest_rpath, diffiter,
checkpoint, write_finaldata)
dest_rpath.setdata() dest_rpath.setdata()
...@@ -61,24 +64,6 @@ class HighLevel: ...@@ -61,24 +64,6 @@ class HighLevel:
dest_rpath.setdata() dest_rpath.setdata()
inc_rpath.setdata() inc_rpath.setdata()
def Restore(rest_time, mirror_base, rel_index, baseinc_tup, target_base):
"""Like Restore.RestoreRecursive but check arguments"""
if (Globals.preserve_hardlinks != 0 and
Hardlink.retrieve_final(rest_time)):
Log("Hard link information found, attempting to preserve "
"hard links.", 4)
SetConnections.UpdateGlobal('preserve_hardlinks', 1)
else: SetConnections.UpdateGlobal('preserve_hardlinks', None)
if not isinstance(target_base, DSRPath):
target_base = DSRPath(target_base.conn, target_base.base,
target_base.index, target_base.data)
if not isinstance(mirror_base, DSRPath):
mirror_base = DSRPath(mirror_base.conn, mirror_base.base,
mirror_base.index, mirror_base.data)
Restore.RestoreRecursive(rest_time, mirror_base, rel_index,
baseinc_tup, target_base)
MakeStatic(HighLevel) MakeStatic(HighLevel)
...@@ -164,7 +149,7 @@ class HLDestinationStruct: ...@@ -164,7 +149,7 @@ class HLDestinationStruct:
def compare(src_rorp, dest_dsrp): def compare(src_rorp, dest_dsrp):
"""Return dest_dsrp if they are different, None if the same""" """Return dest_dsrp if they are different, None if the same"""
if not dest_dsrp: if not dest_dsrp:
dest_dsrp = DSRPath(baserp.conn, baserp.base, src_rorp.index) dest_dsrp = cls.get_dsrp(baserp, src_rorp.index)
if dest_dsrp.lstat(): if dest_dsrp.lstat():
Log("Warning: Found unexpected destination file %s, " Log("Warning: Found unexpected destination file %s, "
"not processing it." % dest_dsrp.path, 2) "not processing it." % dest_dsrp.path, 2)
...@@ -203,8 +188,9 @@ class HLDestinationStruct: ...@@ -203,8 +188,9 @@ class HLDestinationStruct:
def get_dsrp(cls, dest_rpath, index): def get_dsrp(cls, dest_rpath, index):
"""Return initialized dsrp based on dest_rpath with given index""" """Return initialized dsrp based on dest_rpath with given index"""
return DSRPath(source = None, dest_rpath.conn, dsrp = DSRPath(None, dest_rpath.conn, dest_rpath.base, index)
dest_rpath.base, index) if Globals.quoting_enabled: dsrp.quote_path()
return dsrp
def get_finalizer(cls): def get_finalizer(cls):
"""Return finalizer, starting from session info if necessary""" """Return finalizer, starting from session info if necessary"""
...@@ -216,9 +202,13 @@ class HLDestinationStruct: ...@@ -216,9 +202,13 @@ class HLDestinationStruct:
"""Return ITR, starting from state if necessary""" """Return ITR, starting from state if necessary"""
if cls._session_info and cls._session_info.ITR: if cls._session_info and cls._session_info.ITR:
return cls._session_info.ITR return cls._session_info.ITR
else: return IncrementITR(inc_rpath) else:
iitr = IncrementITR(inc_rpath)
iitr.override_changed()
return iitr
def patch_and_finalize(cls, dest_rpath, diffs, checkpoint = 1): def patch_and_finalize(cls, dest_rpath, diffs,
checkpoint = 1, write_finaldata = 1):
"""Apply diffs and finalize""" """Apply diffs and finalize"""
collated = RORPIter.CollateIterators(diffs, cls.initial_dsiter2) collated = RORPIter.CollateIterators(diffs, cls.initial_dsiter2)
finalizer = cls.get_finalizer() finalizer = cls.get_finalizer()
...@@ -242,7 +232,7 @@ class HLDestinationStruct: ...@@ -242,7 +232,7 @@ class HLDestinationStruct:
if checkpoint: SaveState.checkpoint_mirror(finalizer, dsrp) if checkpoint: SaveState.checkpoint_mirror(finalizer, dsrp)
except: cls.handle_last_error(dsrp, finalizer) except: cls.handle_last_error(dsrp, finalizer)
finalizer.Finish() finalizer.Finish()
if Globals.preserve_hardlinks and Globals.rbdir: if Globals.preserve_hardlinks and write_finaldata:
Hardlink.final_writedata() Hardlink.final_writedata()
if checkpoint: SaveState.checkpoint_remove() if checkpoint: SaveState.checkpoint_remove()
...@@ -300,8 +290,7 @@ class HLDestinationStruct: ...@@ -300,8 +290,7 @@ class HLDestinationStruct:
Log.exception(1) Log.exception(1)
if ITR: SaveState.checkpoint_inc_backup(ITR, finalizer, dsrp, 1) if ITR: SaveState.checkpoint_inc_backup(ITR, finalizer, dsrp, 1)
else: SaveState.checkpoint_mirror(finalizer, dsrp, 1) else: SaveState.checkpoint_mirror(finalizer, dsrp, 1)
if Globals.preserve_hardlinks: if Globals.preserve_hardlinks: Hardlink.final_checkpoint(Globals.rbdir)
Hardlink.final_checkpoint(Globals.rbdir)
SaveState.touch_last_file_definitive() SaveState.touch_last_file_definitive()
raise raise
......
trunk/rdiff-backup/src/increment.py
View file @ cea2c3bc
execfile("selection.py") execfile("filename_mapping.py")
####################################################################### #######################################################################
# #
...@@ -85,10 +85,12 @@ class Inc: ...@@ -85,10 +85,12 @@ class Inc:
"""Get new increment rp with given time suffix""" """Get new increment rp with given time suffix"""
addtostr = lambda s: "%s.%s.%s" % (s, timestr, typestr) addtostr = lambda s: "%s.%s.%s" % (s, timestr, typestr)
if rp.index: if rp.index:
return rp.__class__(rp.conn, rp.base, rp.index[:-1] + incrp = rp.__class__(rp.conn, rp.base, rp.index[:-1] +
(addtostr(rp.index[-1]),)) (addtostr(rp.index[-1]),))
else: return rp.__class__(rp.conn, addtostr(rp.base), rp.index) else: incrp = rp.__class__(rp.conn, addtostr(rp.base), rp.index)
if Globals.quoting_enabled: incrp.quote_path()
return incrp
inctime = 0 inctime = 0
while 1: while 1:
inctime = Resume.FindTime(rp.index, inctime) inctime = Resume.FindTime(rp.index, inctime)
...@@ -123,7 +125,7 @@ class IncrementITR(IterTreeReducer): ...@@ -123,7 +125,7 @@ class IncrementITR(IterTreeReducer):
def __init__(self, inc_rpath): def __init__(self, inc_rpath):
"""Set inc_rpath, an rpath of the base of the tree""" """Set inc_rpath, an rpath of the base of the tree"""
self.inc_rpath = inc_rpath self.inc_rpath = inc_rpath
IterTreeReducer.__init__(inc_rpath) IterTreeReducer.__init__(self, inc_rpath)
def start_process(self, index, diff_rorp, dsrp): def start_process(self, index, diff_rorp, dsrp):
"""Initial processing of file """Initial processing of file
...@@ -133,11 +135,21 @@ class IncrementITR(IterTreeReducer): ...@@ -133,11 +135,21 @@ class IncrementITR(IterTreeReducer):
""" """
incpref = self.inc_rpath.new_index(index) incpref = self.inc_rpath.new_index(index)
if Globals.quoting_enabled: incpref.quote_path()
if dsrp.isdir(): if dsrp.isdir():
self.init_dir(dsrp, diff_rorp, incpref) self.init_dir(dsrp, diff_rorp, incpref)
self.setvals(diff_rorp, dsrp, incpref) self.setvals(diff_rorp, dsrp, incpref)
else: self.init_non_dir(dsrp, diff_rorp, incpref) else: self.init_non_dir(dsrp, diff_rorp, incpref)
def override_changed(self):
"""Set changed flag to true
This is used only at the top level of a backup, to make sure
that a marker is created recording every backup session.
"""
self.changed = 1
def setvals(self, diff_rorp, dsrp, incpref): def setvals(self, diff_rorp, dsrp, incpref):
"""Record given values in state dict since in directory """Record given values in state dict since in directory
...@@ -162,7 +174,7 @@ class IncrementITR(IterTreeReducer): ...@@ -162,7 +174,7 @@ class IncrementITR(IterTreeReducer):
""" """
if not (incpref.lstat() and incpref.isdir()): incpref.mkdir() if not (incpref.lstat() and incpref.isdir()): incpref.mkdir()
if diff_rorp and diff_rorp.isreg() and diff_rorp.file: if diff_rorp and diff_rorp.isreg() and diff_rorp.file:
tf = TempFileManager(dsrp) tf = TempFileManager.new(dsrp)
RPathStatic.copy_with_attribs(diff_rorp, tf) RPathStatic.copy_with_attribs(diff_rorp, tf)
tf.set_attached_filetype(diff_rorp.get_attached_filetype()) tf.set_attached_filetype(diff_rorp.get_attached_filetype())
self.directory_replacement = tf self.directory_replacement = tf
...@@ -170,7 +182,7 @@ class IncrementITR(IterTreeReducer): ...@@ -170,7 +182,7 @@ class IncrementITR(IterTreeReducer):
def init_non_dir(self, dsrp, diff_rorp, incpref): def init_non_dir(self, dsrp, diff_rorp, incpref):
"""Process a non directory file (initial pass)""" """Process a non directory file (initial pass)"""
if not diff_rorp: return # no diff, so no change necessary if not diff_rorp: return # no diff, so no change necessary
if diff_rorp.isreg and (dsrp.isreg() or diff_rorp.isflaglinked()): if diff_rorp.isreg() and (dsrp.isreg() or diff_rorp.isflaglinked()):
tf = TempFileManager.new(dsrp) tf = TempFileManager.new(dsrp)
def init_thunk(): def init_thunk():
if diff_rorp.isflaglinked(): if diff_rorp.isflaglinked():
...@@ -180,8 +192,8 @@ class IncrementITR(IterTreeReducer): ...@@ -180,8 +192,8 @@ class IncrementITR(IterTreeReducer):
Inc.Increment_action(tf, dsrp, incpref).execute() Inc.Increment_action(tf, dsrp, incpref).execute()
Robust.make_tf_robustaction(init_thunk, (tf,), (dsrp,)).execute() Robust.make_tf_robustaction(init_thunk, (tf,), (dsrp,)).execute()
else: else:
Robust.chain([Inc.Increment_action(diff_rorp, dsrp, incref), Robust.chain([Inc.Increment_action(diff_rorp, dsrp, incpref),
RORPIter.patchonce_action(none, dsrp, diff_rorp)] RORPIter.patchonce_action(None, dsrp, diff_rorp)]
).execute() ).execute()
self.changed = 1 self.changed = 1
...@@ -207,89 +219,3 @@ class IncrementITR(IterTreeReducer): ...@@ -207,89 +219,3 @@ class IncrementITR(IterTreeReducer):
def make_patch_increment_ITR(inc_rpath, initial_state = None):
"""Return IterTreeReducer that patches and increments"""
def base_init(indexed_tuple):
"""Patch if appropriate, return (a,b) tuple
a is true if found directory and thus didn't take action
if a is false, b is true if some changes were made
if a is true, b is the rp of a temporary file used to hold
the diff_rorp's data (for dir -> normal file change), and
false if none was necessary.
"""
diff_rorp, dsrp = indexed_tuple
incpref = inc_rpath.new_index(indexed_tuple.index)
if dsrp.isdir(): return init_dir(dsrp, diff_rorp, incpref)
else: return init_non_dir(dsrp, diff_rorp, incpref)
def init_dir(dsrp, diff_rorp, incpref):
"""Initial processing of a directory
Make the corresponding directory right away, but wait
until the end to write the replacement. However, if the
diff_rorp contains data, we must write it locally before
continuing, or else that data will be lost in the stream.
"""
if not (incpref.lstat() and incpref.isdir()): incpref.mkdir()
if diff_rorp and diff_rorp.isreg() and diff_rorp.file:
tf = TempFileManager.new(dsrp)
RPathStatic.copy_with_attribs(diff_rorp, tf)
tf.set_attached_filetype(diff_rorp.get_attached_filetype())
return (1, tf)
else: return (1, None)
def init_non_dir(dsrp, diff_rorp, incpref):
"""Initial processing of non-directory
If a reverse diff is called for it is generated by apply
the forwards diff first on a temporary file.
"""
if diff_rorp:
if diff_rorp.isreg() and (dsrp.isreg() or
diff_rorp.isflaglinked()):
tf = TempFileManager.new(dsrp)
def init_thunk():
if diff_rorp.isflaglinked():
Hardlink.link_rp(diff_rorp, tf, dsrp)
else: Rdiff.patch_with_attribs_action(dsrp, diff_rorp,
tf).execute()
Inc.Increment_action(tf, dsrp, incpref).execute()
Robust.make_tf_robustaction(init_thunk, (tf,),
(dsrp,)).execute()
else:
Robust.chain([Inc.Increment_action(diff_rorp, dsrp,
incpref),
RORPIter.patchonce_action(
None, dsrp, diff_rorp)]).execute()
return (None, 1)
return (None, None)
def base_final(base_tuple, base_init_tuple, changed):
"""Patch directory if not done, return true iff made change"""
if base_init_tuple[0]: # was directory
diff_rorp, dsrp = base_tuple
if changed or diff_rorp:
if base_init_tuple[1]: diff_rorp = base_init_tuple[1]
Inc.Increment(diff_rorp, dsrp,
inc_rpath.new_index(base_tuple.index))
if diff_rorp:
RORPIter.patchonce_action(None, dsrp,
diff_rorp).execute()
if isinstance(diff_rorp, TempFile): diff_rorp.delete()
return 1
return None
else: # changed iff base_init_tuple says it was
return base_init_tuple[1]
return IterTreeReducer(base_init, lambda x,y: x or y, None,
base_final, initial_state)
trunk/rdiff-backup/src/lazy.py
View file @ cea2c3bc
...@@ -260,7 +260,7 @@ class IterTreeReducer: ...@@ -260,7 +260,7 @@ class IterTreeReducer:
""" """
index = args[0] index = args[0]
assert type(index) is types.TupleType assert type(index) is types.TupleType, type(index)
if self.index is None: if self.index is None:
self.start_process(*args) self.start_process(*args)
......
trunk/rdiff-backup/src/main.py
View file @ cea2c3bc
This diff is collapsed.
trunk/rdiff-backup/src/manage.py
View file @ cea2c3bc
...@@ -12,37 +12,53 @@ class Manage: ...@@ -12,37 +12,53 @@ class Manage:
"""Return Increments objects given the rdiff-backup data directory""" """Return Increments objects given the rdiff-backup data directory"""
return map(IncObj, Manage.find_incrps_with_base(datadir, "increments")) return map(IncObj, Manage.find_incrps_with_base(datadir, "increments"))
def find_incrps_with_base(dir_rp, basename): def get_file_type(rp):
"""Return list of incfiles with given basename in dir_rp""" """Returns one of "regular", "directory", "missing", or "special"."""
rps = map(dir_rp.append, dir_rp.listdir()) if not rp.lstat(): return "missing"
incrps = filter(RPath.isincfile, rps) elif rp.isdir(): return "directory"
result = filter(lambda rp: rp.getincbase_str() == basename, incrps) elif rp.isreg(): return "regular"
Log("find_incrps_with_base: found %d incs" % len(result), 6) else: return "special"
return result
def describe_root_incs(datadir): def get_inc_type(inc):
"""Return file type increment represents"""
assert inc.isincfile()
type = inc.getinctype()
if type == "dir": return "directory"
elif type == "diff": return "regular"
elif type == "missing": return "missing"
elif type == "snapshot": return Manage.get_file_type(inc)
else: assert None, "Unknown type %s" % (type,)
def describe_incs_parsable(incs, mirror_time, mirrorrp):
"""Return a string parsable by computer describing the increments
Each line is a time in seconds of the increment, and then the
type of the file. It will be sorted oldest to newest. For example:
10000 regular
20000 directory
30000 special
40000 missing
50000 regular <- last will be the current mirror
"""
incpairs = [(Time.stringtotime(inc.getinctime()), inc) for inc in incs]
incpairs.sort()
result = ["%s %s" % (time, Manage.get_inc_type(inc))
for time, inc in incpairs]
result.append("%s %s" % (mirror_time, Manage.get_file_type(mirrorrp)))
return "\n".join(result)
def describe_incs_human(incs, mirror_time, mirrorrp):
"""Return a string describing all the the root increments""" """Return a string describing all the the root increments"""
result = [] incpairs = [(Time.stringtotime(inc.getinctime()), inc) for inc in incs]
currentrps = Manage.find_incrps_with_base(datadir, "current_mirror") incpairs.sort()
if not currentrps:
Log("Warning: no current mirror marker found", 1) result = ["Found %d increments:" % len(incpairs)]
elif len(currentrps) > 1: for time, inc in incpairs:
Log("Warning: multiple mirror markers found", 1) result.append(" %s %s" %
for rp in currentrps: (inc.dirsplit()[1], Time.timetopretty(time)))
result.append("Found mirror marker %s" % rp.path) result.append("Current mirror: %s" % Time.timetopretty(mirror_time))
result.append("Indicating latest mirror taken at %s" %
Time.stringtopretty(rp.getinctime()))
result.append("---------------------------------------------"
"-------------")
# Sort so they are in reverse order by time
time_w_incobjs = map(lambda io: (-io.time, io),
Manage.get_incobjs(datadir))
time_w_incobjs.sort()
incobjs = map(lambda x: x[1], time_w_incobjs)
result.append("Found %d increments:" % len(incobjs))
result.append("\n------------------------------------------\n".join(
map(IncObj.full_description, incobjs)))
return "\n".join(result) return "\n".join(result)
def delete_earlier_than(baserp, time): def delete_earlier_than(baserp, time):
...@@ -53,6 +69,11 @@ class Manage: ...@@ -53,6 +69,11 @@ class Manage:
rdiff-backup-data directory should be the root of the tree. rdiff-backup-data directory should be the root of the tree.
""" """
baserp.conn.Manage.delete_earlier_than_local(baserp, time)
def delete_earlier_than_local(baserp, time):
"""Like delete_earlier_than, but run on local connection for speed"""
assert baserp.conn is Globals.local_connection
def yield_files(rp): def yield_files(rp):
yield rp yield rp
if rp.isdir(): if rp.isdir():
......
trunk/rdiff-backup/src/restore.py
View file @ cea2c3bc
This diff is collapsed.
trunk/rdiff-backup/src/robust.py
View file @ cea2c3bc
...@@ -252,7 +252,7 @@ class TempFile(RPath): ...@@ -252,7 +252,7 @@ class TempFile(RPath):
if self.isdir() and not rp_dest.isdir(): if self.isdir() and not rp_dest.isdir():
# Cannot move a directory directly over another file # Cannot move a directory directly over another file
rp_dest.delete() rp_dest.delete()
if (isinstance(rp_dest, DSRPath) and rp_dest.perms_delayed if (isinstance(rp_dest, DSRPath) and rp_dest.delay_perms
and not self.hasfullperms()): and not self.hasfullperms()):
# If we are moving to a delayed perm directory, delay # If we are moving to a delayed perm directory, delay
# permission change on destination. # permission change on destination.
...@@ -531,7 +531,7 @@ class Resume: ...@@ -531,7 +531,7 @@ class Resume:
Log("Last backup dated %s was aborted, but we aren't " Log("Last backup dated %s was aborted, but we aren't "
"resuming it." % Time.timetopretty(si.time), 2) "resuming it." % Time.timetopretty(si.time), 2)
return None return None
assert 0 assert None
MakeClass(Resume) MakeClass(Resume)
......
trunk/rdiff-backup/src/rpath.py
View file @ cea2c3bc
...@@ -168,6 +168,17 @@ class RPathStatic: ...@@ -168,6 +168,17 @@ class RPathStatic:
rp_dest.data = rp_source.data rp_dest.data = rp_source.data
rp_source.data = {'type': None} rp_source.data = {'type': None}
# If we are moving to a DSRPath, assume that the current times
# are the intended ones. We need to save them now in case
# they are changed later.
if isinstance(rp_dest, DSRPath):
if rp_dest.delay_mtime:
if 'mtime' in rp_dest.data:
rp_dest.setmtime(rp_dest.data['mtime'])
if rp_dest.delay_atime:
if 'atime' in rp_dest.data:
rp_dest.setatime(rp_dest.data['atime'])
def tupled_lstat(filename): def tupled_lstat(filename):
"""Like os.lstat, but return only a tuple, or None if os.error """Like os.lstat, but return only a tuple, or None if os.error
...@@ -413,7 +424,7 @@ class RPath(RORPath): ...@@ -413,7 +424,7 @@ class RPath(RORPath):
self.base = base self.base = base
self.path = apply(os.path.join, (base,) + self.index) self.path = apply(os.path.join, (base,) + self.index)
self.file = None self.file = None
if data: self.data = data if data or base is None: self.data = data
else: self.setdata() else: self.setdata()
def __str__(self): def __str__(self):
...@@ -493,6 +504,12 @@ class RPath(RORPath): ...@@ -493,6 +504,12 @@ class RPath(RORPath):
s = self.conn.reval("lambda path: os.lstat(path).st_rdev", self.path) s = self.conn.reval("lambda path: os.lstat(path).st_rdev", self.path)
return (s >> 8, s & 0xff) return (s >> 8, s & 0xff)
def quote_path(self):
"""Set path from quoted version of index"""
quoted_list = [FilenameMapping.quote(path) for path in self.index]
self.path = apply(os.path.join, [self.base] + quoted_list)
self.setdata()
def chmod(self, permissions): def chmod(self, permissions):
"""Wrapper around os.chmod""" """Wrapper around os.chmod"""
self.conn.os.chmod(self.path, permissions) self.conn.os.chmod(self.path, permissions)
...@@ -594,7 +611,8 @@ class RPath(RORPath): ...@@ -594,7 +611,8 @@ class RPath(RORPath):
if not self.lstat(): return # must have been deleted in meantime if not self.lstat(): return # must have been deleted in meantime
elif self.isdir(): elif self.isdir():
itm = RpathDeleter() itm = RpathDeleter()
for dsrp in Select(self, None).set_iter(): itm(dsrp.index, dsrp) for dsrp in Select(DSRPath(None, self)).set_iter():
itm(dsrp.index, dsrp)
itm.Finish() itm.Finish()
else: self.conn.os.unlink(self.path) else: self.conn.os.unlink(self.path)
self.setdata() self.setdata()
...@@ -616,7 +634,7 @@ class RPath(RORPath): ...@@ -616,7 +634,7 @@ class RPath(RORPath):
self.path.split("/"))) self.path.split("/")))
if self.path[0] == "/": newpath = "/" + newpath if self.path[0] == "/": newpath = "/" + newpath
elif not newpath: newpath = "." elif not newpath: newpath = "."
return self.__class__(self.conn, newpath, ()) return self.newpath(newpath)
def dirsplit(self): def dirsplit(self):
"""Returns a tuple of strings (dirname, basename) """Returns a tuple of strings (dirname, basename)
...@@ -635,10 +653,20 @@ class RPath(RORPath): ...@@ -635,10 +653,20 @@ class RPath(RORPath):
comps = normed.path.split("/") comps = normed.path.split("/")
return "/".join(comps[:-1]), comps[-1] return "/".join(comps[:-1]), comps[-1]
def newpath(self, newpath, index = ()):
"""Return new RPath with the same connection but different path"""
return self.__class__(self.conn, newpath, index)
def append(self, ext): def append(self, ext):
"""Return new RPath with same connection by adjoing ext""" """Return new RPath with same connection by adjoing ext"""
return self.__class__(self.conn, self.base, self.index + (ext,)) return self.__class__(self.conn, self.base, self.index + (ext,))
def append_path(self, ext, new_index = ()):
"""Like append, but add ext to path instead of to index"""
assert not self.index # doesn't make sense if index isn't ()
return self.__class__(self.conn, os.path.join(self.base, ext),
new_index)
def new_index(self, index): def new_index(self, index):
"""Return similar RPath but with new index""" """Return similar RPath but with new index"""
return self.__class__(self.conn, self.base, index) return self.__class__(self.conn, self.base, index)
......
trunk/rdiff-backup/src/selection.py
View file @ cea2c3bc
...@@ -60,21 +60,20 @@ class Select: ...@@ -60,21 +60,20 @@ class Select:
# This re should not match normal filenames, but usually just globs # This re should not match normal filenames, but usually just globs
glob_re = re.compile("(.*[*?[]|ignorecase\\:)", re.I | re.S) glob_re = re.compile("(.*[*?[]|ignorecase\\:)", re.I | re.S)
def __init__(self, rpath, source): def __init__(self, dsrpath, quoted_filenames = None):
"""DSRPIterator initializer. """DSRPIterator initializer. dsrp is the root directory
rpath is the root dir. Source is true if rpath is the root of When files have quoted characters in them, quoted_filenames
the source directory, and false for the mirror directory should be true. Then RPath's index will be the unquoted
version.
""" """
assert isinstance(rpath, RPath) assert isinstance(dsrpath, DSRPath)
self.selection_functions = [] self.selection_functions = []
self.source = source self.dsrpath = dsrpath
if isinstance(rpath, DSRPath): self.dsrpath = rpath
else: self.dsrpath = DSRPath(rpath.conn, rpath.base,
rpath.index, rpath.data)
self.prefix = self.dsrpath.path self.prefix = self.dsrpath.path
self.quoting_on = Globals.quoting_enabled and quoted_filenames
def set_iter(self, starting_index = None, sel_func = None): def set_iter(self, starting_index = None, sel_func = None):
"""Initialize more variables, get ready to iterate """Initialize more variables, get ready to iterate
...@@ -106,7 +105,7 @@ class Select: ...@@ -106,7 +105,7 @@ class Select:
""" """
s = sel_func(dsrpath) s = sel_func(dsrpath)
if s === 0: return if s == 0: return
elif s == 1: # File is included elif s == 1: # File is included
yield dsrpath yield dsrpath
if dsrpath.isdir(): if dsrpath.isdir():
...@@ -122,11 +121,15 @@ class Select: ...@@ -122,11 +121,15 @@ class Select:
def iterate_in_dir(self, dsrpath, rec_func, sel_func): def iterate_in_dir(self, dsrpath, rec_func, sel_func):
"""Iterate the dsrps in directory dsrpath.""" """Iterate the dsrps in directory dsrpath."""
dir_listing = dsrpath.listdir() if self.quoting_on:
dir_listing.sort() for subdir in FilenameMapping.get_quoted_dir_children(dsrpath):
for filename in dir_listing: for dsrp in rec_func(subdir, rec_func, sel_func): yield dsrp
for dsrp in rec_func(dsrpath.append(filename), rec_func, sel_func): else:
yield dsrp dir_listing = dsrpath.listdir()
dir_listing.sort()
for filename in dir_listing:
for dsrp in rec_func(dsrpath.append(filename),
rec_func, sel_func): yield dsrp
def iterate_starting_from(self, dsrpath, rec_func, sel_func): def iterate_starting_from(self, dsrpath, rec_func, sel_func):
"""Like Iterate, but only yield indicies > self.starting_index""" """Like Iterate, but only yield indicies > self.starting_index"""
......
trunk/rdiff-backup/src/ttime.py
View file @ cea2c3bc
...@@ -12,6 +12,7 @@ class Time: ...@@ -12,6 +12,7 @@ class Time:
"""Functions which act on the time""" """Functions which act on the time"""
_interval_conv_dict = {"s": 1, "m": 60, "h": 3600, "D": 86400, _interval_conv_dict = {"s": 1, "m": 60, "h": 3600, "D": 86400,
"W": 7*86400, "M": 30*86400, "Y": 365*86400} "W": 7*86400, "M": 30*86400, "Y": 365*86400}
_integer_regexp = re.compile("^[0-9]+$")
_interval_regexp = re.compile("^([0-9]+)([smhDWMY])") _interval_regexp = re.compile("^([0-9]+)([smhDWMY])")
_genstr_date_regexp1 = re.compile("^(?P<year>[0-9]{4})[-/]" _genstr_date_regexp1 = re.compile("^(?P<year>[0-9]{4})[-/]"
"(?P<month>[0-9]{1,2})[-/](?P<day>[0-9]{1,2})$") "(?P<month>[0-9]{1,2})[-/](?P<day>[0-9]{1,2})$")
...@@ -70,7 +71,7 @@ class Time: ...@@ -70,7 +71,7 @@ class Time:
utc_in_secs = time.mktime(timetuple) - time.altzone utc_in_secs = time.mktime(timetuple) - time.altzone
else: utc_in_secs = time.mktime(timetuple) - time.timezone else: utc_in_secs = time.mktime(timetuple) - time.timezone
return utc_in_secs + cls.tzdtoseconds(timestring[19:]) return long(utc_in_secs) + cls.tzdtoseconds(timestring[19:])
except (TypeError, ValueError, AssertionError): return None except (TypeError, ValueError, AssertionError): return None
def timetopretty(cls, timeinseconds): def timetopretty(cls, timeinseconds):
...@@ -155,8 +156,10 @@ strings, like "2002-04-26T04:22:01-07:00" (strings like ...@@ -155,8 +156,10 @@ strings, like "2002-04-26T04:22:01-07:00" (strings like
"2002-04-26T04:22:01" are also acceptable - rdiff-backup will use the "2002-04-26T04:22:01" are also acceptable - rdiff-backup will use the
current time zone), or ordinary dates like 2/4/1997 or 2001-04-23 current time zone), or ordinary dates like 2/4/1997 or 2001-04-23
(various combinations are acceptable, but the month always precedes (various combinations are acceptable, but the month always precedes
the day). the day).""" % timestr)
""" % timestr)
# Test for straight integer
if cls._integer_regexp.search(timestr): return int(timestr)
# Test for w3-datetime format, possibly missing tzd # Test for w3-datetime format, possibly missing tzd
t = cls.stringtotime(timestr) or cls.stringtotime(timestr+cls.gettzd()) t = cls.stringtotime(timestr) or cls.stringtotime(timestr+cls.gettzd())
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment
GitLab Nexedi Edition | About GitLab | About Nexedi | 沪ICP备2021021310号-2 | 沪ICP备2021021310号-7